From: W. Trevor King Date: Tue, 21 Feb 2012 15:46:57 +0000 (-0500) Subject: Update gallery.py to optionally run as an SCGI server (v0.5). X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=3fa788d411750e2348b4b05186ab4155f6d4baf8;p=blog.git Update gallery.py to optionally run as an SCGI server (v0.5). --- diff --git a/posts/gallery/gallery.py b/posts/gallery/gallery.py index b84a0cd..1f9ad62 100755 --- a/posts/gallery/gallery.py +++ b/posts/gallery/gallery.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2010-2011 W. Trevor King +# Copyright (C) 2010-2012 W. Trevor King # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,42 +38,23 @@ Note that you can store a caption for ```` as plain text in ``.txt``. See RFC 3875 for more details on the the Common Gateway Interface. -""" - -# import logging first so we can timestamp other module imports -import logging - - -LOG = logging.getLogger('gallery') -_ch = logging.StreamHandler() -_ch.setLevel(logging.DEBUG) -_formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') -_ch.setFormatter(_formatter) -LOG.addHandler(_ch) -#LOG.setLevel(logging.DEBUG) -LOG.setLevel(logging.WARNING) -LOG.debug('importing math') -import math -LOG.debug('importing os') -import os -LOG.debug('importing os.path') -import os.path -LOG.debug('importing random') -import random -LOG.debug('importing re') -import re -LOG.debug('importing subprocess') -from subprocess import Popen, PIPE -LOG.debug('importing sys') -import sys +This script can also be run as a Simple Common Gateway Interface +(SCGI) with the ``--scgi`` option. +""" +import logging as _logging +import logging.handlers as _logging_handlers +import math as _math +import os as _os +import os.path as _os_path +import random as _random +import re as _re +import subprocess as _subprocess -LOG.debug('parsing class') +__version__ = '0.5' -__version__ = '0.4' IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.tif', '.tiff', '.png', '.gif'] VIDEO_EXTENSIONS = ['.mov', '.mp4', '.ogv'] @@ -82,6 +63,14 @@ RESPONSES = { # httplib takes half a second to load 404: 'Not Found', } +LOG = _logging.getLogger('gallery.py') +LOG.addHandler(_logging.StreamHandler()) +#LOG.addHandler(_logging_handlers.SysLogHandler()) +LOG.handlers[0].setFormatter( + _logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +LOG.setLevel(_logging.DEBUG) +#LOG.setLevel(_logging.WARNING) + class CommandError(Exception): def __init__(self, command, status, stdout=None, stderr=None): @@ -93,8 +82,13 @@ class CommandError(Exception): self.stdout = stdout self.stderr = stderr -def invoke(args, stdin=None, stdout=PIPE, stderr=PIPE, expect=(0,), - cwd=None, encoding=None): + +class ProcessingComplete(Exception): + pass + + +def invoke(args, stdin=None, stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, + expect=(0,), cwd=None, encoding=None): """ expect should be a tuple of allowed exit codes. cwd should be the directory from which the command will be executed. When @@ -105,7 +99,8 @@ def invoke(args, stdin=None, stdout=PIPE, stderr=PIPE, expect=(0,), cwd = '.' LOG.debug('{}$ {}'.format(cwd, ' '.join(args))) try : - q = Popen(args, stdin=PIPE, stdout=stdout, stderr=stderr, cwd=cwd) + q = _subprocess.Popen(args, stdin=_subprocess.PIPE, stdout=stdout, + stderr=stderr, cwd=cwd) except OSError, e: raise CommandError(args, status=e.args[0], stderr=e) stdout,stderr = q.communicate(input=stdin) @@ -134,7 +129,7 @@ class CGIGalleryServer (object): self._base_path = base_path self._base_url = base_url self._cache_path = cache_path - self._url_regexp = re.compile('^[a-z0-9._/-]*$') + self._url_regexp = _re.compile('^[a-z0-9._/-]*$') self._rows = 3 self._columns = 3 self.header = [] @@ -150,35 +145,38 @@ class CGIGalleryServer (object): header.append('Content-type: {}{}'.format(mime, charset)) return '\n'.join(header) - def _response(self, header=None, content='

It works!

'): + def _response(self, header=None, content='

It works!

', + stream=None): if header is None: header = self._http_header() - sys.stdout.write(header) - sys.stdout.write('\n\n') - sys.stdout.write(content) - sys.exit(0) + stream.write(header) + stream.write('\n\n') + stream.write(content) + raise ProcessingComplete() - def _response_stream(self, header=None, content=None, chunk_size=1024): + def _response_stream(self, header=None, content=None, stream=None, + chunk_size=1024): LOG.debug('streaming response') if header is None: header = self._http_header() - sys.stdout.write(header) - sys.stdout.write('\n\n') - sys.stdout.flush() # flush headers + stream.write(header) + stream.write('\n\n') + stream.flush() # flush headers while True: chunk = content.read(chunk_size) if not chunk: break - sys.stdout.write(chunk) - sys.exit(0) + stream.write(chunk) + raise ProcessingComplete() - def _error(self, status=404, content=None): + def _error(self, status=404, content=None, stream=None): header = self._http_header(status=status) if content is None: content = RESPONSES[status] - self._response(header=header, content=content) + self._response(header=header, content=content, stream=stream) - def validate_url(self, url): + def validate_url(self, url, stream=None): + LOG.debug('validating {}'.format(repr(url))) if url is None: return elif (not self._url_regexp.match(url) or @@ -186,31 +184,46 @@ class CGIGalleryServer (object): '..' in url ): LOG.error('invalid url') - self._error(404) - path = os.path.join(self._base_path, url) - if os.path.exists(path) and not os.path.isdir(path): + self._error(404, stream=stream) + path = _os_path.join(self._base_path, url) + if _os_path.exists(path) and not _os_path.isdir(path): LOG.error('nonexstand directory') - self._error(404) + self._error(404, stream=stream) - def serve(self, url, page=0): + def serve(self, url=None, page=0, stream=None): LOG.info('serving url {} (page {})'.format(url, page)) + try: + if url is None: + self.index(stream=stream) + elif url.endswith('random'): + self.random( + url=url, stream=stream, max_width=500, max_height=500) + elif self.is_cached(url=url): + self.cached(url=url, stream=stream) + elif url.endswith('.png'): + self.thumb(url=url, stream=stream) + else: + self.validate_url(url=url, stream=stream) + self.page(url=url, page=page, stream=stream) + LOG.error('unexpected url type') + self._error(404, stream=stream) + except ProcessingComplete: + pass + + def relative_url(self, url, stream=None): if url is None: - self.index() - elif url.endswith('random'): - self.random(url=url, max_width=500, max_height=500) - elif self.is_cached(url=url): - self.cached(url=url) - elif url.endswith('.png'): - self.thumb(url=url) - else: - self.page(url=url, page=page) - self.validate_url(url=url) - LOG.error('unexpected url type') - self._error(404) + return url + if not url.startswith(self._base_url): + LOG.error('cannot convert {} to a relative URL of {}'.format( + url, self._base_url)) + return self._error(404) + if url == self._base_url: + return None + return url[len(self._base_url):] def _url(self, path): - relpath = os.path.relpath( - os.path.join(self._base_path, path), self._base_path) + relpath = _os_path.relpath( + _os_path.join(self._base_path, path), self._base_path) if relpath == '.': relpath = '' elif path.endswith('/'): @@ -218,9 +231,9 @@ class CGIGalleryServer (object): return '{}{}'.format(self._base_url, relpath) def _label(self, path): - dirname,base = os.path.split(path) + dirname,base = _os_path.split(path) if not base: # directory path ending with '/' - dirname,base = os.path.split(dirname) + dirname,base = _os_path.split(dirname) return base.replace('_', ' ').title() def _link(self, path, text=None): @@ -231,50 +244,50 @@ class CGIGalleryServer (object): def _subdirs(self, path): try: order = [d.strip() for d in - open(os.path.join(path, '_order')).readlines()] + open(_os_path.join(path, '_order')).readlines()] except IOError: order = [] - dirs = sorted(os.listdir(path)) + dirs = sorted(_os.listdir(path)) start = [] for d in order: if d in dirs: start.append(d) dirs.remove(d) for d in start + dirs: - dirpath = os.path.join(path, d) - if os.path.isdir(dirpath): + dirpath = _os_path.join(path, d) + if _os_path.isdir(dirpath): yield dirpath def _images(self, path): - for p in sorted(os.listdir(path)): + for p in sorted(_os.listdir(path)): if p.startswith('.') or p.endswith('~'): continue - picture_path = os.path.join(path, p) + picture_path = _os_path.join(path, p) if is_picture(picture_path): yield picture_path - def index(self): + def index(self, stream=None): LOG.debug('index page') - return self._directory(self._base_path) + return self._directory(self._base_path, stream=stream) def _thumb(self, image, max_width=None, max_height=None): - if not os.path.exists(self._cache_path): - os.makedirs(self._cache_path) - dirname,filename = os.path.split(image) - reldir = os.path.relpath(dirname, self._base_path) - cache_dir = os.path.join(self._cache_path, reldir) - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) + if not _os_path.exists(self._cache_path): + _os.makedirs(self._cache_path) + dirname,filename = _os_path.split(image) + reldir = _os_path.relpath(dirname, self._base_path) + cache_dir = _os_path.join(self._cache_path, reldir) + if not _os_path.isdir(cache_dir): + _os.makedirs(cache_dir) extension = '-{:d}-{:d}.png'.format(max_width, max_height) thumb_filename = image_base(filename)+extension - thumb_url = os.path.join(dirname, thumb_filename) - thumb_path = os.path.join(cache_dir, thumb_filename) - image_path = os.path.join(self._base_path, image) - if not os.path.isfile(image_path): + thumb_url = _os_path.join(dirname, thumb_filename) + thumb_path = _os_path.join(cache_dir, thumb_filename) + image_path = _os_path.join(self._base_path, image) + if not _os_path.isfile(image_path): LOG.error('image path for thumbnail does not exist') return self._error(404) - if (not os.path.isfile(thumb_path) - or os.path.getmtime(image_path) > os.path.getmtime(thumb_path)): + if (not _os_path.isfile(thumb_path) + or _os_path.getmtime(image_path) > _os_path.getmtime(thumb_path)): invoke(['convert', '-format', 'png', '-strip', '-quality', '95', image_path, '-thumbnail', '{:d}x{:d}'.format(max_width, max_height), @@ -284,19 +297,19 @@ class CGIGalleryServer (object): def _mp4(self, video, *args): if not video.endswith('.mov'): LOG.error("can't translate {} to MPEGv4".format(video)) - dirname,filename = os.path.split(video) + dirname,filename = _os_path.split(video) mp4_filename = image_base(filename) + '.mp4' - reldir = os.path.relpath(dirname, self._base_path) - cache_dir = os.path.join(self._cache_path, reldir) - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - mp4_url = os.path.join(dirname, mp4_filename) - mp4_path = os.path.join(cache_dir, mp4_filename) - if not os.path.isfile(video): + reldir = _os_path.relpath(dirname, self._base_path) + cache_dir = _os_path.join(self._cache_path, reldir) + if not _os_path.isdir(cache_dir): + _os.makedirs(cache_dir) + mp4_url = _os_path.join(dirname, mp4_filename) + mp4_path = _os_path.join(cache_dir, mp4_filename) + if not _os_path.isfile(video): LOG.error('source video path does not exist') return self._error(404) - if (not os.path.isfile(mp4_path) - or os.path.getmtime(video) > os.path.getmtime(mp4_path)): + if (not _os_path.isfile(mp4_path) + or _os_path.getmtime(video) > _os_path.getmtime(mp4_path)): arg = ['ffmpeg', '-i', video, '-acodec', 'libfaac', '-aq', '200', '-ac', '1', '-s', '640x480', '-vcodec', 'libx264', '-preset', 'slower', '-vpre', 'ipod640', '-b', '800k', @@ -309,19 +322,19 @@ class CGIGalleryServer (object): def _ogv(self, video, *args): if not video.endswith('.mov'): LOG.error("can't translate {} to Ogg Video".format(video)) - dirname,filename = os.path.split(video) + dirname,filename = _os_path.split(video) ogv_filename = image_base(filename) + '.ogv' - reldir = os.path.relpath(dirname, self._base_path) - cache_dir = os.path.join(self._cache_path, reldir) - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - ogv_url = os.path.join(dirname, ogv_filename) - ogv_path = os.path.join(cache_dir, ogv_filename) - if not os.path.isfile(video): + reldir = _os_path.relpath(dirname, self._base_path) + cache_dir = _os_path.join(self._cache_path, reldir) + if not _os_path.isdir(cache_dir): + _os.makedirs(cache_dir) + ogv_url = _os_path.join(dirname, ogv_filename) + ogv_path = _os_path.join(cache_dir, ogv_filename) + if not _os_path.isfile(video): LOG.error('source video path does not exist') return self._error(404) - if (not os.path.isfile(ogv_path) - or os.path.getmtime(video) > os.path.getmtime(ogv_path)): + if (not _os_path.isfile(ogv_path) + or _os_path.getmtime(video) > _os_path.getmtime(ogv_path)): arg = ['ffmpeg2theora', '--optimize'] arg.extend(args) arg.extend(['--output', ogv_path, video]) @@ -339,7 +352,7 @@ class CGIGalleryServer (object): base_path = image_base(path) for extension in VIDEO_EXTENSIONS: video_path = base_path + extension - if os.path.isfile(video_path): + if _os_path.isfile(video_path): return self._video(video_path, fallback=fallback) return None @@ -398,27 +411,28 @@ class CGIGalleryServer (object): def _image_page(self, image): return image_base(image) + '/' - def random(self, url=None, **kwargs): + def random(self, url=None, stream=None, **kwargs): LOG.debug('random image') if url.endswith('/random'): - base_dir = os.path.join( + base_dir = _os_path.join( self._base_path, url[:(-len('/random'))]) elif url == 'random': base_dir = self._base_path else: - self._error(404) + self._error(404, stream=stream) images = [] - for dirpath,dirnames,filenames in os.walk(base_dir): + for dirpath,dirnames,filenames in _os.walk(base_dir): for filename in filenames: if is_picture(filename): - images.append(os.path.join(dirpath, filename)) + images.append(_os_path.join(dirpath, filename)) if not images: - self._response(content='

no images to choose from

') - image = random.choice(images) + self._response(content='

no images to choose from

', + stream=stream) + image = _random.choice(images) LOG.debug('selected random image {}'.format(image)) page = self._image_page(image) content = self._captioned_video(path=image, href=page) - self._response(content='\n'.join(content)) + self._response(content='\n'.join(content), stream=stream) def is_cached(self, url): for extension in ['.png', '.mp4', '.ogv']: @@ -426,7 +440,7 @@ class CGIGalleryServer (object): return True return False - def cached(self, url): + def cached(self, url, stream=None): LOG.debug('retrieving cached item') if url.endswith('.png'): mime = 'image/png' @@ -437,39 +451,40 @@ class CGIGalleryServer (object): else: raise NotImplementedError() header = self._http_header(mime=mime) - cache_path = os.path.join(self._cache_path, url) + cache_path = _os_path.join(self._cache_path, url) try: - stream = open(cache_path, 'rb') + content = open(cache_path, 'rb') except IOError, e: LOG.error('invalid url') LOG.error(e) self._error(404) if mime in ['video/ogg']: - self._response_stream(header=header, content=stream) - content = stream.read() - self._response(header=header, content=content) + self._response_stream( + header=header, content=content, stream=stream) + content = content.read() + self._response(header=header, content=content, stream=stream) - def page(self, url, page=0): + def page(self, url, page=0, stream=None): LOG.debug('HTML page') if not url.endswith('/'): LOG.error('HTML page URLs must end with a slash') self._error(404) - abspath = os.path.join(self._base_path, url) - if os.path.isdir(abspath): - self._directory(path=abspath, page=page) + abspath = _os_path.join(self._base_path, url) + if _os_path.isdir(abspath): + self._directory(path=abspath, page=page, stream=stream) for extension in IMAGE_EXTENSIONS: file_path = abspath[:-1] + extension - if os.path.isfile(file_path): - self._page(path=file_path) + if _os_path.isfile(file_path): + self._page(path=file_path, stream=stream) LOG.debug('unknown HTML page') - self._error(404) + self._error(404, stream=stream) def _directory_header(self, path): - relpath = os.path.relpath(path, self._base_path) + relpath = _os_path.relpath(path, self._base_path) crumbs = [] dirname = relpath while dirname: - dirname,base = os.path.split(dirname) + dirname,base = _os_path.split(dirname) if base != '.': crumbs.insert(0, base) crumbs.insert(0, '') @@ -480,7 +495,7 @@ class CGIGalleryServer (object): links[i] = self._link(self._base_path, 'Gallery') else: relpath = '/'.join(crumbs[1:i+1]) + '/' - fullpath = os.path.join(self._base_path, relpath) + fullpath = _os_path.join(self._base_path, relpath) links[i] = self._link(path=fullpath) else: if i == 0: @@ -539,11 +554,11 @@ class CGIGalleryServer (object): content.append('') return content - def _directory(self, path, page=0): + def _directory(self, path, page=0, stream=None): LOG.debug('directory page') images = list(self._images(path)) images_per_page = self._rows * self._columns - pages = int(math.ceil(float(len(images)) / images_per_page)) or 1 + pages = int(_math.ceil(float(len(images)) / images_per_page)) or 1 if page < 0 or page >= pages: LOG.error( 'page out of bounds for this gallery 0 <= {:d} < {:d}'.format( @@ -560,11 +575,11 @@ class CGIGalleryServer (object): content.extend(self._directory_images(path, images=images)) content.extend(nav) content.extend(self.footer) - self._response(content='\n'.join(content)) + self._response(content='\n'.join(content), stream=stream) - def _page(self, path): + def _page(self, path, stream=None): LOG.debug('image page') - gallery = os.path.dirname(path) + gallery = _os_path.dirname(path) images = list(self._images(gallery)) images_per_page = self._rows * self._columns i = images.index(path) @@ -585,49 +600,95 @@ class CGIGalleryServer (object): content.extend(self._captioned_video(path)) content.append('') content.extend(self.footer) - self._response(content='\n'.join(content)) + self._response(content='\n'.join(content), stream=stream) -if __name__ == '__main__': - LOG.debug('entering __main__ loop') - - LOG.debug('importing cgi') +def serve_cgi(server): import cgi - LOG.debug('importing cgitb') import cgitb - - LOG.debug('parsing arguments') - - url = None - page = 0 - if '--url' in sys.argv: - i = sys.argv.index('--url') + import sys + + url=None + page=0 + cgitb.enable() + #cgitb.enable(display=0, logdir="/tmp/") + data = cgi.FieldStorage() + if 'p' in data: + p = data['p'] + if isinstance(p, list): + p = p[0] + url = p.value + if 'pp' in data: try: - url = sys.argv[i+1] - except IndexError: + page = int(data['pp'].value) - 1 + except ValueError: pass - if '--page' in sys.argv: - i = sys.argv.index('--page') - page = int(sys.argv[i+1]) - else: - cgitb.enable() - #cgitb.enable(display=0, logdir="/tmp/") - data = cgi.FieldStorage() - if 'p' in data: - p = data['p'] - if isinstance(p, list): - p = p[0] - url = p.value - if 'pp' in data: + server.serve(url=url, page=page, stream=sys.stdout) + +def serve_scgi(server, port=4000): + import scgi + import scgi.scgi_server + import urlparse + + class GalleryHandler(scgi.scgi_server.SCGIHandler): + def produce(self, env, bodysize, input, output): + #LOG.info(HTTP_USER_AGENT REQUEST_METHOD REMOTE_ADDR REQUEST_URI + url = env.get('DOCUMENT_URI', None) + page = 0 + data = urlparse.parse_qs(env.get('QUERY_STRING', '')) + if 'pp' in data: + pp = data['pp'] + if isinstance(pp, list): + pp = pp[0] + try: + page = int(pp) - 1 + except ValueError: + pass try: - page = int(data['pp'].value) - 1 - except ValueError: + url = server.relative_url(url=url, stream=output) + except ProcessingComplete: pass + else: + server.serve(url=url, page=page, stream=output) + + s = scgi.scgi_server.SCGIServer( + handler_class=GalleryHandler, host='localhost', port=port) + LOG.info('serving SCGI') + s.serve() + + +if __name__ == '__main__': + import argparse as _argparse + + parser = _argparse.ArgumentParser(description=__doc__, version=__version__) + parser.add_argument( + '--scgi', default=False, action='store_const', const=True, + help='Run as a SCGI server (vs. serving a single CGI call)') + parser.add_argument( + '--base-path', default='/var/www/localhost/htdocs/gallery/', + help='Path to the root gallery source') + parser.add_argument( + '--base-url', default='/gallery/', + help='URL for the root gallery source') + parser.add_argument( + '--shared-path', default=None, + help=('Optional path to the shared directory containing ' + '`header.shtml` and `footer.shtml`')) + parser.add_argument( + '--cache-path', default='/tmp/gallery-cache', + help='Path to the thumbnail and movie cache directory') + + args = parser.parse_args() s = CGIGalleryServer( - base_path='/var/www/localhost/htdocs/gallery/', - base_url='/gallery/') - shared = '/var/www/localhost/htdocs/shared/' - s.header = [open(os.path.join(shared, 'header.shtml'), 'r').read()] - s.footer = [open(os.path.join(shared, 'footer.shtml'), 'r').read()] - s.serve(url=url, page=page) + base_path=args.base_path, base_url=args.base_url, + cache_path=args.cache_path) + if args.shared_path: + shared = args.shared_path + s.header = [open(_os_path.join(shared, 'header.shtml'), 'r').read()] + s.footer = [open(_os_path.join(shared, 'footer.shtml'), 'r').read()] + + if args.scgi: + serve_scgi(server=s) + else: + serve_cgi(server=s)