From: Armin Ronacher Date: Wed, 17 Sep 2008 19:28:01 +0000 (+0200) Subject: Improved bbcache and documented it. X-Git-Tag: 2.1~26 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=a816bf488997daaefc1d573cfb2aa679b932cbde;p=jinja2.git Improved bbcache and documented it. --HG-- branch : trunk --- diff --git a/CHANGES b/CHANGES index af3ef4c..775b709 100644 --- a/CHANGES +++ b/CHANGES @@ -30,6 +30,8 @@ Version 2.1 - fixed a bug with empty statements in macros. +- implemented a bytecode cache system. (:ref:`bytecode-cache`) + Version 2.0 ----------- (codename jinjavitus, released on July 17th 2008) diff --git a/docs/api.rst b/docs/api.rst index 41949c8..b939eaa 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -250,7 +250,7 @@ others fail. The closest to regular Python behavior is the `StrictUndefined` which disallows all operations beside testing if it's an undefined object. -.. autoclass:: jinja2.runtime.Undefined() +.. autoclass:: jinja2.Undefined() .. attribute:: _undefined_hint @@ -278,9 +278,9 @@ disallows all operations beside testing if it's an undefined object. :attr:`_undefined_exception` with an error message generated from the undefined hints stored on the undefined object. -.. autoclass:: jinja2.runtime.DebugUndefined() +.. autoclass:: jinja2.DebugUndefined() -.. autoclass:: jinja2.runtime.StrictUndefined() +.. autoclass:: jinja2.StrictUndefined() Undefined objects are created by calling :attr:`undefined`. @@ -379,22 +379,62 @@ size by default and templates are automatically reloaded. All loaders are subclasses of :class:`BaseLoader`. If you want to create your own loader, subclass :class:`BaseLoader` and override `get_source`. -.. autoclass:: jinja2.loaders.BaseLoader +.. autoclass:: jinja2.BaseLoader :members: get_source, load Here a list of the builtin loaders Jinja2 provides: -.. autoclass:: jinja2.loaders.FileSystemLoader +.. autoclass:: jinja2.FileSystemLoader -.. autoclass:: jinja2.loaders.PackageLoader +.. autoclass:: jinja2.PackageLoader -.. autoclass:: jinja2.loaders.DictLoader +.. autoclass:: jinja2.DictLoader -.. autoclass:: jinja2.loaders.FunctionLoader +.. autoclass:: jinja2.FunctionLoader -.. autoclass:: jinja2.loaders.PrefixLoader +.. autoclass:: jinja2.PrefixLoader -.. autoclass:: jinja2.loaders.ChoiceLoader +.. autoclass:: jinja2.ChoiceLoader + + +.. _bytecode-cache: + +Bytecode Cache +-------------- + +Jinja 2.1 and higher support external bytecode caching. Bytecode caches make +it possible to store the generated bytecode on the file system or a different +location to avoid parsing the templates on first use. + +This is especially useful if you have a web application that is initialized on +the first request and Jinja compiles many templates at once which slows down +the application. + +To use a bytecode cache, instanciate it and pass it to the :class:`Environment`. + +.. autoclass:: jinja2.BytecodeCache + :members: load_bytecode, dump_bytecode, clear + +.. autoclass:: jinja2.bccache.Bucket + :members: write_bytecode, load_bytecode, bytecode_from_string, + bytecode_to_string, reset + + .. attribute:: environment + + The :class:`Environment` that created the bucket. + + .. attribute:: key + + The unique cache key for this bucket + + .. attribute:: code + + The bytecode if it's loaded, otherwise `None`. + + +Builtin bytecode caches: + +.. autoclass:: jinja2.FileSystemBytecodeCache Utilities @@ -403,13 +443,13 @@ Utilities These helper functions and classes are useful if you add custom filters or functions to a Jinja2 environment. -.. autofunction:: jinja2.filters.environmentfilter +.. autofunction:: jinja2.environmentfilter -.. autofunction:: jinja2.filters.contextfilter +.. autofunction:: jinja2.contextfilter -.. autofunction:: jinja2.utils.environmentfunction +.. autofunction:: jinja2.environmentfunction -.. autofunction:: jinja2.utils.contextfunction +.. autofunction:: jinja2.contextfunction .. function:: escape(s) @@ -420,11 +460,11 @@ functions to a Jinja2 environment. The return value is a :class:`Markup` string. -.. autofunction:: jinja2.utils.clear_caches +.. autofunction:: jinja2.clear_caches -.. autofunction:: jinja2.utils.is_undefined +.. autofunction:: jinja2.is_undefined -.. autoclass:: jinja2.utils.Markup([string]) +.. autoclass:: jinja2.Markup([string]) :members: escape, unescape, striptags .. admonition:: Note @@ -437,13 +477,13 @@ functions to a Jinja2 environment. Exceptions ---------- -.. autoexception:: jinja2.exceptions.TemplateError +.. autoexception:: jinja2.TemplateError -.. autoexception:: jinja2.exceptions.UndefinedError +.. autoexception:: jinja2.UndefinedError -.. autoexception:: jinja2.exceptions.TemplateNotFound +.. autoexception:: jinja2.TemplateNotFound -.. autoexception:: jinja2.exceptions.TemplateSyntaxError +.. autoexception:: jinja2.TemplateSyntaxError .. attribute:: message @@ -466,7 +506,7 @@ Exceptions unicode strings is that Python 2.x is not using unicode for exceptions and tracebacks as well as the compiler. This will change with Python 3. -.. autoexception:: jinja2.exceptions.TemplateAssertionError +.. autoexception:: jinja2.TemplateAssertionError .. _writing-filters: diff --git a/jinja2/__init__.py b/jinja2/__init__.py index 194390a..f7576d1 100644 --- a/jinja2/__init__.py +++ b/jinja2/__init__.py @@ -40,6 +40,9 @@ from jinja2.environment import Environment, Template from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache + # undefined types from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined diff --git a/jinja2/bccache.py b/jinja2/bccache.py index 50532cd..ff3dd69 100644 --- a/jinja2/bccache.py +++ b/jinja2/bccache.py @@ -14,9 +14,11 @@ :copyright: Copyright 2008 by Armin Ronacher. :license: BSD. """ -from os import path +from os import path, listdir, remove import marshal +import tempfile import cPickle as pickle +import fnmatch from cStringIO import StringIO try: from hashlib import sha1 @@ -29,128 +31,185 @@ bc_magic = 'j2' + pickle.dumps(bc_version, 2) class Bucket(object): - """Buckets are used to store the bytecode for one template. It's - initialized by the bytecode cache with the checksum for the code - as well as the unique key. + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. - The bucket then provides method to load the bytecode from file(-like) - objects and strings or dump it again. + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. """ - def __init__(self, cache, environment, key, checksum): - self._cache = cache + def __init__(self, environment, key, checksum): self.environment = environment self.key = key self.checksum = checksum self.reset() def reset(self): - """Resets the bucket (unloads the code).""" + """Resets the bucket (unloads the bytecode).""" self.code = None - def load(self, f): - """Loads bytecode from a f.""" + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" # make sure the magic header is correct magic = f.read(len(bc_magic)) if magic != bc_magic: self.reset() return # the source code of the file changed, we need to reload - checksum = pickle.load(f) + checksum = pickle.load_bytecode(f) if self.checksum != checksum: self.reset() return - # now load the code. Because marshal is not able to load + # now load_bytecode the code. Because marshal is not able to load_bytecode # from arbitrary streams we have to work around that if isinstance(f, file): - self.code = marshal.load(f) + self.code = marshal.load_bytecode(f) else: self.code = marshal.loads(f.read()) - def dump(self, f): - """Dump the bytecode into f.""" + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" if self.code is None: raise TypeError('can\'t write empty bucket') f.write(bc_magic) - pickle.dump(self.checksum, f, 2) + pickle.write_bytecode(self.checksum, f, 2) if isinstance(f, file): - marshal.dump(self.code, f) + marshal.write_bytecode(self.code, f) else: f.write(marshal.dumps(self.code)) - def loads(self, string): + def bytecode_from_string(self, string): """Load bytecode from a string.""" - self.load(StringIO(string)) + self.load_bytecode(StringIO(string)) - def dumps(self): + def bytecode_to_string(self): """Return the bytecode as string.""" out = StringIO() - self.dump(out) + self.write_bytecode(out) return out.getvalue() - def write_back(self): - """Write the bucket back to the cache.""" - self._cache.dump_bucket(self) - class BytecodeCache(object): """To implement your own bytecode cache you have to subclass this class - and override :meth:`load_bucket` and :meth:`dump_bucket`. Both of these - methods are passed a :class:`Bucket` that they have to load or dump. + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with file(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with file(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. """ - def load_bucket(self, bucket): - """Subclasses have to override this method to load bytecode - into a bucket. + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. """ raise NotImplementedError() - def dump_bucket(self, bucket): - """Subclasses have to override this method to write the - bytecode from a bucket back to the cache. + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. """ raise NotImplementedError() - def get_cache_key(self, name): - """Return the unique hash key for this template name.""" - return sha1(name.encode('utf-8')).hexdigest() + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + if isinstance(filename, unicode): + filename = filename.encode('utf-8') + hash.update('|' + filename) + return hash.hexdigest() def get_source_checksum(self, source): - """Return a checksum for the source.""" + """Returns a checksum for the source.""" return sha1(source.encode('utf-8')).hexdigest() - def get_bucket(self, environment, name, source): - """Return a cache bucket.""" - key = self.get_cache_key(name) + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) checksum = self.get_source_checksum(source) - bucket = Bucket(self, environment, key, checksum) - self.load_bucket(bucket) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) return bucket + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified the system temporary items folder is used. -class FileSystemCache(BytecodeCache): - """A bytecode cache that stores bytecode on the filesystem.""" + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. - def __init__(self, directory, pattern='%s.jbc'): + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = tempfile.gettempdir() self.directory = directory self.pattern = pattern def _get_cache_filename(self, bucket): return path.join(self.directory, self.pattern % bucket.key) - def load_bucket(self, bucket): + def load_bytecode(self, bucket): filename = self._get_cache_filename(bucket) if path.exists(filename): f = file(filename, 'rb') try: - bucket.load(f) + bucket.load_bytecode(f) finally: f.close() - def dump_bucket(self, bucket): + def dump_bytecode(self, bucket): filename = self._get_cache_filename(bucket) f = file(filename, 'wb') try: - bucket.dump(f) + bucket.write_bytecode(f) finally: f.close() + + def clear(self): + for filename in filter(listdir(self.directory), self.pattern % '*'): + try: + remove(path.join(self.directory, filename)) + except OSError: + pass diff --git a/jinja2/environment.py b/jinja2/environment.py index 5b77d45..a64b469 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -160,6 +160,8 @@ class Environment(object): If set to a bytecode cache object, this object will provide a cache for the internal Jinja bytecode so that templates don't have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. """ #: if this environment is sandboxed. Modifying this variable won't make diff --git a/jinja2/loaders.py b/jinja2/loaders.py index 662425c..dff95f7 100644 --- a/jinja2/loaders.py +++ b/jinja2/loaders.py @@ -86,22 +86,32 @@ class BaseLoader(object): loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`) will not call this method but `get_source` directly. """ + code = None if globals is None: globals = {} + + # first we try to get the source for this template together + # with the filename and the uptodate function. source, filename, uptodate = self.get_source(environment, name) - code = bucket = None - if environment.bytecode_cache is not None: - bucket = environment.bytecode_cache.get_bucket(environment, name, - source) + # try to load the code from the bytecode cache if there is a + # bytecode cache configured. + bbc = environment.bytecode_cache + if bbc is not None: + bucket = bcc.get_bucket(environment, name, filename, source) code = bucket.code + # if we don't have code so far (not cached, no longer up to + # date) etc. we compile the template if code is None: code = environment.compile(source, name, filename) - if bucket and bucket.code is None: + # if the bytecode cache is available and the bucket doesn't + # have a code so far, we give the bucket the new code and put + # it back to the bytecode cache. + if bbc is not None and bucket.code is None: bucket.code = code - bucket.write_back() + bbc.set_bucket(bucket) return environment.template_class.from_code(environment, code, globals, uptodate)