1 # -*- coding: utf-8 -*-
6 This module implements the bytecode cache system Jinja is optionally
7 using. This is useful if you have very complex template situations and
8 the compiliation of all those templates slow down your application too
11 Situations where this is useful are often forking web applications that
12 are initialized on the first request.
14 :copyright: (c) 2010 by the Jinja Team.
17 from os import path, listdir
21 import cPickle as pickle
23 from cStringIO import StringIO
25 from hashlib import sha1
27 from sha import new as sha1
28 from jinja2.utils import open_if_exists
33 # magic version used to only change with new jinja versions. With 2.6
34 # we change this to also take Python version changes into account. The
35 # reason for this is that Python tends to segfault if fed earlier bytecode
36 # versions because someone thought it would be a good idea to reuse opcodes
37 # or make Python incompatible with earlier versions.
38 bc_magic = 'j2'.encode('ascii') + \
39 pickle.dumps(bc_version, 2) + \
40 pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
44 """Buckets are used to store the bytecode for one template. It's created
45 and initialized by the bytecode cache and passed to the loading functions.
47 The buckets get an internal checksum from the cache assigned and use this
48 to automatically reject outdated cache material. Individual bytecode
49 cache subclasses don't have to care about cache invalidation.
52 def __init__(self, environment, key, checksum):
53 self.environment = environment
55 self.checksum = checksum
59 """Resets the bucket (unloads the bytecode)."""
62 def load_bytecode(self, f):
63 """Loads bytecode from a file or file like object."""
64 # make sure the magic header is correct
65 magic = f.read(len(bc_magic))
69 # the source code of the file changed, we need to reload
70 checksum = pickle.load(f)
71 if self.checksum != checksum:
74 # now load the code. Because marshal is not able to load
75 # from arbitrary streams we have to work around that
76 if isinstance(f, file):
77 self.code = marshal.load(f)
79 self.code = marshal.loads(f.read())
81 def write_bytecode(self, f):
82 """Dump the bytecode into the file or file like object passed."""
84 raise TypeError('can\'t write empty bucket')
86 pickle.dump(self.checksum, f, 2)
87 if isinstance(f, file):
88 marshal.dump(self.code, f)
90 f.write(marshal.dumps(self.code))
92 def bytecode_from_string(self, string):
93 """Load bytecode from a string."""
94 self.load_bytecode(StringIO(string))
96 def bytecode_to_string(self):
97 """Return the bytecode as string."""
99 self.write_bytecode(out)
100 return out.getvalue()
103 class BytecodeCache(object):
104 """To implement your own bytecode cache you have to subclass this class
105 and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
106 these methods are passed a :class:`~jinja2.bccache.Bucket`.
108 A very basic bytecode cache that saves the bytecode on the file system::
112 class MyCache(BytecodeCache):
114 def __init__(self, directory):
115 self.directory = directory
117 def load_bytecode(self, bucket):
118 filename = path.join(self.directory, bucket.key)
119 if path.exists(filename):
120 with open(filename, 'rb') as f:
121 bucket.load_bytecode(f)
123 def dump_bytecode(self, bucket):
124 filename = path.join(self.directory, bucket.key)
125 with open(filename, 'wb') as f:
126 bucket.write_bytecode(f)
128 A more advanced version of a filesystem based bytecode cache is part of
132 def load_bytecode(self, bucket):
133 """Subclasses have to override this method to load bytecode into a
134 bucket. If they are not able to find code in the cache for the
135 bucket, it must not do anything.
137 raise NotImplementedError()
139 def dump_bytecode(self, bucket):
140 """Subclasses have to override this method to write the bytecode
141 from a bucket back to the cache. If it unable to do so it must not
142 fail silently but raise an exception.
144 raise NotImplementedError()
147 """Clears the cache. This method is not used by Jinja2 but should be
148 implemented to allow applications to clear the bytecode cache used
149 by a particular environment.
152 def get_cache_key(self, name, filename=None):
153 """Returns the unique hash key for this template name."""
154 hash = sha1(name.encode('utf-8'))
155 if filename is not None:
156 if isinstance(filename, unicode):
157 filename = filename.encode('utf-8')
158 hash.update('|' + filename)
159 return hash.hexdigest()
161 def get_source_checksum(self, source):
162 """Returns a checksum for the source."""
163 return sha1(source.encode('utf-8')).hexdigest()
165 def get_bucket(self, environment, name, filename, source):
166 """Return a cache bucket for the given template. All arguments are
167 mandatory but filename may be `None`.
169 key = self.get_cache_key(name, filename)
170 checksum = self.get_source_checksum(source)
171 bucket = Bucket(environment, key, checksum)
172 self.load_bytecode(bucket)
175 def set_bucket(self, bucket):
176 """Put the bucket into the cache."""
177 self.dump_bytecode(bucket)
180 class FileSystemBytecodeCache(BytecodeCache):
181 """A bytecode cache that stores bytecode on the filesystem. It accepts
182 two arguments: The directory where the cache items are stored and a
183 pattern string that is used to build the filename.
185 If no directory is specified the system temporary items folder is used.
187 The pattern can be used to have multiple separate caches operate on the
188 same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
189 is replaced with the cache key.
191 >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
193 This bytecode cache supports clearing of the cache using the clear method.
196 def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
197 if directory is None:
198 directory = tempfile.gettempdir()
199 self.directory = directory
200 self.pattern = pattern
202 def _get_cache_filename(self, bucket):
203 return path.join(self.directory, self.pattern % bucket.key)
205 def load_bytecode(self, bucket):
206 f = open_if_exists(self._get_cache_filename(bucket), 'rb')
209 bucket.load_bytecode(f)
213 def dump_bytecode(self, bucket):
214 f = open(self._get_cache_filename(bucket), 'wb')
216 bucket.write_bytecode(f)
221 # imported lazily here because google app-engine doesn't support
222 # write access on the file system and the function does not exist
224 from os import remove
225 files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
226 for filename in files:
228 remove(path.join(self.directory, filename))
233 class MemcachedBytecodeCache(BytecodeCache):
234 """This class implements a bytecode cache that uses a memcache cache for
235 storing the information. It does not enforce a specific memcache library
236 (tummy's memcache or cmemcache) but will accept any class that provides
237 the minimal interface required.
239 Libraries compatible with this class:
241 - `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
242 - `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
243 - `cmemcache <http://gijsbert.org/cmemcache/>`_
245 (Unfortunately the django cache interface is not compatible because it
246 does not support storing binary data, only unicode. You can however pass
247 the underlying cache client to the bytecode cache which is available
248 as `django.core.cache.cache._client`.)
250 The minimal interface for the client passed to the constructor is this:
252 .. class:: MinimalClientInterface
254 .. method:: set(key, value[, timeout])
256 Stores the bytecode in the cache. `value` is a string and
257 `timeout` the timeout of the key. If timeout is not provided
258 a default timeout or no timeout should be assumed, if it's
259 provided it's an integer with the number of seconds the cache
264 Returns the value for the cache key. If the item does not
265 exist in the cache the return value must be `None`.
267 The other arguments to the constructor are the prefix for all keys that
268 is added before the actual cache key and the timeout for the bytecode in
269 the cache system. We recommend a high (or no) timeout.
271 This bytecode cache does not support clearing of used items in the cache.
272 The clear method is a no-operation function.
275 def __init__(self, client, prefix='jinja2/bytecode/', timeout=None):
278 self.timeout = timeout
280 def load_bytecode(self, bucket):
281 code = self.client.get(self.prefix + bucket.key)
283 bucket.bytecode_from_string(code)
285 def dump_bytecode(self, bucket):
286 args = (self.prefix + bucket.key, bucket.bytecode_to_string())
287 if self.timeout is not None:
288 args += (self.timeout,)
289 self.client.set(*args)