From 814f6c25fe2c1afec87c75ca7fcbae54d56bdd64 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 17 Apr 2008 15:52:23 +0200 Subject: [PATCH] simplified loader api and added builtin caching --HG-- branch : trunk --- jinja2/__init__.py | 1 + jinja2/defaults.py | 1 + jinja2/environment.py | 13 +++-- jinja2/filters.py | 30 +++-------- jinja2/loaders.py | 92 ++++++++++++++++++++++++++------ jinja2/utils.py | 121 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 43 deletions(-) diff --git a/jinja2/__init__.py b/jinja2/__init__.py index b3507a3..a59d42c 100644 --- a/jinja2/__init__.py +++ b/jinja2/__init__.py @@ -57,5 +57,6 @@ :license: BSD, see LICENSE for more details. """ from jinja2.environment import Environment +from jinja2.loaders import BaseLoader, FileSystemLoader, DictLoader from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined from jinja2.utils import Markup, escape diff --git a/jinja2/defaults.py b/jinja2/defaults.py index ea8591d..4deb8e5 100644 --- a/jinja2/defaults.py +++ b/jinja2/defaults.py @@ -15,6 +15,7 @@ from jinja.tests import TESTS as DEFAULT_TESTS DEFAULT_NAMESPACE = { 'range': xrange, # fake translators so that {% trans %} is a noop by default + '_': lambda x: x, 'gettext': lambda x: x, 'ngettext': lambda s, p, n: (s, p)[n != 1] } diff --git a/jinja2/environment.py b/jinja2/environment.py index 4b1e70c..b650ac7 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -130,8 +130,8 @@ class Environment(object): """ return self.lexer.tokeniter(source, name) - def compile(self, source, name=None, filename=None, raw=False, - globals=None): + def compile(self, source, name=None, filename=None, globals=None, + raw=False): """Compile a node or source. The name is the load name of the template after it was joined using `join_path` if necessary, filename is the estimated filename of the template on the file @@ -182,7 +182,7 @@ class Environment(object): class Template(object): """Represents a template.""" - def __init__(self, environment, code, globals): + def __init__(self, environment, code, globals, uptodate=None): namespace = {'environment': environment} exec code in namespace self.environment = environment @@ -194,6 +194,7 @@ class Template(object): # debug helpers self._get_debug_info = namespace['get_debug_info'] + self._uptodate = uptodate namespace['__jinja_template__'] = self def render(self, *args, **kwargs): @@ -239,6 +240,12 @@ class Template(object): return template_line return 1 + def is_up_to_date(self): + """Check if the template is still up to date.""" + if self._uptodate is None: + return True + return self._uptodate() + def __repr__(self): return '<%s %r>' % ( self.__class__.__name__, diff --git a/jinja2/filters.py b/jinja2/filters.py index 137f536..176685d 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -21,7 +21,6 @@ from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode from jinja2.runtime import Undefined - _striptags_re = re.compile(r'(|<[^>]*>)') @@ -80,14 +79,6 @@ def do_lower(s): return soft_unicode(s).lower() -def do_escape(s): - """XML escape ``&``, ``<``, ``>``, and ``"`` in a string of data. - - This method will have no effect it the value is already escaped. - """ - return escape(s) - - def do_xmlattr(d, autospace=False): """Create an SGML/XML attribute string based on the items in a dict. All values that are neither `none` nor `undefined` are automatically @@ -400,10 +391,7 @@ def do_int(value, default=0): try: return int(value) except (TypeError, ValueError): - try: - return int(float(value)) - except (TypeError, ValueError): - return default + return default def do_float(value, default=0.0): @@ -468,8 +456,7 @@ def do_slice(value, slices, fill_with=None): If you pass it a second argument it's used to fill missing values on the last iteration. """ - result = [] - seq = list(value) + seq = list(seq) length = len(seq) items_per_slice = length // slices slices_with_extra = length % slices @@ -482,8 +469,7 @@ def do_slice(value, slices, fill_with=None): tmp = seq[start:end] if fill_with is not None and slice_number >= slices_with_extra: tmp.append(fill_with) - result.append(tmp) - return result + yield tmp def do_batch(value, linecount, fill_with=None): @@ -509,14 +495,13 @@ def do_batch(value, linecount, fill_with=None): tmp = [] for item in value: if len(tmp) == linecount: - result.append(tmp) + yield tmp tmp = [] tmp.append(item) if tmp: if fill_with is not None and len(tmp) < linecount: tmp += [fill_with] * (linecount - len(tmp)) - result.append(tmp) - return result + yield tmp def do_round(value, precision=0, method='common'): @@ -594,8 +579,8 @@ FILTERS = { 'replace': do_replace, 'upper': do_upper, 'lower': do_lower, - 'escape': do_escape, - 'e': do_escape, + 'escape': escape, + 'e': escape, 'xmlattr': do_xmlattr, 'capitalize': do_capitalize, 'title': do_title, @@ -613,7 +598,6 @@ FILTERS = { 'random': do_random, 'filesizeformat': do_filesizeformat, 'pprint': do_pprint, - 'indent': do_indent, 'truncate': do_truncate, 'wordwrap': do_wordwrap, 'wordcount': do_wordcount, diff --git a/jinja2/loaders.py b/jinja2/loaders.py index 37c34e1..a78116c 100644 --- a/jinja2/loaders.py +++ b/jinja2/loaders.py @@ -9,51 +9,109 @@ :license: BSD, see LICENSE for more details. """ from os import path +from time import time from jinja2.exceptions import TemplateNotFound from jinja2.environment import Template +from jinja2.utils import LRUCache class BaseLoader(object): - """Baseclass for all loaders.""" + """ + Baseclass for all loaders. Subclass this and override `get_source` to + implement a custom loading mechanism. + + The environment provides a `get_template` method that will automatically + call the loader bound to an environment. + """ + + def __init__(self, cache_size=50, auto_reload=True): + if cache_size > 0: + self.cache = LRUCache(cache_size) + else: + self.cache = None + self.auto_reload = auto_reload def get_source(self, environment, template): - raise TemplateNotFound() + """Get the template source, filename and reload helper for a template. + It's passed the environment and template name and has to return a + tuple in the form ``(source, filename, uptodate)`` or raise a + `TemplateNotFound` error if it can't locate the template. + + The source part of the returned tuple must be the source of the + template as unicode string or a ASCII bytestring. The filename should + be the name of the file on the filesystem if it was loaded from there, + otherwise `None`. The filename is used by python for the tracebacks + if no loader extension is used. + + The last item in the tuple is the `uptodate` function. If auto + reloading is enabled it's always called to check if the template + changed. No arguments are passed so the function must store the + old state somewhere (for example in a closure). If it returns `False` + the template will be reloaded. + """ + raise TemplateNotFound(template) def load(self, environment, name, globals=None): - source, filename = self.get_source(environment, name) - code = environment.compile(source, name, filename, globals=globals) - return Template(environment, code, globals or {}) + """Loads a template. This method should not be overriden by + subclasses unless `get_source` doesn't provide enough flexibility. + """ + if globals is None: + globals = {} + + if self.cache is not None: + template = self.cache.get(name) + if template is not None and (not self.auto_reload or \ + template.is_up_to_date()): + return template + + source, filename, uptodate = self.get_source(environment, name) + code = environment.compile(source, name, filename, globals) + template = Template(environment, code, globals, uptodate) + if self.cache is not None: + self.cache[name] = template + return template class FileSystemLoader(BaseLoader): + """Loads templates from the file system.""" - def __init__(self, path, encoding='utf-8'): - self.path = path + def __init__(self, searchpath, encoding='utf-8', cache_size=50, + auto_reload=True): + BaseLoader.__init__(self, cache_size, auto_reload) + if isinstance(searchpath, basestring): + searchpath = [searchpath] + self.searchpath = searchpath self.encoding = encoding def get_source(self, environment, template): pieces = [] for piece in template.split('/'): if piece == '..': - raise TemplateNotFound() + raise TemplateNotFound(template) elif piece != '.': pieces.append(piece) - filename = path.join(self.path, *pieces) - if not path.isfile(filename): - raise TemplateNotFound(template) - f = file(filename) - try: - return f.read().decode(self.encoding), filename - finally: - f.close() + for searchpath in self.searchpath: + filename = path.join(searchpath, *pieces) + if path.isfile(filename): + f = file(filename) + try: + contents = f.read().decode(self.encoding) + finally: + f.close() + mtime = path.getmtime(filename) + def uptodate(): + return path.getmtime(filename) != mtime + return contents, filename, uptodate + raise TemplateNotFound(template) class DictLoader(BaseLoader): + """Loads a template from a python dict. Used for unittests mostly.""" def __init__(self, mapping): self.mapping = mapping def get_source(self, environment, template): if template in self.mapping: - return self.mapping[template], template + return self.mapping[template], None, None raise TemplateNotFound(template) diff --git a/jinja2/utils.py b/jinja2/utils.py index 63394ce..c030d24 100644 --- a/jinja2/utils.py +++ b/jinja2/utils.py @@ -10,6 +10,8 @@ """ import re import string +from collections import deque +from copy import deepcopy from functools import update_wrapper from itertools import imap @@ -196,3 +198,122 @@ class _MarkupEscapeHelper(object): __repr__ = lambda s: str(repr(escape(s.obj))) __int__ = lambda s: int(s.obj) __float__ = lambda s: float(s.obj) + + +class LRUCache(object): + """A simple LRU Cache implementation.""" + # this is fast for small capacities (something around 200) but doesn't + # scale. But as long as it's only used for the database connections in + # a non request fallback it's fine. + + def __init__(self, capacity): + self.capacity = capacity + self._mapping = {} + self._queue = deque() + + # alias all queue methods for faster lookup + self._popleft = self._queue.popleft + self._pop = self._queue.pop + if hasattr(self._queue, 'remove'): + self._remove = self._queue.remove + self._append = self._queue.append + + def _remove(self, obj): + """Python 2.4 compatibility.""" + for idx, item in enumerate(self._queue): + if item == obj: + del self._queue[idx] + break + + def copy(self): + """Return an shallow copy of the instance.""" + rv = LRUCache(self.capacity) + rv._mapping.update(self._mapping) + rv._queue = self._queue[:] + return rv + + def get(self, key, default=None): + """Return an item from the cache dict or `default`""" + if key in self: + return self[key] + return default + + def setdefault(self, key, default=None): + """ + Set `default` if the key is not in the cache otherwise + leave unchanged. Return the value of this key. + """ + if key in self: + return self[key] + self[key] = default + return default + + def clear(self): + """Clear the cache.""" + self._mapping.clear() + self._queue.clear() + + def __contains__(self, key): + """Check if a key exists in this cache.""" + return key in self._mapping + + def __len__(self): + """Return the current size of the cache.""" + return len(self._mapping) + + def __repr__(self): + return '<%s %r>' % ( + self.__class__.__name__, + self._mapping + ) + + def __getitem__(self, key): + """Get an item from the cache. Moves the item up so that it has the + highest priority then. + + Raise an `KeyError` if it does not exist. + """ + rv = self._mapping[key] + if self._queue[-1] != key: + self._remove(key) + self._append(key) + return rv + + def __setitem__(self, key, value): + """Sets the value for an item. Moves the item up so that it + has the highest priority then. + """ + if key in self._mapping: + self._remove(key) + elif len(self._mapping) == self.capacity: + del self._mapping[self._popleft()] + self._append(key) + self._mapping[key] = value + + def __delitem__(self, key): + """Remove an item from the cache dict. + Raise an `KeyError` if it does not exist. + """ + del self._mapping[key] + self._remove(key) + + def __iter__(self): + """Iterate over all values in the cache dict, ordered by + the most recent usage. + """ + return reversed(self._queue) + + def __reversed__(self): + """Iterate over the values in the cache dict, oldest items + coming first. + """ + return iter(self._queue) + + __copy__ = copy + + def __deepcopy__(self): + """Return a deep copy of the LRU Cache""" + rv = LRUCache(self.capacity) + rv._mapping = deepcopy(self._mapping) + rv._queue = deepcopy(self._queue) + return rv -- 2.26.2