From db69d0a1a16ec4032dce47b46481c730108abb37 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 2 Jun 2007 01:35:53 +0200 Subject: [PATCH] [svn] added baseclasses for loaders, added the memcaching loader, updated documentation for loaders --HG-- branch : trunk --- CHANGES | 6 + Makefile | 3 + docs/generate.py | 26 ++++ docs/src/loaders.txt | 56 ++++++++- jinja/loaders.py | 286 +++++++++++++++++++++++++++++++++++-------- jinja/utils.py | 1 - tests/test_syntax.py | 6 + 7 files changed, 332 insertions(+), 52 deletions(-) diff --git a/CHANGES b/CHANGES index da46f59..29a981d 100644 --- a/CHANGES +++ b/CHANGES @@ -112,6 +112,12 @@ Version 1.1 - fixed extended slicing +- reworked loader layer. All the cached loaders now have "private" non cached + baseclasses so that you can easily mix your own caching layers in. + +- added `MemcachedLoaderMixin` and `MemcachedFileSystemLoader` contributed + by Bryan McLemore. + Version 1.0 ----------- diff --git a/Makefile b/Makefile index 9b1cd17..17f0b32 100644 --- a/Makefile +++ b/Makefile @@ -25,3 +25,6 @@ pylint: release: documentation @(python2.3 setup.py release bdist_egg upload; python2.4 setup.py release bdist_egg upload; python2.5 setup.py release bdist_egg sdist upload) + +test-release: documentation + @(python2.3 setup.py release bdist_egg; python2.4 setup.py release bdist_egg; python2.5 setup.py release bdist_egg sdist) diff --git a/docs/generate.py b/docs/generate.py index f8e8238..6b6b0ac 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -101,6 +101,30 @@ def generate_list_of_loaders(): return '\n\n'.join(result) +def generate_list_of_baseloaders(): + from jinja import loaders as loader_module + + result = [] + loaders = [] + for item in dir(loader_module): + obj = getattr(loader_module, item) + try: + if issubclass(obj, loader_module.BaseLoader) and \ + obj.__name__ != 'BaseLoader' and \ + obj.__name__ not in loader_module.__all__: + loaders.append(obj) + except TypeError: + pass + loaders.sort(key=lambda x: x.__name__.lower()) + + for loader in loaders: + doclines = [] + for line in inspect.getdoc(loader).splitlines(): + doclines.append(' ' + line) + result.append('`%s`\n%s' % (loader.__name__, '\n'.join(doclines))) + + return '\n\n'.join(result) + def generate_environment_doc(): from jinja.environment import Environment return '%s\n\n%s' % ( @@ -115,6 +139,7 @@ PYGMENTS_FORMATTER = HtmlFormatter(style='pastie', cssclass='syntax') LIST_OF_FILTERS = generate_list_of_filters() LIST_OF_TESTS = generate_list_of_tests() LIST_OF_LOADERS = generate_list_of_loaders() +LIST_OF_BASELOADERS = generate_list_of_baseloaders() ENVIRONMENT_DOC = generate_environment_doc() CHANGELOG = file(os.path.join(os.path.dirname(__file__), os.pardir, 'CHANGES'))\ .read().decode('utf-8') @@ -240,6 +265,7 @@ def generate_documentation(data, link_style): data = data.replace('[[list_of_filters]]', LIST_OF_FILTERS)\ .replace('[[list_of_tests]]', LIST_OF_TESTS)\ .replace('[[list_of_loaders]]', LIST_OF_LOADERS)\ + .replace('[[list_of_baseloaders]]', LIST_OF_BASELOADERS)\ .replace('[[environment_doc]]', ENVIRONMENT_DOC)\ .replace('[[changelog]]', CHANGELOG) parts = publish_parts( diff --git a/docs/src/loaders.txt b/docs/src/loaders.txt index 29da849..306b14d 100644 --- a/docs/src/loaders.txt +++ b/docs/src/loaders.txt @@ -7,8 +7,21 @@ This part of the documentation explains how to use and write a template loader. Builtin Loaders =============== +This list contains the builtin loaders you can use without further +modification: + [[list_of_loaders]] +Loader Baseclasses +================== + +With Jinja 1.1 onwards all the loaders have (except of the uncached) now have +baseclasses you can use to mix your own caching layer in. This technique is +explained below. The `BaseLoader` itself is also a loader baseclass but because +it's the baseclass of each loader it's explained in detail below. + +[[list_of_baseloaders]] + Developing Loaders ================== @@ -43,7 +56,13 @@ Here the implementation of a simple loader based on the `BaseLoader` from f.close() The functions `load` and `parse` which are a requirement for a loader are -added automatically by the `BaseLoader`. +added automatically by the `BaseLoader`. Instead of the normal `BaseLoader` +you can use one of the other base loaders that already come with a proper +`get_source` method for further modification. Those loaders however are +new in Jinja 1.1. + +CachedLoaderMixin +----------------- Additionally to the `BaseLoader` there is a mixin class called `CachedLoaderMixin` that implements memory and disk caching of templates. @@ -91,4 +110,37 @@ You don't have to provide the `check_source_changed` method. If it doesn't exist the option `auto_reload` won't have an effect. Also note that the `check_source_changed` method must not raise an exception if the template does not exist but return ``-1``. The return value ``-1`` is considered -"always reload" whereas ``0`` means "do not reload". +"always reload" whereas ``0`` means "do not reload". The default return +value for not existing templates should be ``-1``. + +For the default base classes that come with Jinja 1.1 onwards there exist +also concrete implementations that support caching. The implementation +just mixes in the `CachedLoaderMixin`. + +MemcachedLoaderMixin +-------------------- + +*New in Jinja 1.1* + +The `MemcachedLoaderMixin` class adds support for `memcached`_ caching. +There is only one builtin loader that mixes it in: The +`MemcachedFileSystemLoader`. If you need other loaders with this mixin +you can easily subclass one of the existing base loaders. Here an example +for the `FunctionLoader`: + +.. sourcecode:: python + + from jinja.loaders import FunctionLoader, MemcachedLoaderMixin + + class MemcachedFunctionLoader(MemcachedLoaderMixin, FunctionLoader): + + def __init__(self, loader_func): + BaseFunctionLoader.__init__(self, loader_func) + MemcachedLoaderMixin.__init__(self, + True, # use memcached + 60 * 60 * 24 * 7, # 7 days expiration + ['127.0.0.1:11211'], # the memcached hosts + 'template/' # string prefix for the cache keys + ) + +.. _memcached: http://www.danga.com/memcached/ diff --git a/jinja/loaders.py b/jinja/loaders.py index 27999a2..c6b82f0 100644 --- a/jinja/loaders.py +++ b/jinja/loaders.py @@ -5,7 +5,7 @@ Jinja loader classes. - :copyright: 2007 by Armin Ronacher. + :copyright: 2007 by Armin Ronacher, Bryan McLemore. :license: BSD, see LICENSE for more details. """ @@ -22,7 +22,7 @@ from jinja.utils import CacheDict, raise_syntax_error #: when updating this, update the listing in the jinja package too __all__ = ['FileSystemLoader', 'PackageLoader', 'DictLoader', 'ChoiceLoader', - 'FunctionLoader'] + 'FunctionLoader', 'MemcachedFileSystemLoader'] def get_template_filename(searchpath, name): @@ -142,7 +142,14 @@ class BaseLoader(object): class CachedLoaderMixin(object): """ - Mixin this class to implement simple memory and disk caching. + Mixin this class to implement simple memory and disk caching. The + memcaching just uses a dict in the loader so if you have a global + environment or at least a global loader this can speed things up. + + If the memcaching is enabled you can use (with Jinja 1.1 onwards) + the `clear_memcache` function to clear the cache. + + For memcached support check the `MemcachedLoaderMixin`. """ def __init__(self, use_memcache, cache_size, cache_folder, auto_reload, @@ -160,13 +167,20 @@ class CachedLoaderMixin(object): self.__times = {} self.__lock = Lock() + def clear_memcache(self): + """ + Clears the memcache. + """ + if self.__memcache is not None: + self.__memcache.clear() + def load(self, environment, name, translator): """ Load and translate a template. First we check if there is a cached version of this template in the memory cache. If this is not the cache check for a compiled template in the disk cache - folder. And if none of this is the case we translate the temlate - using the `LoaderMixin.load` function, cache and return it. + folder. And if none of this is the case we translate the temlate, + cache and return it. """ self.__lock.acquire() try: @@ -193,8 +207,7 @@ class CachedLoaderMixin(object): if name in self.__memcache: tmpl = self.__memcache[name] # if auto reload is enabled check if the template changed - if last_change is not None and \ - last_change > self.__times[name]: + if last_change and last_change > self.__times[name]: tmpl = None push_to_memory = True else: @@ -247,7 +260,105 @@ class CachedLoaderMixin(object): self.__lock.release() -class FileSystemLoader(CachedLoaderMixin, BaseLoader): +class MemcachedLoaderMixin(object): + """ + Uses a memcached server to cache the templates. + """ + + def __init__(self, use_memcache, memcache_time=60 * 60 * 24 * 7, + memcache_host=None, item_prefix='template/'): + try: + from memcache import Client + except ImportError: + raise RuntimeError('the %r loader requires an installed ' + 'memcache module' % self.__class__.__name__) + if memcache_host is None: + memcache_host = ['127.0.0.1:11211'] + if use_memcache: + self.__memcache = Client(list(memcache_host)) + self.__memcache_time = memcache_time + else: + self.__memcache = None + self.__item_prefix = item_prefix + self.__lock = Lock() + + def load(self, environment, name, translator): + """ + Load and translate a template. First we check if there is a + cached version of this template in the memory cache. If this is + not the cache check for a compiled template in the disk cache + folder. And if none of this is the case we translate the template, + cache and return it. + """ + self.__lock.acquire() + try: + # caching is only possible for the python translator. skip + # all other translators + if translator is not PythonTranslator: + return super(MemcachedLoaderMixin, self).load( + environment, name, translator) + tmpl = None + push_to_memory = False + + # check if we have something in the memory cache and the + # memory cache is enabled. + if self.__memcache is not None: + bytecode = self.__memcache.get(self.__item_prefix + name) + if bytecode: + tmpl = Template.load(environment, bytecode) + else: + push_to_memory = True + + # if we still have no template we load, parse and translate it. + if tmpl is None: + tmpl = super(MemcachedLoaderMixin, self).load( + environment, name, translator) + + # if memcaching is enabled and the template not loaded + # we add that there. + if push_to_memory: + self.__memcache.set(self.__item_prefix + name, tmpl.dump(), + self.__memcache_time) + return tmpl + finally: + self.__lock.release() + + +class BaseFileSystemLoader(BaseLoader): + """ + Baseclass for the file system loader that does not do any caching. + It exists to avoid redundant code, just don't use it without subclassing. + + How subclassing can work: + + .. sourcecode:: python + + from jinja.loaders import BaseFileSystemLoader + + class MyFileSystemLoader(BaseFileSystemLoader): + def __init__(self): + BaseFileSystemLoader.__init__(self, '/path/to/templates') + + The base file system loader only takes one parameter beside self which + is the path to the templates. + """ + + def __init__(self, searchpath): + self.searchpath = path.abspath(searchpath) + + def get_source(self, environment, name, parent): + filename = get_template_filename(self.searchpath, name) + if path.exists(filename): + f = codecs.open(filename, 'r', environment.template_charset) + try: + return f.read() + finally: + f.close() + else: + raise TemplateNotFound(name) + + +class FileSystemLoader(CachedLoaderMixin, BaseFileSystemLoader): """ Loads templates from the filesystem: @@ -257,7 +368,7 @@ class FileSystemLoader(CachedLoaderMixin, BaseLoader): e = Environment(loader=FileSystemLoader('templates/')) You can pass the following keyword arguments to the loader on - initialisation: + initialization: =================== ================================================= ``searchpath`` String with the path to the templates on the @@ -287,23 +398,13 @@ class FileSystemLoader(CachedLoaderMixin, BaseLoader): def __init__(self, searchpath, use_memcache=False, memcache_size=40, cache_folder=None, auto_reload=True, cache_salt=None): - self.searchpath = path.abspath(searchpath) + BaseFileSystemLoader.__init__(self, searchpath) + if cache_salt is None: cache_salt = self.searchpath CachedLoaderMixin.__init__(self, use_memcache, memcache_size, cache_folder, auto_reload, cache_salt) - def get_source(self, environment, name, parent): - filename = get_template_filename(self.searchpath, name) - if path.exists(filename): - f = codecs.open(filename, 'r', environment.template_charset) - try: - return f.read() - finally: - f.close() - else: - raise TemplateNotFound(name) - def check_source_changed(self, environment, name): filename = get_template_filename(self.searchpath, name) if path.exists(filename): @@ -311,7 +412,84 @@ class FileSystemLoader(CachedLoaderMixin, BaseLoader): return -1 -class PackageLoader(CachedLoaderMixin, BaseLoader): +class MemcachedFileSystemLoader(MemcachedLoaderMixin, BaseFileSystemLoader): + """ + Loads templates from the filesystem and caches them on a memcached + server. + + .. sourcecode:: python + + from jinja import Environment, MemcachedFileSystemLoader + e = Environment(loader=MemcachedFileSystemLoader('templates/', + memcache_host=['192.168.2.250:11211'] + )) + + You can pass the following keyword arguments to the loader on + initialization: + + =================== ================================================= + ``searchpath`` String with the path to the templates on the + filesystem. + ``use_memcache`` Set this to ``True`` to enable memcached caching. + In that case it behaves like a normal + `FileSystemLoader` with disabled caching. + ``memcache_time`` The expire time of a template in the cache. + ``memcache_host`` a list of memcached servers. + ``item_prefix`` The prefix for the items on the server. Defaults + to ``'template/'``. + =================== ================================================= + """ + + def __init__(self, searchpath, use_memcache=True, + memcache_time=60 * 60 * 24 * 7, memcache_host=None, + item_prefix='template/'): + BaseFileSystemLoader.__init__(self, searchpath) + MemcachedLoaderMixin.__init__(self, use_memcache, memcache_time, + memcache_host, item_prefix) + + +class BasePackageLoader(BaseLoader): + """ + Baseclass for the package loader that does not do any caching. + + It accepts two parameters: The name of the package and the path relative + to the package: + + .. sourcecode:: python + + from jinja.loaders import BasePackageLoader + + class MyPackageLoader(BasePackageLoader): + def __init__(self): + BasePackageLoader.__init__(self, 'my_package', 'shared/templates') + + The relative path must use slashes as path delimiters, even on Mac OS + and Microsoft Windows. + + It uses the `pkg_resources` libraries distributed with setuptools for + retrieving the data from the packages. This works for eggs too so you + don't have to mark your egg as non zip safe. + """ + + def __init__(self, package_name, package_path): + try: + import pkg_resources + except ImportError: + raise RuntimeError('setuptools not installed') + self.package_name = package_name + self.package_path = package_path + + def get_source(self, environment, name, parent): + from pkg_resources import resource_exists, resource_string + path = '/'.join([self.package_path] + [p for p in name.split('/') + if p != '..']) + if not resource_exists(self.package_name, path): + raise TemplateNotFound(name) + contents = resource_string(self.package_name, path) + return contents.decode(environment.template_charset) + + +class PackageLoader(CachedLoaderMixin, BasePackageLoader): """ Loads templates from python packages using setuptools. @@ -321,7 +499,7 @@ class PackageLoader(CachedLoaderMixin, BaseLoader): e = Environment(loader=PackageLoader('yourapp', 'template/path')) You can pass the following keyword arguments to the loader on - initialisation: + initialization: =================== ================================================= ``package_name`` Name of the package containing the templates. @@ -361,26 +539,13 @@ class PackageLoader(CachedLoaderMixin, BaseLoader): def __init__(self, package_name, package_path, use_memcache=False, memcache_size=40, cache_folder=None, auto_reload=True, cache_salt=None): - try: - import pkg_resources - except ImportError: - raise RuntimeError('setuptools not installed') - self.package_name = package_name - self.package_path = package_path + BasePackageLoader.__init__(self, package_name, package_path) + if cache_salt is None: cache_salt = package_name + '/' + package_path CachedLoaderMixin.__init__(self, use_memcache, memcache_size, cache_folder, auto_reload, cache_salt) - def get_source(self, environment, name, parent): - from pkg_resources import resource_exists, resource_string - path = '/'.join([self.package_path] + [p for p in name.split('/') - if p != '..']) - if not resource_exists(self.package_name, path): - raise TemplateNotFound(name) - contents = resource_string(self.package_name, path) - return contents.decode(environment.template_charset) - def check_source_changed(self, environment, name): from pkg_resources import resource_exists, resource_filename fn = resource_filename(self.package_name, '/'.join([self.package_path] + @@ -390,7 +555,38 @@ class PackageLoader(CachedLoaderMixin, BaseLoader): return -1 -class FunctionLoader(CachedLoaderMixin, BaseLoader): +class BaseFunctionLoader(BaseLoader): + """ + Baseclass for the function loader that doesn't do any caching. + + It just accepts one parameter which is the function which is called + with the name of the requested template. If the return value is `None` + the loader will raise a `TemplateNotFound` error. + + .. sourcecode:: python + + from jinja.loaders import BaseFunctionLoader + + templates = {...} + + class MyFunctionLoader(BaseFunctionLoader): + def __init__(self): + BaseFunctionLoader(templates.get) + """ + + def __init__(self, loader_func): + self.loader_func = loader_func + + def get_source(self, environment, name, parent): + rv = self.loader_func(name) + if rv is None: + raise TemplateNotFound(name) + if isinstance(rv, str): + return rv.decode(environment.template_charset) + return rv + + +class FunctionLoader(CachedLoaderMixin, BaseFunctionLoader): """ Loads templates by calling a function which has to return a string or `None` if an error occoured. @@ -410,7 +606,7 @@ class FunctionLoader(CachedLoaderMixin, BaseLoader): solid backend. You can pass the following keyword arguments to the loader on - initialisation: + initialization: =================== ================================================= ``loader_func`` Function that takes the name of the template to @@ -446,23 +642,15 @@ class FunctionLoader(CachedLoaderMixin, BaseLoader): def __init__(self, loader_func, getmtime_func=None, use_memcache=False, memcache_size=40, cache_folder=None, auto_reload=True, cache_salt=None): + BaseFunctionLoader.__init__(self, loader_func) # when changing the signature also check the jinja.plugin function # loader instantiation. - self.loader_func = loader_func self.getmtime_func = getmtime_func if auto_reload and getmtime_func is None: auto_reload = False CachedLoaderMixin.__init__(self, use_memcache, memcache_size, cache_folder, auto_reload, cache_salt) - def get_source(self, environment, name, parent): - rv = self.loader_func(name) - if rv is None: - raise TemplateNotFound(name) - if isinstance(rv, str): - return rv.decode(environment.template_charset) - return rv - def check_source_changed(self, environment, name): return self.getmtime_func(name) diff --git a/jinja/utils.py b/jinja/utils.py index d9608eb..42eec0d 100644 --- a/jinja/utils.py +++ b/jinja/utils.py @@ -71,7 +71,6 @@ except NameError: #: function types callable_types = (FunctionType, MethodType) - #: number of maximal range items MAX_RANGE = 1000000 diff --git a/tests/test_syntax.py b/tests/test_syntax.py index 79aaf2f..e6a8714 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -19,6 +19,7 @@ UNARY = '''{{ +3 }}|{{ -3 }}''' COMPARE = '''{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|{{ 2 == 2 }}|{{ 1 <= 1 }}''' LITERALS = '''{{ [] }}|{{ {} }}|{{ '' }}''' BOOL = '''{{ true and false }}|{{ false or true }}|{{ not false }}''' +GROUPING = '''{{ (true and false) or (false and true) and not false }}''' def test_call(): @@ -82,3 +83,8 @@ def test_literals(env): def test_bool(env): tmpl = env.from_string(BOOL) assert tmpl.render() == 'False|True|True' + + +def test_grouping(env): + tmpl = env.from_string(GROUPING) + assert tmpl.render() == 'False' -- 2.26.2