From: Armin Ronacher Date: Fri, 12 Mar 2010 02:17:41 +0000 (+0100) Subject: added a :class:`ModuleLoader` that can load templates from X-Git-Tag: 2.4~24 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=64b08a0a84a0decd3778c93f3f5a82b75585e257;p=jinja2.git added a :class:`ModuleLoader` that can load templates from precompiled sources. The environment now features a method to compile the templates from a configured loader into a zip file or folder. --HG-- branch : trunk extra : rebase_source : 4824f663e4ff58ca3d2176c65fc1b7494e1f0c43 --- diff --git a/CHANGES b/CHANGES index 863ec14..7926e38 100644 --- a/CHANGES +++ b/CHANGES @@ -9,6 +9,10 @@ Version 2.4 pass through a template object if it was passed to it. This makes it possible to import or extend from a template object that was passed to the template. +- added a :class:`ModuleLoader` that can load templates from + precompiled sources. The environment now features a method + to compile the templates from a configured loader into a zip + file or folder. - the _speedups C extension now supports Python 3. diff --git a/jinja2/__init__.py b/jinja2/__init__.py index ed91fc5..2fc0219 100644 --- a/jinja2/__init__.py +++ b/jinja2/__init__.py @@ -38,7 +38,8 @@ from jinja2.environment import Environment, Template # loaders from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ - DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader # bytecode caches from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ @@ -64,6 +65,6 @@ __all__ = [ 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', - 'environmentfilter', 'contextfilter', 'Markup', 'escape', + 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined' ] diff --git a/jinja2/compiler.py b/jinja2/compiler.py index a52b1c7..ccf0811 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -53,11 +53,12 @@ def unoptimize_before_dead_code(): unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure) -def generate(node, environment, name, filename, stream=None): +def generate(node, environment, name, filename, stream=None, + defer_init=False): """Generate the python source for a node tree.""" if not isinstance(node, nodes.Template): raise TypeError('Can\'t compile non template nodes') - generator = CodeGenerator(environment, name, filename, stream) + generator = CodeGenerator(environment, name, filename, stream, defer_init) generator.visit(node) if stream is None: return generator.stream.getvalue() @@ -365,7 +366,8 @@ class CompilerExit(Exception): class CodeGenerator(NodeVisitor): - def __init__(self, environment, name, filename, stream=None): + def __init__(self, environment, name, filename, stream=None, + defer_init=False): if stream is None: stream = StringIO() self.environment = environment @@ -373,6 +375,7 @@ class CodeGenerator(NodeVisitor): self.filename = filename self.stream = stream self.created_block_context = False + self.defer_init = defer_init # aliases for imports self.import_aliases = {} @@ -753,6 +756,10 @@ class CodeGenerator(NodeVisitor): if not unoptimize_before_dead_code: self.writeline('dummy = lambda *x: None') + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ', environment=environment' or '' + # do we have an extends tag at all? If not, we can save some # overhead by just not processing any inheritance code. have_extends = node.find(nodes.Extends) is not None @@ -779,7 +786,7 @@ class CodeGenerator(NodeVisitor): self.writeline('name = %r' % self.name) # generate the root render function. - self.writeline('def root(context, environment=environment):', extra=1) + self.writeline('def root(context%s):' % envenv, extra=1) # process the root frame = Frame() @@ -814,8 +821,8 @@ class CodeGenerator(NodeVisitor): block_frame = Frame() block_frame.inspect(block.body) block_frame.block = name - self.writeline('def block_%s(context, environment=environment):' - % name, block, 1) + self.writeline('def block_%s(context%s):' % (name, envenv), + block, 1) self.indent() undeclared = find_undeclared(block.body, ('self', 'super')) if 'self' in undeclared: diff --git a/jinja2/environment.py b/jinja2/environment.py index 6404ed0..742bc1c 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -415,7 +415,8 @@ class Environment(object): return stream @internalcode - def compile(self, source, name=None, filename=None, raw=False): + def compile(self, source, name=None, filename=None, raw=False, + defer_init=False): """Compile a node or template source code. The `name` parameter is the load name of the template after it was joined using :meth:`join_path` if necessary, not the filename on the file system. @@ -427,6 +428,13 @@ class Environment(object): parameter is `True` the return value will be a string with python code equivalent to the bytecode returned otherwise. This method is mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. """ source_hint = None try: @@ -435,7 +443,8 @@ class Environment(object): source = self._parse(source, name, filename) if self.optimized: source = optimize(source, self) - source = generate(source, self, name, filename) + source = generate(source, self, name, filename, + defer_init=defer_init) if raw: return source if filename is None: @@ -491,6 +500,82 @@ class Environment(object): template = self.from_string(nodes.Template(body, lineno=1)) return TemplateExpression(template, undefined_to_none) + def compile_templates(self, target, extensions=None, filter_func=None, + zip=True, log_function=None): + """Compiles all the templates the loader can find, compiles them + and stores them in `target`. If `zip` is true, a zipfile will be + written, otherwise the templates are stored in a directory. + + `extensions` and `filter_func` are passed to :meth:`list_templates`. + Each template returned will be compiled to the target folder or + zipfile. + + .. versionadded:: 2.4 + """ + from jinja2.loaders import ModuleLoader + if log_function is None: + log_function = lambda x: None + + if zip: + from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED + f = ZipFile(target, 'w', ZIP_DEFLATED) + log_function('Compiling into Zip archive "%s"' % target) + else: + if not os.path.isdir(target): + os.makedirs(target) + log_function('Compiling into folder "%s"' % target) + + try: + for name in self.list_templates(extensions, filter_func): + source, filename, _ = self.loader.get_source(self, name) + try: + code = self.compile(source, name, filename, True, True) + except TemplateSyntaxError, e: + log_function('Could not compile "%s": %s' % (name, e)) + continue + module = ModuleLoader.get_module_filename(name) + if zip: + info = ZipInfo(module) + info.external_attr = 0755 << 16L + f.writestr(info, code) + else: + f = open(filename, 'w') + try: + f.write(code) + finally: + f.close() + log_function('Compiled "%s" as %s' % (name, module)) + finally: + if zip: + f.close() + + log_function('Finished compiling templates') + + def list_templates(self, extensions=None, filter_func=None): + """Returns a list of templates for this environment. This requires + that the loader supports the loader's + :meth:`~BaseLoader.list_templates` method. + + If there are other files in the template folder besides the + actual templates, the returned list can be filtered. There are two + ways: either `extensions` is set to a list of file extensions for + templates, or a `filter_func` can be provided which is a callable that + is passed a template name and should return `True` if it should end up + in the result list. + + If the loader does not support that, a :exc:`TypeError` is raised. + """ + x = self.loader.list_templates() + if extensions is not None: + if filter_func is not None: + raise TypeError('either extensions or filter_func ' + 'can be passed, but not both') + filter_func = lambda x: '.' in x and \ + x.rsplit('.', 1)[1] in extensions + if filter_func is not None: + x = filter(filter_func, x) + return x + def handle_exception(self, exc_info=None, rendered=False, source_hint=None): """Exception handling helper. This is used internally to either raise rewritten exceptions or return a rendered traceback for the template. @@ -679,16 +764,31 @@ class Template(object): """Creates a template object from compiled code and the globals. This is used by the loaders and environment to create a template object. """ - t = object.__new__(cls) namespace = { - 'environment': environment, - '__jinja_template__': t + 'environment': environment, + '__file__': code.co_filename } exec code in namespace + rv = cls._from_namespace(environment, namespace, globals) + rv._uptodate = uptodate + return rv + + @classmethod + def from_module_dict(cls, environment, module_dict, globals): + """Creates a template object from a module. This is used by the + module loader to create a template object. + + .. versionadded:: 2.4 + """ + return cls._from_namespace(environment, module_dict, globals) + + @classmethod + def _from_namespace(cls, environment, namespace, globals): + t = object.__new__(cls) t.environment = environment t.globals = globals t.name = namespace['name'] - t.filename = code.co_filename + t.filename = namespace['__file__'] t.blocks = namespace['blocks'] # render function and module @@ -697,7 +797,11 @@ class Template(object): # debug and loader helpers t._debug_info = namespace['debug_info'] - t._uptodate = uptodate + t._uptodate = None + + # store the reference + namespace['environment'] = environment + namespace['__jinja_template__'] = t return t diff --git a/jinja2/loaders.py b/jinja2/loaders.py index cfc738b..5c82b79 100644 --- a/jinja2/loaders.py +++ b/jinja2/loaders.py @@ -8,6 +8,10 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ +import os +import sys +import weakref +from types import ModuleType from os import path try: from hashlib import sha1 @@ -59,6 +63,12 @@ class BaseLoader(object): return source, path, lambda: mtime == getmtime(path) """ + #: if set to `False` it indicates that the loader cannot provide access + #: to the source of templates. + #: + #: .. versionadded:: 2.4 + has_source_access = True + def get_source(self, environment, template): """Get the template source, filename and reload helper for a template. It's passed the environment and template name and has to return a @@ -77,8 +87,17 @@ class BaseLoader(object): old state somewhere (for example in a closure). If it returns `False` the template will be reloaded. """ + if not self.has_source_access: + raise RuntimeError('%s cannot provide access to the source' % + self.__class__.__name__) raise TemplateNotFound(template) + def list_templates(self): + """Iterates over all templates. If the loader does not support that + it should raise a :exc:`TypeError` which is the default behavior. + """ + raise TypeError('this loader cannot iterate over all templates') + @internalcode def load(self, environment, name, globals=None): """Loads a template. This method looks up the template in the cache @@ -160,6 +179,20 @@ class FileSystemLoader(BaseLoader): return contents, filename, uptodate raise TemplateNotFound(template) + def list_templates(self): + found = set() + for searchpath in self.searchpath: + for dirpath, dirnames, filenames in os.walk(searchpath): + for filename in filenames: + template = os.path.join(dirpath, filename) \ + [len(searchpath):].strip(os.path.sep) \ + .replace(os.path.sep, '/') + if template[:2] == './': + template = template[2:] + if template not in found: + found.add(template) + return sorted(found) + class PackageLoader(BaseLoader): """Load templates from python eggs or packages. It is constructed with @@ -206,6 +239,26 @@ class PackageLoader(BaseLoader): source = self.provider.get_resource_string(self.manager, p) return source.decode(self.encoding), filename, uptodate + def list_templates(self): + path = self.package_path + if path[:2] == './': + path = path[2:] + elif path == '.': + path = '' + offset = len(path) + results = [] + def _walk(path): + for filename in self.provider.resource_listdir(path): + fullname = path + '/' + filename + if self.provider.resource_isdir(fullname): + for item in _walk(fullname): + results.append(item) + else: + results.append(fullname[offset:].lstrip('/')) + _walk(path) + results.sort() + return results + class DictLoader(BaseLoader): """Loads a template from a python dict. It's passed a dict of unicode @@ -225,6 +278,9 @@ class DictLoader(BaseLoader): return source, None, lambda: source != self.mapping.get(template) raise TemplateNotFound(template) + def list_templates(self): + return sorted(self.mapping) + class FunctionLoader(BaseLoader): """A loader that is passed a function which does the loading. The @@ -288,6 +344,13 @@ class PrefixLoader(BaseLoader): # (the one that includes the prefix) raise TemplateNotFound(template) + def list_templates(self): + result = [] + for prefix, loader in self.mapping.iteritems(): + for template in loader.list_templates(): + result.append(prefix + self.delimiter + template) + return result + class ChoiceLoader(BaseLoader): """This loader works like the `PrefixLoader` just that no prefix is @@ -313,3 +376,81 @@ class ChoiceLoader(BaseLoader): except TemplateNotFound: pass raise TemplateNotFound(template) + + def list_templates(self): + found = set() + for loader in self.loaders: + found.update(loader.list_templates()) + return sorted(found) + + +class _TemplateModule(ModuleType): + + def __init__(self, module): + if isinstance(module, basestring): + super(_TemplateModule, self).__init__(module) + else: + super(_TemplateModule, self).__init__(module.__name__) + self.__dict__.update(module.__dict__) + + +class ModuleLoader(BaseLoader): + """This loader loads templates from precompiled templates. + + Example usage: + + >>> loader = ChoiceLoader([ + ... ModuleLoader('/path/to/compiled/templates'), + ... FileSystemLoader('/path/to/templates') + ... ]) + """ + + has_source_access = False + + def __init__(self, path): + package_name = '_jinja2_module_templates_%x' % id(self) + + # create a fake module that looks for the templates in the + # path given. + mod = _TemplateModule(package_name) + if isinstance(path, basestring): + path = [path] + else: + path = list(path) + mod.__path__ = path + + sys.modules[package_name] = weakref.proxy(mod, + lambda x: sys.modules.pop(package_name, None)) + + # the only strong reference, the sys.modules entry is weak + # so that the garbage collector can remove it once the + # loader that created it goes out of business. + self.module = mod + self.package_name = package_name + + @staticmethod + def get_template_key(name): + return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest() + + @staticmethod + def get_module_filename(name): + return ModuleLoader.get_template_key(name) + '.py' + + @internalcode + def load(self, environment, name, globals=None): + key = self.get_template_key(name) + module = '%s.%s' % (self.package_name, key) + mod = getattr(self.module, module, None) + if mod is None: + try: + mod = _TemplateModule(__import__(module, None, + None, ['root'])) + except ImportError: + raise TemplateNotFound(name) + + # remove the entry from sys.modules, we only want the attribute + # on the module object we have stored on the loader. + sys.modules.pop(module, None) + + return environment.template_class.from_module_dict( + environment, mod.__dict__, globals) diff --git a/jinja2/testsuite/__init__.py b/jinja2/testsuite/__init__.py index 7099355..1f10ef6 100644 --- a/jinja2/testsuite/__init__.py +++ b/jinja2/testsuite/__init__.py @@ -38,6 +38,18 @@ class JinjaTestCase(unittest.TestCase): ### use only these methods for testing. If you need standard ### unittest method, wrap them! + def setup(self): + pass + + def teardown(self): + pass + + def setUp(self): + self.setup() + + def tearDown(self): + self.teardown() + def assert_equal(self, a, b): return self.assertEqual(a, b) diff --git a/jinja2/testsuite/loader.py b/jinja2/testsuite/loader.py index 5f61f7a..a46bda2 100644 --- a/jinja2/testsuite/loader.py +++ b/jinja2/testsuite/loader.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for more details. """ import os +import gc +import sys import time import tempfile import unittest @@ -98,7 +100,46 @@ class LoaderTestCase(JinjaTestCase): self.assert_raises(TemplateNotFound, split_template_path, '../foo') +class ModuleLoaderTestCase(JinjaTestCase): + archive = None + + def setup(self): + super(ModuleLoaderTestCase, self).setup() + self.reg_env = Environment(loader=prefix_loader) + self.archive = tempfile.mkstemp(suffix='.zip')[1] + self.reg_env.compile_templates(self.archive) + self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive)) + + def teardown(self): + super(ModuleLoaderTestCase, self).teardown() + os.remove(self.archive) + self.archive = None + + def test_module_loader(self): + tmpl1 = self.reg_env.get_template('a/test.html') + tmpl2 = self.mod_env.get_template('a/test.html') + assert tmpl1.render() == tmpl2.render() + + tmpl1 = self.reg_env.get_template('b/justdict.html') + tmpl2 = self.mod_env.get_template('b/justdict.html') + assert tmpl1.render() == tmpl2.render() + + def test_weak_references(self): + tmpl = self.mod_env.get_template('a/test.html') + key = loaders.ModuleLoader.get_template_key('a/test.html') + name = self.mod_env.loader.module.__name__ + + assert hasattr(self.mod_env.loader.module, key) + assert name in sys.modules + + # unset all, ensure the module is gone from sys.modules + self.mod_env = tmpl = None + gc.collect() + assert name not in sys.modules + + def suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(LoaderTestCase)) + suite.addTest(unittest.makeSuite(ModuleLoaderTestCase)) return suite