From 487a41889f51ba9fc4d14d2ccaeb4ff2d58e6623 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 1 Mar 2007 19:01:07 +0100 Subject: [PATCH] [svn] checked in todays jinja changes --HG-- branch : trunk --- jinja/datastructure.py | 34 +++++++++++++-- jinja/environment.py | 47 ++++++++++++++------ jinja/exceptions.py | 6 +++ jinja/filters.py | 6 +-- jinja/lexer.py | 10 +---- jinja/loaders.py | 18 +++++--- jinja/translators/python.py | 86 ++++++++++++++++++++++++++----------- jinja/utils.py | 34 +++++++++++++++ 8 files changed, 179 insertions(+), 62 deletions(-) create mode 100644 jinja/utils.py diff --git a/jinja/datastructure.py b/jinja/datastructure.py index 96bce49..a86d0ea 100644 --- a/jinja/datastructure.py +++ b/jinja/datastructure.py @@ -17,6 +17,12 @@ except NameError: def _reversed(c): return c[::-1] +# sets +try: + set +except NameError: + from sets import Set as set + class UndefinedType(object): """ @@ -77,6 +83,16 @@ class Deferred(object): return self.factory(context, name) +class Markup(unicode): + """ + Mark a string as safe for XML. If the environment uses the + auto_escape option values marked as `Markup` aren't escaped. + """ + + +safe_types = set([Markup, int, long, float]) + + class Context(object): """ Dict like object. @@ -84,8 +100,11 @@ class Context(object): def __init__(self, _environment_, *args, **kwargs): self.environment = _environment_ - self._stack = [self.environment.globals, dict(*args, **kwargs), {}] - self.globals, _, self.current = self._stack + self._stack = [self.environment.globals, dict(*args, **kwargs), {}, {}] + self.globals, self.initial, self.current = self._stack + + # cache object used for filters and tests + self.cache = {} def pop(self): if len(self._stack) <= 2: @@ -116,7 +135,12 @@ class Context(object): if name in d: rv = d[name] if isinstance(rv, Deferred): - d[name] = rv = rv(self, name) + rv = rv(self, name) + # never tough the globals! + if d is self.globals: + self.initial[name] = rv + else: + d[name] = rv return rv return Undefined @@ -142,7 +166,7 @@ class LoopContext(object): """ jinja_allowed_attributes = ['index', 'index0', 'length', 'parent', - 'even', 'odd'] + 'even', 'odd', 'revindex0', 'revindex'] def __init__(self, seq, parent, loop_function): self.loop_function = loop_function @@ -164,6 +188,8 @@ class LoopContext(object): iterated = property(lambda s: s._stack[-1]['index'] > -1) index0 = property(lambda s: s._stack[-1]['index']) index = property(lambda s: s._stack[-1]['index'] + 1) + revindex0 = property(lambda s: s._stack[-1]['length'] - s._stack[-1]['index'] - 1) + revindex = property(lambda s: s._stack[-1]['length'] - s._stack[-1]['index']) length = property(lambda s: s._stack[-1]['length']) even = property(lambda s: s._stack[-1]['index'] % 2 == 0) odd = property(lambda s: s._stack[-1]['index'] % 2 == 1) diff --git a/jinja/environment.py b/jinja/environment.py index b8d05b4..ffce6b3 100644 --- a/jinja/environment.py +++ b/jinja/environment.py @@ -8,11 +8,13 @@ :copyright: 2006 by Armin Ronacher. :license: BSD, see LICENSE for more details. """ +import re from jinja.lexer import Lexer from jinja.parser import Parser from jinja.loaders import LoaderWrapper from jinja.datastructure import Undefined -from jinja.exceptions import FilterNotFound, TestNotFound +from jinja.utils import escape +from jinja.exceptions import FilterNotFound, TestNotFound, SecurityException from jinja.defaults import DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE @@ -28,6 +30,8 @@ class Environment(object): variable_end_string='}}', comment_start_string='{#', comment_end_string='#}', + trim_blocks=False, + auto_escape=False, template_charset='utf-8', charset='utf-8', namespace=None, @@ -42,6 +46,7 @@ class Environment(object): self.variable_end_string = variable_end_string self.comment_start_string = comment_start_string self.comment_end_string = comment_end_string + self.trim_blocks = trim_blocks # other stuff self.template_charset = template_charset @@ -49,6 +54,7 @@ class Environment(object): self.loader = loader self.filters = filters is None and DEFAULT_FILTERS.copy() or filters self.tests = tests is None and DEFAULT_TESTS.copy() or tests + self.auto_escape = auto_escape # global namespace self.globals = namespace is None and DEFAULT_NAMESPACE.copy() \ @@ -75,6 +81,11 @@ class Environment(object): from jinja.translators.python import PythonTranslator return PythonTranslator.process(self, Parser(self, source).parse()) + def get_template(self, filename): + """Load a template from a filename. Only works + if a proper loader is set.""" + return self._loader.load(filename) + def to_unicode(self, value): """ Convert a value to unicode with the rules defined on the environment. @@ -87,28 +98,36 @@ class Environment(object): except UnicodeError: return str(value).decode(self.charset, 'ignore') - def apply_filters(self, value, filtercache, context, filters): + def apply_filters(self, value, context, filters): """ Apply a list of filters on the variable. """ for key in filters: - if key in filtercache: - func = filtercache[key] + if key in context.cache: + func = context.cache[key] else: filtername, args = key if filtername not in self.filters: raise FilterNotFound(filtername) - filtercache[key] = func = self.filters[filtername](*args) + context.cache[key] = func = self.filters[filtername](*args) value = func(self, context, value) return value - def perform_test(self, context, testname, value): + def perform_test(self, context, testname, args, value, invert): """ Perform a test on a variable. """ - if testname not in self.tests: - raise TestNotFound(testname) - return bool(self.tests[testname](self, context, value)) + key = (testname, args) + if key in context.cache: + func = context.cache[key] + else: + if testname not in self.tests: + raise TestNotFound(testname) + context.cache[key] = func = self.tests[testname](*args) + rv = func(self, context, value) + if invert: + return not rv + return bool(rv) def get_attribute(self, obj, name): """ @@ -121,7 +140,7 @@ class Environment(object): r = getattr(obj, 'jinja_allowed_attributes', None) if r is not None: if name not in r: - raise AttributeError() + raise SecurityException('unsafe attributed %r accessed' % name) return rv return Undefined @@ -150,10 +169,10 @@ class Environment(object): """ As long as no write_var function is passed to the template evaluator the source generated by the python translator will - call this function for all variables. You can use this to - enable output escaping etc or just ensure that None and - Undefined values are rendered as empty strings. + call this function for all variables. """ - if value is None or value is Undefined: + if value is Undefined: return u'' + elif self.auto_escape: + return escape(value, True) return unicode(value) diff --git a/jinja/exceptions.py b/jinja/exceptions.py index d5f2af7..da634ed 100644 --- a/jinja/exceptions.py +++ b/jinja/exceptions.py @@ -14,6 +14,12 @@ class TemplateError(RuntimeError): pass +class SecurityException(TemplateError): + """ + Raise if the template designer tried to do something dangerous. + """ + + class FilterNotFound(KeyError, TemplateError): """ Raised if a filter does not exist. diff --git a/jinja/filters.py b/jinja/filters.py index 1d0330a..e45849d 100644 --- a/jinja/filters.py +++ b/jinja/filters.py @@ -10,6 +10,7 @@ """ from random import choice from urllib import urlencode, quote +from jinja.utils import escape try: @@ -83,10 +84,7 @@ def do_escape(s, attribute=False): XML escape &, <, and > in a string of data. If attribute is True it also converts ``"`` to ``"`` """ - s = s.replace("&", "&").replace("<", "<").replace(">", ">") - if attribute: - s = s.replace('"', """) - return s + return escape(s, attribute) do_escape = stringfilter(do_escape) diff --git a/jinja/lexer.py b/jinja/lexer.py index 2d56adb..a2595fa 100644 --- a/jinja/lexer.py +++ b/jinja/lexer.py @@ -81,13 +81,6 @@ class Lexer(object): (c('(.*?)(?:%s)' % '|'.join([ '(?P<%s_begin>%s)' % (n, e(r)) for n, r in root_tag_rules ])), ('data', '#bygroup'), '#bygroup'), - #(c('(.*?)(?:(?P' + - # e(environment.comment_start_string) + - # ')|(?P' + - # e(environment.block_start_string) + - # ')|(?P' + - # e(environment.variable_start_string) + - # '))'), ('data', '#bygroup'), '#bygroup'), (c('.+'), 'data', None) ], 'comment_begin': [ @@ -96,7 +89,8 @@ class Lexer(object): (c('(.)'), (Failure('Missing end of comment tag'),), None) ], 'block_begin': [ - (c(e(environment.block_end_string)), 'block_end', '#pop') + (c(e(environment.block_end_string) + + (environment.trim_blocks and '\\n?' or '')), 'block_end', '#pop') ] + tag_rules, 'variable_begin': [ (c(e(environment.variable_end_string)), 'variable_end', diff --git a/jinja/loaders.py b/jinja/loaders.py index b2903e3..a7a1221 100644 --- a/jinja/loaders.py +++ b/jinja/loaders.py @@ -32,19 +32,20 @@ class LoaderWrapper(object): def __init__(self, environment, loader): self.environment = environment self.loader = loader + if self.loader is None: + self.get_source = self.parse = self.load = self._loader_missing + self.available = False + else: + self.available = True def get_source(self, name, parent=None): - """ - Retrieve the sourcecode of a template. - """ + """Retrieve the sourcecode of a template.""" # just ascii chars are allowed as template names name = str(name) return self.loader.get_source(self.environment, name, parent) def parse(self, name, parent=None): - """ - Retreive a template and parse it. - """ + """Retreive a template and parse it.""" # just ascii chars are allowed as template names name = str(name) return self.loader.parse(self.environment, name, parent) @@ -59,6 +60,11 @@ class LoaderWrapper(object): name = str(name) return self.loader.load(self.environment, name, translator) + def _loader_missing(self, *args, **kwargs): + """Helper method that overrides all other methods if no + loader is defined.""" + raise RuntimeError('no loader defined') + class FileSystemLoader(object): """ diff --git a/jinja/translators/python.py b/jinja/translators/python.py index a8e3fde..bb15a2d 100644 --- a/jinja/translators/python.py +++ b/jinja/translators/python.py @@ -31,14 +31,29 @@ class Template(object): Represents a finished template. """ - def __init__(self, environment, generate_func): + def __init__(self, environment, code): self.environment = environment - self.generate_func = generate_func + self.code = code + self.generate_func = None + + def dump(self, filename): + """Dump the template into python bytecode.""" + from marshal import dumps + return dumps(self.code) + + def load(environment, data): + """Load the template from python bytecode.""" + from marshal import loads + code = loads(data) + return Template(environment, code) + load = staticmethod(load) def render(self, *args, **kwargs): - """ - Render a template. - """ + """Render a template.""" + if self.generate_func is None: + ns = {} + exec self.code in ns + self.generate_func = ns['generate'] result = [] ctx = Context(self.environment, *args, **kwargs) self.generate_func(ctx, result.append) @@ -114,10 +129,8 @@ class PythonTranslator(Translator): def process(environment, node): translator = PythonTranslator(environment, node) - source = translator.translate() - ns = {} - exec source in ns - return Template(environment, ns['generate']) + return Template(environment, + compile(translator.translate(), node.filename, 'exec')) process = staticmethod(process) # -- private methods @@ -149,8 +162,6 @@ class PythonTranslator(Translator): Handle the overall template node. This node is the first node and ensures that we get the bootstrapping code. It also knows about inheritance information. It only occours as outer node, never in the tree itself. - - Nevertheless we call indent here to simplify futur changes. """ # if there is a parent template we parse the parent template and # update the blocks there. Once this is done we drop the current @@ -164,7 +175,7 @@ class PythonTranslator(Translator): block.replace(node.blocks[block.name]) node = tmpl - lines = [self.indent( + lines = [ 'from jinja.datastructure import Undefined, LoopContext, CycleContext\n\n' 'def generate(context, write):\n' ' # BOOTSTRAPPING CODE\n' @@ -175,12 +186,12 @@ class PythonTranslator(Translator): ' call_function = environment.call_function\n' ' call_function_simple = environment.call_function_simple\n' ' finish_var = environment.finish_var\n' - ' write_var = lambda x: write(finish_var(x))\n' - ' filtercache = {}\n\n' + ' write_var = lambda x: write(finish_var(x))\n\n' ' # TEMPLATE CODE' - )] + ] self.indention += 1 lines.append(self.handle_node_list(node)) + self.indention -= 1 return '\n'.join(lines) @@ -281,9 +292,9 @@ class PythonTranslator(Translator): write('if not %r in context.current:' % name) self.indention += 1 if node.seq.__class__ in (ast.Tuple, ast.List): - write('context.current[%r] = CycleContext([%s])' % ( + write('context.current[%r] = CycleContext(%s)' % ( name, - ', '.join([self.handle_node(n) for n in node.seq.nodes]) + _to_tuple([self.handle_node(n) for n in node.seq.nodes]) )) hardcoded = True else: @@ -370,23 +381,46 @@ class PythonTranslator(Translator): # the semantic for the is operator is different. # for jinja the is operator performs tests and must # be the only operator - if node.ops[0][0] == 'is': + if node.ops[0][0] in ('is', 'is not'): if len(node.ops) > 1: raise TemplateSyntaxError('is operator must not be chained', node.lineno) - elif node.ops[0][1].__class__ is not ast.Name: - raise TemplateSyntaxError('is operator requires a test name', + elif node.ops[0][1].__class__ is ast.Name: + args = [] + name = node.ops[0][1].name + elif node.ops[0][1].__class__ is ast.CallFunc: + n = node.ops[0][1] + if n.node.__class__ is not ast.Name: + raise TemplateSyntaxError('invalid test. test must ' + 'be a hardcoded function name ' + 'from the test namespace', + n.lineno) + name = n.node.name + args = [] + for arg in n.args: + if arg.__class__ is ast.Keyword: + raise TemplateSyntaxError('keyword arguments for ' + 'tests are not supported.', + n.lineno) + args.append(self.handle_node(arg)) + if n.star_args is not None or n.dstar_args is not None: + raise TemplateSynaxError('*args / **kwargs is not supported ' + 'for tests', n.lineno) + else: + raise TemplateSyntaxError('is operator requires a test name' ' as operand', node.lineno) - return 'perform_test(context, %r, %s)' % ( - node.ops[0][1].name, - self.handle_node(node.expr) - ) + return 'perform_test(context, %r, %s, %s, %s)' % ( + name, + _to_tuple(args), + self.handle_node(node.expr), + node.ops[0][0] == 'is not' + ) # normal operators buf = [] buf.append(self.handle_node(node.expr)) for op, n in node.ops: - if op == 'is': + if op in ('is', 'is not'): raise TemplateSyntaxError('is operator must not be chained', node.lineno) buf.append(op) @@ -465,7 +499,7 @@ class PythonTranslator(Translator): 'hardcoded function name from the ' 'filter namespace', n.lineno) - return 'apply_filters(%s, filtercache, context, %s)' % ( + return 'apply_filters(%s, context, %s)' % ( self.handle_node(node.nodes[0]), _to_tuple(filters) ) diff --git a/jinja/utils.py b/jinja/utils.py new file mode 100644 index 0000000..de3f570 --- /dev/null +++ b/jinja/utils.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" + jinja.utils + ~~~~~~~~~~~ + + Utility functions. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +import re +from jinja.datastructure import safe_types + + +_escape_pairs = { + '&': '&', + '<': '<', + '>': '>', + '"': '"' +} + +_escape_res = ( + re.compile('(&|<|>|")'), + re.compile('(&|<|>)') +) + +def escape(x, attribute=False): + """ + Escape an object x which is converted to unicode first. + """ + if type(x) in safe_types: + return x + return _escape_res[not attribute].sub(lambda m: _escape_pairs[m.group()], + unicode(x)) -- 2.26.2