From 8efc5226e16926f5cd92b0e41663c48d3844b886 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 8 Apr 2008 14:47:40 +0200 Subject: [PATCH] more compiler stuff --HG-- branch : trunk --- jinja2/compiler.py | 202 +++++++++++++++++++++++++++++++---- jinja2/datastructure.py | 10 +- jinja2/nodes.py | 231 ++++++++++++---------------------------- jinja2/parser.py | 7 +- jinja2/runtime.py | 35 +++--- test.py | 13 +-- 6 files changed, 287 insertions(+), 211 deletions(-) diff --git a/jinja2/compiler.py b/jinja2/compiler.py index 330db56..41577c8 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -8,6 +8,7 @@ :copyright: Copyright 2008 by Armin Ronacher. :license: GNU GPL. """ +from copy import copy from random import randrange from operator import xor from cStringIO import StringIO @@ -74,7 +75,9 @@ class Frame(object): def __init__(self, parent=None): self.identifiers = Identifiers() + self.toplevel = False self.parent = parent + self.block = parent and parent.block or None if parent is not None: self.identifiers.declared.update( parent.identifiers.declared | @@ -83,6 +86,12 @@ class Frame(object): parent.identifiers.declared_parameter ) + def copy(self): + """Create a copy of the current one.""" + rv = copy(self) + rv.identifiers = copy(self) + return rv + def inspect(self, nodes): """Walk the node and check for identifiers.""" visitor = FrameIdentifierVisitor(self.identifiers) @@ -113,7 +122,7 @@ class FrameIdentifierVisitor(NodeVisitor): self.identifiers.declared_locally.add(node.name) # stop traversing at instructions that have their own scope. - visit_Block = visit_Call = visit_FilterBlock = \ + visit_Block = visit_CallBlock = visit_FilterBlock = \ visit_For = lambda s, n: None @@ -139,8 +148,8 @@ class CodeGenerator(NodeVisitor): def indent(self): self.indentation += 1 - def outdent(self): - self.indentation -= 1 + def outdent(self, step=1): + self.indentation -= step def blockvisit(self, nodes, frame, force_generator=False): self.indent() @@ -170,6 +179,27 @@ class CodeGenerator(NodeVisitor): self.new_lines = 1 self._last_line = node.lineno + def signature(self, node, frame, have_comma=True): + have_comma = have_comma and [True] or [] + def touch_comma(): + if have_comma: + self.write(', ') + else: + have_comma.append(True) + + for arg in node.args: + touch_comma() + self.visit(arg, frame) + for kwarg in node.kwargs: + touch_comma() + self.visit(kwarg, frame) + if node.dyn_args: + touch_comma() + self.visit(node.dyn_args, frame) + if node.dyn_kwargs: + touch_comma() + self.visit(node.dyn_kwargs, frame) + def pull_locals(self, frame, no_indent=False): if not no_indent: self.indent() @@ -184,27 +214,35 @@ class CodeGenerator(NodeVisitor): assert frame is None, 'no root frame allowed' self.writeline('from jinja2.runtime import *') self.writeline('filename = %r' % self.filename) - self.writeline('context = TemplateContext(global_context, ' + self.writeline('template_context = TemplateContext(global_context, ' 'make_undefined, filename)') - # generate the body render function. - self.writeline('def body(context=context):', extra=1) + # generate the root render function. + self.writeline('def root(context=template_context):', extra=1) + self.indent() + self.writeline('parent_root = None') + self.outdent() frame = Frame() frame.inspect(node.body) + frame.toplevel = True self.pull_locals(frame) self.blockvisit(node.body, frame, True) - # top level changes to locals are pushed back to the - # context of *this* template for include. + # make sure that the parent root is called. self.indent() - self.writeline('context.from_locals(locals())') - self.outdent() + self.writeline('if parent_root is not None:') + self.indent() + self.writeline('for event in parent_root(context):') + self.indent() + self.writeline('yield event') + self.outdent(3) # at this point we now have the blocks collected and can visit them too. for name, block in self.blocks.iteritems(): block_frame = Frame() block_frame.inspect(block.body) - self.writeline('def block_%s(context=context):' % name, block, 1) + block_frame.block = name + self.writeline('def block_%s(context):' % name, block, 1) self.pull_locals(block_frame) self.blockvisit(block.body, block_frame, True) @@ -215,23 +253,32 @@ class CodeGenerator(NodeVisitor): node.name, node.lineno, self.filename) self.blocks[node.name] = node - self.writeline('for event in block_%s():' % node.name) + self.writeline('for event in block_%s(context):' % node.name) self.indent() self.writeline('yield event') self.outdent() def visit_Extends(self, node, frame): """Calls the extender.""" - self.writeline('extends(', node, 1) - self.visit(node.template) + if not frame.toplevel: + raise TemplateAssertionError('cannot use extend from a non ' + 'top-level scope', node.lineno, + self.filename) + self.writeline('if parent_root is not None:') + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'extended multiple times') + self.outdent() + self.writeline('parent_root = extends(', node, 1) + self.visit(node.template, frame) self.write(', globals())') def visit_For(self, node, frame): loop_frame = frame.inner() loop_frame.inspect(node.iter_child_nodes()) - loop_frame.identifiers.add_special('loop') extended_loop = bool(node.else_) or \ 'loop' in loop_frame.identifiers.undeclared + loop_frame.identifiers.add_special('loop') # make sure we "backup" overridden, local identifiers # TODO: we should probably optimize this and check if the @@ -257,9 +304,16 @@ class CodeGenerator(NodeVisitor): self.writeline('if l_loop is None:') self.blockvisit(node.else_, loop_frame) - # reset the aliases and clean them up + # reset the aliases and clean up + delete = set('l_' + x for x in loop_frame.identifiers.declared_locally + | loop_frame.identifiers.declared_parameter) + if extended_loop: + delete.add('l_loop') for name, alias in aliases.iteritems(): - self.writeline('l_%s = %s; del %s' % (name, alias, alias)) + self.writeline('l_%s = %s' % (name, alias)) + delete.add(alias) + delete.discard('l_' + name) + self.writeline('del %s' % ', '.join(delete)) def visit_If(self, node, frame): self.writeline('if ', node) @@ -270,6 +324,31 @@ class CodeGenerator(NodeVisitor): self.writeline('else:') self.blockvisit(node.else_, frame) + def visit_Macro(self, node, frame): + macro_frame = frame.inner() + macro_frame.inspect(node.body) + args = ['l_' + x.name for x in node.args] + if 'arguments' in macro_frame.identifiers.undeclared: + accesses_arguments = True + args.append('l_arguments') + else: + accesses_arguments = False + self.writeline('def macro(%s):' % ', '.join(args), node) + self.indent() + self.writeline('if 0: yield None') + self.outdent() + self.blockvisit(node.body, frame) + self.newline() + if frame.toplevel: + self.write('context[%r] = ' % node.name) + arg_tuple = ', '.join(repr(x.name) for x in node.args) + if len(node.args) == 1: + arg_tuple += ',' + self.write('l_%s = Macro(macro, %r, (%s), %s)' % ( + node.name, node.name, + arg_tuple, accesses_arguments + )) + def visit_ExprStmt(self, node, frame): self.newline(node) self.visit(node, frame) @@ -320,9 +399,27 @@ class CodeGenerator(NodeVisitor): self.visit(argument, frame) self.write(idx == 0 and ',)' or ')') + def visit_Assign(self, node, frame): + self.newline(node) + # toplevel assignments however go into the local namespace and + # the current template's context. We create a copy of the frame + # here and add a set so that the Name visitor can add the assigned + # names here. + if frame.toplevel: + assignment_frame = frame.copy() + assignment_frame.assigned_names = set() + else: + assignment_frame = frame + self.visit(node.target, assignment_frame) + self.write(' = ') + self.visit(node.node, frame) + if frame.toplevel: + for name in assignment_frame.assigned_names: + self.writeline('context[%r] = l_%s' % (name, name)) + def visit_Name(self, node, frame): - # at this point we should only have locals left as the - # blocks, macros and template body ensure that they are set. + if frame.toplevel and node.ctx == 'store': + frame.assigned_names.add(node.name) self.write('l_' + node.name) def visit_Const(self, node, frame): @@ -333,6 +430,15 @@ class CodeGenerator(NodeVisitor): else: self.write(repr(val)) + def visit_Tuple(self, node, frame): + self.write('(') + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(idx == 0 and ',)' or ')') + def binop(operator): def visitor(self, node, frame): self.write('(') @@ -373,8 +479,62 @@ class CodeGenerator(NodeVisitor): self.visit(node.expr, frame) def visit_Subscript(self, node, frame): - self.write('subscript(') + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write('[') + self.visit(node.arg, frame) + self.write(']') + return + try: + const = node.arg.as_const() + have_const = True + except nodes.Impossible: + have_const = False + if have_const: + if isinstance(const, (int, long, float)): + self.visit(node.node, frame) + self.write('[%s]' % const) + return + self.write('subscribe(') self.visit(node.node, frame) self.write(', ') - self.visit(node.arg, frame) + if have_const: + self.write(repr(const)) + else: + self.visit(node.arg, frame) + self.write(', make_undefined)') + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(':') + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(':') + self.visit(node.step, frame) + + def visit_Filter(self, node, frame): + for filter in node.filters: + self.write('context.filters[%r](' % filter.name) + self.visit(node.node, frame) + for filter in reversed(node.filters): + self.signature(filter, frame) + self.write(')') + + def visit_Test(self, node, frame): + self.write('context.tests[%r](') + self.visit(node.node, frame) + self.signature(node, frame) self.write(')') + + def visit_Call(self, node, frame): + self.visit(node.node, frame) + self.write('(') + self.signature(node, frame, False) + self.write(')') + + def visit_Keyword(self, node, frame): + self.visit(node.key, frame) + self.write('=') + self.visit(node.value, frame) diff --git a/jinja2/datastructure.py b/jinja2/datastructure.py index 11c2671..6b684c4 100644 --- a/jinja2/datastructure.py +++ b/jinja2/datastructure.py @@ -96,11 +96,11 @@ class TokenStream(object): def look(self): """Look at the next token.""" - old_token = self.current - next = self.next() - self.push(old_token) - self.push(next) - return next + old_token = self.next() + result = self.current + self.push(result) + self.current = old_token + return result def skip(self, n): """Got n tokens ahead.""" diff --git a/jinja2/nodes.py b/jinja2/nodes.py index e6e68a1..dc8cc0b 100644 --- a/jinja2/nodes.py +++ b/jinja2/nodes.py @@ -35,15 +35,14 @@ _uaop_to_func = { } - - class Impossible(Exception): - """ - Raised if the node could not perform a requested action. - """ + """Raised if the node could not perform a requested action.""" class NodeType(type): + """A metaclass for nodes that handles the field and attribute + inheritance. fields and attributes from the parent class are + automatically forwarded to the child.""" def __new__(cls, name, bases, d): for attr in 'fields', 'attributes': @@ -57,9 +56,7 @@ class NodeType(type): class Node(object): - """ - Baseclass for all Jinja nodes. - """ + """Baseclass for all Jinja nodes.""" __metaclass__ = NodeType fields = () attributes = ('lineno',) @@ -115,11 +112,10 @@ class Node(object): yield result def set_ctx(self, ctx): - """ - Reset the context of a node and all child nodes. Per default the parser - will all generate nodes that have a 'load' context as it's the most common - one. This method is used in the parser to set assignment targets and - other nodes to a store context. + """Reset the context of a node and all child nodes. Per default the + parser will all generate nodes that have a 'load' context as it's the + most common one. This method is used in the parser to set assignment + targets and other nodes to a store context. """ todo = deque([self]) while todo: @@ -137,28 +133,21 @@ class Node(object): class Stmt(Node): - """ - Base node for all statements. - """ + """Base node for all statements.""" class Helper(Node): - """ - Nodes that exist in a specific context only. - """ + """Nodes that exist in a specific context only.""" class Template(Node): - """ - Node that represents a template. - """ + """Node that represents a template.""" fields = ('body',) class Output(Stmt): - """ - A node that holds multiple expressions which are then printed out. This - is used both for the `print` statement and the regular template data. + """A node that holds multiple expressions which are then printed out. + This is used both for the `print` statement and the regular template data. """ fields = ('nodes',) @@ -179,112 +168,81 @@ class Output(Stmt): class Extends(Stmt): - """ - Represents an extends statement. - """ + """Represents an extends statement.""" fields = ('template',) class For(Stmt): - """ - A node that represents a for loop - """ + """A node that represents a for loop""" fields = ('target', 'iter', 'body', 'else_', 'recursive') class If(Stmt): - """ - A node that represents an if condition. - """ + """A node that represents an if condition.""" fields = ('test', 'body', 'else_') class Macro(Stmt): - """ - A node that represents a macro. - """ - fields = ('name', 'args', 'defaults', 'dyn_args', 'dyn_kwargs', 'body') + """A node that represents a macro.""" + fields = ('name', 'args', 'defaults', 'body') class CallBlock(Stmt): - """ - A node that represents am extended macro call. - """ + """A node that represents am extended macro call.""" fields = ('call', 'body') class Set(Stmt): - """ - Allows defining own variables. - """ + """Allows defining own variables.""" fields = ('name', 'expr') class FilterBlock(Stmt): - """ - Node for filter sections. - """ + """Node for filter sections.""" fields = ('body', 'filters') class Block(Stmt): - """ - A node that represents a block. - """ + """A node that represents a block.""" fields = ('name', 'body') class Include(Stmt): - """ - A node that represents the include tag. - """ + """A node that represents the include tag.""" fields = ('template', 'target') class Trans(Stmt): - """ - A node for translatable sections. - """ + """A node for translatable sections.""" fields = ('singular', 'plural', 'indicator', 'replacements') class ExprStmt(Stmt): - """ - A statement that evaluates an expression to None. - """ + """A statement that evaluates an expression to None.""" fields = ('node',) class Assign(Stmt): - """ - Assigns an expression to a target. - """ + """Assigns an expression to a target.""" fields = ('target', 'node') class Expr(Node): - """ - Baseclass for all expressions. - """ + """Baseclass for all expressions.""" def as_const(self): - """ - Return the value of the expression as constant or raise `Impossible` - if this was not possible. + """Return the value of the expression as constant or raise + `Impossible` if this was not possible. """ raise Impossible() def can_assign(self): - """ - Check if it's possible to assign something to this node. - """ + """Check if it's possible to assign something to this node.""" return False class BinExpr(Expr): - """ - Baseclass for all binary expressions. - """ + """Baseclass for all binary expressions.""" fields = ('left', 'right') operator = None @@ -293,14 +251,11 @@ class BinExpr(Expr): try: return f(self.left.as_const(), self.right.as_const()) except: - print self.left, f, self.right raise Impossible() class UnaryExpr(Expr): - """ - Baseclass for all unary expressions. - """ + """Baseclass for all unary expressions.""" fields = ('node',) operator = None @@ -313,9 +268,7 @@ class UnaryExpr(Expr): class Name(Expr): - """ - any name such as {{ foo }} - """ + """any name such as {{ foo }}""" fields = ('name', 'ctx') def can_assign(self): @@ -323,15 +276,11 @@ class Name(Expr): class Literal(Expr): - """ - Baseclass for literals. - """ + """Baseclass for literals.""" class Const(Literal): - """ - any constat such as {{ "foo" }} - """ + """any constat such as {{ "foo" }}""" fields = ('value',) def as_const(self): @@ -339,8 +288,7 @@ class Const(Literal): class Tuple(Literal): - """ - For loop unpacking and some other things like multiple arguments + """For loop unpacking and some other things like multiple arguments for subscripts. """ fields = ('items', 'ctx') @@ -356,9 +304,7 @@ class Tuple(Literal): class List(Literal): - """ - any list literal such as {{ [1, 2, 3] }} - """ + """any list literal such as {{ [1, 2, 3] }}""" fields = ('items',) def as_const(self): @@ -366,9 +312,7 @@ class List(Literal): class Dict(Literal): - """ - any dict literal such as {{ {1: 2, 3: 4} }} - """ + """any dict literal such as {{ {1: 2, 3: 4} }}""" fields = ('items',) def as_const(self): @@ -376,19 +320,20 @@ class Dict(Literal): class Pair(Helper): - """ - A key, value pair for dicts. - """ + """A key, value pair for dicts.""" fields = ('key', 'value') def as_const(self): return self.key.as_const(), self.value.as_const() +class Keyword(Helper): + """A key, value pair for keyword arguments.""" + fields = ('key', 'value') + + class CondExpr(Expr): - """ - {{ foo if bar else baz }} - """ + """{{ foo if bar else baz }}""" fields = ('test', 'expr1', 'expr2') def as_const(self): @@ -398,37 +343,27 @@ class CondExpr(Expr): class Filter(Expr): - """ - {{ foo|bar|baz }} - """ + """{{ foo|bar|baz }}""" fields = ('node', 'filters') class FilterCall(Expr): - """ - {{ |bar() }} - """ + """{{ |bar() }}""" fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') class Test(Expr): - """ - {{ foo is lower }} - """ + """{{ foo is lower }}""" fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') class Call(Expr): - """ - {{ foo(bar) }} - """ + """{{ foo(bar) }}""" fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') class Subscript(Expr): - """ - {{ foo.bar }} and {{ foo['bar'] }} etc. - """ + """{{ foo.bar }} and {{ foo['bar'] }} etc.""" fields = ('node', 'arg', 'ctx') def as_const(self): @@ -442,16 +377,12 @@ class Subscript(Expr): class Slice(Expr): - """ - 1:2:3 etc. - """ + """1:2:3 etc.""" fields = ('start', 'stop', 'step') class Concat(Expr): - """ - For {{ foo ~ bar }}. Concatenates strings. - """ + """For {{ foo ~ bar }}. Concatenates strings.""" fields = ('nodes',) def as_const(self): @@ -459,72 +390,52 @@ class Concat(Expr): class Compare(Expr): - """ - {{ foo == bar }}, {{ foo >= bar }} etc. - """ + """{{ foo == bar }}, {{ foo >= bar }} etc.""" fields = ('expr', 'ops') class Operand(Helper): - """ - Operator + expression. - """ + """Operator + expression.""" fields = ('op', 'expr') class Mul(BinExpr): - """ - {{ foo * bar }} - """ + """{{ foo * bar }}""" operator = '*' class Div(BinExpr): - """ - {{ foo / bar }} - """ + """{{ foo / bar }}""" operator = '/' class FloorDiv(BinExpr): - """ - {{ foo // bar }} - """ + """{{ foo // bar }}""" operator = '//' class Add(BinExpr): - """ - {{ foo + bar }} - """ + """{{ foo + bar }}""" operator = '+' class Sub(BinExpr): - """ - {{ foo - bar }} - """ + """{{ foo - bar }}""" operator = '-' class Mod(BinExpr): - """ - {{ foo % bar }} - """ + """{{ foo % bar }}""" operator = '%' class Pow(BinExpr): - """ - {{ foo ** bar }} - """ + """{{ foo ** bar }}""" operator = '**' class And(BinExpr): - """ - {{ foo and bar }} - """ + """{{ foo and bar }}""" operator = 'and' def as_const(self): @@ -532,9 +443,7 @@ class And(BinExpr): class Or(BinExpr): - """ - {{ foo or bar }} - """ + """{{ foo or bar }}""" operator = 'or' def as_const(self): @@ -542,21 +451,15 @@ class Or(BinExpr): class Not(UnaryExpr): - """ - {{ not foo }} - """ + """{{ not foo }}""" operator = 'not' class Neg(UnaryExpr): - """ - {{ -foo }} - """ + """{{ -foo }}""" operator = '-' class Pos(UnaryExpr): - """ - {{ +foo }} - """ + """{{ +foo }}""" operator = '+' diff --git a/jinja2/parser.py b/jinja2/parser.py index 96ba0a0..426e554 100644 --- a/jinja2/parser.py +++ b/jinja2/parser.py @@ -59,7 +59,7 @@ class Parser(object): self.stream.next() return self.parse_call_block() lineno = self.stream.current - expr = self.parse_expression() + expr = self.parse_tuple() if self.stream.current.type == 'assign': result = self.parse_assign(expr) else: @@ -202,6 +202,7 @@ class Parser(object): if self.stream.current.type is 'assign': self.stream.next() defaults.append(self.parse_expression()) + args.append(arg) self.stream.expect('rparen') node.body = self.parse_statements(('endmacro',), drop_needle=True) return node @@ -555,8 +556,8 @@ class Parser(object): self.stream.look().type is 'assign': key = self.stream.current.value self.stream.skip(2) - kwargs.append(nodes.Pair(key, self.parse_expression(), - lineno=key.lineno)) + kwargs.append(nodes.Keyword(key, self.parse_expression(), + lineno=key.lineno)) else: ensure(not kwargs) args.append(self.parse_expression()) diff --git a/jinja2/runtime.py b/jinja2/runtime.py index 8032823..8a1bdba 100644 --- a/jinja2/runtime.py +++ b/jinja2/runtime.py @@ -14,34 +14,45 @@ except ImportError: defaultdict = None -__all__ = ['extends', 'TemplateContext'] +__all__ = ['extends', 'subscribe', 'TemplateContext'] def extends(template, namespace): - """ - This loads a template (and evaluates it) and replaces the blocks. - """ + """This loads a template (and evaluates it) and replaces the blocks.""" + + +def subscribe(obj, argument, undefined_factory): + """Get an item or attribute of an object.""" + try: + return getattr(obj, argument) + except AttributeError: + try: + return obj[argument] + except LookupError: + return undefined_factory(attr=argument) class TemplateContext(dict): def __init__(self, globals, undefined_factory, filename): - dict.__init__(self, globals) + dict.__init__(self) + self.globals = globals self.undefined_factory = undefined_factory self.filename = filename + self.filters = {} + self.tests = {} # if there is a default dict, dict has a __missing__ method we can use. if defaultdict is None: def __getitem__(self, name): if name in self: return self[name] + elif name in self.globals: + return self.globals[name] return self.undefined_factory(name) else: def __missing__(self, key): - return self.undefined_factory(key) - - def from_locals(self, mapping): - """Update the template context from locals.""" - for key, value in mapping.iteritems(): - if key[:2] == 'l_': - self[key[:-2]] = value + try: + return self.globals[key] + except: + return self.undefined_factory(key) diff --git a/test.py b/test.py index d2f0653..7b5d78e 100644 --- a/test.py +++ b/test.py @@ -4,12 +4,13 @@ from jinja2.compiler import generate env = Environment() ast = env.parse(""" -Hello {{ name }}!. How is it {{ "going" }}. {{ [1, 2, 3] }}? -{% for name in user_names %} - {{ loop.index }}: {{ name }} - {% if loop.index % 2 == 0 %}...{% endif %} -{% endfor %} -Name again: {{ name }}! +{% (a, b), c = foo() %} +{% macro foo(a, b, c=42) %} + 42 {{ arguments }} +{% endmacro %} +{% block body %} + {% bar = 23 %} +{% endblock %} """) print ast print -- 2.26.2