--- /dev/null
+# -*- coding: utf-8 -*-
+"""
+ jinja2.compiler
+ ~~~~~~~~~~~~~~~
+
+ Compiles nodes into python code.
+
+ :copyright: Copyright 2008 by Armin Ronacher.
+ :license: GNU GPL.
+"""
+from random import randrange
+from operator import xor
+from cStringIO import StringIO
+from jinja2 import nodes
+from jinja2.visitor import NodeVisitor, NodeTransformer
+from jinja2.exceptions import TemplateAssertionError
+
+
+operators = {
+ 'eq': '==',
+ 'ne': '!=',
+ 'gt': '>',
+ 'gteq': '>=',
+ 'lt': '<',
+ 'lteq': '<=',
+ 'in': 'in',
+ 'notin': 'not in'
+}
+
+
+def generate(node, filename, stream=None):
+ is_child = node.find(nodes.Extends) is not None
+ generator = CodeGenerator(is_child, filename, stream)
+ generator.visit(node)
+ if stream is None:
+ return generator.stream.getvalue()
+
+
+class Identifiers(object):
+ """Tracks the status of identifiers in frames."""
+
+ def __init__(self):
+ # variables that are known to be declared (probably from outer
+ # frames or because they are special for the frame)
+ self.declared = set()
+
+ # names that are accessed without being explicitly declared by
+ # this one or any of the outer scopes. Names can appear both in
+ # declared and undeclared.
+ self.undeclared = set()
+
+ # names that are declared locally
+ self.declared_locally = set()
+
+ # names that are declared by parameters
+ self.declared_parameter = set()
+
+ def add_special(self, name):
+ """Register a special name like `loop`."""
+ self.undeclared.discard(name)
+ self.declared.add(name)
+
+ def is_declared(self, name):
+ """Check if a name is declared in this or an outer scope."""
+ return name in self.declared or name in self.declared_locally or \
+ name in self.declared_parameter
+
+ def find_shadowed(self):
+ """Find all the shadowed names."""
+ return self.declared & (self.declared_locally | self.declared_parameter)
+
+
+class Frame(object):
+
+ def __init__(self, parent=None):
+ self.identifiers = Identifiers()
+ self.parent = parent
+ if parent is not None:
+ self.identifiers.declared.update(
+ parent.identifiers.declared |
+ parent.identifiers.undeclared |
+ parent.identifiers.declared_locally |
+ parent.identifiers.declared_parameter
+ )
+
+ def inspect(self, nodes):
+ """Walk the node and check for identifiers."""
+ visitor = FrameIdentifierVisitor(self.identifiers)
+ for node in nodes:
+ visitor.visit(node)
+
+ def inner(self):
+ """Return an inner frame."""
+ return Frame(self)
+
+
+class FrameIdentifierVisitor(NodeVisitor):
+ """A visitor for `Frame.inspect`."""
+
+ def __init__(self, identifiers):
+ self.identifiers = identifiers
+
+ def visit_Name(self, node):
+ """All assignments to names go through this function."""
+ if node.ctx in ('store', 'param'):
+ self.identifiers.declared_locally.add(node.name)
+ elif node.ctx == 'load':
+ if not self.identifiers.is_declared(node.name):
+ self.identifiers.undeclared.add(node.name)
+
+ def visit_Macro(self, node):
+ """Macros set local."""
+ self.identifiers.declared_locally.add(node.name)
+
+ # stop traversing at instructions that have their own scope.
+ visit_Block = visit_Call = visit_FilterBlock = \
+ visit_For = lambda s, n: None
+
+
+class CodeGenerator(NodeVisitor):
+
+ def __init__(self, is_child, filename, stream=None):
+ if stream is None:
+ stream = StringIO()
+ self.is_child = is_child
+ self.filename = filename
+ self.stream = stream
+ self.blocks = {}
+ self.indentation = 0
+ self.new_lines = 0
+ self.last_identifier = 0
+ self._last_line = 0
+ self._first_write = True
+
+ def temporary_identifier(self):
+ self.last_identifier += 1
+ return 't%d' % self.last_identifier
+
+ def indent(self):
+ self.indentation += 1
+
+ def outdent(self):
+ self.indentation -= 1
+
+ def blockvisit(self, nodes, frame, force_generator=False):
+ self.indent()
+ if force_generator:
+ self.writeline('if 0: yield None')
+ for node in nodes:
+ self.visit(node, frame)
+ self.outdent()
+
+ def write(self, x):
+ if self.new_lines:
+ if not self._first_write:
+ self.stream.write('\n' * self.new_lines)
+ self._first_write = False
+ self.stream.write(' ' * self.indentation)
+ self.new_lines = 0
+ self.stream.write(x)
+
+ def writeline(self, x, node=None, extra=0):
+ self.newline(node, extra)
+ self.write(x)
+
+ def newline(self, node=None, extra=0):
+ self.new_lines = max(self.new_lines, 1 + extra)
+ if node is not None and node.lineno != self._last_line:
+ self.write('# line: %s' % node.lineno)
+ self.new_lines = 1
+ self._last_line = node.lineno
+
+ def pull_locals(self, frame, no_indent=False):
+ if not no_indent:
+ self.indent()
+ for name in frame.identifiers.undeclared:
+ self.writeline('l_%s = context[%r]' % (name, name))
+ if not no_indent:
+ self.outdent()
+
+ # -- Visitors
+
+ def visit_Template(self, node, frame=None):
+ assert frame is None, 'no root frame allowed'
+ self.writeline('from jinja2.runtime import *')
+ self.writeline('filename = %r' % self.filename)
+ self.writeline('context = TemplateContext(global_context, '
+ 'make_undefined, filename)')
+
+ # generate the body render function.
+ self.writeline('def body(context=context):', extra=1)
+ frame = Frame()
+ frame.inspect(node.body)
+ self.pull_locals(frame)
+ self.blockvisit(node.body, frame, True)
+
+ # top level changes to locals are pushed back to the
+ # context of *this* template for include.
+ self.indent()
+ self.writeline('context.from_locals(locals())')
+ self.outdent()
+
+ # at this point we now have the blocks collected and can visit them too.
+ for name, block in self.blocks.iteritems():
+ block_frame = Frame()
+ block_frame.inspect(block.body)
+ self.writeline('def block_%s(context=context):' % name, block, 1)
+ self.pull_locals(block_frame)
+ self.blockvisit(block.body, block_frame, True)
+
+ def visit_Block(self, node, frame):
+ """Call a block and register it for the template."""
+ if node.name in self.blocks:
+ raise TemplateAssertionError("the block '%s' was already defined" %
+ node.name, node.lineno,
+ self.filename)
+ self.blocks[node.name] = node
+ self.writeline('for event in block_%s():' % node.name)
+ self.indent()
+ self.writeline('yield event')
+ self.outdent()
+
+ def visit_Extends(self, node, frame):
+ """Calls the extender."""
+ self.writeline('extends(', node, 1)
+ self.visit(node.template)
+ self.write(', globals())')
+
+ def visit_For(self, node, frame):
+ loop_frame = frame.inner()
+ loop_frame.inspect(node.iter_child_nodes())
+ loop_frame.identifiers.add_special('loop')
+ extended_loop = bool(node.else_) or \
+ 'loop' in loop_frame.identifiers.undeclared
+
+ # make sure we "backup" overridden, local identifiers
+ # TODO: we should probably optimize this and check if the
+ # identifier is in use afterwards.
+ aliases = {}
+ for name in loop_frame.identifiers.find_shadowed():
+ aliases[name] = ident = self.temporary_identifier()
+ self.writeline('%s = l_%s' % (ident, name))
+
+ self.pull_locals(loop_frame, True)
+
+ self.newline(node)
+ if node.else_:
+ self.writeline('l_loop = None')
+ self.write('for ')
+ self.visit(node.target, loop_frame)
+ self.write(extended_loop and ', l_loop in looper(' or ' in ')
+ self.visit(node.iter, loop_frame)
+ self.write(extended_loop and '):' or ':')
+ self.blockvisit(node.body, loop_frame)
+
+ if node.else_:
+ self.writeline('if l_loop is None:')
+ self.blockvisit(node.else_, loop_frame)
+
+ # reset the aliases and clean them up
+ for name, alias in aliases.iteritems():
+ self.writeline('l_%s = %s; del %s' % (name, alias, alias))
+
+ def visit_If(self, node, frame):
+ self.writeline('if ', node)
+ self.visit(node.test, frame)
+ self.write(':')
+ self.blockvisit(node.body, frame)
+ if node.else_:
+ self.writeline('else:')
+ self.blockvisit(node.else_, frame)
+
+ def visit_ExprStmt(self, node, frame):
+ self.newline(node)
+ self.visit(node, frame)
+
+ def visit_Output(self, node, frame):
+ self.newline(node)
+
+ # try to evaluate as many chunks as possible into a static
+ # string at compile time.
+ body = []
+ for child in node.nodes:
+ try:
+ const = unicode(child.as_const())
+ except:
+ body.append(child)
+ continue
+ if body and isinstance(body[-1], list):
+ body[-1].append(const)
+ else:
+ body.append([const])
+
+ # if we have less than 3 nodes we just yield them
+ if len(body) < 3:
+ for item in body:
+ if isinstance(item, list):
+ self.writeline('yield %s' % repr(u''.join(item)))
+ else:
+ self.newline(item)
+ self.write('yield unicode(')
+ self.visit(item, frame)
+ self.write(')')
+
+ # otherwise we create a format string as this is faster in that case
+ else:
+ format = []
+ arguments = []
+ for item in body:
+ if isinstance(item, list):
+ format.append(u''.join(item).replace('%', '%%'))
+ else:
+ format.append('%s')
+ arguments.append(item)
+ self.writeline('yield %r %% (' % u''.join(format))
+ idx = -1
+ for idx, argument in enumerate(arguments):
+ if idx:
+ self.write(', ')
+ self.visit(argument, frame)
+ self.write(idx == 0 and ',)' or ')')
+
+ def visit_Name(self, node, frame):
+ # at this point we should only have locals left as the
+ # blocks, macros and template body ensure that they are set.
+ self.write('l_' + node.name)
+
+ def visit_Const(self, node, frame):
+ val = node.value
+ if isinstance(val, float):
+ # XXX: add checks for infinity and nan
+ self.write(str(val))
+ else:
+ self.write(repr(val))
+
+ def binop(operator):
+ def visitor(self, node, frame):
+ self.write('(')
+ self.visit(node.left, frame)
+ self.write(' %s ' % operator)
+ self.visit(node.right, frame)
+ self.write(')')
+ return visitor
+
+ def uaop(operator):
+ def visitor(self, node, frame):
+ self.write('(' + operator)
+ self.visit(node.node)
+ self.write(')')
+ return visitor
+
+ visit_Add = binop('+')
+ visit_Sub = binop('-')
+ visit_Mul = binop('*')
+ visit_Div = binop('/')
+ visit_FloorDiv = binop('//')
+ visit_Pow = binop('**')
+ visit_Mod = binop('%')
+ visit_And = binop('and')
+ visit_Or = binop('or')
+ visit_Pos = uaop('+')
+ visit_Neg = uaop('-')
+ visit_Not = uaop('not ')
+ del binop, uaop
+
+ def visit_Compare(self, node, frame):
+ self.visit(node.expr, frame)
+ for op in node.ops:
+ self.visit(op, frame)
+
+ def visit_Operand(self, node, frame):
+ self.write(' %s ' % operators[node.op])
+ self.visit(node.expr, frame)
+
+ def visit_Subscript(self, node, frame):
+ self.write('subscript(')
+ self.visit(node.node, frame)
+ self.write(', ')
+ self.visit(node.arg, frame)
+ self.write(')')
if token.type == 'eof':
self._stream.close()
raise StopIteration()
- self._stream.next()
+ self._stream.next(False)
return token
for x in xrange(n):
self.next()
- def next(self):
- """Go one token ahead."""
- if self._pushed:
- self.current = self._pushed.popleft()
- elif self.current.type != 'eof':
- try:
- self.current = self._next()
- except StopIteration:
- self.close()
+ def next(self, skip_eol=True):
+ """Go one token ahead and return the old one"""
+ rv = self.current
+ while 1:
+ if self._pushed:
+ self.current = self._pushed.popleft()
+ elif self.current.type is not 'eof':
+ try:
+ self.current = self._next()
+ except StopIteration:
+ self.close()
+ if not skip_eol or self.current.type is not 'eol':
+ break
+ return rv
def close(self):
"""Close the stream."""
"""
-class TemplateError(RuntimeError):
+class TemplateError(Exception):
pass
"""
def __init__(self, message, lineno, filename):
- SyntaxError.__init__(self, message)
+ SyntaxError.__init__(self, '%s (line %s)' % (message, lineno))
+ self.message = message
self.lineno = lineno
self.filename = filename
+class TemplateAssertionError(AssertionError, TemplateSyntaxError):
+
+ def __init__(self, message, lineno, filename):
+ AssertionError.__init__(self, message)
+ TemplateSyntaxError.__init__(self, message, lineno, filename)
+
+
class TemplateRuntimeError(TemplateError):
"""
Raised by the template engine if a tag encountered an error when
string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)')
integer_re = re.compile(r'\d+')
-name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
+name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
float_re = re.compile(r'\d+\.\d+')
+eol_re = re.compile(r'(\s*$\s*)+(?m)')
# set of used keywords
-keywords = set(['and', 'block', 'elif', 'else', 'endblock',
+keywords = set(['and', 'block', 'elif', 'else', 'endblock', 'print',
'endfilter', 'endfor', 'endif', 'endmacro', 'endraw',
'endtrans', 'extends', 'filter', 'for', 'if', 'in',
'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw',
- 'recursive', 'set', 'trans', 'call', 'endcall',
- 'true', 'false', 'none'])
+ 'recursive', 'set', 'trans', 'call', 'endcall'])
# bind operators to token types
operators = {
reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
assert len(operators) == len(reverse_operators), 'operators dropped'
-operator_re = re.compile('(%s)' % '|'.join([re.escape(x) for x in
- sorted(operators, key=lambda x: -len(x))]))
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+ sorted(operators, key=lambda x: -len(x))))
simple_escapes = {
'a': '\a',
# lexing rules for tags
tag_rules = [
+ (eol_re, 'eol', None),
(whitespace_re, None, None),
(float_re, 'float', None),
(integer_re, 'integer', None),
- (c('%s' % '|'.join(sorted(keywords, key=lambda x: -len(x)))),
+ (c(r'\b(?:%s)\b' % '|'.join(sorted(keywords, key=lambda x: -len(x)))),
'keyword', None),
(name_re, 'name', None),
(string_re, 'string', None),
}
-def set_ctx(node, ctx):
- """
- Reset the context of a node and all child nodes. Per default the parser
- will all generate nodes that have a 'load' context as it's the most common
- one. This method is used in the parser to set assignment targets and
- other nodes to a store context.
- """
- todo = deque([node])
- while todo:
- node = todo.popleft()
- if 'ctx' in node._fields:
- node.ctx = ctx
- todo.extend(node.iter_child_nodes())
class Impossible(Exception):
class NodeType(type):
def __new__(cls, name, bases, d):
- for attr in '_fields', '_attributes':
+ for attr in 'fields', 'attributes':
storage = []
for base in bases:
storage.extend(getattr(base, attr, ()))
class Node(object):
"""
- Base jinja node.
+ Baseclass for all Jinja nodes.
"""
__metaclass__ = NodeType
- _fields = ()
- _attributes = ('lineno',)
+ fields = ()
+ attributes = ('lineno',)
def __init__(self, *args, **kw):
if args:
- if len(args) != len(self._fields):
- if not self._fields:
+ if len(args) != len(self.fields):
+ if not self.fields:
raise TypeError('%r takes 0 arguments' %
self.__class__.__name__)
raise TypeError('%r takes 0 or %d argument%s' % (
self.__class__.__name__,
- len(self._fields),
- len(self._fields) != 1 and 's' or ''
+ len(self.fields),
+ len(self.fields) != 1 and 's' or ''
))
- for name, arg in izip(self._fields, args):
+ for name, arg in izip(self.fields, args):
setattr(self, name, arg)
- for attr in self._attributes:
+ for attr in self.attributes:
setattr(self, attr, kw.pop(attr, None))
if kw:
raise TypeError('unknown keyword argument %r' %
iter(kw).next())
def iter_fields(self):
- for name in self._fields:
+ """Iterate over all fields."""
+ for name in self.fields:
try:
yield name, getattr(self, name)
except AttributeError:
pass
def iter_child_nodes(self):
+ """Iterate over all child nodes."""
for field, item in self.iter_fields():
if isinstance(item, list):
for n in item:
elif isinstance(item, Node):
yield item
+ def find(self, node_type):
+ """Find the first node of a given type."""
+ for result in self.find_all(node_type):
+ return result
+
+ def find_all(self, node_type):
+ """Find all the nodes of a given type."""
+ for child in self.iter_child_nodes():
+ if isinstance(child, node_type):
+ yield child
+ for result in child.find_all(node_type):
+ yield result
+
+ def set_ctx(self, ctx):
+ """
+ Reset the context of a node and all child nodes. Per default the parser
+ will all generate nodes that have a 'load' context as it's the most common
+ one. This method is used in the parser to set assignment targets and
+ other nodes to a store context.
+ """
+ todo = deque([self])
+ while todo:
+ node = todo.popleft()
+ if 'ctx' in node.fields:
+ node.ctx = ctx
+ todo.extend(node.iter_child_nodes())
+
def __repr__(self):
return '%s(%s)' % (
self.__class__.__name__,
', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
- arg in self._fields)
+ arg in self.fields)
)
"""
Node that represents a template.
"""
- _fields = ('body',)
+ fields = ('body',)
class Output(Stmt):
A node that holds multiple expressions which are then printed out. This
is used both for the `print` statement and the regular template data.
"""
- _fields = ('nodes',)
+ fields = ('nodes',)
+
+ def optimized_nodes(self):
+ """Try to optimize the nodes."""
+ buffer = []
+ for node in self.nodes:
+ try:
+ const = unicode(node.as_const())
+ except:
+ buffer.append(node)
+ else:
+ if buffer and isinstance(buffer[-1], unicode):
+ buffer[-1] += const
+ else:
+ buffer.append(const)
+ return buffer
class Extends(Stmt):
"""
Represents an extends statement.
"""
- _fields = ('extends',)
+ fields = ('template',)
class For(Stmt):
"""
A node that represents a for loop
"""
- _fields = ('target', 'iter', 'body', 'else_', 'recursive')
+ fields = ('target', 'iter', 'body', 'else_', 'recursive')
class If(Stmt):
"""
A node that represents an if condition.
"""
- _fields = ('test', 'body', 'else_')
+ fields = ('test', 'body', 'else_')
class Macro(Stmt):
"""
A node that represents a macro.
"""
- _fields = ('name', 'arguments', 'body')
+ fields = ('name', 'args', 'defaults', 'dyn_args', 'dyn_kwargs', 'body')
class CallBlock(Stmt):
"""
A node that represents am extended macro call.
"""
- _fields = ('expr', 'body')
+ fields = ('call', 'body')
class Set(Stmt):
"""
Allows defining own variables.
"""
- _fields = ('name', 'expr')
+ fields = ('name', 'expr')
class FilterBlock(Stmt):
"""
Node for filter sections.
"""
- _fields = ('body', 'filters')
+ fields = ('body', 'filters')
class Block(Stmt):
"""
A node that represents a block.
"""
- _fields = ('name', 'body')
+ fields = ('name', 'body')
class Include(Stmt):
"""
A node that represents the include tag.
"""
- _fields = ('template',)
+ fields = ('template', 'target')
class Trans(Stmt):
"""
A node for translatable sections.
"""
- _fields = ('singular', 'plural', 'indicator', 'replacements')
+ fields = ('singular', 'plural', 'indicator', 'replacements')
class ExprStmt(Stmt):
"""
A statement that evaluates an expression to None.
"""
- _fields = ('node',)
+ fields = ('node',)
class Assign(Stmt):
"""
Assigns an expression to a target.
"""
- _fields = ('target', 'node')
+ fields = ('target', 'node')
class Expr(Node):
"""
Baseclass for all binary expressions.
"""
- _fields = ('left', 'right')
+ fields = ('left', 'right')
operator = None
def as_const(self):
"""
Baseclass for all unary expressions.
"""
- _fields = ('node',)
+ fields = ('node',)
operator = None
def as_const(self):
"""
any name such as {{ foo }}
"""
- _fields = ('name', 'ctx')
+ fields = ('name', 'ctx')
def can_assign(self):
- return True
+ return self.name not in ('true', 'false', 'none')
class Literal(Expr):
"""
any constat such as {{ "foo" }}
"""
- _fields = ('value',)
+ fields = ('value',)
def as_const(self):
return self.value
For loop unpacking and some other things like multiple arguments
for subscripts.
"""
- _fields = ('items', 'ctx')
+ fields = ('items', 'ctx')
def as_const(self):
return tuple(x.as_const() for x in self.items)
"""
any list literal such as {{ [1, 2, 3] }}
"""
- _fields = ('items',)
+ fields = ('items',)
def as_const(self):
return [x.as_const() for x in self.items]
"""
any dict literal such as {{ {1: 2, 3: 4} }}
"""
- _fields = ('items',)
+ fields = ('items',)
def as_const(self):
return dict(x.as_const() for x in self.items)
"""
A key, value pair for dicts.
"""
- _fields = ('key', 'value')
+ fields = ('key', 'value')
def as_const(self):
return self.key.as_const(), self.value.as_const()
"""
{{ foo if bar else baz }}
"""
- _fields = ('test', 'expr1', 'expr2')
+ fields = ('test', 'expr1', 'expr2')
def as_const(self):
if self.test.as_const():
"""
{{ foo|bar|baz }}
"""
- _fields = ('node', 'filters')
+ fields = ('node', 'filters')
class FilterCall(Expr):
"""
{{ |bar() }}
"""
- _fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+ fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
class Test(Expr):
"""
{{ foo is lower }}
"""
- _fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+ fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
class Call(Expr):
"""
{{ foo(bar) }}
"""
- _fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+ fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
class Subscript(Expr):
"""
{{ foo.bar }} and {{ foo['bar'] }} etc.
"""
- _fields = ('node', 'arg', 'ctx')
+ fields = ('node', 'arg', 'ctx')
def as_const(self):
try:
"""
1:2:3 etc.
"""
- _fields = ('start', 'stop', 'step')
+ fields = ('start', 'stop', 'step')
class Concat(Expr):
"""
For {{ foo ~ bar }}. Concatenates strings.
"""
- _fields = ('nodes',)
+ fields = ('nodes',)
def as_const(self):
return ''.join(unicode(x.as_const()) for x in self.nodes)
"""
{{ foo == bar }}, {{ foo >= bar }} etc.
"""
- _fields = ('expr', 'ops')
+ fields = ('expr', 'ops')
class Operand(Helper):
"""
Operator + expression.
"""
- _fields = ('op', 'expr')
+ fields = ('op', 'expr')
class Mul(BinExpr):
operator = 'not'
-class NegExpr(UnaryExpr):
+class Neg(UnaryExpr):
"""
{{ -foo }}
"""
operator = '-'
-class PosExpr(UnaryExpr):
+class Pos(UnaryExpr):
"""
{{ +foo }}
"""
__all__ = ['Parser']
-_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'include'])
+_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
+ 'macro', 'include'])
_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq', 'in'])
-_tuple_edge_tokens = set(['rparen', 'block_end', 'variable_end', 'in',
- 'semicolon', 'recursive'])
_statement_end_tokens = set(['elif', 'else', 'endblock', 'endfilter',
- 'endfor', 'endif', 'endmacro',
- 'endcall', 'block_end'])
+ 'endfor', 'endif', 'endmacro', 'variable_end',
+ 'in', 'recursive', 'endcall', 'block_end'])
class Parser(object):
self.source = source
self.filename = filename
self.closed = False
- self.blocks = set()
self.no_variable_block = self.environment.lexer.no_variable_block
self.stream = environment.lexer.tokenize(source, filename)
elif token_type is 'call':
self.stream.next()
return self.parse_call_block()
- lineno = self.stream.current.lineno
+ lineno = self.stream.current
expr = self.parse_expression()
if self.stream.current.type == 'assign':
- return self.parse_assign(expr)
+ result = self.parse_assign(expr)
+ else:
+ result = nodes.ExprStmt(expr, lineno=lineno)
self.end_statement()
- return nodes.ExprStmt(expr, lineno=lineno)
+ return result
def parse_assign(self, target):
"""Parse an assign statement."""
self.filename)
expr = self.parse_tuple()
self.end_statement()
- nodes.set_ctx(target, 'store')
+ target.set_ctx('store')
return nodes.Assign(target, expr, lineno=lineno)
def parse_statements(self, end_tokens, drop_needle=False):
else:
result = []
while self.stream.current.type not in end_tokens:
+ if self.stream.current.type is 'block_end':
+ self.stream.next()
+ result.extend(self.subparse(end_tokens))
+ break
result.append(self.parse_statement())
if drop_needle:
self.stream.next()
"""Parse a for loop."""
lineno = self.stream.expect('for').lineno
target = self.parse_tuple(simplified=True)
- nodes.set_ctx(target, 'store')
+ if not target.can_assign():
+ raise TemplateSyntaxError("can't assign to '%s'" %
+ target, target.lineno,
+ self.filename)
+ target.set_ctx('store')
self.stream.expect('in')
iter = self.parse_tuple()
if self.stream.current.type is 'recursive':
else:
recursive = False
body = self.parse_statements(('endfor', 'else'))
- token_type = self.stream.current.type
- self.stream.next()
- if token_type is 'endfor':
+ if self.stream.next().type is 'endfor':
else_ = []
else:
else_ = self.parse_statements(('endfor',), drop_needle=True)
return nodes.For(target, iter, body, else_, False, lineno=lineno)
def parse_if(self):
- pass
+ """Parse an if construct."""
+ node = result = nodes.If(lineno=self.stream.expect('if').lineno)
+ while 1:
+ # TODO: exclude conditional expressions here
+ node.test = self.parse_tuple()
+ node.body = self.parse_statements(('elif', 'else', 'endif'))
+ token_type = self.stream.next().type
+ if token_type is 'elif':
+ new_node = nodes.If(lineno=self.stream.current.lineno)
+ node.else_ = [new_node]
+ node = new_node
+ continue
+ elif token_type is 'else':
+ node.else_ = self.parse_statements(('endif',),
+ drop_needle=True)
+ else:
+ node.else_ = []
+ break
+ return result
def parse_block(self):
- pass
+ node = nodes.Block(lineno=self.stream.expect('block').lineno)
+ node.name = self.stream.expect('name').value
+ node.body = self.parse_statements(('endblock',), drop_needle=True)
+ return node
def parse_extends(self):
- pass
+ node = nodes.Extends(lineno=self.stream.expect('extends').lineno)
+ node.template = self.parse_expression()
+ self.end_statement()
+ return node
def parse_include(self):
- pass
+ node = nodes.Include(lineno=self.stream.expect('include').lineno)
+ expr = self.parse_expression()
+ if self.stream.current.type is 'assign':
+ self.stream.next()
+ if not expr.can_assign():
+ raise TemplateSyntaxError('can\'t assign imported template '
+ 'to this expression.', expr.lineno,
+ self.filename)
+ expr.set_ctx('store')
+ node.target = expr
+ node.template = self.parse_expression()
+ else:
+ node.target = None
+ node.template = expr
+ self.end_statement()
+ return node
def parse_call_block(self):
- pass
+ node = nodes.CallBlock(lineno=self.stream.expect('call').lineno)
+ node.call = self.parse_call()
+ node.body = self.parse_statements(('endcall',), drop_needle=True)
+ return node
+
+ def parse_macro(self):
+ node = nodes.Macro(lineno=self.stream.expect('macro').lineno)
+ node.name = self.stream.expect('name').value
+ self.stream.expect('lparen')
+ node.args = args = []
+ node.defaults = defaults = []
+ while self.stream.current.type is not 'rparen':
+ if args:
+ self.stream.expect('comma')
+ token = self.stream.expect('name')
+ arg = nodes.Name(token.value, 'param', lineno=token.lineno)
+ if not arg.can_assign():
+ raise TemplateSyntaxError("can't assign to '%s'" %
+ arg.name, arg.lineno,
+ self.filename)
+ if self.stream.current.type is 'assign':
+ self.stream.next()
+ defaults.append(self.parse_expression())
+ self.stream.expect('rparen')
+ node.body = self.parse_statements(('endmacro',), drop_needle=True)
+ return node
+
+ def parse_print(self):
+ node = nodes.Output(lineno=self.stream.expect('print').lineno)
+ node.nodes = []
+ while self.stream.current.type not in _statement_end_tokens:
+ if node.nodes:
+ self.stream.expect('comma')
+ node.nodes.append(self.parse_expression())
+ self.end_statement()
+ return node
def parse_expression(self):
"""Parse an expression."""
while self.stream.current.type is 'div':
self.stream.next()
right = self.parse_floordiv()
- left = nodes.Floor(left, right, lineno=lineno)
+ left = nodes.Div(left, right, lineno=lineno)
lineno = self.stream.current.lineno
return left
if token_type is 'not':
self.stream.next()
node = self.parse_unary()
- return nodes.Neg(node, lineno=lineno)
+ return nodes.Not(node, lineno=lineno)
if token_type is 'sub':
self.stream.next()
node = self.parse_unary()
- return nodes.Sub(node, lineno=lineno)
+ return nodes.Neg(node, lineno=lineno)
if token_type is 'add':
self.stream.next()
node = self.parse_unary()
while 1:
if args:
self.stream.expect('comma')
- if self.stream.current.type in _tuple_edge_tokens:
+ if self.stream.current.type in _statement_end_tokens:
break
args.append(parse())
if self.stream.current.type is not 'comma':
def parse_subscript(self, node):
token = self.stream.next()
if token.type is 'dot':
- if token.type not in ('name', 'integer'):
+ attr_token = self.stream.current
+ if attr_token.type not in ('name', 'integer'):
raise TemplateSyntaxError('expected name or number',
- token.lineno, self.filename)
- arg = nodes.Const(token.value, lineno=token.lineno)
+ attr_token.lineno, self.filename)
+ arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
self.stream.next()
elif token.type is 'lbracket':
args = []
self.stream.next()
elif token.type is 'variable_begin':
self.stream.next()
- add_data(self.parse_tuple())
+ want_comma = False
+ while not self.stream.current.type in _statement_end_tokens:
+ if want_comma:
+ self.stream.expect('comma')
+ add_data(self.parse_expression())
+ want_comma = True
self.stream.expect('variable_end')
elif token.type is 'block_begin':
flush_data()
--- /dev/null
+# -*- coding: utf-8 -*-
+"""
+ jinja2.runtime
+ ~~~~~~~~~~~~~~
+
+ Runtime helpers.
+
+ :copyright: Copyright 2008 by Armin Ronacher.
+ :license: GNU GPL.
+"""
+try:
+ from collections import defaultdict
+except ImportError:
+ defaultdict = None
+
+
+__all__ = ['extends', 'TemplateContext']
+
+
+def extends(template, namespace):
+ """
+ This loads a template (and evaluates it) and replaces the blocks.
+ """
+
+
+class TemplateContext(dict):
+
+ def __init__(self, globals, undefined_factory, filename):
+ dict.__init__(self, globals)
+ self.undefined_factory = undefined_factory
+ self.filename = filename
+
+ # if there is a default dict, dict has a __missing__ method we can use.
+ if defaultdict is None:
+ def __getitem__(self, name):
+ if name in self:
+ return self[name]
+ return self.undefined_factory(name)
+ else:
+ def __missing__(self, key):
+ return self.undefined_factory(key)
+
+ def from_locals(self, mapping):
+ """Update the template context from locals."""
+ for key, value in mapping.iteritems():
+ if key[:2] == 'l_':
+ self[key[:-2]] = value
--- /dev/null
+# -*- coding: utf-8 -*-
+"""
+ jinja2.visitor
+ ~~~~~~~~~~~~~~
+
+ This module implements a visitor for the nodes.
+
+ :copyright: Copyright 2008 by Armin Ronacher.
+ :license: GNU GPL.
+"""
+from jinja2.nodes import Node
+
+
+class NodeVisitor(object):
+ """
+ Walks the abstract syntax tree and call visitor functions for every node
+ found. The visitor functions may return values which will be forwarded
+ by the `visit` method.
+
+ Per default the visitor functions for the nodes are ``'visit_'`` +
+ class name of the node. So a `TryFinally` node visit function would
+ be `visit_TryFinally`. This behavior can be changed by overriding
+ the `get_visitor` function. If no visitor function exists for a node
+ (return value `None`) the `generic_visit` visitor is used instead.
+ """
+
+ def get_visitor(self, node):
+ """
+ Return the visitor function for this node or `None` if no visitor
+ exists for this node. In that case the generic visit function is
+ used instead.
+ """
+ method = 'visit_' + node.__class__.__name__
+ return getattr(self, method, None)
+
+ def visit(self, node, *args, **kwargs):
+ """Visit a node."""
+ f = self.get_visitor(node)
+ if f is not None:
+ return f(node, *args, **kwargs)
+ return self.generic_visit(node, *args, **kwargs)
+
+ def generic_visit(self, node, *args, **kwargs):
+ """Called if no explicit visitor function exists for a node."""
+ for node in node.iter_child_nodes():
+ self.visit(node, *args, **kwargs)
+
+
+class NodeTransformer(NodeVisitor):
+ """
+ Walks the abstract syntax tree and allows modifications of nodes.
+
+ The `NodeTransformer` will walk the AST and use the return value of the
+ visitor functions to replace or remove the old node. If the return
+ value of the visitor function is `None` the node will be removed
+ from the previous location otherwise it's replaced with the return
+ value. The return value may be the original node in which case no
+ replacement takes place.
+ """
+
+ def generic_visit(self, node, *args, **kwargs):
+ for field, old_value in node.iter_fields():
+ if isinstance(old_value, list):
+ new_values = []
+ for value in old_value:
+ if isinstance(value, Node):
+ value = self.visit(value, *args, **kwargs)
+ if value is None:
+ continue
+ elif not isinstance(value, Node):
+ new_values.extend(value)
+ continue
+ new_values.append(value)
+ old_value[:] = new_values
+ elif isinstance(old_value, Node):
+ new_node = self.visit(old_value, *args, **kwargs)
+ if new_node is None:
+ delattr(node, field)
+ else:
+ setattr(node, field, new_node)
+ return node
--- /dev/null
+from jinja2 import Environment
+from jinja2.compiler import generate
+
+
+env = Environment()
+ast = env.parse("""
+Hello {{ name }}!. How is it {{ "going" }}. {{ [1, 2, 3] }}?
+{% for name in user_names %}
+ {{ loop.index }}: {{ name }}
+ {% if loop.index % 2 == 0 %}...{% endif %}
+{% endfor %}
+Name again: {{ name }}!
+""")
+print ast
+print
+print generate(ast, "foo.html")
projects / jinja2.git / commitdiff