From e791c2a281ddf0cd0411ea82a1cd9c6888d8f741 Mon Sep 17 00:00:00 2001
From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Mon, 7 Apr 2008 18:39:54 +0200
Subject: [PATCH] added first working pieces of compiler

--HG--
branch : trunk
---
 jinja2/compiler.py      | 380 ++++++++++++++++++++++++++++++++++++++++
 jinja2/datastructure.py |  25 +--
 jinja2/exceptions.py    |  12 +-
 jinja2/lexer.py         |  15 +-
 jinja2/nodes.py         | 151 +++++++++-------
 jinja2/parser.py        | 140 ++++++++++++---
 jinja2/runtime.py       |  47 +++++
 jinja2/visitor.py       |  81 +++++++++
 test.py                 |  16 ++
 9 files changed, 761 insertions(+), 106 deletions(-)
 create mode 100644 jinja2/compiler.py
 create mode 100644 jinja2/runtime.py
 create mode 100644 jinja2/visitor.py
 create mode 100644 test.py

diff --git a/jinja2/compiler.py b/jinja2/compiler.py
new file mode 100644
index 0000000..330db56
--- /dev/null
+++ b/jinja2/compiler.py
@@ -0,0 +1,380 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.compiler
+    ~~~~~~~~~~~~~~~
+
+    Compiles nodes into python code.
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: GNU GPL.
+"""
+from random import randrange
+from operator import xor
+from cStringIO import StringIO
+from jinja2 import nodes
+from jinja2.visitor import NodeVisitor, NodeTransformer
+from jinja2.exceptions import TemplateAssertionError
+
+
+operators = {
+    'eq':       '==',
+    'ne':       '!=',
+    'gt':       '>',
+    'gteq':     '>=',
+    'lt':       '<',
+    'lteq':     '<=',
+    'in':       'in',
+    'notin':    'not in'
+}
+
+
+def generate(node, filename, stream=None):
+    is_child = node.find(nodes.Extends) is not None
+    generator = CodeGenerator(is_child, filename, stream)
+    generator.visit(node)
+    if stream is None:
+        return generator.stream.getvalue()
+
+
+class Identifiers(object):
+    """Tracks the status of identifiers in frames."""
+
+    def __init__(self):
+        # variables that are known to be declared (probably from outer
+        # frames or because they are special for the frame)
+        self.declared = set()
+
+        # names that are accessed without being explicitly declared by
+        # this one or any of the outer scopes.  Names can appear both in
+        # declared and undeclared.
+        self.undeclared = set()
+
+        # names that are declared locally
+        self.declared_locally = set()
+
+        # names that are declared by parameters
+        self.declared_parameter = set()
+
+    def add_special(self, name):
+        """Register a special name like `loop`."""
+        self.undeclared.discard(name)
+        self.declared.add(name)
+
+    def is_declared(self, name):
+        """Check if a name is declared in this or an outer scope."""
+        return name in self.declared or name in self.declared_locally or \
+               name in self.declared_parameter
+
+    def find_shadowed(self):
+        """Find all the shadowed names."""
+        return self.declared & (self.declared_locally | self.declared_parameter)
+
+
+class Frame(object):
+
+    def __init__(self, parent=None):
+        self.identifiers = Identifiers()
+        self.parent = parent
+        if parent is not None:
+            self.identifiers.declared.update(
+                parent.identifiers.declared |
+                parent.identifiers.undeclared |
+                parent.identifiers.declared_locally |
+                parent.identifiers.declared_parameter
+            )
+
+    def inspect(self, nodes):
+        """Walk the node and check for identifiers."""
+        visitor = FrameIdentifierVisitor(self.identifiers)
+        for node in nodes:
+            visitor.visit(node)
+
+    def inner(self):
+        """Return an inner frame."""
+        return Frame(self)
+
+
+class FrameIdentifierVisitor(NodeVisitor):
+    """A visitor for `Frame.inspect`."""
+
+    def __init__(self, identifiers):
+        self.identifiers = identifiers
+
+    def visit_Name(self, node):
+        """All assignments to names go through this function."""
+        if node.ctx in ('store', 'param'):
+            self.identifiers.declared_locally.add(node.name)
+        elif node.ctx == 'load':
+            if not self.identifiers.is_declared(node.name):
+                self.identifiers.undeclared.add(node.name)
+
+    def visit_Macro(self, node):
+        """Macros set local."""
+        self.identifiers.declared_locally.add(node.name)
+
+    # stop traversing at instructions that have their own scope.
+    visit_Block = visit_Call = visit_FilterBlock = \
+        visit_For = lambda s, n: None
+
+
+class CodeGenerator(NodeVisitor):
+
+    def __init__(self, is_child, filename, stream=None):
+        if stream is None:
+            stream = StringIO()
+        self.is_child = is_child
+        self.filename = filename
+        self.stream = stream
+        self.blocks = {}
+        self.indentation = 0
+        self.new_lines = 0
+        self.last_identifier = 0
+        self._last_line = 0
+        self._first_write = True
+
+    def temporary_identifier(self):
+        self.last_identifier += 1
+        return 't%d' % self.last_identifier
+
+    def indent(self):
+        self.indentation += 1
+
+    def outdent(self):
+        self.indentation -= 1
+
+    def blockvisit(self, nodes, frame, force_generator=False):
+        self.indent()
+        if force_generator:
+            self.writeline('if 0: yield None')
+        for node in nodes:
+            self.visit(node, frame)
+        self.outdent()
+
+    def write(self, x):
+        if self.new_lines:
+            if not self._first_write:
+                self.stream.write('\n' * self.new_lines)
+            self._first_write = False
+            self.stream.write('    ' * self.indentation)
+            self.new_lines = 0
+        self.stream.write(x)
+
+    def writeline(self, x, node=None, extra=0):
+        self.newline(node, extra)
+        self.write(x)
+
+    def newline(self, node=None, extra=0):
+        self.new_lines = max(self.new_lines, 1 + extra)
+        if node is not None and node.lineno != self._last_line:
+            self.write('# line: %s' % node.lineno)
+            self.new_lines = 1
+            self._last_line = node.lineno
+
+    def pull_locals(self, frame, no_indent=False):
+        if not no_indent:
+            self.indent()
+        for name in frame.identifiers.undeclared:
+            self.writeline('l_%s = context[%r]' % (name, name))
+        if not no_indent:
+            self.outdent()
+
+    # -- Visitors
+
+    def visit_Template(self, node, frame=None):
+        assert frame is None, 'no root frame allowed'
+        self.writeline('from jinja2.runtime import *')
+        self.writeline('filename = %r' % self.filename)
+        self.writeline('context = TemplateContext(global_context, '
+                       'make_undefined, filename)')
+
+        # generate the body render function.
+        self.writeline('def body(context=context):', extra=1)
+        frame = Frame()
+        frame.inspect(node.body)
+        self.pull_locals(frame)
+        self.blockvisit(node.body, frame, True)
+
+        # top level changes to locals are pushed back to the
+        # context of *this* template for include.
+        self.indent()
+        self.writeline('context.from_locals(locals())')
+        self.outdent()
+
+        # at this point we now have the blocks collected and can visit them too.
+        for name, block in self.blocks.iteritems():
+            block_frame = Frame()
+            block_frame.inspect(block.body)
+            self.writeline('def block_%s(context=context):' % name, block, 1)
+            self.pull_locals(block_frame)
+            self.blockvisit(block.body, block_frame, True)
+
+    def visit_Block(self, node, frame):
+        """Call a block and register it for the template."""
+        if node.name in self.blocks:
+            raise TemplateAssertionError("the block '%s' was already defined" %
+                                         node.name, node.lineno,
+                                         self.filename)
+        self.blocks[node.name] = node
+        self.writeline('for event in block_%s():' % node.name)
+        self.indent()
+        self.writeline('yield event')
+        self.outdent()
+
+    def visit_Extends(self, node, frame):
+        """Calls the extender."""
+        self.writeline('extends(', node, 1)
+        self.visit(node.template)
+        self.write(', globals())')
+
+    def visit_For(self, node, frame):
+        loop_frame = frame.inner()
+        loop_frame.inspect(node.iter_child_nodes())
+        loop_frame.identifiers.add_special('loop')
+        extended_loop = bool(node.else_) or \
+                        'loop' in loop_frame.identifiers.undeclared
+
+        # make sure we "backup" overridden, local identifiers
+        # TODO: we should probably optimize this and check if the
+        # identifier is in use afterwards.
+        aliases = {}
+        for name in loop_frame.identifiers.find_shadowed():
+            aliases[name] = ident = self.temporary_identifier()
+            self.writeline('%s = l_%s' % (ident, name))
+
+        self.pull_locals(loop_frame, True)
+
+        self.newline(node)
+        if node.else_:
+            self.writeline('l_loop = None')
+        self.write('for ')
+        self.visit(node.target, loop_frame)
+        self.write(extended_loop and ', l_loop in looper(' or ' in ')
+        self.visit(node.iter, loop_frame)
+        self.write(extended_loop and '):' or ':')
+        self.blockvisit(node.body, loop_frame)
+
+        if node.else_:
+            self.writeline('if l_loop is None:')
+            self.blockvisit(node.else_, loop_frame)
+
+        # reset the aliases and clean them up
+        for name, alias in aliases.iteritems():
+            self.writeline('l_%s = %s; del %s' % (name, alias, alias))
+
+    def visit_If(self, node, frame):
+        self.writeline('if ', node)
+        self.visit(node.test, frame)
+        self.write(':')
+        self.blockvisit(node.body, frame)
+        if node.else_:
+            self.writeline('else:')
+            self.blockvisit(node.else_, frame)
+
+    def visit_ExprStmt(self, node, frame):
+        self.newline(node)
+        self.visit(node, frame)
+
+    def visit_Output(self, node, frame):
+        self.newline(node)
+
+        # try to evaluate as many chunks as possible into a static
+        # string at compile time.
+        body = []
+        for child in node.nodes:
+            try:
+                const = unicode(child.as_const())
+            except:
+                body.append(child)
+                continue
+            if body and isinstance(body[-1], list):
+                body[-1].append(const)
+            else:
+                body.append([const])
+
+        # if we have less than 3 nodes we just yield them
+        if len(body) < 3:
+            for item in body:
+                if isinstance(item, list):
+                    self.writeline('yield %s' % repr(u''.join(item)))
+                else:
+                    self.newline(item)
+                    self.write('yield unicode(')
+                    self.visit(item, frame)
+                    self.write(')')
+
+        # otherwise we create a format string as this is faster in that case
+        else:
+            format = []
+            arguments = []
+            for item in body:
+                if isinstance(item, list):
+                    format.append(u''.join(item).replace('%', '%%'))
+                else:
+                    format.append('%s')
+                    arguments.append(item)
+            self.writeline('yield %r %% (' % u''.join(format))
+            idx = -1
+            for idx, argument in enumerate(arguments):
+                if idx:
+                    self.write(', ')
+                self.visit(argument, frame)
+            self.write(idx == 0 and ',)' or ')')
+
+    def visit_Name(self, node, frame):
+        # at this point we should only have locals left as the
+        # blocks, macros and template body ensure that they are set.
+        self.write('l_' + node.name)
+
+    def visit_Const(self, node, frame):
+        val = node.value
+        if isinstance(val, float):
+            # XXX: add checks for infinity and nan
+            self.write(str(val))
+        else:
+            self.write(repr(val))
+
+    def binop(operator):
+        def visitor(self, node, frame):
+            self.write('(')
+            self.visit(node.left, frame)
+            self.write(' %s ' % operator)
+            self.visit(node.right, frame)
+            self.write(')')
+        return visitor
+
+    def uaop(operator):
+        def visitor(self, node, frame):
+            self.write('(' + operator)
+            self.visit(node.node)
+            self.write(')')
+        return visitor
+
+    visit_Add = binop('+')
+    visit_Sub = binop('-')
+    visit_Mul = binop('*')
+    visit_Div = binop('/')
+    visit_FloorDiv = binop('//')
+    visit_Pow = binop('**')
+    visit_Mod = binop('%')
+    visit_And = binop('and')
+    visit_Or = binop('or')
+    visit_Pos = uaop('+')
+    visit_Neg = uaop('-')
+    visit_Not = uaop('not ')
+    del binop, uaop
+
+    def visit_Compare(self, node, frame):
+        self.visit(node.expr, frame)
+        for op in node.ops:
+            self.visit(op, frame)
+
+    def visit_Operand(self, node, frame):
+        self.write(' %s ' % operators[node.op])
+        self.visit(node.expr, frame)
+
+    def visit_Subscript(self, node, frame):
+        self.write('subscript(')
+        self.visit(node.node, frame)
+        self.write(', ')
+        self.visit(node.arg, frame)
+        self.write(')')
diff --git a/jinja2/datastructure.py b/jinja2/datastructure.py
index 2777df5..11c2671 100644
--- a/jinja2/datastructure.py
+++ b/jinja2/datastructure.py
@@ -59,7 +59,7 @@ class TokenStreamIterator(object):
         if token.type == 'eof':
             self._stream.close()
             raise StopIteration()
-        self._stream.next()
+        self._stream.next(False)
         return token
 
 
@@ -107,15 +107,20 @@ class TokenStream(object):
         for x in xrange(n):
             self.next()
 
-    def next(self):
-        """Go one token ahead."""
-        if self._pushed:
-            self.current = self._pushed.popleft()
-        elif self.current.type != 'eof':
-            try:
-                self.current = self._next()
-            except StopIteration:
-                self.close()
+    def next(self, skip_eol=True):
+        """Go one token ahead and return the old one"""
+        rv = self.current
+        while 1:
+            if self._pushed:
+                self.current = self._pushed.popleft()
+            elif self.current.type is not 'eof':
+                try:
+                    self.current = self._next()
+                except StopIteration:
+                    self.close()
+            if not skip_eol or self.current.type is not 'eol':
+                break
+        return rv
 
     def close(self):
         """Close the stream."""
diff --git a/jinja2/exceptions.py b/jinja2/exceptions.py
index 9ec5c27..e345f03 100644
--- a/jinja2/exceptions.py
+++ b/jinja2/exceptions.py
@@ -10,7 +10,7 @@
 """
 
 
-class TemplateError(RuntimeError):
+class TemplateError(Exception):
     pass
 
 
@@ -30,11 +30,19 @@ class TemplateSyntaxError(SyntaxError, TemplateError):
     """
 
     def __init__(self, message, lineno, filename):
-        SyntaxError.__init__(self, message)
+        SyntaxError.__init__(self, '%s (line %s)' % (message, lineno))
+        self.message = message
         self.lineno = lineno
         self.filename = filename
 
 
+class TemplateAssertionError(AssertionError, TemplateSyntaxError):
+
+    def __init__(self, message, lineno, filename):
+        AssertionError.__init__(self, message)
+        TemplateSyntaxError.__init__(self, message, lineno, filename)
+
+
 class TemplateRuntimeError(TemplateError):
     """
     Raised by the template engine if a tag encountered an error when
diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index 515508b..6e9fc89 100644
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -34,17 +34,17 @@ whitespace_re = re.compile(r'\s+(?um)')
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)')
 integer_re = re.compile(r'\d+')
-name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
+name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
 float_re = re.compile(r'\d+\.\d+')
+eol_re = re.compile(r'(\s*$\s*)+(?m)')
 
 
 # set of used keywords
-keywords = set(['and', 'block', 'elif', 'else', 'endblock',
+keywords = set(['and', 'block', 'elif', 'else', 'endblock', 'print',
                 'endfilter', 'endfor', 'endif', 'endmacro', 'endraw',
                 'endtrans', 'extends', 'filter', 'for', 'if', 'in',
                 'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw',
-                'recursive', 'set', 'trans', 'call', 'endcall',
-                'true', 'false', 'none'])
+                'recursive', 'set', 'trans', 'call', 'endcall'])
 
 # bind operators to token types
 operators = {
@@ -78,8 +78,8 @@ operators = {
 
 reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
 assert len(operators) == len(reverse_operators), 'operators dropped'
-operator_re = re.compile('(%s)' % '|'.join([re.escape(x) for x in
-                         sorted(operators, key=lambda x: -len(x))]))
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+                         sorted(operators, key=lambda x: -len(x))))
 
 simple_escapes = {
     'a':    '\a',
@@ -229,10 +229,11 @@ class Lexer(object):
 
         # lexing rules for tags
         tag_rules = [
+            (eol_re, 'eol', None),
             (whitespace_re, None, None),
             (float_re, 'float', None),
             (integer_re, 'integer', None),
-            (c('%s' % '|'.join(sorted(keywords, key=lambda x: -len(x)))),
+            (c(r'\b(?:%s)\b' % '|'.join(sorted(keywords, key=lambda x: -len(x)))),
              'keyword', None),
             (name_re, 'name', None),
             (string_re, 'string', None),
diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 974b4ed..e6e68a1 100644
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -35,19 +35,6 @@ _uaop_to_func = {
 }
 
 
-def set_ctx(node, ctx):
-    """
-    Reset the context of a node and all child nodes.  Per default the parser
-    will all generate nodes that have a 'load' context as it's the most common
-    one.  This method is used in the parser to set assignment targets and
-    other nodes to a store context.
-    """
-    todo = deque([node])
-    while todo:
-        node = todo.popleft()
-        if 'ctx' in node._fields:
-            node.ctx = ctx
-        todo.extend(node.iter_child_nodes())
 
 
 class Impossible(Exception):
@@ -59,7 +46,7 @@ class Impossible(Exception):
 class NodeType(type):
 
     def __new__(cls, name, bases, d):
-        for attr in '_fields', '_attributes':
+        for attr in 'fields', 'attributes':
             storage = []
             for base in bases:
                 storage.extend(getattr(base, attr, ()))
@@ -71,39 +58,41 @@ class NodeType(type):
 
 class Node(object):
     """
-    Base jinja node.
+    Baseclass for all Jinja nodes.
     """
     __metaclass__ = NodeType
-    _fields = ()
-    _attributes = ('lineno',)
+    fields = ()
+    attributes = ('lineno',)
 
     def __init__(self, *args, **kw):
         if args:
-            if len(args) != len(self._fields):
-                if not self._fields:
+            if len(args) != len(self.fields):
+                if not self.fields:
                     raise TypeError('%r takes 0 arguments' %
                                     self.__class__.__name__)
                 raise TypeError('%r takes 0 or %d argument%s' % (
                     self.__class__.__name__,
-                    len(self._fields),
-                    len(self._fields) != 1 and 's' or ''
+                    len(self.fields),
+                    len(self.fields) != 1 and 's' or ''
                 ))
-            for name, arg in izip(self._fields, args):
+            for name, arg in izip(self.fields, args):
                 setattr(self, name, arg)
-        for attr in self._attributes:
+        for attr in self.attributes:
             setattr(self, attr, kw.pop(attr, None))
         if kw:
             raise TypeError('unknown keyword argument %r' %
                             iter(kw).next())
 
     def iter_fields(self):
-        for name in self._fields:
+        """Iterate over all fields."""
+        for name in self.fields:
             try:
                 yield name, getattr(self, name)
             except AttributeError:
                 pass
 
     def iter_child_nodes(self):
+        """Iterate over all child nodes."""
         for field, item in self.iter_fields():
             if isinstance(item, list):
                 for n in item:
@@ -112,11 +101,38 @@ class Node(object):
             elif isinstance(item, Node):
                 yield item
 
+    def find(self, node_type):
+        """Find the first node of a given type."""
+        for result in self.find_all(node_type):
+            return result
+
+    def find_all(self, node_type):
+        """Find all the nodes of a given type."""
+        for child in self.iter_child_nodes():
+            if isinstance(child, node_type):
+                yield child
+            for result in child.find_all(node_type):
+                yield result
+
+    def set_ctx(self, ctx):
+        """
+        Reset the context of a node and all child nodes.  Per default the parser
+        will all generate nodes that have a 'load' context as it's the most common
+        one.  This method is used in the parser to set assignment targets and
+        other nodes to a store context.
+        """
+        todo = deque([self])
+        while todo:
+            node = todo.popleft()
+            if 'ctx' in node.fields:
+                node.ctx = ctx
+            todo.extend(node.iter_child_nodes())
+
     def __repr__(self):
         return '%s(%s)' % (
             self.__class__.__name__,
             ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
-                      arg in self._fields)
+                      arg in self.fields)
         )
 
 
@@ -136,7 +152,7 @@ class Template(Node):
     """
     Node that represents a template.
     """
-    _fields = ('body',)
+    fields = ('body',)
 
 
 class Output(Stmt):
@@ -144,91 +160,106 @@ class Output(Stmt):
     A node that holds multiple expressions which are then printed out.  This
     is used both for the `print` statement and the regular template data.
     """
-    _fields = ('nodes',)
+    fields = ('nodes',)
+
+    def optimized_nodes(self):
+        """Try to optimize the nodes."""
+        buffer = []
+        for node in self.nodes:
+            try:
+                const = unicode(node.as_const())
+            except:
+                buffer.append(node)
+            else:
+                if buffer and isinstance(buffer[-1], unicode):
+                    buffer[-1] += const
+                else:
+                    buffer.append(const)
+        return buffer
 
 
 class Extends(Stmt):
     """
     Represents an extends statement.
     """
-    _fields = ('extends',)
+    fields = ('template',)
 
 
 class For(Stmt):
     """
     A node that represents a for loop
     """
-    _fields = ('target', 'iter', 'body', 'else_', 'recursive')
+    fields = ('target', 'iter', 'body', 'else_', 'recursive')
 
 
 class If(Stmt):
     """
     A node that represents an if condition.
     """
-    _fields = ('test', 'body', 'else_')
+    fields = ('test', 'body', 'else_')
 
 
 class Macro(Stmt):
     """
     A node that represents a macro.
     """
-    _fields = ('name', 'arguments', 'body')
+    fields = ('name', 'args', 'defaults', 'dyn_args', 'dyn_kwargs', 'body')
 
 
 class CallBlock(Stmt):
     """
     A node that represents am extended macro call.
     """
-    _fields = ('expr', 'body')
+    fields = ('call', 'body')
 
 
 class Set(Stmt):
     """
     Allows defining own variables.
     """
-    _fields = ('name', 'expr')
+    fields = ('name', 'expr')
 
 
 class FilterBlock(Stmt):
     """
     Node for filter sections.
     """
-    _fields = ('body', 'filters')
+    fields = ('body', 'filters')
 
 
 class Block(Stmt):
     """
     A node that represents a block.
     """
-    _fields = ('name', 'body')
+    fields = ('name', 'body')
 
 
 class Include(Stmt):
     """
     A node that represents the include tag.
     """
-    _fields = ('template',)
+    fields = ('template', 'target')
 
 
 class Trans(Stmt):
     """
     A node for translatable sections.
     """
-    _fields = ('singular', 'plural', 'indicator', 'replacements')
+    fields = ('singular', 'plural', 'indicator', 'replacements')
 
 
 class ExprStmt(Stmt):
     """
     A statement that evaluates an expression to None.
     """
-    _fields = ('node',)
+    fields = ('node',)
 
 
 class Assign(Stmt):
     """
     Assigns an expression to a target.
     """
-    _fields = ('target', 'node')
+    fields = ('target', 'node')
 
 
 class Expr(Node):
@@ -254,7 +285,7 @@ class BinExpr(Expr):
     """
     Baseclass for all binary expressions.
     """
-    _fields = ('left', 'right')
+    fields = ('left', 'right')
     operator = None
 
     def as_const(self):
@@ -270,7 +301,7 @@ class UnaryExpr(Expr):
     """
     Baseclass for all unary expressions.
     """
-    _fields = ('node',)
+    fields = ('node',)
     operator = None
 
     def as_const(self):
@@ -285,10 +316,10 @@ class Name(Expr):
     """
     any name such as {{ foo }}
     """
-    _fields = ('name', 'ctx')
+    fields = ('name', 'ctx')
 
     def can_assign(self):
-        return True
+        return self.name not in ('true', 'false', 'none')
 
 
 class Literal(Expr):
@@ -301,7 +332,7 @@ class Const(Literal):
     """
     any constat such as {{ "foo" }}
     """
-    _fields = ('value',)
+    fields = ('value',)
 
     def as_const(self):
         return self.value
@@ -312,7 +343,7 @@ class Tuple(Literal):
     For loop unpacking and some other things like multiple arguments
     for subscripts.
     """
-    _fields = ('items', 'ctx')
+    fields = ('items', 'ctx')
 
     def as_const(self):
         return tuple(x.as_const() for x in self.items)
@@ -328,7 +359,7 @@ class List(Literal):
     """
     any list literal such as {{ [1, 2, 3] }}
     """
-    _fields = ('items',)
+    fields = ('items',)
 
     def as_const(self):
         return [x.as_const() for x in self.items]
@@ -338,7 +369,7 @@ class Dict(Literal):
     """
     any dict literal such as {{ {1: 2, 3: 4} }}
     """
-    _fields = ('items',)
+    fields = ('items',)
 
     def as_const(self):
         return dict(x.as_const() for x in self.items)
@@ -348,7 +379,7 @@ class Pair(Helper):
     """
     A key, value pair for dicts.
     """
-    _fields = ('key', 'value')
+    fields = ('key', 'value')
 
     def as_const(self):
         return self.key.as_const(), self.value.as_const()
@@ -358,7 +389,7 @@ class CondExpr(Expr):
     """
     {{ foo if bar else baz }}
     """
-    _fields = ('test', 'expr1', 'expr2')
+    fields = ('test', 'expr1', 'expr2')
 
     def as_const(self):
         if self.test.as_const():
@@ -370,35 +401,35 @@ class Filter(Expr):
     """
     {{ foo|bar|baz }}
     """
-    _fields = ('node', 'filters')
+    fields = ('node', 'filters')
 
 
 class FilterCall(Expr):
     """
     {{ |bar() }}
     """
-    _fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+    fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
 
 
 class Test(Expr):
     """
     {{ foo is lower }}
     """
-    _fields = ('name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
 
 
 class Call(Expr):
     """
     {{ foo(bar) }}
     """
-    _fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
+    fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')
 
 
 class Subscript(Expr):
     """
     {{ foo.bar }} and {{ foo['bar'] }} etc.
     """
-    _fields = ('node', 'arg', 'ctx')
+    fields = ('node', 'arg', 'ctx')
 
     def as_const(self):
         try:
@@ -414,14 +445,14 @@ class Slice(Expr):
     """
     1:2:3 etc.
     """
-    _fields = ('start', 'stop', 'step')
+    fields = ('start', 'stop', 'step')
 
 
 class Concat(Expr):
     """
     For {{ foo ~ bar }}.  Concatenates strings.
     """
-    _fields = ('nodes',)
+    fields = ('nodes',)
 
     def as_const(self):
         return ''.join(unicode(x.as_const()) for x in self.nodes)
@@ -431,14 +462,14 @@ class Compare(Expr):
     """
     {{ foo == bar }}, {{ foo >= bar }} etc.
     """
-    _fields = ('expr', 'ops')
+    fields = ('expr', 'ops')
 
 
 class Operand(Helper):
     """
     Operator + expression.
     """
-    _fields = ('op', 'expr')
+    fields = ('op', 'expr')
 
 
 class Mul(BinExpr):
@@ -517,14 +548,14 @@ class Not(UnaryExpr):
     operator = 'not'
 
 
-class NegExpr(UnaryExpr):
+class Neg(UnaryExpr):
     """
     {{ -foo }}
     """
     operator = '-'
 
 
-class PosExpr(UnaryExpr):
+class Pos(UnaryExpr):
     """
     {{ +foo }}
     """
diff --git a/jinja2/parser.py b/jinja2/parser.py
index 0c4edfc..96ba0a0 100644
--- a/jinja2/parser.py
+++ b/jinja2/parser.py
@@ -14,13 +14,12 @@ from jinja2.exceptions import TemplateSyntaxError
 
 __all__ = ['Parser']
 
-_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'include'])
+_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
+                                 'macro', 'include'])
 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq', 'in'])
-_tuple_edge_tokens = set(['rparen', 'block_end', 'variable_end', 'in',
-                         'semicolon', 'recursive'])
 _statement_end_tokens = set(['elif', 'else', 'endblock', 'endfilter',
-                             'endfor', 'endif', 'endmacro',
-                             'endcall', 'block_end'])
+                             'endfor', 'endif', 'endmacro', 'variable_end',
+                             'in', 'recursive', 'endcall', 'block_end'])
 
 
 class Parser(object):
@@ -39,7 +38,6 @@ class Parser(object):
         self.source = source
         self.filename = filename
         self.closed = False
-        self.blocks = set()
         self.no_variable_block = self.environment.lexer.no_variable_block
         self.stream = environment.lexer.tokenize(source, filename)
 
@@ -60,12 +58,14 @@ class Parser(object):
         elif token_type is 'call':
             self.stream.next()
             return self.parse_call_block()
-        lineno = self.stream.current.lineno
+        lineno = self.stream.current
         expr = self.parse_expression()
         if self.stream.current.type == 'assign':
-            return self.parse_assign(expr)
+            result = self.parse_assign(expr)
+        else:
+            result = nodes.ExprStmt(expr, lineno=lineno)
         self.end_statement()
-        return nodes.ExprStmt(expr, lineno=lineno)
+        return result
 
     def parse_assign(self, target):
         """Parse an assign statement."""
@@ -76,7 +76,7 @@ class Parser(object):
                                       self.filename)
         expr = self.parse_tuple()
         self.end_statement()
-        nodes.set_ctx(target, 'store')
+        target.set_ctx('store')
         return nodes.Assign(target, expr, lineno=lineno)
 
     def parse_statements(self, end_tokens, drop_needle=False):
@@ -95,6 +95,10 @@ class Parser(object):
         else:
             result = []
             while self.stream.current.type not in end_tokens:
+                if self.stream.current.type is 'block_end':
+                    self.stream.next()
+                    result.extend(self.subparse(end_tokens))
+                    break
                 result.append(self.parse_statement())
         if drop_needle:
             self.stream.next()
@@ -104,7 +108,11 @@ class Parser(object):
         """Parse a for loop."""
         lineno = self.stream.expect('for').lineno
         target = self.parse_tuple(simplified=True)
-        nodes.set_ctx(target, 'store')
+        if not target.can_assign():
+            raise TemplateSyntaxError("can't assign to '%s'" %
+                                      target, target.lineno,
+                                      self.filename)
+        target.set_ctx('store')
         self.stream.expect('in')
         iter = self.parse_tuple()
         if self.stream.current.type is 'recursive':
@@ -113,28 +121,100 @@ class Parser(object):
         else:
             recursive = False
         body = self.parse_statements(('endfor', 'else'))
-        token_type = self.stream.current.type
-        self.stream.next()
-        if token_type is 'endfor':
+        if self.stream.next().type is 'endfor':
             else_ = []
         else:
             else_ = self.parse_statements(('endfor',), drop_needle=True)
         return nodes.For(target, iter, body, else_, False, lineno=lineno)
 
     def parse_if(self):
-        pass
+        """Parse an if construct."""
+        node = result = nodes.If(lineno=self.stream.expect('if').lineno)
+        while 1:
+            # TODO: exclude conditional expressions here
+            node.test = self.parse_tuple()
+            node.body = self.parse_statements(('elif', 'else', 'endif'))
+            token_type = self.stream.next().type
+            if token_type is 'elif':
+                new_node = nodes.If(lineno=self.stream.current.lineno)
+                node.else_ = [new_node]
+                node = new_node
+                continue
+            elif token_type is 'else':
+                node.else_ = self.parse_statements(('endif',),
+                                                   drop_needle=True)
+            else:
+                node.else_ = []
+            break
+        return result
 
     def parse_block(self):
-        pass
+        node = nodes.Block(lineno=self.stream.expect('block').lineno)
+        node.name = self.stream.expect('name').value
+        node.body = self.parse_statements(('endblock',), drop_needle=True)
+        return node
 
     def parse_extends(self):
-        pass
+        node = nodes.Extends(lineno=self.stream.expect('extends').lineno)
+        node.template = self.parse_expression()
+        self.end_statement()
+        return node
 
     def parse_include(self):
-        pass
+        node = nodes.Include(lineno=self.stream.expect('include').lineno)
+        expr = self.parse_expression()
+        if self.stream.current.type is 'assign':
+            self.stream.next()
+            if not expr.can_assign():
+                raise TemplateSyntaxError('can\'t assign imported template '
+                                          'to this expression.', expr.lineno,
+                                          self.filename)
+            expr.set_ctx('store')
+            node.target = expr
+            node.template = self.parse_expression()
+        else:
+            node.target = None
+            node.template = expr
+        self.end_statement()
+        return node
 
     def parse_call_block(self):
-        pass
+        node = nodes.CallBlock(lineno=self.stream.expect('call').lineno)
+        node.call = self.parse_call()
+        node.body = self.parse_statements(('endcall',), drop_needle=True)
+        return node
+
+    def parse_macro(self):
+        node = nodes.Macro(lineno=self.stream.expect('macro').lineno)
+        node.name = self.stream.expect('name').value
+        self.stream.expect('lparen')
+        node.args = args = []
+        node.defaults = defaults = []
+        while self.stream.current.type is not 'rparen':
+            if args:
+                self.stream.expect('comma')
+            token = self.stream.expect('name')
+            arg = nodes.Name(token.value, 'param', lineno=token.lineno)
+            if not arg.can_assign():
+                raise TemplateSyntaxError("can't assign to '%s'" %
+                                          arg.name, arg.lineno,
+                                          self.filename)
+            if self.stream.current.type is 'assign':
+                self.stream.next()
+                defaults.append(self.parse_expression())
+        self.stream.expect('rparen')
+        node.body = self.parse_statements(('endmacro',), drop_needle=True)
+        return node
+
+    def parse_print(self):
+        node = nodes.Output(lineno=self.stream.expect('print').lineno)
+        node.nodes = []
+        while self.stream.current.type not in _statement_end_tokens:
+            if node.nodes:
+                self.stream.expect('comma')
+            node.nodes.append(self.parse_expression())
+        self.end_statement()
+        return node
 
     def parse_expression(self):
         """Parse an expression."""
@@ -237,7 +317,7 @@ class Parser(object):
         while self.stream.current.type is 'div':
             self.stream.next()
             right = self.parse_floordiv()
-            left = nodes.Floor(left, right, lineno=lineno)
+            left = nodes.Div(left, right, lineno=lineno)
             lineno = self.stream.current.lineno
         return left
 
@@ -277,11 +357,11 @@ class Parser(object):
         if token_type is 'not':
             self.stream.next()
             node = self.parse_unary()
-            return nodes.Neg(node, lineno=lineno)
+            return nodes.Not(node, lineno=lineno)
         if token_type is 'sub':
             self.stream.next()
             node = self.parse_unary()
-            return nodes.Sub(node, lineno=lineno)
+            return nodes.Neg(node, lineno=lineno)
         if token_type is 'add':
             self.stream.next()
             node = self.parse_unary()
@@ -330,7 +410,7 @@ class Parser(object):
         while 1:
             if args:
                 self.stream.expect('comma')
-            if self.stream.current.type in _tuple_edge_tokens:
+            if self.stream.current.type in _statement_end_tokens:
                 break
             args.append(parse())
             if self.stream.current.type is not 'comma':
@@ -389,10 +469,11 @@ class Parser(object):
     def parse_subscript(self, node):
         token = self.stream.next()
         if token.type is 'dot':
-            if token.type not in ('name', 'integer'):
+            attr_token = self.stream.current
+            if attr_token.type not in ('name', 'integer'):
                 raise TemplateSyntaxError('expected name or number',
-                                          token.lineno, self.filename)
-            arg = nodes.Const(token.value, lineno=token.lineno)
+                                          attr_token.lineno, self.filename)
+            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
             self.stream.next()
         elif token.type is 'lbracket':
             args = []
@@ -547,7 +628,12 @@ class Parser(object):
                 self.stream.next()
             elif token.type is 'variable_begin':
                 self.stream.next()
-                add_data(self.parse_tuple())
+                want_comma = False
+                while not self.stream.current.type in _statement_end_tokens:
+                    if want_comma:
+                        self.stream.expect('comma')
+                    add_data(self.parse_expression())
+                    want_comma = True
                 self.stream.expect('variable_end')
             elif token.type is 'block_begin':
                 flush_data()
diff --git a/jinja2/runtime.py b/jinja2/runtime.py
new file mode 100644
index 0000000..8032823
--- /dev/null
+++ b/jinja2/runtime.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.runtime
+    ~~~~~~~~~~~~~~
+
+    Runtime helpers.
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: GNU GPL.
+"""
+try:
+    from collections import defaultdict
+except ImportError:
+    defaultdict = None
+
+
+__all__ = ['extends', 'TemplateContext']
+
+
+def extends(template, namespace):
+    """
+    This loads a template (and evaluates it) and replaces the blocks.
+    """
+
+
+class TemplateContext(dict):
+
+    def __init__(self, globals, undefined_factory, filename):
+        dict.__init__(self, globals)
+        self.undefined_factory = undefined_factory
+        self.filename = filename
+
+    # if there is a default dict, dict has a __missing__ method we can use.
+    if defaultdict is None:
+        def __getitem__(self, name):
+            if name in self:
+                return self[name]
+            return self.undefined_factory(name)
+    else:
+        def __missing__(self, key):
+            return self.undefined_factory(key)
+
+    def from_locals(self, mapping):
+        """Update the template context from locals."""
+        for key, value in mapping.iteritems():
+            if key[:2] == 'l_':
+                self[key[:-2]] = value
diff --git a/jinja2/visitor.py b/jinja2/visitor.py
new file mode 100644
index 0000000..760fca5
--- /dev/null
+++ b/jinja2/visitor.py
@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja2.visitor
+    ~~~~~~~~~~~~~~
+
+    This module implements a visitor for the nodes.
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: GNU GPL.
+"""
+from jinja2.nodes import Node
+
+
+class NodeVisitor(object):
+    """
+    Walks the abstract syntax tree and call visitor functions for every node
+    found.  The visitor functions may return values which will be forwarded
+    by the `visit` method.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `get_visitor` function.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+    """
+
+    def get_visitor(self, node):
+        """
+        Return the visitor function for this node or `None` if no visitor
+        exists for this node.  In that case the generic visit function is
+        used instead.
+        """
+        method = 'visit_' + node.__class__.__name__
+        return getattr(self, method, None)
+
+    def visit(self, node, *args, **kwargs):
+        """Visit a node."""
+        f = self.get_visitor(node)
+        if f is not None:
+            return f(node, *args, **kwargs)
+        return self.generic_visit(node, *args, **kwargs)
+
+    def generic_visit(self, node, *args, **kwargs):
+        """Called if no explicit visitor function exists for a node."""
+        for node in node.iter_child_nodes():
+            self.visit(node, *args, **kwargs)
+
+
+class NodeTransformer(NodeVisitor):
+    """
+    Walks the abstract syntax tree and allows modifications of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor functions to replace or remove the old node.  If the return
+    value of the visitor function is `None` the node will be removed
+    from the previous location otherwise it's replaced with the return
+    value.  The return value may be the original node in which case no
+    replacement takes place.
+    """
+
+    def generic_visit(self, node, *args, **kwargs):
+        for field, old_value in node.iter_fields():
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, Node):
+                        value = self.visit(value, *args, **kwargs)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, Node):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, Node):
+                new_node = self.visit(old_value, *args, **kwargs)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
diff --git a/test.py b/test.py
new file mode 100644
index 0000000..d2f0653
--- /dev/null
+++ b/test.py
@@ -0,0 +1,16 @@
+from jinja2 import Environment
+from jinja2.compiler import generate
+
+
+env = Environment()
+ast = env.parse("""
+Hello {{ name }}!.  How is it {{ "going" }}. {{ [1, 2, 3] }}?
+{% for name in user_names %}
+    {{ loop.index }}: {{ name }}
+    {% if loop.index % 2 == 0 %}...{% endif %}
+{% endfor %}
+Name again: {{ name }}!
+""")
+print ast
+print
+print generate(ast, "foo.html")
-- 
2.26.2