From 1cc232c9de127a0e8b7c786528eb0a192067d14d Mon Sep 17 00:00:00 2001
From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Fri, 7 Sep 2007 17:52:41 +0200
Subject: [PATCH] [svn] merged newparser into trunk

--HG--
branch : trunk
---
 CHANGES                         |   63 ++
 docs/src/builtins.txt           |   22 +-
 docs/src/designerdoc.txt        |   36 +-
 docs/src/scopes.txt             |   23 +
 jdebug.py                       |    8 +-
 jinja/__init__.py               |    8 +-
 jinja/_debugger.c               |   10 +-
 jinja/contrib/__init__.py       |   11 +
 jinja/contrib/_djangosupport.py |  208 ++++
 jinja/contrib/djangosupport.py  |  124 +++
 jinja/datastructure.py          |  221 +++--
 jinja/debugger.py               |   59 +-
 jinja/defaults.py               |   12 +-
 jinja/environment.py            |   20 +-
 jinja/filters.py                |   51 +-
 jinja/lexer.py                  |  201 +++-
 jinja/loaders.py                |   46 +-
 jinja/nodes.py                  |  627 +++++++++---
 jinja/parser.py                 | 1644 +++++++++++++++++++------------
 jinja/plugin.py                 |    5 +
 jinja/tests.py                  |   23 +-
 jinja/translators/__init__.py   |    2 +-
 jinja/translators/python.py     |  596 ++++++-----
 jinja/utils.py                  |   48 +-
 tests/runtime/bigtable.py       |    6 +-
 tests/runtime/exception.py      |   26 +-
 tests/test_lexer.py             |   23 +
 tests/test_loaders.py           |   21 +-
 tests/test_macros.py            |    7 +
 tests/test_security.py          |    4 +-
 tests/test_syntax.py            |  116 ++-
 tests/test_tests.py             |   11 +-
 tests/test_various.py           |   21 +-
 33 files changed, 2971 insertions(+), 1332 deletions(-)
 create mode 100644 jinja/contrib/__init__.py
 create mode 100644 jinja/contrib/_djangosupport.py
 create mode 100644 jinja/contrib/djangosupport.py

diff --git a/CHANGES b/CHANGES
index b30fa9b..73d1609 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,69 @@ Version 1.2
 
 - added `sameas` test function.
 
+- standalone parser. Jinja does not use the python parser any more and will
+  continue having the same semantics in any future python versions. This
+  was done in order to simplify updating Jinja to 2.6 and 3.0 and to support
+  non python syntax.
+
+- added support for ``expr1 if test else expr2`` (conditional expressions)
+
+- ``foo.0`` as alias for ``foo[0]`` is possible now. This is mainly for
+  django compatibility.
+
+- the filter operators has a much higher priority now which makes it
+  possible to do ``foo|filter + bar|filter``.
+
+- new AST. the return value of `Environment.parse` is now a Jinja AST and not
+  a Jinja-Python AST. This is also the only backwards incompatible change but
+  should not affect many users because this feature is more or less
+  undocumented and has few use cases.
+
+- tuple syntax returns tuples now and not lists any more.
+
+- the print directive and ``{{ variable }}`` syntax now accepts and implicit
+  tuple like the `for` and `cycle` tags. (``{{ 1, 2 }}`` is an implicit alias
+  for ``{{ (1, 2) }}` like ``{% for a, b in seq %}`` is for
+  ``{% for (a, b) in seq %}``.
+
+- tests called with *one* parameter don't need parentheses. This gives a more
+  natural syntax for the `sameas` test and some others:
+  ``{{ foo is sameas bar }}`` instead of ``{{ foo is sameas(bar) }}``. If you
+  however want to pass more than one argument you have to use parentheses
+  because ``{{ foo is sometest bar, baz }}`` is handled as
+  ``{{ (foo is sometest(bar), baz) }}``, so as tuple expression.
+
+- removed support for octal character definitions in strings such as
+  ``'\14'``, use ``'\x0c'`` now.
+
+- added regular expression literal. ``@/expr/flags`` equals
+  ``re.compile(r'(?flags)expr')``. This is useful for the `matching` test and
+  probably some others.
+
+- added set literal. We do not use python3's {1, 2} syntax because
+  this conflicts with the dict literal. To be compatible with the regex
+  literal we use ``@(1, 2)`` instead.
+
+- fixed bug in `get_attribute` that disallowed retreiving attributes of objects
+  without a `__class__` such as `_sre.SRE_Pattern`.
+
+- addded `django.contrib.jinja` which provides advanced support for django.
+  (thanks Bryan McLemore)
+
+- debugger is now able to rewrite the whole traceback, not only the first
+  frame. (requires the optional debugger c module which is compiled
+  automatically on installation if possible)
+
+- if the set that is postfixed with a bang (!) it acts like the python 3
+  "nonlocal" keyword. This means that you can now override variables
+  defined in the outer scope from within a loop.
+
+- ``foo + bar`` is now a simpler alternative to ``foo|string + bar|string``
+
+- `PackageLoader` can now work without pkg_resources too
+
+- added `getattribute` and `getitem` filter.
+
 
 Version 1.1
 -----------
diff --git a/docs/src/builtins.txt b/docs/src/builtins.txt
index 09d9da8..0708077 100644
--- a/docs/src/builtins.txt
+++ b/docs/src/builtins.txt
@@ -27,9 +27,11 @@ with the arguments ``'foo'`` and ``'bar'``, and pass the result to the filter
 
 .. admonition:: note
 
-    The filter operator has a pretty low priority. If you want to add fitered
-    values you have to put them into parentheses. The same applies if you want
-    to access attributes or return values:
+    *Jinja 1.0 and 1.1 notice*
+
+    The filter operator has a pretty low priority in Jinja 1.0 and 1.1. If you
+    want to add fitered values you have to put them into parentheses. The same
+    applies if you want to access attributes or return values:
 
     .. sourcecode:: jinja
 
@@ -43,6 +45,9 @@ with the arguments ``'foo'`` and ``'bar'``, and pass the result to the filter
         wrong:
             {{ foo|filter.attribute }}
 
+    This changed in Jinja 1.2, from that version one the filter operator has
+    the highest priority so you can do ``foo|filter + bar|filter``.
+
 *new in Jinja 1.1*:
 
 Because the application can provide additional filters you can get a documentation
@@ -84,6 +89,17 @@ of all the provided tests by calling ``debug.tests()``:
     {{ debug.tests(False) }}
         -> same as above but without the builtin ones.
 
+*new in Jinja 1.2*:
+
+If a test function expects one or no argument you can leave out the parentheses.
+Previously this was only possible for text functions without arguments:
+
+.. sourcecode:: jinja
+
+    {{ foo is matching @/\s+/ }}
+        is the same as
+    {{ foo is matching(@/\s+/) }}
+
 
 Global Functions
 ================
diff --git a/docs/src/designerdoc.txt b/docs/src/designerdoc.txt
index d7d4f6b..c0ab8f6 100644
--- a/docs/src/designerdoc.txt
+++ b/docs/src/designerdoc.txt
@@ -194,6 +194,31 @@ You can also use comparison operators:
     in some situations it might be a good thing to have the abilities to create
     them.
 
+Literals
+========
+
+For most of the builtin python types, literals exist in Jinja. The following
+table shows which syntax elements are supported:
+
+    ======================= ===================================================
+    ``"text" / 'text'``     work like python's unicode literals (u'text').
+    ``42``                  integer literls.
+    ``42.0``                float literals (exponents are not supported and
+                            before and after the dot digits must be present)
+    ``[1, 'two', none]``    list literal
+    ``(), (1,), (1, 2)``    tuple literals. (tuples work like lists but consume
+                            less memory and are not modifyable.)
+    ``{'foo': 'bar'}``      dictionary literal
+    ``@/expr/flags``        regular expression literals. ``@/expr/flags`` is
+                            equivalent to ``re.compile('(?flags)expr')`` in
+                            python.
+    ``@(1, 2, 3)``          set literal. ``@(1, 2, 3)`` in Jinja is is equal to
+                            ``set([1, 2, 3])`` in python.
+    ``true / false``        corresponds to `True` and `False` in python.
+    ``none``                corresponds to `None` in python.
+    ``undefined``           special Jinja undefined singleton.
+    ======================= ===================================================
+
 Operators
 =========
 
@@ -291,7 +316,7 @@ create a macro from it:
 
     {% macro show_user user %}
       <h1>{{ user.name|e }}</h1>
-      <div class="test">
+      <div class="text">
         {{ user.description }}
       </div>
     {% endmacro %}
@@ -311,7 +336,7 @@ You can also specify more than one value:
     {% macro show_dialog title, text %}
       <div class="dialog">
         <h1>{{ title|e }}</h1>
-        <div class="test">{{ text|e }}</div>
+        <div class="text">{{ text|e }}</div>
       </div>
     {% endmacro %}
 
@@ -491,12 +516,7 @@ The following keywords exist and cannot be used as identifiers:
     `and`, `block`, `cycle`, `elif`, `else`, `endblock`, `endfilter`,
     `endfor`, `endif`, `endmacro`, `endraw`, `endtrans`, `extends`, `filter`,
     `for`, `if`, `in`, `include`, `is`, `macro`, `not`, `or`, `pluralize`,
-    `print`, `raw`, `recursive`, `set`, `trans`, `call` +, `endcall` +
-
-keywords marked with a plus sign can be used until Jinja 1.3 as
-identifiers because they were introduced after the Jinja 1.0 release and
-can cause backwards compatiblity problems otherwise. You should not use
-them any more.
+    `print`, `raw`, `recursive`, `set`, `trans`, `call`, `endcall`
 
 If you want to use such a name you have to prefix or suffix it or use
 alternative names:
diff --git a/docs/src/scopes.txt b/docs/src/scopes.txt
index 205c8b4..52a30bf 100644
--- a/docs/src/scopes.txt
+++ b/docs/src/scopes.txt
@@ -159,3 +159,26 @@ In order to check if a value is defined you can use the `defined` test:
 
     It raises exceptions as soon as you either render it or want to iterate
     over it or try to access attributes etc.
+
+
+Overriding Variables Of Outer Scopes
+====================================
+
+*New in Jinja 1.2*
+
+Normally you cannot override a variable from an outer scope, you can just hide
+it. There is however a way to override a variable from an outer scope using the
+`set` tag, postfixed with a bang (!):
+
+.. source:: jinja
+
+    {% set last_item = none %}
+    {% for item in seq %}
+        {% set last_item = item! %}
+    {% endfor %}
+
+After the iteration `last_item` will point to the item of the last iteration.
+
+If `last_item` was not defined in the outer scope it wouldn't exist now because
+in that situation `set`, even with a postfixed bang just behaves like setting a
+scope variable.
diff --git a/jdebug.py b/jdebug.py
index 487ec66..22ea06d 100644
--- a/jdebug.py
+++ b/jdebug.py
@@ -36,11 +36,13 @@ if os.environ.get('JDEBUG_SOURCEPRINT'):
 def p(x=None, f=None):
     if x is None and f is not None:
         x = e.loader.get_source(f)
-    print PythonTranslator(e, Parser(e, x, f).parse()).translate()
+    print PythonTranslator(e, Parser(e, x, f).parse(), None).translate()
 
 def l(x):
-    for item in e.lexer.tokenize(x):
-        print '%5s  %-20s  %r' % item
+    for token in e.lexer.tokenize(x):
+        print '%5s  %-20s  %r' % (item.lineno,
+                                  item.type,
+                                  item.value)
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:
diff --git a/jinja/__init__.py b/jinja/__init__.py
index 8178f5c..3c09362 100644
--- a/jinja/__init__.py
+++ b/jinja/__init__.py
@@ -59,12 +59,12 @@
 
 from jinja.environment import Environment
 from jinja.datastructure import Markup
-from jinja.utils import from_string
 from jinja.plugin import jinja_plugin_factory as template_plugin_factory
 from jinja.loaders import FileSystemLoader, PackageLoader, DictLoader, \
-     ChoiceLoader, FunctionLoader
+     ChoiceLoader, FunctionLoader, MemcachedFileSystemLoader
+from jinja.utils import from_string
 
 
 __all__ = ['Environment', 'Markup', 'FileSystemLoader', 'PackageLoader',
-           'DictLoader', 'ChoiceLoader', 'FunctionLoader', 'from_string',
-           'template_plugin_factory']
+           'DictLoader', 'ChoiceLoader', 'FunctionLoader',
+           'MemcachedFileSystemLoader', 'from_string']
diff --git a/jinja/_debugger.c b/jinja/_debugger.c
index 87ef3c3..50462e1 100644
--- a/jinja/_debugger.c
+++ b/jinja/_debugger.c
@@ -28,18 +28,18 @@ tb_set_next(PyObject *self, PyObject *args)
 
 	if (!PyArg_ParseTuple(args, "O!O:tb_set_next", &PyTraceBack_Type, &tb, &next))
 		return NULL;
-	if (next == Py_None) {
+	if (next == Py_None)
 		next = NULL;
-	} else if (!PyTraceBack_Check(next)) {
+	else if (!PyTraceBack_Check(next)) {
 		PyErr_SetString(PyExc_TypeError,
 				"tb_set_next arg 2 must be traceback or None");
 		return NULL;
-	} else {
-		Py_INCREF(next);
 	}
+	else
+		Py_INCREF(next);
 
 	old = tb->tb_next;
-	tb->tb_next = (PyTracebackObject *)next;
+	tb->tb_next = (PyTracebackObject*)next;
 	Py_XDECREF(old);
 
 	Py_INCREF(Py_None);
diff --git a/jinja/contrib/__init__.py b/jinja/contrib/__init__.py
new file mode 100644
index 0000000..1770052
--- /dev/null
+++ b/jinja/contrib/__init__.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja.contrib
+    ~~~~~~~~~~~~~
+
+    This module collections various third-party helper functions and classes
+    that are useful for frameworks etc.
+
+    :copyright: 2007 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
diff --git a/jinja/contrib/_djangosupport.py b/jinja/contrib/_djangosupport.py
new file mode 100644
index 0000000..ff38110
--- /dev/null
+++ b/jinja/contrib/_djangosupport.py
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja.contrib._djangosupport
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Django suport layer. This module is a metamodule, do never import it
+    directly or access any of the functions defined here.
+
+    The public interface is `jinja.contrib.djangosupport` and
+    `django.contrib.jinja`. See the docstring of `jinja.contrib.djangosupport`
+    for more details.
+
+    :copyright: 2007 by Armin Ronacher, Bryan McLemore.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+import new
+from django.conf import settings
+from django.template.context import get_standard_processors
+from django.http import HttpResponse
+from django import contrib
+
+from jinja import Environment, FileSystemLoader, ChoiceLoader
+from jinja.loaders import MemcachedFileSystemLoader
+
+
+exported = ['render_to_response', 'render_to_string', 'convert_django_filter']
+
+
+#: used environment
+env = None
+
+
+#: default filters
+DEFAULT_FILTERS = [
+    'django.template.defaultfilters.date',
+    'django.template.defaultfilters.timesince',
+    'django.template.defaultfilters.linebreaks',
+    'django.contrib.humanize.templatetags.humanize.intcomma'
+]
+
+
+def configure(convert_filters=DEFAULT_FILTERS, loader=None, **options):
+    """
+    Initialize the system.
+    """
+    global env
+
+    if env is not None:
+        raise RuntimeError('jinja already configured')
+
+    # setup environment
+    if loader is None:
+        loaders = [FileSystemLoader(l) for l in settings.TEMPLATE_DIRS]
+        if not loaders:
+            loader = None
+        elif len(loaders) == 1:
+            loader = loaders[0]
+        else:
+            loader = ChoiceLoader(loaders)
+    env = Environment(loader=loader, **options)
+
+    # convert requested filters
+    for name in convert_filters:
+        env.filters[name] = convert_django_filter(name)
+
+    # import templatetags of installed apps
+    for app in settings.INSTALLED_APPS:
+        try:
+            __import__(app + '.templatetags')
+        except ImportError:
+            pass
+
+    # setup the django.contrib.jinja module
+    setup_django_module()
+
+
+def setup_django_module():
+    """
+    create a new Jinja module for django.
+    """
+    from jinja.contrib import djangosupport
+    module = contrib.jinja = sys.modules['django.contrib.jinja'] = \
+             new.module('django.contrib.jinja')
+    module.env = env
+    module.__doc__ = djangosupport.__doc__
+    module.register = Library()
+    public_names = module.__all__ = ['register', 'env']
+    get_name = globals().get
+    for name in exported:
+        setattr(module, name, get_name(name))
+        public_names.append(name)
+
+
+def render_to_response(template, context={}, request=None,
+                       mimetype=None):
+    """This function will take a few variables and spit out a full webpage."""
+    content = render_to_string(template, context, request)
+    if mimetype is None:
+        mimetype = settings.DEFUALT_CONTENT_TYPE
+    return HttpResponse(content, content_type)
+
+
+def render_to_string(template, context={}, request=None):
+    """Render a template to a string."""
+    assert env is not None, 'Jinja not configured for django'
+    if request is not None:
+        context['request'] = request
+        for processor in get_standard_processors():
+            context.update(processor(request))
+    template = env.get_template(template)
+    return template.render(context)
+
+
+def convert_django_filter(f):
+    """Convert a django filter into a Jinja filter."""
+    if isinstance(f, str):
+        p = f.split('.')
+        f = getattr(__import__('.'.join(p[:-1]), None, None, ['']), p[-1])
+    def filter_factory(*args):
+        def wrapped(env, ctx, value):
+            return f(value, *args)
+        return wrapped
+    try:
+        filter_factory.__name__ = f.__name__
+        filter_factory.__doc__ = f.__doc__
+    except:
+        pass
+    return filter_factory
+
+
+class Library(object):
+    """
+    Continues a general feel of wrapping all the registration
+    methods for easy importing.
+
+    This is available in `django.contrib.jinja` as `register`.
+
+    For more details see the docstring of the `django.contrib.jinja` module.
+    """
+    __slots__ = ()
+
+    def object(obj, name=None):
+        """Register a new global."""
+        if name is None:
+            name = getattr(obj, '__name__')
+        env.globals[name] = obj
+        return func
+
+    def filter(func, name=None):
+        """Register a new filter function."""
+        if name is None:
+            name = func.__name__
+        env.filters[name] = func
+        return func
+
+    def test(func, name):
+        """Register a new test function."""
+        if name is None:
+            name = func.__name__
+        env.tests[name] = func
+        return func
+
+    def context_inclusion(func, template, name=None):
+        """
+        Similar to the inclusion tag from django this one expects func to be a
+        function with a similar argument list to func(context, *args, **kwargs)
+
+        It passed in the current context allowing the function to edit it or read
+        from it.  the function must return a dict with which to pass into the
+        renderer.  Normally expected is an altered dictionary.
+
+        Note processors are NOT ran on this context.
+        """
+        def wrapper(env, context, *args, **kwargs):
+            context = func(context.to_dict(), *args, **kwargs)
+            return render_to_string(template, context)
+        wrapper.jinja_context_callable = True
+        if name is None:
+            name = func.__name__
+        try:
+            wrapper.__name__ = func.__name__
+            wrapper.__doc__ = func.__doc__
+        except:
+            pass
+        env.globals[name] = wrapper
+
+    def clean_inclusion(func, template, name=None, run_processors=False):
+        """
+        Similar to above however it won't pass the context into func().
+        Also the returned context will have the context processors run upon it.
+        """
+        def wrapper(env, context, *args, **kwargs):
+            if run_processors:
+                request = context['request']
+            else:
+                request = None
+            context = func({}, *args, **kwargs)
+            return render_to_string(template, context, request)
+        wrapper.jinja_context_callable = True
+        if name is None:
+            name = func.__name__
+        try:
+            wrapper.__name__ = func.__name__
+            wrapper.__doc__ = func.__doc__
+        except:
+            pass
+        env.globals[name] = wrapper
diff --git a/jinja/contrib/djangosupport.py b/jinja/contrib/djangosupport.py
new file mode 100644
index 0000000..9898a82
--- /dev/null
+++ b/jinja/contrib/djangosupport.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+"""
+    jinja.contrib.djangosupport
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Support for the django framework. This module is quite magical because it
+    just exports one single function, the `configure` function which is used
+    to create a new Jinja environment and setup a special module called
+    `django.contrib.jinja` which exports a couple of functions useful for Jinja.
+
+    Quickstart
+    ==========
+
+    To get started execute the following code at the bottom of your settings.py
+    or in some general application file such as urls.py or a central module. The
+    only thing that matters is that it's executed right *after* the settings
+    were set up and *before* `django.contrib.jinja` is imported::
+
+        from jinja.contrib import djangosupport
+        djangosupport.configure()
+
+    What this does is setting up a Jinja environment for this django instance
+    with loaders for `TEMPLATE_DIRS` etc.  It also converts a couple of default
+    django filters such as `date` and `timesince` which are not available in
+    Jinja per default.  If you want to change the list you can provide others
+    by passing a list with filter import names as `convert_filters` keyword
+    argument.
+
+    All other keyword arguments are forwarded to the environment.  If you want
+    to provide a loader yourself pass it a loader keyword argument.
+
+    Rendering Templates
+    ===================
+
+    To render a template you can use the functions `render_to_string` or
+    `render_to_response` from the `django.contrib.jinja` module::
+
+        from django.contrib.jinja import render_to_response
+        resp = render_to_response('Hello {{ username }}!', {
+            'username':     req.session['username']
+        }, req)
+
+    `render_to_string` and `render_to_response` take at least the name of
+    the template as argument, then the optional dict which will become the
+    context.  If you also provide a request object as third argument the
+    context processors will be applied.
+
+    `render_to_response` also takes a forth parameter which can be the
+    content type which defaults to `DEFAULT_CONTENT_TYPE`.
+
+    Converting Filters
+    ==================
+
+    One of the useful objects provided by `django.contrib.jinja` is the
+    `register` object which can be used to register filters, tests and
+    global objects.  You can also convert any filter django provides in
+    a Jinja filter using `convert_django_filter`::
+
+        from django.contrib.jinja import register, convert_django_filter
+        from django.template.defaultfilters import floatformat
+
+        register.filter(convert_django_filter(floatformat), 'floatformat')
+
+    Available methods on the `register` object:
+
+    ``object (obj[, name])``
+        Register a new global as name or with the object's name.
+        Returns the function object unchanged so that you can use
+        it as decorator if no name is provided.
+
+    ``filter (func[, name])``
+        Register a function as filter with the name provided or
+        the object's name as filtername.
+        Returns the function object unchanged so that you can use
+        it as decorator if no name is provided.
+
+    ``test (func[, name])``
+        Register a function as test with the name provided or the
+        object's name as testname.
+        Returns the function object unchanged so that you can use
+        it as decorator if no name is provided.
+
+    ``context_inclusion (func, template[, name])``
+        Register a function with a name provided or the func object's
+        name in the global namespace that acts as subrender function.
+
+        func is called with the callers context as dict and the
+        arguments and keywords argument of the inclusion function.
+        The function should then process the context and return a
+        new context or the same context object. Afterwards the
+        template is rendered with this context.
+
+        Example::
+
+            def add_author(context, author=None):
+                if author is not None:
+                    author = Author.objects.get(name=author)
+                context['author'] = author
+                return context
+
+            register.context_inclusion(add_author, 'author_details.html',
+                                       'render_author_details')
+
+        You can use it in the template like this then::
+
+            {{ render_author_details('John Doe') }}
+
+    ``clean_inclusion (func, template[, name[, run_processors]]) ``
+        Works like `context_inclusion` but doesn't use the calles
+        context but an empty context. If `run_processors` is `True`
+        it will lookup the context for a `request` object and pass
+        it to the render function to apply context processors.
+
+    :copyright: 2007 by Armin Ronacher, Bryan McLemore.
+    :license: BSD, see LICENSE for more details.
+"""
+try:
+    __import__('django')
+except ImportError:
+    raise ImportError('installed django required for djangosupport')
+else:
+    from jinja.contrib._djangosupport import configure
+
+__all__ = ['configure']
diff --git a/jinja/datastructure.py b/jinja/datastructure.py
index 11d0f98..b7cff18 100644
--- a/jinja/datastructure.py
+++ b/jinja/datastructure.py
@@ -11,6 +11,8 @@
 
 from jinja.exceptions import TemplateSyntaxError, TemplateRuntimeError
 
+_missing = object()
+
 
 def contextcallable(f):
     """
@@ -262,6 +264,16 @@ class Context(BaseContext):
                 result[key] = value
         return result
 
+    def set_nonlocal(self, name, value):
+        """
+        Set a value in an outer scope.
+        """
+        for layer in self.stack[:0:-1]:
+            if name in layer:
+                layer[name] = value
+                return
+        self.current[name] = value
+
     def translate_func(self):
         """
         The translation function for this context. It takes
@@ -443,108 +455,171 @@ class StateTest(object):
     functions that replace some lambda expressions
     """
 
-    def __init__(self, func, error_message):
+    def __init__(self, func, msg):
         self.func = func
-        self.error_message = error_message
+        self.msg = msg
+
+    def __call__(self, token):
+        return self.func(token)
+
+    def expect_token(*types, **kw):
+        """Scans until one of the given tokens is found."""
+        msg = kw.pop('msg', None)
+        if kw:
+            raise TypeError('unexpected keyword argument %r' % iter(kw).next())
+        if len(types) == 1:
+            if msg is None:
+                msg = "expected '%s'" % types[0]
+            return StateTest(lambda t: t.type == types[0], msg)
+        if msg is None:
+            msg = 'expected one of %s' % ', '.join(["'%s'" % type
+                                                    for type in types])
+        return StateTest(lambda t: t.type in types, msg)
+    expect_token = staticmethod(expect_token)
 
-    def __call__(self, p, t, d):
-        return self.func(p, t, d)
 
-    def expect_token(token_name, error_message=None):
-        """Scans until a token types is found."""
-        return StateTest(lambda p, t, d: t == token_name, 'expected ' +
-                         (error_message or token_name))
-    expect_token = staticmethod(expect_token)
+class Token(object):
+    """
+    Token class.
+    """
+    __slots__ = ('lineno', 'type', 'value')
 
-    def expect_name(*names):
-        """Scans until one of the given names is found."""
-        if len(names) == 1:
-            name = names[0]
-            return StateTest(lambda p, t, d: t == 'name' and d == name,
-                             "expected '%s'" % name)
-        else:
-            return StateTest(lambda p, t, d: t == 'name' and d in names,
-                             'expected one of %s' % ','.join(["'%s'" % name
-                             for name in names]))
-    expect_name = staticmethod(expect_name)
+    def __init__(self, lineno, type, value):
+        self.lineno = lineno
+        self.type = intern(str(type))
+        self.value = value
+
+    def __str__(self):
+        from jinja.lexer import keywords, reverse_operators
+        if self.type in keywords:
+            return self.type
+        elif self.type in reverse_operators:
+            return reverse_operators[self.type]
+        return self.value
+
+    def __repr__(self):
+        return 'Token(%r, %r, %r)' % (
+            self.lineno,
+            self.type,
+            self.value
+        )
+
+
+class TokenStreamIterator(object):
+    """
+    The iterator for tokenstreams. Iterate over the stream
+    until the eof token is reached.
+    """
+
+    def __init__(self, stream):
+        self._stream = stream
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        token = self._stream.current
+        if token.type == 'eof':
+            raise StopIteration()
+        self._stream.next()
+        return token
 
 
 class TokenStream(object):
     """
-    A token stream works like a normal generator just that
-    it supports pushing tokens back to the stream.
+    A token stream wraps a generator and supports pushing tokens back.
+    It also provides some functions to expect tokens and similar stuff.
     """
 
     def __init__(self, generator, filename):
         self._next = generator.next
         self._pushed = []
-        self.last = (1, 'initial', '')
+        self.current = Token(1, 'initial', '')
         self.filename = filename
+        self.next()
+
+    def __iter__(self):
+        return TokenStreamIterator(self)
 
     def bound(self):
         """Return True if the token stream is bound to a parser."""
         return self.parser is not None
     bound = property(bound, doc=bound.__doc__)
 
-    def __iter__(self):
-        """Return self in order to mark this is iterator."""
-        return self
+    def lineno(self):
+        """The current line number."""
+        return self.current.lineno
+    lineno = property(lineno, doc=lineno.__doc__)
 
     def __nonzero__(self):
         """Are we at the end of the tokenstream?"""
-        if self._pushed:
-            return True
-        try:
-            self.push(self.next())
-        except StopIteration:
-            return False
-        return True
+        return bool(self._pushed) or self.current.type != 'eof'
 
     eos = property(lambda x: not x.__nonzero__(), doc=__nonzero__.__doc__)
 
-    def next(self):
-        """Return the next token from the stream."""
-        if self._pushed:
-            rv = self._pushed.pop()
-        else:
-            rv = self._next()
-        self.last = rv
-        return rv
-
     def look(self):
-        """Pop and push a token, return it."""
-        token = self.next()
-        self.push(*token)
-        return token
+        """See what's the next token."""
+        if self._pushed:
+            return self._pushed[-1]
+        old_token = self.current
+        self.next()
+        new_token = self.current
+        self.current = old_token
+        self.push(new_token)
+        return new_token
+
+    def push(self, token):
+        """Push a token back to the stream."""
+        self._pushed.append(token)
+
+    def skip(self, n):
+        """Got n tokens ahead."""
+        for x in xrange(n):
+            self.next()
 
-    def fetch_until(self, test, drop_needle=False):
-        """Fetch tokens until a function matches."""
+    def next(self):
+        """Go one token ahead."""
+        if self._pushed:
+            self.current = self._pushed.pop()
+        elif self.current.type != 'eof':
+            try:
+                self.current = self._next()
+            except StopIteration:
+                self.close()
+
+    def read_whitespace(self):
+        """Read all the whitespace, up to the next tag."""
+        lineno = self.current.lineno
+        buf = []
+        while self.current.type == 'data' and not \
+              self.current.value.strip():
+            buf.append(self.current.value)
+            self.next()
+        if buf:
+            return Token(lineno, 'data', u''.join(buf))
+
+    def close(self):
+        """Close the stream."""
+        self.current = Token(self.current.lineno, 'eof', '')
+        self._next = None
+
+    def expect(self, token_type, token_value=_missing):
+        """Expect a given token type and return it"""
+        if self.current.type != token_type:
+            raise TemplateSyntaxError("expected token %r, got %r" %
+                                      (token_type, self.current.type),
+                                      self.current.lineno,
+                                      self.filename)
+        elif token_value is not _missing and \
+             self.current.value != token_value:
+            raise TemplateSyntaxError("expected %r, got %r" %
+                                      (token_value, self.current.value),
+                                      self.current.lineno,
+                                      self.filename)
         try:
-            while True:
-                token = self.next()
-                if test(*token):
-                    if not drop_needle:
-                        self.push(*token)
-                    return
-                else:
-                    yield token
-        except StopIteration:
-            if isinstance(test, StateTest):
-                msg = ': ' + test.error_message
-            else:
-                msg = ''
-            raise TemplateSyntaxError('end of stream' + msg,
-                                      self.last[0], self.filename)
-
-    def drop_until(self, test, drop_needle=False):
-        """Fetch tokens until a function matches and drop all
-        tokens."""
-        for token in self.fetch_until(test, drop_needle):
-            pass
-
-    def push(self, lineno, token, data):
-        """Push an yielded token back to the stream."""
-        self._pushed.append((lineno, token, data))
+            return self.current
+        finally:
+            self.next()
 
 
 class TemplateStream(object):
diff --git a/jinja/debugger.py b/jinja/debugger.py
index 9078b09..932fb1c 100644
--- a/jinja/debugger.py
+++ b/jinja/debugger.py
@@ -47,11 +47,11 @@ except ImportError:
     has_extended_debugger = False
 
 # we need the RUNTIME_EXCEPTION_OFFSET to skip the not used frames
-from jinja.utils import RUNTIME_EXCEPTION_OFFSET
+from jinja.utils import reversed, RUNTIME_EXCEPTION_OFFSET
 
 
-def fake_template_exception(exc_type, exc_value, traceback, filename, lineno,
-                            source, context_or_env):
+def fake_template_exception(exc_type, exc_value, tb, filename, lineno,
+                            source, context_or_env, tb_back=None):
     """
     Raise an exception "in a template". Return a traceback
     object. This is used for runtime debugging, not compile time.
@@ -105,14 +105,17 @@ def fake_template_exception(exc_type, exc_value, traceback, filename, lineno,
 
     # if we have an extended debugger we set the tb_next flag so that
     # we don't loose the higher stack items.
-    if has_extended_debugger and traceback is not None:
-        tb_set_next(exc_info[2].tb_next, traceback.tb_next)
+    if has_extended_debugger:
+        if tb_back is not None:
+            tb_set_next(tb_back, exc_info[2])
+        if tb is not None:
+            tb_set_next(exc_info[2].tb_next, tb.tb_next)
 
     # otherwise just return the exc_info from the simple debugger
     return exc_info
 
 
-def translate_exception(template, context, exc_type, exc_value, traceback):
+def translate_exception(template, context, exc_type, exc_value, tb):
     """
     Translate an exception and return the new traceback.
     """
@@ -120,21 +123,35 @@ def translate_exception(template, context, exc_type, exc_value, traceback):
     # step to get the frame of the current template. The frames before
     # are the toolchain used to render that thing.
     for x in xrange(RUNTIME_EXCEPTION_OFFSET):
-        traceback = traceback.tb_next
-
-    # the next thing we do is matching the current error line against the
-    # debugging table to get the correct source line. If we can't find the
-    # filename and line number we return the traceback object unaltered.
-    error_line = traceback.tb_lineno
-    for code_line, tmpl_filename, tmpl_line in template._debug_info[::-1]:
-        if code_line <= error_line:
-            break
-    else:
-        return traceback
-
-    return fake_template_exception(exc_type, exc_value, traceback,
-                                   tmpl_filename, tmpl_line,
-                                   template._source, context)
+        tb = tb.tb_next
+
+    result_tb = prev_tb = None
+    initial_tb = tb
+
+    # translate all the jinja frames in this traceback
+    while tb is not None:
+        if tb.tb_frame.f_globals.get('__jinja_template__'):
+            debug_info = tb.tb_frame.f_globals['debug_info']
+
+            # the next thing we do is matching the current error line against the
+            # debugging table to get the correct source line. If we can't find the
+            # filename and line number we return the traceback object unaltered.
+            error_line = tb.tb_lineno
+            for code_line, tmpl_filename, tmpl_line in reversed(debug_info):
+                if code_line <= error_line:
+                    source = tb.tb_frame.f_globals['template_source']
+                    tb = fake_template_exception(exc_type, exc_value, tb,
+                                                 tmpl_filename, tmpl_line,
+                                                 source, context, prev_tb)[-1]
+                    break
+        if result_tb is None:
+            result_tb = tb
+        prev_tb = tb
+        tb = tb.tb_next
+
+    # under some conditions we cannot translate any frame. in that
+    # situation just return the original traceback.
+    return (exc_type, exc_value, result_tb or intial_tb)
 
 
 def raise_syntax_error(exception, env, source=None):
diff --git a/jinja/defaults.py b/jinja/defaults.py
index 0a00015..8d5025d 100644
--- a/jinja/defaults.py
+++ b/jinja/defaults.py
@@ -10,17 +10,7 @@
 """
 from jinja.filters import FILTERS as DEFAULT_FILTERS
 from jinja.tests import TESTS as DEFAULT_TESTS
-from jinja.utils import debug_helper, safe_range, generate_lorem_ipsum, \
-     watch_changes, render_included
+from jinja.utils import NAMESPACE as DEFAULT_NAMESPACE
 
 
 __all__ = ['DEFAULT_FILTERS', 'DEFAULT_TESTS', 'DEFAULT_NAMESPACE']
-
-
-DEFAULT_NAMESPACE = {
-    'range':                safe_range,
-    'debug':                debug_helper,
-    'lipsum':               generate_lorem_ipsum,
-    'watchchanges':         watch_changes,
-    'rendertemplate':       render_included
-}
diff --git a/jinja/environment.py b/jinja/environment.py
index 38a22bb..c815f59 100644
--- a/jinja/environment.py
+++ b/jinja/environment.py
@@ -52,6 +52,7 @@ class Environment(object):
                  tests=None,
                  context_class=Context,
                  undefined_singleton=SilentUndefined,
+                 disable_regexps=False,
                  friendly_traceback=True,
                  translator_factory=None):
         """
@@ -103,6 +104,7 @@ class Environment(object):
                                   details.
         `undefined_singleton`     The singleton value that is used for missing
                                   variables. *new in Jinja 1.1*
+        `disable_regexps`         Disable support for regular expresssions.
         `friendly_traceback`      Set this to `False` to disable the developer
                                   friendly traceback rewriting. Whenever an
                                   runtime or syntax error occours jinja will
@@ -143,6 +145,7 @@ class Environment(object):
         self.default_filters = default_filters or []
         self.context_class = context_class
         self.undefined_singleton = undefined_singleton
+        self.disable_regexps = disable_regexps
         self.friendly_traceback = friendly_traceback
 
         # global namespace
@@ -198,19 +201,17 @@ class Environment(object):
         """
         from jinja.translators.python import PythonTranslator
         try:
-            rv = PythonTranslator.process(self, Parser(self, source).parse())
+            rv = PythonTranslator.process(self, Parser(self, source).parse(),
+                                          source)
         except TemplateSyntaxError, e:
             # on syntax errors rewrite the traceback if wanted
             if not self.friendly_traceback:
                 raise
             from jinja.debugger import raise_syntax_error
-            __traceback_hide__ = True
+            if __debug__:
+                __traceback_hide__ = True
             raise_syntax_error(e, self, source)
         else:
-            # everything went well. attach the source and return it
-            # the attached source is used by the traceback system for
-            # debugging porposes
-            rv._source = source
             return rv
 
     def get_template(self, filename):
@@ -285,7 +286,7 @@ class Environment(object):
             value = func(self, context, value)
         return value
 
-    def perform_test(self, context, testname, args, value, invert):
+    def perform_test(self, context, testname, args, value):
         """
         Perform a test on a variable.
         """
@@ -301,10 +302,7 @@ class Environment(object):
             if testname not in self.tests:
                 raise TestNotFound(testname)
             context.cache[key] = func = self.tests[testname](*args)
-        rv = func(self, context, value)
-        if invert:
-            return not rv
-        return bool(rv)
+        return not not func(self, context, value)
 
     def get_attribute(self, obj, name):
         """
diff --git a/jinja/filters.py b/jinja/filters.py
index a3bf93f..af05a0e 100644
--- a/jinja/filters.py
+++ b/jinja/filters.py
@@ -12,9 +12,10 @@ import re
 from random import choice
 from operator import itemgetter
 from urllib import urlencode, quote
-from jinja.utils import urlize, escape, reversed, sorted, groupby
+from jinja.utils import urlize, escape, reversed, sorted, groupby, \
+     get_attribute
 from jinja.datastructure import TemplateData
-from jinja.exceptions import FilterArgumentError
+from jinja.exceptions import FilterArgumentError, SecurityException
 
 
 _striptags_re = re.compile(r'(<!--.*?-->|<[^>]+>)')
@@ -886,6 +887,48 @@ def do_groupby(attribute):
     return wrapped
 
 
+def do_getattribute(attribute):
+    """
+    Get one attribute from an object. Normally you don't have to use this
+    filter because the attribute and subscript expressions try to either
+    get an attribute of an object or an item. In some situations it could
+    be that there is an item *and* an attribute with the same name. In that
+    situation only the item is returned, never the attribute.
+
+    .. sourcecode:: jinja
+
+        {{ foo.bar }} -> {{ foo|getattribute('bar') }}
+
+    *New in Jinja 1.2*
+    """
+    def wrapped(env, context, value):
+        try:
+            return get_attribute(value, attribute)
+        except (SecurityException, AttributeError):
+            return env.undefined_singleton
+    return wrapped
+
+
+def do_getitem(key):
+    """
+    This filter basically works like the normal subscript expression but
+    it doesn't fall back to attribute lookup. If an item does not exist for
+    an object undefined is returned.
+
+    .. sourcecode:: jinja
+
+        {{ foo.bar }} -> {{ foo|getitem('bar') }}
+
+    *New in Jinja 1.2*
+    """
+    def wrapped(env, context, value):
+        try:
+            return value[key]
+        except (TypeError, KeyError, IndexError, AttributeError):
+            return env.undefined_singleton
+    return wrapped
+
+
 FILTERS = {
     'replace':              do_replace,
     'upper':                do_upper,
@@ -933,5 +976,7 @@ FILTERS = {
     'abs':                  do_abs,
     'round':                do_round,
     'sort':                 do_sort,
-    'groupby':              do_groupby
+    'groupby':              do_groupby,
+    'getattribute':         do_getattribute,
+    'getitem':              do_getitem
 }
diff --git a/jinja/lexer.py b/jinja/lexer.py
index 2c452c5..18b7484 100644
--- a/jinja/lexer.py
+++ b/jinja/lexer.py
@@ -23,9 +23,10 @@
     :license: BSD, see LICENSE for more details.
 """
 import re
-from jinja.datastructure import TokenStream
+import unicodedata
+from jinja.datastructure import TokenStream, Token
 from jinja.exceptions import TemplateSyntaxError
-from jinja.utils import set
+from jinja.utils import set, sorted
 from weakref import WeakValueDictionary
 
 
@@ -42,25 +43,138 @@ whitespace_re = re.compile(r'\s+(?m)')
 name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)')
-number_re = re.compile(r'\d+(\.\d+)*')
-
-operator_re = re.compile('(%s)' % '|'.join([
-    isinstance(x, unicode) and str(x) or re.escape(x) for x in [
-    # math operators
-    '+', '-', '**', '*', '//', '/', '%',
-    # braces and parenthesis
-    '[', ']', '(', ')', '{', '}',
-    # attribute access and comparison / logical operators
-    '.', ':', ',', '|', '==', '<=', '>=', '<', '>', '!=', '=',
-    ur'or\b', ur'and\b', ur'not\b', ur'in\b', ur'is\b'
-]]))
+integer_re = re.compile(r'\d+')
+float_re = re.compile(r'\d+\.\d+')
+regex_re = re.compile(r'@/([^/\\]*(?:\\.[^/\\]*)*)*/[a-z]*(?ms)')
+
 
 # set of used keywords
 keywords = set(['and', 'block', 'cycle', 'elif', 'else', 'endblock',
                 'endfilter', 'endfor', 'endif', 'endmacro', 'endraw',
                 'endtrans', 'extends', 'filter', 'for', 'if', 'in',
                 'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw',
-                'recursive', 'set', 'trans', 'print'])
+                'recursive', 'set', 'trans', 'print', 'call', 'endcall'])
+
+# bind operators to token types
+operators = {
+    '+':            'add',
+    '-':            'sub',
+    '/':            'div',
+    '//':           'floordiv',
+    '*':            'mul',
+    '%':            'mod',
+    '**':           'pow',
+    '~':            'tilde',
+    '!':            'bang',
+    '@':            'at',
+    '[':            'lbracket',
+    ']':            'rbracket',
+    '(':            'lparen',
+    ')':            'rparen',
+    '{':            'lbrace',
+    '}':            'rbrace',
+    '==':           'eq',
+    '!=':           'ne',
+    '>':            'gt',
+    '>=':           'gteq',
+    '<':            'lt',
+    '<=':           'lteq',
+    '=':            'assign',
+    '.':            'dot',
+    ':':            'colon',
+    '|':            'pipe',
+    ',':            'comma'
+}
+
+reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
+assert len(operators) == len(reverse_operators), 'operators dropped'
+operator_re = re.compile('(%s)' % '|'.join([re.escape(x) for x in
+                         sorted(operators, key=lambda x: -len(x))]))
+
+
+def unescape_string(lineno, filename, s):
+    r"""
+    Unescape a string. Supported escapes:
+        \a, \n, \r\, \f, \v, \\, \", \', \0
+
+        \x00, \u0000, \U00000000, \N{...}
+
+    Not supported are \101 because imho redundant.
+    """
+    result = []
+    write = result.append
+    simple_escapes = {
+        'a':    '\a',
+        'n':    '\n',
+        'r':    '\r',
+        'f':    '\f',
+        't':    '\t',
+        'v':    '\v',
+        '\\':   '\\',
+        '"':    '"',
+        "'":    "'",
+        '0':    '\x00'
+    }
+    unicode_escapes = {
+        'x':    2,
+        'u':    4,
+        'U':    8
+    }
+    chariter = iter(s)
+    next_char = chariter.next
+
+    try:
+        for char in chariter:
+            if char == '\\':
+                char = next_char()
+                if char in simple_escapes:
+                    write(simple_escapes[char])
+                elif char in unicode_escapes:
+                    seq = [next_char() for x in xrange(unicode_escapes[char])]
+                    try:
+                        write(unichr(int(''.join(seq), 16)))
+                    except ValueError:
+                        raise TemplateSyntaxError('invalid unicode codepoint',
+                                                  lineno, filename)
+                elif char == 'N':
+                    if next_char() != '{':
+                        raise TemplateSyntaxError('no name for codepoint',
+                                                  lineno, filename)
+                    seq = []
+                    while True:
+                        char = next_char()
+                        if char == '}':
+                            break
+                        seq.append(char)
+                    try:
+                        write(unicodedata.lookup(u''.join(seq)))
+                    except KeyError:
+                        raise TemplateSyntaxError('unknown character name',
+                                                  lineno, filename)
+                else:
+                    write('\\' + char)
+            else:
+                write(char)
+    except StopIteration:
+        raise TemplateSyntaxError('invalid string escape', lineno, filename)
+    return u''.join(result)
+
+
+def unescape_regex(s):
+    """
+    Unescape rules for regular expressions.
+    """
+    buffer = []
+    write = buffer.append
+    in_escape = False
+    for char in s:
+        if in_escape:
+            in_escape = False
+            if char not in safe_chars:
+                write('\\' + char)
+                continue
+        write(char)
+    return u''.join(buffer)
 
 
 class Failure(object):
@@ -121,10 +235,12 @@ class Lexer(object):
         # lexing rules for tags
         tag_rules = [
             (whitespace_re, None, None),
-            (number_re, 'number', None),
+            (float_re, 'float', None),
+            (integer_re, 'integer', None),
             (name_re, 'name', None),
-            (operator_re, 'operator', None),
-            (string_re, 'string', None)
+            (string_re, 'string', None),
+            (regex_re, 'regex', None),
+            (operator_re, 'operator', None)
         ]
 
         #: if variables and blocks have the same delimiters we won't
@@ -218,18 +334,45 @@ class Lexer(object):
 
     def tokenize(self, source, filename=None):
         """
-        Simple tokenize function that yields ``(position, type, contents)``
-        tuples. Wrap the generator returned by this function in a
-        `TokenStream` to get real token instances and be able to push tokens
-        back to the stream. That's for example done by the parser.
-
-        Additionally non keywords are escaped.
+        Works like `tokeniter` but returns a tokenstream of tokens and not a
+        generator or token tuples. Additionally all token values are already
+        converted into types and postprocessed. For example keywords are
+        already keyword tokens, not named tokens, comments are removed,
+        integers and floats converted, strings unescaped etc.
         """
         def generate():
             for lineno, token, value in self.tokeniter(source, filename):
-                if token == 'name' and value not in keywords:
-                    value += '_'
-                yield lineno, token, value
+                if token in ('comment_begin', 'comment', 'comment_end'):
+                    continue
+                elif token == 'data':
+                    try:
+                        value = str(value)
+                    except UnicodeError:
+                        pass
+                elif token == 'name':
+                    value = str(value)
+                    if value in keywords:
+                        token = value
+                        value = ''
+                elif token == 'string':
+                    value = unescape_string(lineno, filename, value[1:-1])
+                    try:
+                        value = str(value)
+                    except UnicodeError:
+                        pass
+                elif token == 'regex':
+                    args = value[value.rfind('/') + 1:]
+                    value = unescape_regex(value[2:-(len(args) + 1)])
+                    if args:
+                        value = '(?%s)%s' % (args, value)
+                elif token == 'integer':
+                    value = int(value)
+                elif token == 'float':
+                    value = float(value)
+                elif token == 'operator':
+                    token = operators[value]
+                    value = ''
+                yield Token(lineno, token, value)
         return TokenStream(generate(), filename)
 
     def tokeniter(self, source, filename=None):
@@ -237,8 +380,8 @@ class Lexer(object):
         This method tokenizes the text and returns the tokens in a generator.
         Use this method if you just want to tokenize a template. The output
         you get is not compatible with the input the jinja parser wants. The
-        parser uses the `tokenize` function with returns a `TokenStream` with
-        some escaped tokens.
+        parser uses the `tokenize` function with returns a `TokenStream` and
+        keywords instead of just names.
         """
         source = '\n'.join(source.splitlines())
         pos = 0
diff --git a/jinja/loaders.py b/jinja/loaders.py
index 95d9d8d..4a5c7f8 100644
--- a/jinja/loaders.py
+++ b/jinja/loaders.py
@@ -42,13 +42,11 @@ def get_cachename(cachepath, name, salt=None):
                              (name, salt or '')).hexdigest())
 
 
-
 def _loader_missing(*args, **kwargs):
     """Helper function for `LoaderWrapper`."""
     raise RuntimeError('no loader defined')
 
 
-
 class LoaderWrapper(object):
     """
     Wraps a loader so that it's bound to an environment.
@@ -476,24 +474,54 @@ class BasePackageLoader(BaseLoader):
 
     It uses the `pkg_resources` libraries distributed with setuptools for
     retrieving the data from the packages. This works for eggs too so you
-    don't have to mark your egg as non zip safe.
+    don't have to mark your egg as non zip safe. If pkg_resources is not
+    available it just falls back to path joining relative to the package.
     """
 
-    def __init__(self, package_name, package_path):
+    def __init__(self, package_name, package_path, force_native=False):
         try:
             import pkg_resources
         except ImportError:
             raise RuntimeError('setuptools not installed')
         self.package_name = package_name
         self.package_path = package_path
+        self.force_native = force_native
 
-    def get_source(self, environment, name, parent):
-        from pkg_resources import resource_exists, resource_string
-        path = '/'.join([self.package_path] + [p for p in name.split('/')
+    def _get_load_func(self):
+        if hasattr(self, '_load_func'):
+            return self._load_func
+        try:
+            from pkg_resources import resource_exists, resource_string
+            if self.force_native:
+                raise ImportError()
+        except ImportError:
+            basepath = path.dirname(__import__(self.package_name, None, None,
+                                    ['__file__']).__file__)
+            def load_func(name):
+                filename = path.join(basepath, *(
+                    self.package_path.split('/') +
+                    [p for p in name.split('/') if p != '..'])
+                )
+                if path.exists(filename):
+                    f = file(filename)
+                    try:
+                        return f.read()
+                    finally:
+                        f.close()
+        else:
+            def load_func(name):
+                path = '/'.join([self.package_path] + [p for p in name.split('/')
                          if p != '..'])
-        if not resource_exists(self.package_name, path):
+                if resource_exists(self.package_name, path):
+                    return resource_string(self.package_name, path)
+        self._load_func = load_func
+        return load_func
+
+    def get_source(self, environment, name, parent):
+        load_func = self._get_load_func()
+        contents = load_func(name)
+        if contents is None:
             raise TemplateNotFound(name)
-        contents = resource_string(self.package_name, path)
         return contents.decode(environment.template_charset)
 
 
diff --git a/jinja/nodes.py b/jinja/nodes.py
index bd60ca5..9dd4584 100644
--- a/jinja/nodes.py
+++ b/jinja/nodes.py
@@ -12,73 +12,52 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
-from compiler import ast
+from itertools import chain
 from copy import copy
 
 
-def inc_lineno(offset, tree):
-    """
-    Increment the linenumbers of all nodes in tree with offset.
-    """
-    todo = [tree]
-    while todo:
-        node = todo.pop()
-        if node.lineno:
-            node.lineno += offset - 1
-        else:
-            node.lineno = offset
-        todo.extend(node.getChildNodes())
-
-
 def get_nodes(nodetype, tree, exclude_root=True):
     """
     Get all nodes from nodetype in the tree excluding the
     node passed if `exclude_root` is `True` (default).
     """
     if exclude_root:
-        todo = tree.getChildNodes()
+        todo = tree.get_child_nodes()
     else:
         todo = [tree]
     while todo:
         node = todo.pop()
         if node.__class__ is nodetype:
             yield node
-        todo.extend(node.getChildNodes())
+        todo.extend(node.get_child_nodes())
 
 
-def get_nodes_parentinfo(nodetype, tree):
+class NotPossible(NotImplementedError):
     """
-    Like `get_nodes` but it yields tuples in the form ``(parent, node)``.
-    If a node is a direct ancestor of `tree` parent will be `None`.
-
-    Always excludes the root node.
+    If a given node cannot do something.
     """
-    todo = [tree]
-    while todo:
-        node = todo.pop()
-        if node is tree:
-            parent = None
-        else:
-            parent = node
-        for child in node.getChildNodes():
-            if child.__class__ is nodetype:
-                yield parent, child
-            todo.append(child)
 
 
-class Node(ast.Node):
+class Node(object):
     """
     Jinja node.
     """
 
+    def __init__(self, lineno=None, filename=None):
+        self.lineno = lineno
+        self.filename = filename
+
     def get_items(self):
         return []
 
-    def getChildren(self):
-        return self.get_items()
+    def get_child_nodes(self):
+        return [x for x in self.get_items() if isinstance(x, Node)]
 
-    def getChildNodes(self):
-        return [x for x in self.get_items() if isinstance(x, ast.Node)]
+    def allows_assignments(self):
+        return False
+
+    def __repr__(self):
+        return 'Node()'
 
 
 class Text(Node):
@@ -86,35 +65,17 @@ class Text(Node):
     Node that represents normal text.
     """
 
-    def __init__(self, lineno, text):
-        self.lineno = lineno
+    def __init__(self, text, variables, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.text = text
-
-    def get_items(self):
-        return [self.text]
-
-    def __repr__(self):
-        return 'Text(%r)' % (self.text,)
-
-
-class DynamicText(Node):
-    """
-    Note that represents normal text with string formattings.
-    Those are used for texts that contain variables. The attribute
-    `variables` contains a list of Print tags and nothing else.
-    """
-
-    def __init__(self, lineno, format_string, variables):
-        self.lineno = lineno
-        self.format_string = format_string
         self.variables = variables
 
     def get_items(self):
-        return [self.format_string] + list(self.variables)
+        return [self.text] + list(self.variables)
 
     def __repr__(self):
-        return 'DynamicText(%r, %r)' % (
-            self.format_string,
+        return 'Text(%r, %r)' % (
+            self.text,
             self.variables
         )
 
@@ -124,36 +85,34 @@ class NodeList(list, Node):
     A node that stores multiple childnodes.
     """
 
-    def __init__(self, lineno, data=None):
-        self.lineno = lineno
-        list.__init__(self, data or ())
+    def __init__(self, data, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
+        list.__init__(self, data)
 
-    getChildren = getChildNodes = lambda s: list(s) + s.get_items()
+    def get_items(self):
+        return list(self)
 
     def __repr__(self):
         return 'NodeList(%s)' % list.__repr__(self)
 
 
-class Template(NodeList):
+class Template(Node):
     """
     Node that represents a template.
     """
 
-    def __init__(self, filename, body, extends):
-        if body.__class__ is not NodeList:
-            body = (body,)
-        NodeList.__init__(self, 1, body)
+    def __init__(self, extends, body, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.extends = extends
-        self.filename = filename
+        self.body = body
 
     def get_items(self):
-        return self.extends is not None and [self.extends] or []
+        return [self.extends, self.body]
 
     def __repr__(self):
-        return 'Template(%r, %r, %s)' % (
-            self.filename,
+        return 'Template(%r, %r)' % (
             self.extends,
-            list.__repr__(self)
+            self.body
         )
 
 
@@ -162,8 +121,9 @@ class ForLoop(Node):
     A node that represents a for loop
     """
 
-    def __init__(self, lineno, item, seq, body, else_, recursive):
-        self.lineno = lineno
+    def __init__(self, item, seq, body, else_, recursive, lineno=None,
+                 filename=None):
+        Node.__init__(self, lineno, filename)
         self.item = item
         self.seq = seq
         self.body = body
@@ -188,8 +148,8 @@ class IfCondition(Node):
     A node that represents an if condition.
     """
 
-    def __init__(self, lineno, tests, else_):
-        self.lineno = lineno
+    def __init__(self, tests, else_, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.tests = tests
         self.else_ = else_
 
@@ -212,8 +172,8 @@ class Cycle(Node):
     A node that represents the cycle statement.
     """
 
-    def __init__(self, lineno, seq):
-        self.lineno = lineno
+    def __init__(self, seq, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.seq = seq
 
     def get_items(self):
@@ -228,15 +188,15 @@ class Print(Node):
     A node that represents variable tags and print calls.
     """
 
-    def __init__(self, lineno, variable):
-        self.lineno = lineno
-        self.variable = variable
+    def __init__(self, expr, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
+        self.expr = expr
 
     def get_items(self):
-        return [self.variable]
+        return [self.expr]
 
     def __repr__(self):
-        return 'Print(%r)' % (self.variable,)
+        return 'Print(%r)' % (self.expr,)
 
 
 class Macro(Node):
@@ -244,19 +204,14 @@ class Macro(Node):
     A node that represents a macro.
     """
 
-    def __init__(self, lineno, name, arguments, body):
-        self.lineno = lineno
+    def __init__(self, name, arguments, body, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.name = name
         self.arguments = arguments
         self.body = body
 
     def get_items(self):
-        result = [self.name]
-        if self.arguments:
-            for item in self.arguments:
-                result.extend(item)
-        result.append(self.body)
-        return result
+        return [self.name] + list(chain(*self.arguments)) + [self.body]
 
     def __repr__(self):
         return 'Macro(%r, %r, %r)' % (
@@ -271,8 +226,8 @@ class Call(Node):
     A node that represents am extended macro call.
     """
 
-    def __init__(self, lineno, expr, body):
-        self.lineno = lineno
+    def __init__(self, expr, body, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.expr = expr
         self.body = body
 
@@ -288,21 +243,23 @@ class Call(Node):
 
 class Set(Node):
     """
-    Allow defining own variables.
+    Allows defining own variables.
     """
 
-    def __init__(self, lineno, name, expr):
-        self.lineno = lineno
+    def __init__(self, name, expr, scope_local, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.name = name
         self.expr = expr
+        self.scope_local = scope_local
 
     def get_items(self):
-        return [self.name, self.expr]
+        return [self.name, self.expr, self.scope_local]
 
     def __repr__(self):
-        return 'Set(%r, %r)' % (
+        return 'Set(%r, %r, %r)' % (
             self.name,
-            self.expr
+            self.expr,
+            self.scope_local
         )
 
 
@@ -311,8 +268,8 @@ class Filter(Node):
     Node for filter sections.
     """
 
-    def __init__(self, lineno, body, filters):
-        self.lineno = lineno
+    def __init__(self, body, filters, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.body = body
         self.filters = filters
 
@@ -331,17 +288,21 @@ class Block(Node):
     A node that represents a block.
     """
 
-    def __init__(self, lineno, name, body):
-        self.lineno = lineno
+    def __init__(self, name, body, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.name = name
         self.body = body
 
     def replace(self, node):
         """
-        Replace the current data with the data of another block node.
+        Replace the current data with the copied data of another block
+        node.
         """
         assert node.__class__ is Block
-        self.__dict__.update(node.__dict__)
+        self.lineno = node.lineno
+        self.filename = node.filename
+        self.name = node.name
+        self.body = copy(node.body)
 
     def clone(self):
         """
@@ -359,29 +320,13 @@ class Block(Node):
         )
 
 
-class Extends(Node):
-    """
-    A node that represents the extends tag.
-    """
-
-    def __init__(self, lineno, template):
-        self.lineno = lineno
-        self.template = template
-
-    def get_items(self):
-        return [self.template]
-
-    def __repr__(self):
-        return 'Extends(%r)' % self.template
-
-
 class Include(Node):
     """
     A node that represents the include tag.
     """
 
-    def __init__(self, lineno, template):
-        self.lineno = lineno
+    def __init__(self, template, lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.template = template
 
     def get_items(self):
@@ -398,8 +343,9 @@ class Trans(Node):
     A node for translatable sections.
     """
 
-    def __init__(self, lineno, singular, plural, indicator, replacements):
-        self.lineno = lineno
+    def __init__(self, singular, plural, indicator, replacements,
+                 lineno=None, filename=None):
+        Node.__init__(self, lineno, filename)
         self.singular = singular
         self.plural = plural
         self.indicator = indicator
@@ -419,3 +365,428 @@ class Trans(Node):
             self.indicator,
             self.replacements
         )
+
+
+class Expression(Node):
+    """
+    Baseclass for all expressions.
+    """
+
+
+class BinaryExpression(Expression):
+    """
+    Baseclass for all binary expressions.
+    """
+
+    def __init__(self, left, right, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.left = left
+        self.right = right
+
+    def get_items(self):
+        return [self.left, self.right]
+
+    def __repr__(self):
+        return '%s(%r, %r)' % (
+            self.__class__.__name__,
+            self.left,
+            self.right
+        )
+
+
+class UnaryExpression(Expression):
+    """
+    Baseclass for all unary expressions.
+    """
+
+    def __init__(self, node, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.node = node
+
+    def get_items(self):
+        return [self.node]
+
+    def __repr__(self):
+        return '%s(%r)' % (
+            self.__class__.__name__,
+            self.node
+        )
+
+
+class ConstantExpression(Expression):
+    """
+    any constat such as {{ "foo" }}
+    """
+
+    def __init__(self, value, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.value = value
+
+    def get_items(self):
+        return [self.value]
+
+    def __repr__(self):
+        return 'ConstantExpression(%r)' % (self.value,)
+
+
+class UndefinedExpression(Expression):
+    """
+    represents the special 'undefined' value.
+    """
+
+    def __repr__(self):
+        return 'UndefinedExpression()'
+
+
+class RegexExpression(Expression):
+    """
+    represents the regular expression literal.
+    """
+
+    def __init__(self, value, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.value = value
+
+    def get_items(self):
+        return [self.value]
+
+    def __repr__(self):
+        return 'RegexExpression(%r)' % (self.value,)
+
+
+class NameExpression(Expression):
+    """
+    any name such as {{ foo }}
+    """
+
+    def __init__(self, name, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.name = name
+
+    def get_items(self):
+        return [self.name]
+
+    def allows_assignments(self):
+        return self.name != '_'
+
+    def __repr__(self):
+        return 'NameExpression(%r)' % self.name
+
+
+class ListExpression(Expression):
+    """
+    any list literal such as {{ [1, 2, 3] }}
+    """
+
+    def __init__(self, items, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.items = items
+
+    def get_items(self):
+        return list(self.items)
+
+    def __repr__(self):
+        return 'ListExpression(%r)' % (self.items,)
+
+
+class DictExpression(Expression):
+    """
+    any dict literal such as {{ {1: 2, 3: 4} }}
+    """
+
+    def __init__(self, items, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.items = items
+
+    def get_items(self):
+        return list(chain(*self.items))
+
+    def __repr__(self):
+        return 'DictExpression(%r)' % (self.items,)
+
+
+class SetExpression(Expression):
+    """
+    any set literal such as {{ @(1, 2, 3) }}
+    """
+
+    def __init__(self, items, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.items = items
+
+    def get_items(self):
+        return self.items[:]
+
+    def __repr__(self):
+        return 'SetExpression(%r)' % (self.items,)
+
+
+class ConditionalExpression(Expression):
+    """
+    {{ foo if bar else baz }}
+    """
+
+    def __init__(self, test, expr1, expr2, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.test = test
+        self.expr1 = expr1
+        self.expr2 = expr2
+
+    def get_items(self):
+        return [self.test, self.expr1, self.expr2]
+
+    def __repr__(self):
+        return 'ConstantExpression(%r, %r, %r)' % (
+            self.test,
+            self.expr1,
+            self.expr2
+        )
+
+
+class FilterExpression(Expression):
+    """
+    {{ foo|bar|baz }}
+    """
+
+    def __init__(self, node, filters, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.node = node
+        self.filters = filters
+
+    def get_items(self):
+        result = [self.node]
+        for filter, args in self.filters:
+            result.append(filter)
+            result.extend(args)
+        return result
+
+    def __repr__(self):
+        return 'FilterExpression(%r, %r)' % (
+            self.node,
+            self.filters
+        )
+
+
+class TestExpression(Expression):
+    """
+    {{ foo is lower }}
+    """
+
+    def __init__(self, node, name, args, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.node = node
+        self.name = name
+        self.args = args
+
+    def get_items(self):
+        return [self.node, self.name] + list(self.args)
+
+    def __repr__(self):
+        return 'TestExpression(%r, %r, %r)' % (
+            self.node,
+            self.name,
+            self.args
+        )
+
+
+class CallExpression(Expression):
+    """
+    {{ foo(bar) }}
+    """
+
+    def __init__(self, node, args, kwargs, dyn_args, dyn_kwargs,
+                 lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.node = node
+        self.args = args
+        self.kwargs = kwargs
+        self.dyn_args = dyn_args
+        self.dyn_kwargs = dyn_kwargs
+
+    def get_items(self):
+        return [self.node, self.args, self.kwargs, self.dyn_args,
+                self.dyn_kwargs]
+
+    def __repr__(self):
+        return 'CallExpression(%r, %r, %r, %r, %r)' % (
+            self.node,
+            self.args,
+            self.kwargs,
+            self.dyn_args,
+            self.dyn_kwargs
+        )
+
+
+class SubscriptExpression(Expression):
+    """
+    {{ foo.bar }} and {{ foo['bar'] }} etc.
+    """
+
+    def __init__(self, node, arg, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.node = node
+        self.arg = arg
+
+    def get_items(self):
+        return [self.node, self.arg]
+
+    def __repr__(self):
+        return 'SubscriptExpression(%r, %r)' % (
+            self.node,
+            self.arg
+        )
+
+
+class SliceExpression(Expression):
+    """
+    1:2:3 etc.
+    """
+
+    def __init__(self, start, stop, step, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.start = start
+        self.stop = stop
+        self.step = step
+
+    def get_items(self):
+        return [self.start, self.stop, self.step]
+
+    def __repr__(self):
+        return 'SliceExpression(%r, %r, %r)' % (
+            self.start,
+            self.stop,
+            self.step
+        )
+
+
+class TupleExpression(Expression):
+    """
+    For loop unpacking and some other things like multiple arguments
+    for subscripts.
+    """
+
+    def __init__(self, items, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.items = items
+
+    def get_items(self):
+        return list(self.items)
+
+    def allows_assignments(self):
+        for item in self.items:
+            if not item.allows_assignments():
+                return False
+        return True
+
+    def __repr__(self):
+        return 'TupleExpression(%r)' % (self.items,)
+
+
+class ConcatExpression(Expression):
+    """
+    For {{ foo ~ bar }}. Because of various reasons (especially because
+    unicode conversion takes place for the left and right expression and
+    is better optimized that way)
+    """
+
+    def __init__(self, args, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.args = args
+
+    def get_items(self):
+        return list(self.args)
+
+    def __repr__(self):
+        return 'ConcatExpression(%r)' % (self.items,)
+
+
+class CompareExpression(Expression):
+    """
+    {{ foo == bar }}, {{ foo >= bar }} etc.
+    """
+
+    def __init__(self, expr, ops, lineno=None, filename=None):
+        Expression.__init__(self, lineno, filename)
+        self.expr = expr
+        self.ops = ops
+
+    def get_items(self):
+        return [self.expr] + list(chain(*self.ops))
+
+    def __repr__(self):
+        return 'CompareExpression(%r, %r)' % (
+            self.expr,
+            self.ops
+        )
+
+
+class MulExpression(BinaryExpression):
+    """
+    {{ foo * bar }}
+    """
+
+
+class DivExpression(BinaryExpression):
+    """
+    {{ foo / bar }}
+    """
+
+
+class FloorDivExpression(BinaryExpression):
+    """
+    {{ foo // bar }}
+    """
+
+
+class AddExpression(BinaryExpression):
+    """
+    {{ foo + bar }}
+    """
+
+
+class SubExpression(BinaryExpression):
+    """
+    {{ foo - bar }}
+    """
+
+
+class ModExpression(BinaryExpression):
+    """
+    {{ foo % bar }}
+    """
+
+
+class PowExpression(BinaryExpression):
+    """
+    {{ foo ** bar }}
+    """
+
+
+class AndExpression(BinaryExpression):
+    """
+    {{ foo and bar }}
+    """
+
+
+class OrExpression(BinaryExpression):
+    """
+    {{ foo or bar }}
+    """
+
+
+class NotExpression(UnaryExpression):
+    """
+    {{ not foo }}
+    """
+
+
+class NegExpression(UnaryExpression):
+    """
+    {{ -foo }}
+    """
+
+
+class PosExpression(UnaryExpression):
+    """
+    {{ +foo }}
+    """
diff --git a/jinja/parser.py b/jinja/parser.py
index aba3725..624386c 100644
--- a/jinja/parser.py
+++ b/jinja/parser.py
@@ -14,7 +14,6 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
-from compiler import ast, parse
 from jinja import nodes
 from jinja.datastructure import StateTest
 from jinja.exceptions import TemplateSyntaxError
@@ -25,20 +24,27 @@ __all__ = ['Parser']
 
 
 # general callback functions for the parser
-end_of_block = StateTest.expect_token('block_end', 'end of block tag')
-end_of_variable = StateTest.expect_token('variable_end', 'end of variable')
-end_of_comment = StateTest.expect_token('comment_end', 'end of comment')
+end_of_block = StateTest.expect_token('block_end',
+                                      msg='expected end of block tag')
+end_of_variable = StateTest.expect_token('variable_end',
+                                         msg='expected end of variable')
+end_of_comment = StateTest.expect_token('comment_end',
+                                        msg='expected end of comment')
 
 # internal tag callbacks
-switch_for = StateTest.expect_name('else', 'endfor')
-end_of_for = StateTest.expect_name('endfor')
-switch_if = StateTest.expect_name('else', 'elif', 'endif')
-end_of_if = StateTest.expect_name('endif')
-end_of_filter = StateTest.expect_name('endfilter')
-end_of_macro = StateTest.expect_name('endmacro')
-end_of_call = StateTest.expect_name('endcall_')
-end_of_block_tag = StateTest.expect_name('endblock')
-end_of_trans = StateTest.expect_name('endtrans')
+switch_for = StateTest.expect_token('else', 'endfor')
+end_of_for = StateTest.expect_token('endfor')
+switch_if = StateTest.expect_token('else', 'elif', 'endif')
+end_of_if = StateTest.expect_token('endif')
+end_of_filter = StateTest.expect_token('endfilter')
+end_of_macro = StateTest.expect_token('endmacro')
+end_of_call = StateTest.expect_token('endcall')
+end_of_block_tag = StateTest.expect_token('endblock')
+end_of_trans = StateTest.expect_token('endtrans')
+
+# this ends a tuple
+tuple_edge_tokens = set(['rparen', 'block_end', 'variable_end', 'in',
+                         'recursive'])
 
 
 class Parser(object):
@@ -49,8 +55,6 @@ class Parser(object):
     """
 
     def __init__(self, environment, source, filename=None):
-        #XXX: with Jinja 1.3 call becomes a keyword. Add it also
-        #     to the lexer.py file.
         self.environment = environment
         if isinstance(source, str):
             source = source.decode(environment.template_charset, 'ignore')
@@ -58,28 +62,29 @@ class Parser(object):
             filename = filename.encode('utf-8')
         self.source = source
         self.filename = filename
+        self.closed = False
 
-        #: if this template has a parent template it's stored here
-        #: after parsing
-        self.extends = None
         #: set for blocks in order to keep them unique
         self.blocks = set()
 
         #: mapping of directives that require special treatment
         self.directives = {
-            'raw':          self.handle_raw_directive,
-            'for':          self.handle_for_directive,
-            'if':           self.handle_if_directive,
-            'cycle':        self.handle_cycle_directive,
-            'set':          self.handle_set_directive,
-            'filter':       self.handle_filter_directive,
-            'print':        self.handle_print_directive,
-            'macro':        self.handle_macro_directive,
-            'call_':        self.handle_call_directive,
-            'block':        self.handle_block_directive,
-            'extends':      self.handle_extends_directive,
-            'include':      self.handle_include_directive,
-            'trans':        self.handle_trans_directive
+            # "fake" directives that just trigger errors
+            'raw':          self.parse_raw_directive,
+            'extends':      self.parse_extends_directive,
+
+            # real directives
+            'for':          self.parse_for_loop,
+            'if':           self.parse_if_condition,
+            'cycle':        self.parse_cycle_directive,
+            'call':         self.parse_call_directive,
+            'set':          self.parse_set_directive,
+            'filter':       self.parse_filter_directive,
+            'print':        self.parse_print_directive,
+            'macro':        self.parse_macro_directive,
+            'block':        self.parse_block_directive,
+            'include':      self.parse_include_directive,
+            'trans':        self.parse_trans_directive
         }
 
         #: set of directives that are only available in a certain
@@ -92,733 +97,1074 @@ class Parser(object):
         #: get the `no_variable_block` flag
         self.no_variable_block = self.environment.lexer.no_variable_block
 
-        self.tokenstream = environment.lexer.tokenize(source, filename)
+        self.stream = environment.lexer.tokenize(source, filename)
 
-    def handle_raw_directive(self, lineno, gen):
+    def parse_raw_directive(self):
         """
         Handle fake raw directive. (real raw directives are handled by
         the lexer. But if there are arguments to raw or the end tag
         is missing the parser tries to resolve this directive. In that
         case present the user a useful error message.
         """
-        args = list(gen)
-        if args:
+        if self.stream:
             raise TemplateSyntaxError('raw directive does not support '
                                       'any arguments.', lineno,
                                       self.filename)
         raise TemplateSyntaxError('missing end tag for raw directive.',
                                   lineno, self.filename)
 
-    def handle_for_directive(self, lineno, gen):
+    def parse_extends_directive(self):
+        """
+        Handle the extends directive used for inheritance.
+        """
+        raise TemplateSyntaxError('mispositioned extends tag. extends must '
+                                  'be the first tag of a template.',
+                                  self.stream.lineno, self.filename)
+
+    def parse_for_loop(self):
         """
         Handle a for directive and return a ForLoop node
         """
-        recursive = []
-        def wrapgen():
-            """Wrap the generator to check if we have a recursive for loop."""
-            for token in gen:
-                if token[1:] == ('name', 'recursive'):
-                    try:
-                        item = gen.next()
-                    except StopIteration:
-                        recursive.append(True)
-                        return
-                    yield token
-                    yield item
-                else:
-                    yield token
-        ast = self.parse_python(lineno, wrapgen(), 'for %s:pass')
+        token = self.stream.expect('for')
+        item = self.parse_tuple_expression(simplified=True)
+        if not item.allows_assignments():
+            raise TemplateSyntaxError('cannot assign to expression',
+                                      token.lineno, self.filename)
+
+        self.stream.expect('in')
+        seq = self.parse_tuple_expression()
+        if self.stream.current.type == 'recursive':
+            self.stream.next()
+            recursive = True
+        else:
+            recursive = False
+        self.stream.expect('block_end')
+
         body = self.subparse(switch_for)
 
         # do we have an else section?
-        if self.tokenstream.next()[2] == 'else':
-            self.close_remaining_block()
+        if self.stream.current.type == 'else':
+            self.stream.next()
+            self.stream.expect('block_end')
             else_ = self.subparse(end_of_for, True)
         else:
+            self.stream.next()
             else_ = None
-        self.close_remaining_block()
+        self.stream.expect('block_end')
 
-        return nodes.ForLoop(lineno, ast.assign, ast.list, body, else_,
-                             bool(recursive))
+        return nodes.ForLoop(item, seq, body, else_, recursive,
+                             token.lineno, self.filename)
 
-    def handle_if_directive(self, lineno, gen):
+    def parse_if_condition(self):
         """
         Handle if/else blocks.
         """
-        ast = self.parse_python(lineno, gen, 'if %s:pass')
-        tests = [(ast.tests[0][0], self.subparse(switch_if))]
+        token = self.stream.expect('if')
+        expr = self.parse_expression()
+        self.stream.expect('block_end')
+        tests = [(expr, self.subparse(switch_if))]
+        else_ = None
 
         # do we have an else section?
         while True:
-            lineno, token, needle = self.tokenstream.next()
-            if needle == 'else':
-                self.close_remaining_block()
+            if self.stream.current.type == 'else':
+                self.stream.next()
+                self.stream.expect('block_end')
                 else_ = self.subparse(end_of_if, True)
-                break
-            elif needle == 'elif':
-                gen = self.tokenstream.fetch_until(end_of_block, True)
-                ast = self.parse_python(lineno, gen, 'if %s:pass')
-                tests.append((ast.tests[0][0], self.subparse(switch_if)))
+            elif self.stream.current.type == 'elif':
+                self.stream.next()
+                expr = self.parse_expression()
+                self.stream.expect('block_end')
+                tests.append((expr, self.subparse(switch_if)))
+                continue
             else:
-                else_ = None
-                break
-        self.close_remaining_block()
+                self.stream.next()
+            break
+        self.stream.expect('block_end')
 
-        return nodes.IfCondition(lineno, tests, else_)
+        return nodes.IfCondition(tests, else_, token.lineno, self.filename)
 
-    def handle_cycle_directive(self, lineno, gen):
+    def parse_cycle_directive(self):
         """
         Handle {% cycle foo, bar, baz %}.
         """
-        ast = self.parse_python(lineno, gen, '_cycle((%s))')
-        # ast is something like Discard(CallFunc(Name('_cycle'), ...))
-        # skip that.
-        return nodes.Cycle(lineno, ast.expr.args[0])
+        token = self.stream.expect('cycle')
+        expr = self.parse_tuple_expression()
+        self.stream.expect('block_end')
+        return nodes.Cycle(expr, token.lineno, self.filename)
 
-    def handle_set_directive(self, lineno, gen):
+    def parse_set_directive(self):
         """
         Handle {% set foo = 'value of foo' %}.
         """
-        try:
-            name = gen.next()
-            if name[1] != 'name' or gen.next()[1:] != ('operator', '='):
-                raise ValueError()
-        except (StopIteration, ValueError):
-            raise TemplateSyntaxError('invalid syntax for set', lineno,
-                                      self.filename)
-        ast = self.parse_python(lineno, gen, '(%s)')
-        # disallow keywords
-        if not name[2].endswith('_'):
-            raise TemplateSyntaxError('illegal use of keyword %r '
-                                      'as identifier in set statement.' %
-                                      name[2], lineno, self.filename)
-        return nodes.Set(lineno, str(name[2][:-1]), ast.expr)
+        token = self.stream.expect('set')
+        name = self.stream.expect('name')
+        self.test_name(name.value)
+        self.stream.expect('assign')
+        value = self.parse_expression()
+        if self.stream.current.type == 'bang':
+            self.stream.next()
+            scope_local = False
+        else:
+            scope_local = True
+        self.stream.expect('block_end')
+        return nodes.Set(name.value, value, scope_local,
+                         token.lineno, self.filename)
 
-    def handle_filter_directive(self, lineno, gen):
+    def parse_filter_directive(self):
         """
         Handle {% filter foo|bar %} directives.
         """
-        ast = self.parse_python(lineno, gen, '_filter(dummy|%s)')
+        token = self.stream.expect('filter')
+        filters = []
+        while self.stream.current.type != 'block_end':
+            if filters:
+                self.stream.expect('pipe')
+            token = self.stream.expect('name')
+            args = []
+            if self.stream.current.type == 'lparen':
+                while self.stream.current.type != 'rparen':
+                    if args:
+                        self.stream.expect('comma')
+                    args.append(self.parse_expression())
+                self.stream.expect('rparen')
+            filters.append((token.value, args))
+        self.stream.expect('block_end')
         body = self.subparse(end_of_filter, True)
-        self.close_remaining_block()
-        return nodes.Filter(lineno, body, ast.expr.args[0].nodes[1:])
-
-    def handle_print_directive(self, lineno, gen):
-        """
-        Handle {{ foo }} and {% print foo %}.
-        """
-        ast = self.parse_python(lineno, gen, 'print_(%s)')
-        # ast is something like Discard(CallFunc(Name('print_'), ...))
-        # so just use the args
-        arguments = ast.expr.args
-        # we only accept one argument
-        if len(arguments) != 1:
-            raise TemplateSyntaxError('invalid argument count for print; '
-                                      'print requires exactly one argument, '
-                                      'got %d.' % len(arguments), lineno,
-                                      self.filename)
-        return nodes.Print(lineno, arguments[0])
+        self.stream.expect('block_end')
+        return nodes.Filter(body, filters, token.lineno, self.filename)
 
-    def handle_macro_directive(self, lineno, gen):
+    def parse_print_directive(self):
+        """
+        Handle {% print foo %}.
+        """
+        token = self.stream.expect('print')
+        expr = self.parse_tuple_expression()
+        node = nodes.Print(expr, token.lineno, self.filename)
+        self.stream.expect('block_end')
+        return node
+
+    def parse_macro_directive(self):
         """
         Handle {% macro foo bar, baz %} as well as
         {% macro foo(bar, baz) %}.
         """
-        try:
-            macro_name = gen.next()
-        except StopIteration:
-            raise TemplateSyntaxError('macro requires a name', lineno,
-                                      self.filename)
-        if macro_name[1] != 'name':
-            raise TemplateSyntaxError('expected \'name\', got %r' %
-                                      macro_name[1], lineno,
-                                      self.filename)
-        # disallow keywords as identifiers
-        elif not macro_name[2].endswith('_'):
-            raise TemplateSyntaxError('illegal use of keyword %r '
-                                      'as macro name.' % macro_name[2],
-                                      lineno, self.filename)
-
-        # make the punctuation around arguments optional
-        arg_list = list(gen)
-        if arg_list and arg_list[0][1:] == ('operator', '(') and \
-                        arg_list[-1][1:] == ('operator', ')'):
-            arg_list = arg_list[1:-1]
-
-        ast = self.parse_python(lineno, arg_list, 'def %s(%%s):pass' %
-                                str(macro_name[2][:-1]))
+        token = self.stream.expect('macro')
+        macro_name = self.stream.expect('name')
+        self.test_name(macro_name.value)
+        if self.stream.current.type == 'lparen':
+            self.stream.next()
+            needle_token = 'rparen'
+        else:
+            needle_token = 'block_end'
+
+        args = []
+        while self.stream.current.type != needle_token:
+            if args:
+                self.stream.expect('comma')
+            name = self.stream.expect('name').value
+            self.test_name(name)
+            if self.stream.current.type == 'assign':
+                self.stream.next()
+                default = self.parse_expression()
+            else:
+                default = None
+            args.append((name, default))
+
+        self.stream.next()
+        if needle_token == 'rparen':
+            self.stream.expect('block_end')
+
         body = self.subparse(end_of_macro, True)
-        self.close_remaining_block()
+        self.stream.expect('block_end')
 
-        if ast.varargs or ast.kwargs:
-            raise TemplateSyntaxError('variable length macro signature '
-                                      'not allowed.', lineno,
-                                      self.filename)
-        if ast.argnames:
-            defaults = [None] * (len(ast.argnames) - len(ast.defaults)) + \
-                       ast.defaults
-            args = []
-            for idx, argname in enumerate(ast.argnames):
-                # disallow keywords as argument names
-                if not argname.endswith('_'):
-                    raise TemplateSyntaxError('illegal use of keyword %r '
-                                              'as macro argument.' % argname,
-                                              lineno, self.filename)
-                args.append((argname[:-1], defaults[idx]))
-        else:
-            args = None
-        return nodes.Macro(lineno, ast.name, args, body)
+        return nodes.Macro(macro_name.value, args, body, token.lineno,
+                           self.filename)
 
-    def handle_call_directive(self, lineno, gen):
+    def parse_call_directive(self):
         """
         Handle {% call foo() %}...{% endcall %}
         """
-        expr = self.parse_python(lineno, gen, '(%s)').expr
-        if expr.__class__ is not ast.CallFunc:
-            raise TemplateSyntaxError('call requires a function or macro '
-                                      'call as only argument.', lineno,
-                                      self.filename)
+        token = self.stream.expect('call')
+        expr = self.parse_call_expression()
+        self.stream.expect('block_end')
         body = self.subparse(end_of_call, True)
-        self.close_remaining_block()
-        return nodes.Call(lineno, expr, body)
+        self.stream.expect('block_end')
+        return nodes.Call(expr, body, token.lineno, self.filename)
 
-    def handle_block_directive(self, lineno, gen):
+    def parse_block_directive(self):
         """
         Handle block directives used for inheritance.
         """
-        tokens = list(gen)
-        if not tokens:
-            raise TemplateSyntaxError('block requires a name', lineno,
-                                      self.filename)
-        block_name = tokens.pop(0)
-        if block_name[1] != 'name':
-            raise TemplateSyntaxError('expected \'name\', got %r' %
-                                      block_name[1], lineno, self.filename)
-        # disallow keywords
-        if not block_name[2].endswith('_'):
-            raise TemplateSyntaxError('illegal use of keyword %r '
-                                      'as block name.' % block_name[2],
-                                      lineno, self.filename)
-        name = block_name[2][:-1]
+        token = self.stream.expect('block')
+        name = self.stream.expect('name').value
+
         # check if this block does not exist by now.
         if name in self.blocks:
             raise TemplateSyntaxError('block %r defined twice' %
                                        name, lineno, self.filename)
         self.blocks.add(name)
 
-        if tokens:
-            body = nodes.NodeList(lineno, [nodes.Print(lineno,
-                   self.parse_python(lineno, tokens, '(%s)').expr)])
+        if self.stream.current.type != 'block_end':
+            lineno = self.stream.lineno
+            body = nodes.NodeList([self.parse_variable_tag()], lineno,
+                                  self.filename)
+            self.stream.expect('block_end')
         else:
             # otherwise parse the body and attach it to the block
+            self.stream.expect('block_end')
             body = self.subparse(end_of_block_tag, True)
-            self.close_remaining_block()
-        return nodes.Block(lineno, name, body)
+            self.stream.expect('block_end')
+        return nodes.Block(name, body, token.lineno, self.filename)
 
-    def handle_extends_directive(self, lineno, gen):
-        """
-        Handle the extends directive used for inheritance.
-        """
-        tokens = list(gen)
-        if len(tokens) != 1 or tokens[0][1] != 'string':
-            raise TemplateSyntaxError('extends requires a string', lineno,
-                                      self.filename)
-        if self.extends is not None:
-            raise TemplateSyntaxError('extends called twice', lineno,
-                                      self.filename)
-        self.extends = nodes.Extends(lineno, tokens[0][2][1:-1])
-
-    def handle_include_directive(self, lineno, gen):
+    def parse_include_directive(self):
         """
         Handle the include directive used for template inclusion.
         """
-        tokens = list(gen)
-        # hardcoded include (faster because copied into the bytecode)
-        if len(tokens) == 1 and tokens[0][1] == 'string':
-            return nodes.Include(lineno, str(tokens[0][2][1:-1]))
-        raise TemplateSyntaxError('invalid syntax for include '
-                                  'directive. Requires a hardcoded '
-                                  'string', lineno,
-                                  self.filename)
+        token = self.stream.expect('include')
+        template = self.stream.expect('string').value
+        self.stream.expect('block_end')
+        return nodes.Include(template, token.lineno, self.filename)
 
-    def handle_trans_directive(self, lineno, gen):
+    def parse_trans_directive(self):
         """
         Handle translatable sections.
         """
-        def process_variable(lineno, token, name):
-            if token != 'name':
-                raise TemplateSyntaxError('can only use variable not '
-                                          'constants or expressions in '
-                                          'translation variable blocks.',
-                                          lineno, self.filename)
-            # plural name without trailing "_"? that's a keyword
-            if not name.endswith('_'):
-                raise TemplateSyntaxError('illegal use of keyword \'%s\' as '
-                                          'identifier in translatable block.'
-                                          % name, lineno, self.filename)
-            name = name[:-1]
-            if name not in replacements:
-                raise TemplateSyntaxError('unregistered translation variable'
-                                          " '%s'." % name, lineno,
-                                          self.filename)
-            # check that we don't have an expression here, thus the
-            # next token *must* be a variable_end token (or a
-            # block_end token when in no_variable_block mode)
-            next_token = self.tokenstream.next()[1]
-            if next_token != 'variable_end' and not \
-               (self.no_variable_block and next_token == 'block_end'):
-                raise TemplateSyntaxError('you cannot use variable '
-                                          'expressions inside translatable '
-                                          'tags. apply filters in the '
-                                          'trans header.', lineno,
-                                          self.filename)
-            buf.append('%%(%s)s' % name)
+        trans_token = self.stream.expect('trans')
 
-        # save the initial line number for the resulting node
-        flineno = lineno
-        try:
-            # check for string translations
-            try:
-                lineno, token, data = gen.next()
-            except StopIteration:
-                # no dynamic replacements
-                replacements = {}
-                first_var = None
+        # string based translations {% trans "foo" %}
+        if self.stream.current.type == 'string':
+            text = self.stream.expect('string')
+            self.stream.expect('block_end')
+            return nodes.Trans(text.value, None, None, None,
+                               trans_token.lineno, self.filename)
+
+        # block based translations
+        replacements = {}
+        plural_var = None
+
+        while self.stream.current.type != 'block_end':
+            if replacements:
+                self.stream.expect('comma')
+            name = self.stream.expect('name')
+            if self.stream.current.type == 'assign':
+                self.stream.next()
+                value = self.parse_expression()
             else:
-                if token == 'string':
-                    # check that there are not any more elements
-                    try:
-                        gen.next()
-                    except StopIteration:
-                        # XXX: this looks fishy
-                        data = data[1:-1].encode('utf-8').decode('string-escape')
-                        return nodes.Trans(lineno, data.decode('utf-8'), None,
-                                           None, None)
-                    raise TemplateSyntaxError('string based translations '
-                                              'require at most one argument.',
-                                              lineno, self.filename)
-                # create a new generator with the popped item as first one
-                def wrapgen(oldgen):
-                    yield lineno, token, data
-                    for item in oldgen:
-                        yield item
-                gen = wrapgen(gen)
-
-                # block based translations
-                first_var = None
-                replacements = {}
-                for arg in self.parse_python(lineno, gen,
-                                             '_trans(%s)').expr.args:
-                    if arg.__class__ not in (ast.Keyword, ast.Name):
-                        raise TemplateSyntaxError('translation tags need expl'
-                                                  'icit names for values.',
-                                                  lineno, self.filename)
-                    # disallow keywords
-                    if not arg.name.endswith('_'):
-                        raise TemplateSyntaxError("illegal use of keyword '%s"
-                                                  '\' as identifier.' %
-                                                  arg.name, lineno,
-                                                  self.filename)
-                    # remove the last "_" before writing
-                    name = arg.name[:-1]
-                    if first_var is None:
-                        first_var = name
-                    # if it's a keyword use the expression as value,
-                    # otherwise just reuse the name node.
-                    replacements[name] = getattr(arg, 'expr', arg)
-
-            # look for endtrans/pluralize
-            buf = singular = []
-            plural = indicator = None
-
-            while True:
-                lineno, token, data = self.tokenstream.next()
-                # comments
-                if token == 'comment_begin':
-                    self.tokenstream.drop_until(end_of_comment, True)
-                # nested variables
-                elif token == 'variable_begin':
-                    process_variable(*self.tokenstream.next())
-                # nested blocks are not supported, just look for end blocks
-                elif token == 'block_begin':
-                    _, block_token, block_name = self.tokenstream.next()
-                    if block_token != 'name' or \
-                       block_name not in ('pluralize', 'endtrans'):
-                        # if we have a block name check if it's a real
-                        # directive or a not existing one (which probably
-                        # is a typo)
-                        if block_token == 'name':
-                            # if this block is a context directive the
-                            # designer has probably misspelled endtrans
-                            # with endfor or something like that. raise
-                            # a nicer error message
-                            if block_name in self.context_directives:
-                                raise TemplateSyntaxError('unexpected directi'
-                                                          "ve '%s' found" %
-                                                          block_name, lineno,
-                                                          self.filename)
-                            # if's not a directive, probably misspelled
-                            # endtrans. Raise the "unknown directive"
-                            # exception rather than the "not allowed"
-                            if block_name not in self.directives:
-                                if block_name.endswith('_'):
-                                    # if we don't have a variable block we
-                                    # have to process this as variable.
-                                    if self.no_variable_block:
-                                        process_variable(_, block_token,
-                                                         block_name)
-                                        continue
-                                    block_name = block_name[:-1]
-                                raise TemplateSyntaxError('unknown directive'
-                                                          " '%s'" % block_name,
-                                                          lineno,
-                                                          self.filename)
-                        # we have something different and are in the
-                        # special no_variable_block mode. process this
-                        # as variable
-                        elif self.no_variable_block:
-                            process_variable(_, block_token, block_name)
-                            continue
-                        # if it's indeed a known directive we better
-                        # raise an exception informing the user about
-                        # the fact that we don't support blocks in
-                        # translatable sections.
-                        raise TemplateSyntaxError('directives in translatable'
-                                                  ' sections are not '
-                                                  'allowed', lineno,
-                                                  self.filename)
-                    # pluralize
-                    if block_name == 'pluralize':
-                        if plural is not None:
-                            raise TemplateSyntaxError('translation blocks '
-                                                      'support at most one '
-                                                      'plural block',
-                                                      lineno, self.filename)
-                        _, plural_token, plural_name = self.tokenstream.next()
-                        if plural_token == 'block_end':
-                            indicator = first_var
-                        elif plural_token == 'name':
-                            # disallow keywords
-                            if not plural_name.endswith('_'):
-                                raise TemplateSyntaxError('illegal use of '
-                                                          "keyword '%s' as "
-                                                          'identifier.' %
-                                                          plural_name,
-                                                          lineno,
-                                                          self.filename)
-                            plural_name = plural_name[:-1]
-                            if plural_name not in replacements:
-                                raise TemplateSyntaxError('unregistered '
-                                                          'translation '
-                                                          "variable '%s'" %
-                                                          plural_name, lineno,
-                                                          self.filename)
-                            elif self.tokenstream.next()[1] != 'block_end':
-                                raise TemplateSyntaxError('pluralize takes '
-                                                          'at most one '
-                                                          'argument', lineno,
-                                                          self.filename)
-                            indicator = plural_name
-                        else:
-                            raise TemplateSyntaxError('pluralize requires no '
-                                                      'argument or a variable'
-                                                      ' name.', lineno,
-                                                      self.filename)
-                        plural = buf = []
-                    # end translation
-                    elif block_name == 'endtrans':
-                        self.close_remaining_block()
-                        break
-                # normal data
-                else:
-                    buf.append(data.replace('%', '%%'))
+                value = nodes.NameExpression(name.value, name.lineno,
+                                             self.filename)
+            if name.value in replacements:
+                raise TemplateSyntaxError('translation variable %r '
+                                          'is defined twice' % name.value,
+                                          name.lineno, self.filename)
+            replacements[name.value] = value
+            if plural_var is None:
+                plural_var = name.value
+        self.stream.expect('block_end')
+
+        def process_variable():
+            var_name = self.stream.expect('name')
+            if var_name.value not in replacements:
+                raise TemplateSyntaxError('unregistered translation variable'
+                                          " '%s'." % var_name.value,
+                                          var_name.lineno, self.filename)
+            buf.append('%%(%s)s' % var_name.value)
 
-        except StopIteration:
-            raise TemplateSyntaxError('unexpected end of translation section',
-                                      self.tokenstream.last[0], self.filename)
+        buf = singular = []
+        plural = None
+
+        while True:
+            token = self.stream.current
+            if token.type == 'data':
+                buf.append(token.value.replace('%', '%%'))
+                self.stream.next()
+            elif token.type == 'variable_begin':
+                self.stream.next()
+                process_variable()
+                self.stream.expect('variable_end')
+            elif token.type == 'block_begin':
+                self.stream.next()
+                if plural is None and self.stream.current.type == 'pluralize':
+                    self.stream.next()
+                    if self.stream.current.type == 'name':
+                        plural_var = self.stream.expect('name').value
+                    plural = buf = []
+                elif self.stream.current.type == 'endtrans':
+                    self.stream.next()
+                    self.stream.expect('block_end')
+                    break
+                else:
+                    if self.no_variable_block:
+                        process_variable()
+                    else:
+                        raise TemplateSyntaxError('blocks are not allowed '
+                                                  'in trans tags',
+                                                  self.stream.lineno,
+                                                  self.filename)
+                self.stream.expect('block_end')
+            else:
+                assert False, 'something very strange happened'
 
         singular = u''.join(singular)
         if plural is not None:
             plural = u''.join(plural)
-        return nodes.Trans(flineno, singular, plural, indicator,
-                           replacements or None)
-
-    def parse_python(self, lineno, gen, template):
-        """
-        Convert the passed generator into a flat string representing
-        python sourcecode and return an ast node or raise a
-        TemplateSyntaxError.
-        """
-        tokens = []
-        for t_lineno, t_token, t_data in gen:
-            if t_token == 'string':
-                # because some libraries have problems with unicode
-                # objects we do some lazy unicode processing here.
-                # if a string is ASCII only we yield it as string
-                # in other cases as unicode. This works around
-                # problems with datetimeobj.strftime()
-                # also escape newlines in strings
-                t_data = t_data.replace('\n', '\\n')
-                try:
-                    str(t_data)
-                except UnicodeError:
-                    tokens.append('u' + t_data)
-                    continue
-            tokens.append(t_data)
-        source = '\xef\xbb\xbf' + (template % (u' '.join(tokens)).
-                                   encode('utf-8'))
-        try:
-            ast = parse(source, 'exec')
-        except SyntaxError, e:
-            raise TemplateSyntaxError('invalid syntax in expression',
-                                      lineno + (e.lineno or 0),
+        return nodes.Trans(singular, plural, plural_var, replacements,
+                           trans_token.lineno, self.filename)
+
+    def parse_expression(self):
+        """
+        Parse one expression from the stream.
+        """
+        return self.parse_conditional_expression()
+
+    def parse_subscribed_expression(self):
+        """
+        Like parse_expression but parses slices too. Because this
+        parsing function requires a border the two tokens rbracket
+        and comma mark the end of the expression in some situations.
+        """
+        lineno = self.stream.lineno
+
+        if self.stream.current.type == 'colon':
+            self.stream.next()
+            args = [None]
+        else:
+            node = self.parse_expression()
+            if self.stream.current.type != 'colon':
+                return node
+            self.stream.next()
+            args = [node]
+
+        if self.stream.current.type == 'colon':
+            args.append(None)
+        elif self.stream.current.type not in ('rbracket', 'comma'):
+            args.append(self.parse_expression())
+        else:
+            args.append(None)
+
+        if self.stream.current.type == 'colon':
+            self.stream.next()
+            if self.stream.current.type not in ('rbracket', 'comma'):
+                args.append(self.parse_expression())
+            else:
+                args.append(None)
+        else:
+            args.append(None)
+
+        return nodes.SliceExpression(*(args + [lineno, self.filename]))
+
+    def parse_conditional_expression(self):
+        """
+        Parse a conditional expression (foo if bar else baz)
+        """
+        lineno = self.stream.lineno
+        expr1 = self.parse_or_expression()
+        while self.stream.current.type == 'if':
+            self.stream.next()
+            expr2 = self.parse_or_expression()
+            self.stream.expect('else')
+            expr3 = self.parse_conditional_expression()
+            expr1 = nodes.ConditionalExpression(expr2, expr1, expr3,
+                                                lineno, self.filename)
+            lineno = self.stream.lineno
+        return expr1
+
+    def parse_or_expression(self):
+        """
+        Parse something like {{ foo or bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_and_expression()
+        while self.stream.current.type == 'or':
+            self.stream.next()
+            right = self.parse_and_expression()
+            left = nodes.OrExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_and_expression(self):
+        """
+        Parse something like {{ foo and bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_compare_expression()
+        while self.stream.current.type == 'and':
+            self.stream.next()
+            right = self.parse_compare_expression()
+            left = nodes.AndExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_compare_expression(self):
+        """
+        Parse something like {{ foo == bar }}.
+        """
+        known_operators = set(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq', 'in'])
+        lineno = self.stream.lineno
+        expr = self.parse_add_expression()
+        ops = []
+        while True:
+            if self.stream.current.type in known_operators:
+                op = self.stream.current.type
+                self.stream.next()
+                ops.append([op, self.parse_add_expression()])
+                lineno = self.stream.lineno
+            elif self.stream.current.type == 'not' and \
+                 self.stream.look().type == 'in':
+                self.stream.skip(2)
+                ops.append(['not in', self.parse_add_expression()])
+            else:
+                break
+        if not ops:
+            return expr
+        return nodes.CompareExpression(expr, ops)
+
+    def parse_add_expression(self):
+        """
+        Parse something like {{ foo + bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_sub_expression()
+        while self.stream.current.type == 'add':
+            self.stream.next()
+            right = self.parse_sub_expression()
+            left = nodes.AddExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_sub_expression(self):
+        """
+        Parse something like {{ foo - bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_concat_expression()
+        while self.stream.current.type == 'sub':
+            self.stream.next()
+            right = self.parse_concat_expression()
+            left = nodes.SubExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_concat_expression(self):
+        """
+        Parse something like {{ foo ~ bar }}.
+        """
+        lineno = self.stream.lineno
+        args = [self.parse_mul_expression()]
+        while self.stream.current.type == 'tilde':
+            self.stream.next()
+            args.append(self.parse_mul_expression())
+        if len(args) == 1:
+            return args[0]
+        return nodes.ConcatExpression(args, lineno, self.filename)
+
+    def parse_mul_expression(self):
+        """
+        Parse something like {{ foo * bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_div_expression()
+        while self.stream.current.type == 'mul':
+            self.stream.next()
+            right = self.parse_div_expression()
+            left = nodes.MulExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_div_expression(self):
+        """
+        Parse something like {{ foo / bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_floor_div_expression()
+        while self.stream.current.type == 'div':
+            self.stream.next()
+            right = self.parse_floor_div_expression()
+            left = nodes.DivExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_floor_div_expression(self):
+        """
+        Parse something like {{ foo // bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_mod_expression()
+        while self.stream.current.type == 'floordiv':
+            self.stream.next()
+            right = self.parse_mod_expression()
+            left = nodes.FloorDivExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_mod_expression(self):
+        """
+        Parse something like {{ foo % bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_pow_expression()
+        while self.stream.current.type == 'mod':
+            self.stream.next()
+            right = self.parse_pow_expression()
+            left = nodes.ModExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_pow_expression(self):
+        """
+        Parse something like {{ foo ** bar }}.
+        """
+        lineno = self.stream.lineno
+        left = self.parse_unary_expression()
+        while self.stream.current.type == 'pow':
+            self.stream.next()
+            right = self.parse_unary_expression()
+            left = nodes.PowExpression(left, right, lineno, self.filename)
+            lineno = self.stream.lineno
+        return left
+
+    def parse_unary_expression(self):
+        """
+        Parse all kinds of unary expressions.
+        """
+        if self.stream.current.type == 'not':
+            return self.parse_not_expression()
+        elif self.stream.current.type == 'sub':
+            return self.parse_neg_expression()
+        elif self.stream.current.type == 'add':
+            return self.parse_pos_expression()
+        return self.parse_primary_expression()
+
+    def parse_not_expression(self):
+        """
+        Parse something like {{ not foo }}.
+        """
+        token = self.stream.expect('not')
+        node = self.parse_unary_expression()
+        return nodes.NotExpression(node, token.lineno, self.filename)
+
+    def parse_neg_expression(self):
+        """
+        Parse something like {{ -foo }}.
+        """
+        token = self.stream.expect('sub')
+        node = self.parse_unary_expression()
+        return nodes.NegExpression(node, token.lineno, self.filename)
+
+    def parse_pos_expression(self):
+        """
+        Parse something like {{ +foo }}.
+        """
+        token = self.stream.expect('add')
+        node = self.parse_unary_expression()
+        return nodes.PosExpression(node, token.lineno, self.filename)
+
+    def parse_primary_expression(self, parse_postfix=True):
+        """
+        Parse a primary expression such as a name or literal.
+        """
+        current = self.stream.current
+        if current.type == 'name':
+            if current.value in ('true', 'false'):
+                node = self.parse_bool_expression()
+            elif current.value == 'none':
+                node = self.parse_none_expression()
+            elif current.value == 'undefined':
+                node = self.parse_undefined_expression()
+            elif current.value == '_':
+                node = self.parse_gettext_call()
+            else:
+                node = self.parse_name_expression()
+        elif current.type in ('integer', 'float'):
+            node = self.parse_number_expression()
+        elif current.type == 'string':
+            node = self.parse_string_expression()
+        elif current.type == 'regex':
+            node = self.parse_regex_expression()
+        elif current.type == 'lparen':
+            node = self.parse_paren_expression()
+        elif current.type == 'lbracket':
+            node = self.parse_list_expression()
+        elif current.type == 'lbrace':
+            node = self.parse_dict_expression()
+        elif current.type == 'at':
+            node = self.parse_set_expression()
+        else:
+            raise TemplateSyntaxError("unexpected token '%s'" %
+                                      self.stream.current,
+                                      self.stream.current.lineno,
                                       self.filename)
-        assert len(ast.node.nodes) == 1, 'get %d nodes, 1 expected' % \
-                                         len(ast.node.nodes)
-        result = ast.node.nodes[0]
-        nodes.inc_lineno(lineno, result)
-        return result
+        if parse_postfix:
+            node = self.parse_postfix_expression(node)
+        return node
 
-    def parse(self):
+    def parse_tuple_expression(self, enforce=False, simplified=False):
         """
-        Parse the template and return a Template node. Also unescape the
-        names escaped by the lexer (unused python keywords) and set the
-        filename attributes for all nodes in the tree.
-        """
-        body = self.subparse(None)
-        def walk(nodes_, stack):
-            for node in nodes_:
-                # all names excluding keywords have an trailing underline.
-                # if we find a name without trailing underline that's a
-                # keyword and this code raises an error. else strip the
-                # underline again
-                if node.__class__ in (ast.AssName, ast.Name, ast.Keyword):
-                    if not node.name.endswith('_'):
-                        raise TemplateSyntaxError('illegal use of keyword %r '
-                                                  'as identifier.' %
-                                                  node.name, node.lineno,
-                                                  self.filename)
-                    node.name = node.name[:-1]
-                # same for attributes
-                elif node.__class__ is ast.Getattr:
-                    if not node.attrname.endswith('_'):
-                        raise TemplateSyntaxError('illegal use of keyword %r '
-                                                  'as attribute name.' %
-                                                  node.name, node.lineno,
-                                                  self.filename)
-                    node.attrname = node.attrname[:-1]
-                # if we have a ForLoop we ensure that nobody patches existing
-                # object using "for foo.bar in seq"
-                elif node.__class__ is nodes.ForLoop:
-                    def check(node):
-                        if node.__class__ not in (ast.AssName, ast.AssTuple):
-                            raise TemplateSyntaxError('can\'t assign to '
-                                                      'expression.',
-                                                      node.lineno,
-                                                      self.filename)
-                        for n in node.getChildNodes():
-                            check(n)
-                    check(node.item)
-                # ensure that in child templates block are either top level
-                # or located inside of another block tag.
-                elif self.extends is not None and \
-                     node.__class__ is nodes.Block:
-                    if stack[-1] is not body:
-                        for n in stack:
-                            if n.__class__ is nodes.Block:
-                                break
-                        else:
-                            raise TemplateSyntaxError('misplaced block %r, '
-                                                      'blocks in child '
-                                                      'templates must be '
-                                                      'either top level or '
-                                                      'located in a block '
-                                                      'tag.' % node.name,
-                                                      node.lineno,
-                                                      self.filename)
-                # now set the filename and continue working on the childnodes
-                node.filename = self.filename
-                stack.append(node)
-                walk(node.getChildNodes(), stack)
-                stack.pop()
-        walk([body], [body])
-        return nodes.Template(self.filename, body, self.extends)
+        Parse multiple expressions into a tuple. This can also return
+        just one expression which is not a tuple. If you want to enforce
+        a tuple, pass it enforce=True.
+        """
+        lineno = self.stream.lineno
+        if simplified:
+            parse = self.parse_primary_expression
+        else:
+            parse = self.parse_expression
+        args = []
+        is_tuple = False
+        while True:
+            if args:
+                self.stream.expect('comma')
+            if self.stream.current.type in tuple_edge_tokens:
+                break
+            args.append(parse())
+            if self.stream.current.type == 'comma':
+                is_tuple = True
+            else:
+                break
+        if not is_tuple and args:
+            if enforce:
+                raise TemplateSyntaxError('tuple expected', lineno,
+                                          self.filename)
+            return args[0]
+        return nodes.TupleExpression(args, lineno, self.filename)
+
+    def parse_bool_expression(self):
+        """
+        Parse a boolean literal.
+        """
+        token = self.stream.expect('name')
+        if token.value == 'true':
+            value = True
+        elif token.value == 'false':
+            value = False
+        else:
+            raise TemplateSyntaxError("expected boolean literal",
+                                      token.lineno, self.filename)
+        return nodes.ConstantExpression(value, token.lineno, self.filename)
+
+    def parse_none_expression(self):
+        """
+        Parse a none literal.
+        """
+        token = self.stream.expect('name', 'none')
+        return nodes.ConstantExpression(None, token.lineno, self.filename)
+
+    def parse_undefined_expression(self):
+        """
+        Parse an undefined literal.
+        """
+        token = self.stream.expect('name', 'undefined')
+        return nodes.UndefinedExpression(token.lineno, self.filename)
+
+    def parse_gettext_call(self):
+        """
+        parse {{ _('foo') }}.
+        """
+        # XXX: check if only one argument was passed and if
+        # it is a string literal. Maybe that should become a special
+        # expression anyway.
+        token = self.stream.expect('name', '_')
+        node = nodes.NameExpression(token.value, token.lineno, self.filename)
+        return self.parse_call_expression(node)
+
+    def parse_name_expression(self):
+        """
+        Parse any name.
+        """
+        token = self.stream.expect('name')
+        self.test_name(token.value)
+        return nodes.NameExpression(token.value, token.lineno, self.filename)
+
+    def parse_number_expression(self):
+        """
+        Parse a number literal.
+        """
+        token = self.stream.current
+        if token.type not in ('integer', 'float'):
+            raise TemplateSyntaxError('integer or float literal expected',
+                                      token.lineno, self.filename)
+        self.stream.next()
+        return nodes.ConstantExpression(token.value, token.lineno, self.filename)
+
+    def parse_string_expression(self):
+        """
+        Parse a string literal.
+        """
+        token = self.stream.expect('string')
+        return nodes.ConstantExpression(token.value, token.lineno, self.filename)
+
+    def parse_regex_expression(self):
+        """
+        Parse a regex literal.
+        """
+        token = self.stream.expect('regex')
+        return nodes.RegexExpression(token.value, token.lineno, self.filename)
+
+    def parse_paren_expression(self):
+        """
+        Parse a parenthized expression.
+        """
+        self.stream.expect('lparen')
+        try:
+            return self.parse_tuple_expression()
+        finally:
+            self.stream.expect('rparen')
+
+    def parse_list_expression(self):
+        """
+        Parse something like {{ [1, 2, "three"] }}
+        """
+        token = self.stream.expect('lbracket')
+        items = []
+        while self.stream.current.type != 'rbracket':
+            if items:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'rbracket':
+                break
+            items.append(self.parse_expression())
+        self.stream.expect('rbracket')
+
+        return nodes.ListExpression(items, token.lineno, self.filename)
+
+    def parse_dict_expression(self):
+        """
+        Parse something like {{ {1: 2, 3: 4} }}
+        """
+        token = self.stream.expect('lbrace')
+        items = []
+        while self.stream.current.type != 'rbrace':
+            if items:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'rbrace':
+                break
+            key = self.parse_expression()
+            self.stream.expect('colon')
+            value = self.parse_expression()
+            items.append((key, value))
+        self.stream.expect('rbrace')
+
+        return nodes.DictExpression(items, token.lineno, self.filename)
+
+    def parse_set_expression(self):
+        """
+        Parse something like {{ @(1, 2, 3) }}.
+        """
+        token = self.stream.expect('at')
+        self.stream.expect('lparen')
+        items = []
+        while self.stream.current.type != 'rparen':
+            if items:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'rparen':
+                break
+            items.append(self.parse_expression())
+        self.stream.expect('rparen')
+
+        return nodes.SetExpression(items, token.lineno, self.filename)
+
+    def parse_postfix_expression(self, node):
+        """
+        Parse a postfix expression such as a filter statement or a
+        function call.
+        """
+        while True:
+            current = self.stream.current.type
+            if current == 'dot' or current == 'lbracket':
+                node = self.parse_subscript_expression(node)
+            elif current == 'lparen':
+                node = self.parse_call_expression(node)
+            elif current == 'pipe':
+                node = self.parse_filter_expression(node)
+            elif current == 'is':
+                node = self.parse_test_expression(node)
+            else:
+                break
+        return node
+
+    def parse_subscript_expression(self, node):
+        """
+        Parse a subscript statement. Gets attributes and items from an
+        object.
+        """
+        lineno = self.stream.lineno
+        if self.stream.current.type == 'dot':
+            self.stream.next()
+            token = self.stream.current
+            if token.type in ('name', 'integer'):
+                arg = nodes.ConstantExpression(token.value, token.lineno,
+                                               self.filename)
+            else:
+                raise TemplateSyntaxError('expected name or number',
+                                          token.lineno, self.filename)
+            self.stream.next()
+        elif self.stream.current.type == 'lbracket':
+            self.stream.next()
+            args = []
+            while self.stream.current.type != 'rbracket':
+                if args:
+                    self.stream.expect('comma')
+                args.append(self.parse_subscribed_expression())
+            self.stream.expect('rbracket')
+            if len(args) == 1:
+                arg = args[0]
+            else:
+                arg = nodes.TupleExpression(args, lineno, self.filename)
+        else:
+            raise TemplateSyntaxError('expected subscript expression',
+                                      self.lineno, self.filename)
+        return nodes.SubscriptExpression(node, arg, lineno, self.filename)
+
+    def parse_call_expression(self, node=None):
+        """
+        Parse a call.
+        """
+        if node is None:
+            node = self.parse_primary_expression(parse_postfix=False)
+        token = self.stream.expect('lparen')
+        args = []
+        kwargs = []
+        dyn_args = None
+        dyn_kwargs = None
+        require_comma = False
+
+        def ensure(expr):
+            if not expr:
+                raise TemplateSyntaxError('invalid syntax for function '
+                                          'declaration', token.lineno,
+                                          self.filename)
+
+        while self.stream.current.type != 'rparen':
+            if require_comma:
+                self.stream.expect('comma')
+            if self.stream.current.type == 'mul':
+                ensure(dyn_args is None and dyn_kwargs is None)
+                self.stream.next()
+                dyn_args = self.parse_expression()
+            elif self.stream.current.type == 'pow':
+                ensure(dyn_kwargs is None)
+                self.stream.next()
+                dyn_kwargs = self.parse_expression()
+            else:
+                ensure(dyn_args is None and dyn_kwargs is None)
+                if self.stream.current.type == 'name' and \
+                    self.stream.look().type == 'assign':
+                    key = self.stream.current.value
+                    self.stream.skip(2)
+                    kwargs.append((key, self.parse_expression()))
+                else:
+                    ensure(not kwargs)
+                    args.append(self.parse_expression())
+
+            require_comma = True
+        self.stream.expect('rparen')
+
+        return nodes.CallExpression(node, args, kwargs, dyn_args,
+                                    dyn_kwargs, token.lineno,
+                                    self.filename)
+
+    def parse_filter_expression(self, node):
+        """
+        Parse filter calls.
+        """
+        lineno = self.stream.lineno
+        filters = []
+        while self.stream.current.type == 'pipe':
+            self.stream.next()
+            token = self.stream.expect('name')
+            args = []
+            if self.stream.current.type == 'lparen':
+                self.stream.next()
+                while self.stream.current.type != 'rparen':
+                    if args:
+                        self.stream.expect('comma')
+                    args.append(self.parse_expression())
+                self.stream.expect('rparen')
+            filters.append((token.value, args))
+        return nodes.FilterExpression(node, filters, lineno, self.filename)
+
+    def parse_test_expression(self, node):
+        """
+        Parse test calls.
+        """
+        token = self.stream.expect('is')
+        if self.stream.current.type == 'not':
+            self.stream.next()
+            negated = True
+        else:
+            negated = False
+        name = self.stream.expect('name').value
+        args = []
+        if self.stream.current.type == 'lparen':
+            self.stream.next()
+            while self.stream.current.type != 'rparen':
+                if args:
+                    self.stream.expect('comma')
+                args.append(self.parse_expression())
+            self.stream.expect('rparen')
+        elif self.stream.current.type in ('name', 'string', 'integer',
+                                          'float', 'lparen', 'lbracket',
+                                          'lbrace', 'regex'):
+            args.append(self.parse_expression())
+        node = nodes.TestExpression(node, name, args, token.lineno,
+                                    self.filename)
+        if negated:
+            node = nodes.NotExpression(node, token.lineno, self.filename)
+        return node
+
+    def test_name(self, name):
+        """
+        Test if a name is not a special constant
+        """
+        if name in ('true', 'false', 'none', 'undefined', '_'):
+            raise TemplateSyntaxError('expected name not special constant',
+                                      self.stream.lineno, self.filename)
 
     def subparse(self, test, drop_needle=False):
         """
         Helper function used to parse the sourcecode until the test
         function which is passed a tuple in the form (lineno, token, data)
         returns True. In that case the current token is pushed back to
-        the tokenstream and the generator ends.
+        the stream and the generator ends.
 
         The test function is only called for the first token after a
         block tag. Variable tags are *not* aliases for {% print %} in
         that case.
 
         If drop_needle is True the needle_token is removed from the
-        tokenstream.
-        """
-        def finish():
-            """Helper function to remove unused nodelists."""
-            if data_buffer:
-                flush_data_buffer()
-            if len(result) == 1:
-                return result[0]
-            return result
-
-        def flush_data_buffer():
-            """Helper function to write the contents of the buffer
-            to the result nodelist."""
-            format_string = []
-            insertions = []
-            for item in data_buffer:
-                if item[0] == 'variable':
-                    p = self.handle_print_directive(*item[1:])
-                    format_string.append('%s')
-                    insertions.append(p)
+        stream.
+        """
+        if self.closed:
+            raise RuntimeError('parser is closed')
+        result = []
+        buffer = []
+        next = self.stream.next
+        lineno = self.stream.lineno
+
+        def assemble_list():
+            push_buffer()
+            return nodes.NodeList(result, lineno, self.filename)
+
+        def push_variable():
+            buffer.append((True, self.parse_tuple_expression()))
+
+        def push_data():
+            buffer.append((False, self.stream.expect('data')))
+
+        def push_buffer():
+            if not buffer:
+                return
+            template = []
+            variables = []
+            for is_var, data in buffer:
+                if is_var:
+                    template.append('%s')
+                    variables.append(data)
                 else:
-                    format_string.append(item[2].replace('%', '%%'))
-            # if we do not have insertions yield it as text node
-            if not insertions:
-                result.append(nodes.Text(data_buffer[0][1],
-                              (u''.join(format_string)).replace('%%', '%')))
-            # if we do not have any text data we yield some variable nodes
-            elif len(insertions) == len(format_string):
-                result.extend(insertions)
-            # otherwise we go with a dynamic text node
-            else:
-                result.append(nodes.DynamicText(data_buffer[0][1],
-                                                u''.join(format_string),
-                                                insertions))
-            # clear the buffer
-            del data_buffer[:]
-
-        def process_variable(gen):
-            data_buffer.append(('variable', lineno, tuple(gen)))
-
-        lineno = self.tokenstream.last[0]
-        result = nodes.NodeList(lineno)
-        data_buffer = []
-        for lineno, token, data in self.tokenstream:
-            # comments
-            if token == 'comment_begin':
-                self.tokenstream.drop_until(end_of_comment, True)
-
-            # this token marks the begin or a variable section.
-            # parse everything till the end of it.
-            elif token == 'variable_begin':
-                gen = self.tokenstream.fetch_until(end_of_variable, True)
-                process_variable(gen)
-
-            # this token marks the start of a block. like for variables
-            # just parse everything until the end of the block
-            elif token == 'block_begin':
-                # if we have something in the buffer we write the
-                # data back
-                if data_buffer:
-                    flush_data_buffer()
-
-                node = None
-                gen = self.tokenstream.fetch_until(end_of_block, True)
-                try:
-                    lineno, token, data = gen.next()
-                except StopIteration:
-                    raise TemplateSyntaxError('unexpected end of block',
-                                              lineno, self.filename)
-
-                # first token *must* be a name token
-                if token != 'name':
-                    # well, not always. if we have a lexer without variable
-                    # blocks configured we process these tokens as variable
-                    # block.
+                    template.append(data.value.replace('%', '%%'))
+            result.append(nodes.Text(u''.join(template), variables,
+                                     buffer[0][1].lineno, self.filename))
+            del buffer[:]
+
+        def push_node(node):
+            push_buffer()
+            result.append(node)
+
+        while self.stream:
+            token_type = self.stream.current.type
+            if token_type == 'variable_begin':
+                next()
+                push_variable()
+                self.stream.expect('variable_end')
+            elif token_type == 'block_begin':
+                next()
+                if test is not None and test(self.stream.current):
+                    if drop_needle:
+                        next()
+                    return assemble_list()
+                handler = self.directives.get(self.stream.current.type)
+                if handler is None:
                     if self.no_variable_block:
-                        process_variable([(lineno, token, data)] + list(gen))
+                        push_variable()
+                        self.stream.expect('block_end')
+                    elif self.stream.current.type in self.context_directives:
+                        raise TemplateSyntaxError('unexpected directive %r.' %
+                                                  self.stream.current.type,
+                                                  lineno, self.filename)
                     else:
-                        raise TemplateSyntaxError('unexpected %r token (%r)' %
-                                                  (token, data), lineno,
-                                                  self.filename)
-
-                # if a test function is passed to subparse we check if we
-                # reached the end of such a requested block.
-                if test is not None and test(lineno, token, data):
-                    if not drop_needle:
-                        self.tokenstream.push(lineno, token, data)
-                    return finish()
-
-                # the first token tells us which directive we want to call.
-                # if if doesn't match any existing directive it's like a
-                # template syntax error.
-                if data in self.directives:
-                    node = self.directives[data](lineno, gen)
-                # context depending directive found
-                elif data in self.context_directives:
-                    raise TemplateSyntaxError('unexpected directive %r' %
-                                              str(data), lineno,
-                                              self.filename)
-                # keyword or unknown name with trailing slash
+                        raise TemplateSyntaxError('unknown directive %r.' %
+                                                  self.stream.current.type,
+                                                  lineno, self.filename)
                 else:
-                    # non name token in no_variable_block mode.
-                    if token != 'name' and self.no_variable_block:
-                        process_variable([(lineno, token, data)] +
-                                             list(gen))
-                        continue
-                    if data.endswith('_'):
-                        # it was a non keyword identifier and we have
-                        # no variable tag. sounds like we should process
-                        # this as variable tag
-                        if self.no_variable_block:
-                            process_variable([(lineno, token, data)] +
-                                             list(gen))
-                            continue
-                        # otherwise strip the trailing underscore for the
-                        # exception that is raised
-                        data = data[:-1]
-                    raise TemplateSyntaxError('unknown directive %r' %
-                                              str(data), lineno,
-                                              self.filename)
-                # some tags like the extends tag do not output nodes.
-                # so just skip that.
-                if node is not None:
-                    result.append(node)
-
-            # here the only token we should get is "data". all other
-            # tokens just exist in block or variable sections. (if the
-            # tokenizer is not brocken)
-            elif token in 'data':
-                data_buffer.append(('text', lineno, data))
-
-            # so this should be unreachable code
+                    node = handler()
+                    if node is not None:
+                        push_node(node)
+            elif token_type == 'data':
+                push_data()
+
+            # this should be unreachable code
             else:
-                raise AssertionError('unexpected token %r (%r)' % (token,
-                                                                   data))
+                assert False, "unexpected token %r" % self.stream.current
 
-        # still here and a test function is provided? raise and error
         if test is not None:
-            # if the callback is a state test lambda wrapper we
-            # can use the `error_message` property to get the error
-            if isinstance(test, StateTest):
-                msg = ': ' + test.error_message
-            else:
-                msg = ''
-            raise TemplateSyntaxError('unexpected end of template' + msg,
-                                      lineno, self.filename)
-        return finish()
+            msg = isinstance(test, StateTest) and ': ' + test.message or ''
+            raise TemplateSyntaxError('unexpected end of stream' + msg,
+                                      self.stream.lineno, self.filename)
 
-    def close_remaining_block(self):
+        return assemble_list()
+
+    def parse(self):
         """
-        If we opened a block tag because one of our tags requires an end
-        tag we can use this method to drop the rest of the block from
-        the stream. If the next token isn't the block end we throw an
-        error.
+        Parse the template and return a Template node. This also does some
+        post processing sanitizing and parses for an extends tag.
         """
-        lineno, _, tagname = self.tokenstream.last
+        if self.closed:
+            raise RuntimeError('parser is closed')
+
         try:
-            lineno, token, data = self.tokenstream.next()
-        except StopIteration:
-            raise TemplateSyntaxError('missing closing tag', lineno,
-                                      self.filename)
-        if token != 'block_end':
-            print token, data, list(self.tokenstream)
-            raise TemplateSyntaxError('expected empty %s-directive but '
-                                      'found additional arguments.' %
-                                      tagname, lineno, self.filename)
+            # get the leading whitespace, if we are not in a child
+            # template we push that back to the stream later.
+            leading_whitespace = self.stream.read_whitespace()
+
+            # parse an optional extends which *must* be the first node
+            # of a template.
+            if self.stream.current.type == 'block_begin' and \
+               self.stream.look().type == 'extends':
+                self.stream.skip(2)
+                extends = self.stream.expect('string').value
+                self.stream.expect('block_end')
+            else:
+                extends = None
+                if leading_whitespace:
+                    self.stream.push(leading_whitespace)
+
+            body = self.subparse(None)
+            def walk(nodelist, stack):
+                for node in nodelist:
+                    if extends is not None and \
+                         node.__class__ is nodes.Block and \
+                         stack[-1] is not body:
+                        for n in stack:
+                            if n.__class__ is nodes.Block:
+                                break
+                        else:
+                            raise TemplateSyntaxError('misplaced block %r, '
+                                                      'blocks in child '
+                                                      'templates must be '
+                                                      'either top level or '
+                                                      'located in a block '
+                                                      'tag.' % node.name,
+                                                      node.lineno,
+                                                      self.filename)
+                    stack.append(node)
+                    walk(node.get_child_nodes(), stack)
+                    stack.pop()
+            walk([body], [body])
+            return nodes.Template(extends, body, 1, self.filename)
+        finally:
+            self.close()
+
+    def close(self):
+        """Clean up soon."""
+        self.closed = True
+        self.stream = self.directives = self.stream = self.blocks = \
+            self.environment = None
diff --git a/jinja/plugin.py b/jinja/plugin.py
index 415a2e0..19bc284 100644
--- a/jinja/plugin.py
+++ b/jinja/plugin.py
@@ -14,6 +14,7 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+from warnings import warn
 from jinja.environment import Environment
 from jinja.loaders import FunctionLoader, FileSystemLoader, PackageLoader
 from jinja.exceptions import TemplateNotFound
@@ -121,6 +122,10 @@ def jinja_plugin_factory(options):
                         not being provided this won't have an effect.
     =================== =================================================
     """
+    warn(DeprecationWarning('general plugin interface implementation '
+                            'deprecated because not an accepted '
+                            'standard.'))
+
     if 'environment' in options:
         env = options['environment']
         if not len(options) == 1:
diff --git a/jinja/tests.py b/jinja/tests.py
index 4cbe65a..25f7992 100644
--- a/jinja/tests.py
+++ b/jinja/tests.py
@@ -92,22 +92,25 @@ def test_matching(regex):
 
     .. sourcecode:: jinja
 
-        {% if var is matching('^\\d+$') %}
+        {% if var is matching @/^\d+$/ %}
             var looks like a number
         {% else %}
             var doesn't really look like a number
         {% endif %}
     """
-    if isinstance(regex, unicode):
-        regex = re.compile(regex, re.U)
-    elif isinstance(regex, str):
-        regex = re.compile(regex)
-    elif type(regex) is not regex_type:
-        regex = None
     def wrapped(environment, context, value):
-        if regex is None:
-            return False
-        return regex.search(value) is not None
+        if type(regex) is regex_type:
+            regex_ = regex
+        else:
+            if environment.disable_regexps:
+                raise RuntimeError('regular expressions disabled.')
+            if isinstance(regex, unicode):
+                regex_ = re.compile(regex, re.U)
+            elif isinstance(regex, str):
+                regex_ = re.compile(regex)
+            else:
+                return False
+        return regex_.search(value) is not None
     return wrapped
 
 
diff --git a/jinja/translators/__init__.py b/jinja/translators/__init__.py
index 7576a0e..ba55148 100644
--- a/jinja/translators/__init__.py
+++ b/jinja/translators/__init__.py
@@ -15,7 +15,7 @@ class Translator(object):
     Base class of all translators.
     """
 
-    def process(environment, tree):
+    def process(environment, tree, source=None):
         """
         Process the given ast with the rules defined in
         environment and return a translated version of it.
diff --git a/jinja/translators/python.py b/jinja/translators/python.py
index 907f4fc..f760743 100644
--- a/jinja/translators/python.py
+++ b/jinja/translators/python.py
@@ -19,12 +19,6 @@
     Implementation Details
     ======================
 
-    Some of the semantics are handled directly in the translator in order
-    to speed up the parsing process. For example the semantics for the
-    `is` operator are handled here, changing this would require and
-    additional traversing of the node tree in the parser. Thus the actual
-    translation process can raise a `TemplateSyntaxError` too.
-
     It might sound strange but the translator tries to keep the generated
     code readable as much as possible. This simplifies debugging the Jinja
     core a lot. The additional processing overhead is just relevant for
@@ -36,7 +30,6 @@
 """
 import re
 import sys
-from compiler import ast
 from jinja import nodes
 from jinja.nodes import get_nodes
 from jinja.parser import Parser
@@ -74,11 +67,6 @@ class Template(object):
         self.environment = environment
         self.code = code
         self.generate_func = None
-        #: holds debug information
-        self._debug_info = None
-        #: unused in loader environments but used when a template
-        #: is loaded from a string in order to be able to debug those too
-        self._source = None
 
     def dump(self, stream=None):
         """Dump the template into python bytecode."""
@@ -102,6 +90,7 @@ class Template(object):
 
     def render(self, *args, **kwargs):
         """Render a template."""
+        __traceback_hide__ = True
         ctx = self._prepare(*args, **kwargs)
         try:
             return capture_generator(self.generate_func(ctx))
@@ -130,7 +119,6 @@ class Template(object):
             ns = {'environment': env}
             exec self.code in ns
             self.generate_func = ns['generate']
-            self._debug_info = ns['debug_info']
         return env.context_class(env, *args, **kwargs)
 
     def _debug(self, ctx, exc_type, exc_value, traceback):
@@ -154,20 +142,29 @@ class PythonTranslator(Translator):
     Pass this translator a ast tree to get valid python code.
     """
 
-    def __init__(self, environment, node):
+    def __init__(self, environment, node, source):
         self.environment = environment
         self.node = node
+        self.source = source
+        self.closed = False
 
-        #: mapping to constants in the environment that are not
-        #: converted to context lookups. this should improve
-        #: performance and avoid accidentally screwing things up
-        #: by rebinding essential constants.
-        self.constants = {
-            'true':                 'True',
-            'false':                'False',
-            'none':                 'None',
-            'undefined':            'undefined_singleton'
-        }
+        #: current level of indention
+        self.indention = 0
+        #: each {% cycle %} tag has a unique ID which increments
+        #: automatically for each tag.
+        self.last_cycle_id = 0
+        #: set of used shortcuts jinja has to make local automatically
+        self.used_shortcuts = set(['undefined_singleton'])
+        #: set of used datastructures jinja has to import
+        self.used_data_structures = set()
+        #: set of used utils jinja has to import
+        self.used_utils = set()
+        #: flags for runtime error
+        self.require_runtime_error = False
+        #: do wee need a "set" object?
+        self.need_set_import = False
+        #: flag for regular expressions
+        self.compiled_regular_expressions = {}
 
         #: bind the nodes to the callback functions. There are
         #: some missing! A few are specified in the `unhandled`
@@ -175,75 +172,62 @@ class PythonTranslator(Translator):
         #: will not appear in the jinja parser output because
         #: they are filtered out.
         self.handlers = {
-            # jinja nodes
-            nodes.Template:         self.handle_template,
-            nodes.Text:             self.handle_template_text,
-            nodes.DynamicText:      self.handle_dynamic_text,
-            nodes.NodeList:         self.handle_node_list,
-            nodes.ForLoop:          self.handle_for_loop,
-            nodes.IfCondition:      self.handle_if_condition,
-            nodes.Cycle:            self.handle_cycle,
-            nodes.Print:            self.handle_print,
-            nodes.Macro:            self.handle_macro,
-            nodes.Call:             self.handle_call,
-            nodes.Set:              self.handle_set,
-            nodes.Filter:           self.handle_filter,
-            nodes.Block:            self.handle_block,
-            nodes.Include:          self.handle_include,
-            nodes.Trans:            self.handle_trans,
-            # python nodes
-            ast.Name:               self.handle_name,
-            ast.AssName:            self.handle_assign_name,
-            ast.Compare:            self.handle_compare,
-            ast.Const:              self.handle_const,
-            ast.Subscript:          self.handle_subscript,
-            ast.Getattr:            self.handle_getattr,
-            ast.AssTuple:           self.handle_ass_tuple,
-            ast.Bitor:              self.handle_bitor,
-            ast.CallFunc:           self.handle_call_func,
-            ast.Add:                self.handle_add,
-            ast.Sub:                self.handle_sub,
-            ast.Div:                self.handle_div,
-            ast.FloorDiv:           self.handle_floor_div,
-            ast.Mul:                self.handle_mul,
-            ast.Mod:                self.handle_mod,
-            ast.UnaryAdd:           self.handle_unary_add,
-            ast.UnarySub:           self.handle_unary_sub,
-            ast.Power:              self.handle_power,
-            ast.Dict:               self.handle_dict,
-            ast.List:               self.handle_list,
-            ast.Tuple:              self.handle_list,
-            ast.And:                self.handle_and,
-            ast.Or:                 self.handle_or,
-            ast.Not:                self.handle_not,
-            ast.Slice:              self.handle_slice,
-            ast.Sliceobj:           self.handle_sliceobj
+            # block nodes
+            nodes.Template:                 self.handle_template,
+            nodes.Text:                     self.handle_template_text,
+            nodes.NodeList:                 self.handle_node_list,
+            nodes.ForLoop:                  self.handle_for_loop,
+            nodes.IfCondition:              self.handle_if_condition,
+            nodes.Cycle:                    self.handle_cycle,
+            nodes.Print:                    self.handle_print,
+            nodes.Macro:                    self.handle_macro,
+            nodes.Call:                     self.handle_call,
+            nodes.Set:                      self.handle_set,
+            nodes.Filter:                   self.handle_filter,
+            nodes.Block:                    self.handle_block,
+            nodes.Include:                  self.handle_include,
+            nodes.Trans:                    self.handle_trans,
+
+            # expression nodes
+            nodes.NameExpression:           self.handle_name,
+            nodes.CompareExpression:        self.handle_compare,
+            nodes.TestExpression:           self.handle_test,
+            nodes.ConstantExpression:       self.handle_const,
+            nodes.RegexExpression:          self.handle_regex,
+            nodes.SubscriptExpression:      self.handle_subscript,
+            nodes.FilterExpression:         self.handle_filter_expr,
+            nodes.CallExpression:           self.handle_call_expr,
+            nodes.AddExpression:            self.handle_add,
+            nodes.SubExpression:            self.handle_sub,
+            nodes.ConcatExpression:         self.handle_concat,
+            nodes.DivExpression:            self.handle_div,
+            nodes.FloorDivExpression:       self.handle_floor_div,
+            nodes.MulExpression:            self.handle_mul,
+            nodes.ModExpression:            self.handle_mod,
+            nodes.PosExpression:            self.handle_pos,
+            nodes.NegExpression:            self.handle_neg,
+            nodes.PowExpression:            self.handle_pow,
+            nodes.DictExpression:           self.handle_dict,
+            nodes.SetExpression:            self.handle_set_expr,
+            nodes.ListExpression:           self.handle_list,
+            nodes.TupleExpression:          self.handle_tuple,
+            nodes.UndefinedExpression:      self.handle_undefined,
+            nodes.AndExpression:            self.handle_and,
+            nodes.OrExpression:             self.handle_or,
+            nodes.NotExpression:            self.handle_not,
+            nodes.SliceExpression:          self.handle_slice,
+            nodes.ConditionalExpression:    self.handle_conditional_expr
         }
 
-        #: mapping of unsupported syntax elements.
-        #: the value represents the feature name that appears
-        #: in the exception.
-        self.unsupported = {ast.ListComp: 'list comprehension'}
-
-        #: because of python2.3 compatibility add generator
-        #: expressions only to the list of unused features
-        #: if it exists.
-        if hasattr(ast, 'GenExpr'):
-            self.unsupported[ast.GenExpr] = 'generator expression'
-
-        #: if expressions are unsupported too (so far)
-        if hasattr(ast, 'IfExp'):
-            self.unsupported[ast.IfExp] = 'inline if expression'
-
     # -- public methods
 
-    def process(environment, node):
+    def process(environment, node, source=None):
         """
         The only public method. Creates a translator instance,
         translates the code and returns it in form of an
         `Template` instance.
         """
-        translator = PythonTranslator(environment, node)
+        translator = PythonTranslator(environment, node, source)
         filename = node.filename or '<template>'
         source = translator.translate()
         return Template(environment, compile(source, filename, 'exec'))
@@ -277,87 +261,34 @@ class PythonTranslator(Translator):
             node.lineno
         )
 
-    def filter(self, s, filter_nodes):
-        """
-        Apply a filter on an object that already is a python expression.
-        Used to avoid redundant code in bitor and the filter directive.
-        """
-        filters = []
-        for n in filter_nodes:
-            if n.__class__ is ast.CallFunc:
-                if n.node.__class__ is not ast.Name:
-                    raise TemplateSyntaxError('invalid filter. filter must '
-                                              'be a hardcoded function name '
-                                              'from the filter namespace',
-                                              n.lineno,
-                                              n.filename)
-                args = []
-                for arg in n.args:
-                    if arg.__class__ is ast.Keyword:
-                        raise TemplateSyntaxError('keyword arguments for '
-                                                  'filters are not supported.',
-                                                  n.lineno,
-                                                  n.filename)
-                    args.append(self.handle_node(arg))
-                if n.star_args is not None or n.dstar_args is not None:
-                    raise TemplateSyntaxError('*args / **kwargs is not supported '
-                                              'for filters', n.lineno,
-                                              n.filename)
-                filters.append('(%r, %s)' % (
-                    n.node.name,
-                    self.to_tuple(args)
-                ))
-            elif n.__class__ is ast.Name:
-                filters.append('(%r, ())' % n.name)
-            else:
-                raise TemplateSyntaxError('invalid filter. filter must be a '
-                                          'hardcoded function name from the '
-                                          'filter namespace',
-                                          n.lineno, n.filename)
-        self.used_shortcuts.add('apply_filters')
-        return 'apply_filters(%s, context, %s)' % (s, self.to_tuple(filters))
-
     def handle_node(self, node):
         """
         Handle one node. Resolves the correct callback functions defined
-        in the callback mapping. This also raises the `TemplateSyntaxError`
-        for unsupported syntax elements such as list comprehensions.
+        in the callback mapping.
         """
+        if self.closed:
+            raise RuntimeError('translator is closed')
         if node.__class__ in self.handlers:
             out = self.handlers[node.__class__](node)
-        elif node.__class__ in self.unsupported:
-            raise TemplateSyntaxError('unsupported syntax element %r found.'
-                                      % self.unsupported[node.__class__],
-                                      node.lineno, node.filename)
         else:
             raise AssertionError('unhandled node %r' % node.__class__)
         return out
 
-    def reset(self):
+    def close(self):
         """
-        Reset translation variables such as indention or cycle id
+        Clean up stuff.
         """
-        #: current level of indention
-        self.indention = 0
-        #: each {% cycle %} tag has a unique ID which increments
-        #: automatically for each tag.
-        self.last_cycle_id = 0
-        #: set of used shortcuts jinja has to make local automatically
-        self.used_shortcuts = set(['undefined_singleton'])
-        #: set of used datastructures jinja has to import
-        self.used_data_structures = set()
-        #: set of used utils jinja has to import
-        self.used_utils = set()
-        #: flags for runtime error
-        self.require_runtime_error = False
+        self.closed = True
+        self.handlers = self.node = self.environment = None
 
     def translate(self):
         """
-        Translate the node defined in the constructor after
-        resetting the translator.
+        Translate the node defined in the constructor.
         """
-        self.reset()
-        return self.handle_node(self.node)
+        try:
+            return self.handle_node(self.node)
+        finally:
+            self.close()
 
     # -- jinja nodes
 
@@ -391,12 +322,12 @@ class PythonTranslator(Translator):
             # the master layout template is loaded. This can be used
             # for further processing. The output of those nodes does
             # not appear in the final template.
-            requirements += [child for child in node.getChildNodes()
+            requirements += [child for child in node.get_child_nodes()
                              if child.__class__ not in (nodes.Text,
-                             nodes.Block, nodes.Extends)]
+                             nodes.Block)]
 
             # load the template we inherit from and add not known blocks
-            parent = self.environment.loader.parse(node.extends.template,
+            parent = self.environment.loader.parse(node.extends,
                                                    node.filename)
             # look up all block nodes in the current template and
             # add them to the override dict.
@@ -407,7 +338,7 @@ class PythonTranslator(Translator):
                 # an overwritten block for the parent template. handle that
                 # override in the template and register it in the deferred
                 # block dict.
-                if n.name in overwrites and not n in already_registered_block:
+                if n.name in overwrites and n not in already_registered_block:
                     blocks.setdefault(n.name, []).append(n.clone())
                     n.replace(overwrites[n.name])
                     already_registered_block.add(n)
@@ -422,7 +353,7 @@ class PythonTranslator(Translator):
             requirement_lines.append('    if 0: yield None\n')
 
         # handle body in order to get the used shortcuts
-        body_lines = [self.handle_node(n) for n in node]
+        body_code = self.handle_node(node.body)
 
         # same for blocks in callables
         block_lines = []
@@ -465,18 +396,28 @@ class PythonTranslator(Translator):
         if self.used_data_structures:
             lines.append('from jinja.datastructure import %s' % ', '.
                          join(self.used_data_structures))
+        if self.need_set_import:
+            lines.append('from jinja.utils import set')
+
+        # compile regular expressions
+        if self.compiled_regular_expressions:
+            lines.append('import re')
+            lines.append('\n# Compile used regular expressions')
+            for regex, name in self.compiled_regular_expressions.iteritems():
+                lines.append('%s = re.compile(%r)' % (name, regex))
+
         lines.append(
             '\n# Aliases for some speedup\n'
             '%s\n\n'
+            '# Marker for Jinja templates\n'
+            '__jinja_template__ = True\n\n'
             '# Name for disabled debugging\n'
             '__name__ = %r\n\n'
             'def generate(context):\n'
             '    assert environment is context.environment' % (
                 '\n'.join([
                     '%s = environment.%s' % (item, item) for item in
-                    ['get_attribute', 'perform_test', 'apply_filters',
-                     'call_function', 'call_function_simple', 'finish_var',
-                     'undefined_singleton'] if item in self.used_shortcuts
+                    self.used_shortcuts
                 ]),
                 outer_filename
             )
@@ -485,7 +426,7 @@ class PythonTranslator(Translator):
         # the template body
         if requirements:
             lines.append('    for item in bootstrap(context): pass')
-        lines.extend(body_lines)
+        lines.append(body_code)
         lines.append('    if 0: yield None\n')
 
         # now write the bootstrapping (requirements) core if there is one
@@ -536,36 +477,49 @@ class PythonTranslator(Translator):
                 result.append(line)
 
         # now print file mapping and debug info
+        # the debug info:
+        #   debug_info          binds template line numbers to generated
+        #                       source lines. this information is always
+        #                       present and part of the bytecode.
+        #   template_source     only available if loaded from string to
+        #                       get debug source code. Because this is
+        #                       dumped too it's a bad idea to dump templates
+        #                       loaded from a string.
         result.append('\n# Debug Information')
         file_mapping = file_mapping.items()
         file_mapping.sort(lambda a, b: cmp(a[1], b[1]))
         for filename, file_id in file_mapping:
             result.append('%s = %r' % (file_id, filename))
         result.append('debug_info = %s' % self.to_tuple(debug_mapping))
+        result.append('template_source = %r' % self.source)
         return '\n'.join(result)
 
     def handle_template_text(self, node):
         """
         Handle data around nodes.
         """
-        return self.indent(self.nodeinfo(node)) + '\n' +\
-               self.indent('yield %r' % node.text)
+        # special case: no variables
+        if not node.variables:
+            return self.indent(self.nodeinfo(node)) + '\n' + \
+                   self.indent('yield %r' % node.text.replace('%%', '%'))
 
-    def handle_dynamic_text(self, node):
-        """
-        Like `handle_template_text` but for nodes of the type
-        `DynamicText`.
-        """
+        # special case: one variable, no text
+        self.used_shortcuts.add('finish_var')
+        if len(node.variables) == 1 and node.text == '%s':
+            return self.indent(self.nodeinfo(node)) + '\n' + \
+                   self.indent('yield finish_var(%s, context)' %
+                               self.handle_node(node.variables[0]))
+
+        # all other cases
         buf = []
         write = lambda x: buf.append(self.indent(x))
-        self.used_shortcuts.add('finish_var')
 
         write(self.nodeinfo(node))
-        write('yield %r %% (' % node.format_string)
+        write('yield %r %% (' % node.text)
         self.indention += 1
         for var in node.variables:
             write(self.nodeinfo(var))
-            write('finish_var(%s, context)' % self.handle_node(var.variable) + ',')
+            write('finish_var(%s, context)' % self.handle_node(var) + ',')
         self.indention -= 1
         write(')')
 
@@ -680,10 +634,11 @@ class PythonTranslator(Translator):
         write('if %r not in context.current:' % name)
         self.indention += 1
         write(self.nodeinfo(node))
-        if node.seq.__class__ in (ast.Tuple, ast.List):
+        if node.seq.__class__ in (nodes.TupleExpression,
+                                  nodes.ListExpression):
             write('context.current[%r] = CycleContext(%s)' % (
                 name,
-                self.to_tuple([self.handle_node(n) for n in node.seq.nodes])
+                self.to_tuple([self.handle_node(n) for n in node.seq.items])
             ))
             hardcoded = True
         else:
@@ -711,22 +666,12 @@ class PythonTranslator(Translator):
         self.used_shortcuts.add('finish_var')
         return self.indent(self.nodeinfo(node)) + '\n' +\
                self.indent('yield finish_var(%s, context)' %
-                           self.handle_node(node.variable))
+                           self.handle_node(node.expr))
 
     def handle_macro(self, node):
         """
         Handle macro declarations.
         """
-        # sanity check for name
-        if node.name == '_' or \
-           node.name in self.constants:
-            raise TemplateSyntaxError('cannot override %r' % node.name,
-                                      node.lineno, node.filename)
-        if node.name in self.constants:
-            raise TemplateSyntaxError('you cannot name a macro %r',
-                                      node.lineno,
-                                      node.filename)
-
         buf = []
         write = lambda x: buf.append(self.indent(x))
 
@@ -815,7 +760,7 @@ class PythonTranslator(Translator):
         write('context.pop()')
         write('if 0: yield None')
         self.indention -= 1
-        write('yield ' + self.handle_call_func(node.expr,
+        write('yield ' + self.handle_call_expr(node.expr,
               {'caller': 'buffereater(call)'}))
         self.used_utils.add('buffereater')
 
@@ -825,11 +770,12 @@ class PythonTranslator(Translator):
         """
         Handle variable assignments.
         """
-        if node.name == '_' or node.name in self.constants:
-            raise TemplateSyntaxError('cannot override %r' % node.name,
-                                      node.lineno, node.filename)
+        if node.scope_local:
+            tmpl = 'context[%r] = %s'
+        else:
+            tmpl = 'context.set_nonlocal(%r, %s)'
         return self.indent(self.nodeinfo(node)) + '\n' + \
-               self.indent('context[%r] = %s' % (
+               self.indent(tmpl % (
             node.name,
             self.handle_node(node.expr)
         ))
@@ -850,8 +796,13 @@ class PythonTranslator(Translator):
         write('context.pop()')
         write('if 0: yield None')
         self.indention -= 1
-        write('yield %s' % self.filter('buffereater(filtered)()',
-                                       node.filters))
+        self.used_shortcuts.add('apply_filters')
+        write('yield apply_filters(buffereater(filtered)(), context, %s)' %
+            self.to_tuple(['(%r, %s)' % (
+                name,
+                self.to_tuple(map(self.handle_node, args))
+            ) for name, args in node.filters])
+        )
         self.used_utils.add('buffereater')
         return '\n'.join(buf)
 
@@ -885,7 +836,7 @@ class PythonTranslator(Translator):
         """
         tmpl = self.environment.loader.parse(node.template,
                                              node.filename)
-        return self.handle_node_list(tmpl)
+        return self.handle_node(tmpl.body)
 
     def handle_trans(self, node):
         """
@@ -911,23 +862,11 @@ class PythonTranslator(Translator):
 
     # -- python nodes
 
-    def handle_assign_name(self, node):
-        """
-        Handle name assignments.
-        """
-        if node.name == '_' or \
-           node.name in self.constants:
-            raise TemplateSyntaxError('cannot override %r' % node.name,
-                                      node.lineno, node.filename)
-        return 'context[%r]' % node.name
-
     def handle_name(self, node):
         """
         Handle name assignments and name retreivement.
         """
-        if node.name in self.constants:
-            return self.constants[node.name]
-        elif node.name == '_':
+        if node.name == '_':
             return 'context.translate_func'
         return 'context[%r]' % node.name
 
@@ -935,128 +874,103 @@ class PythonTranslator(Translator):
         """
         Any sort of comparison
         """
-        # the semantic for the is operator is different.
-        # for jinja the is operator performs tests and must
-        # be the only operator
-        if node.ops[0][0] in ('is', 'is not'):
-            if len(node.ops) > 1:
-                raise TemplateSyntaxError('is operator must not be chained',
-                                          node.lineno,
-                                          node.filename)
-            elif node.ops[0][1].__class__ is ast.Name:
-                args = []
-                name = node.ops[0][1].name
-            elif node.ops[0][1].__class__ is ast.CallFunc:
-                n = node.ops[0][1]
-                if n.node.__class__ is not ast.Name:
-                    raise TemplateSyntaxError('invalid test. test must '
-                                              'be a hardcoded function name '
-                                              'from the test namespace',
-                                              n.lineno,
-                                              n.filename)
-                name = n.node.name
-                args = []
-                for arg in n.args:
-                    if arg.__class__ is ast.Keyword:
-                        raise TemplateSyntaxError('keyword arguments for '
-                                                  'tests are not supported.',
-                                                  n.lineno,
-                                                  n.filename)
-                    args.append(self.handle_node(arg))
-                if n.star_args is not None or n.dstar_args is not None:
-                    raise TemplateSyntaxError('*args / **kwargs is not supported '
-                                              'for tests', n.lineno,
-                                              n.filename)
-            else:
-                raise TemplateSyntaxError('is operator requires a test name'
-                                          ' as operand', node.lineno,
-                                          node.filename)
-            self.used_shortcuts.add('perform_test')
-            return 'perform_test(context, %r, %s, %s, %s)' % (
-                    name,
-                    self.to_tuple(args),
-                    self.handle_node(node.expr),
-                    node.ops[0][0] == 'is not'
-                )
-
-        # normal operators
+        ops = {
+            'eq':       '==',
+            'ne':       '!=',
+            'lt':       '<',
+            'lteq':     '<=',
+            'gt':       '>',
+            'gteq':     '>=',
+            'in':       'in',
+            'not in':   'not in'
+        }
         buf = []
         buf.append(self.handle_node(node.expr))
         for op, n in node.ops:
-            if op in ('is', 'is not'):
-                raise TemplateSyntaxError('is operator must not be chained',
-                                          node.lineno, node.filename)
-            buf.append(op)
+            buf.append(ops[op])
             buf.append(self.handle_node(n))
         return ' '.join(buf)
 
+    def handle_test(self, node):
+        """
+        Handle test calls.
+        """
+        self.used_shortcuts.add('perform_test')
+        return 'perform_test(context, %r, %s, %s)' % (
+            node.name,
+            self.to_tuple([self.handle_node(n) for n in node.args]),
+            self.handle_node(node.node)
+        )
+
     def handle_const(self, node):
         """
         Constant values in expressions.
         """
         return repr(node.value)
 
-    def handle_subscript(self, node):
+    def handle_regex(self, node):
         """
-        Handle variable based attribute access foo['bar'].
+        Regular expression literals.
         """
-        if len(node.subs) != 1:
-            raise TemplateSyntaxError('attribute access requires one '
-                                      'argument', node.lineno,
-                                      node.filename)
-        assert node.flags != 'OP_DELETE', 'wtf? do we support that?'
-        if node.subs[0].__class__ is ast.Sliceobj:
-            return '%s%s' % (
-                self.handle_node(node.expr),
-                self.handle_node(node.subs[0])
-            )
-        self.used_shortcuts.add('get_attribute')
-        return 'get_attribute(%s, %s)' % (
-            self.handle_node(node.expr),
-            self.handle_node(node.subs[0])
-        )
+        if self.environment.disable_regexps:
+            raise TemplateSyntaxError('regular expressions disabled.')
+        if node.value in self.compiled_regular_expressions:
+            return self.compiled_regular_expressions[node.value]
+        name = 'regex_%d' % len(self.compiled_regular_expressions)
+        self.compiled_regular_expressions[node.value] = name
+        return name
 
-    def handle_getattr(self, node):
+    def handle_subscript(self, node):
         """
-        Handle hardcoded attribute access.
+        Handle variable based attribute access foo['bar'].
         """
         self.used_shortcuts.add('get_attribute')
-        return 'get_attribute(%s, %r)' % (
-            self.handle_node(node.expr),
-            node.attrname
+        if node.arg.__class__ is nodes.SliceExpression:
+            rv = self.handle_slice(node.arg, getslice_test=True)
+            if rv is not None:
+                return self.handle_node(node.node) + rv
+        return 'get_attribute(%s, %s)' % (
+            self.handle_node(node.node),
+            self.handle_node(node.arg)
         )
 
-    def handle_ass_tuple(self, node):
+    def handle_tuple(self, node):
         """
         Tuple unpacking loops.
         """
-        return self.to_tuple([self.handle_node(n) for n in node.nodes])
+        return self.to_tuple([self.handle_node(n) for n in node.items])
 
-    def handle_bitor(self, node):
+    def handle_filter_expr(self, node):
         """
         We use the pipe operator for filtering.
         """
-        return self.filter(self.handle_node(node.nodes[0]), node.nodes[1:])
+        self.used_shortcuts.add('apply_filters')
+        return 'apply_filters(%s, context, %s)' % (
+            self.handle_node(node.node),
+            self.to_tuple(['(%r, %s)' % (
+                name,
+                self.to_tuple(map(self.handle_node, args))
+            ) for name, args in node.filters])
+        )
 
-    def handle_call_func(self, node, extra_kwargs=None):
+    def handle_call_expr(self, node, extra_kwargs=None):
         """
         Handle function calls.
         """
         args = []
         kwargs = {}
-        star_args = dstar_args = None
-        if node.star_args is not None:
-            star_args = self.handle_node(node.star_args)
-        if node.dstar_args is not None:
-            dstar_args = self.handle_node(node.dstar_args)
+        dyn_args = dyn_kwargs = None
+        if node.dyn_args is not None:
+            dyn_args = self.handle_node(node.dyn_args)
+        if node.dyn_kwargs is not None:
+            dyn_kwargs = self.handle_node(node.dyn_kwargs)
         for arg in node.args:
-            if arg.__class__ is ast.Keyword:
-                kwargs[arg.name] = self.handle_node(arg.expr)
-            else:
-                args.append(self.handle_node(arg))
+            args.append(self.handle_node(arg))
+        for name, arg in node.kwargs:
+            kwargs[name] = self.handle_node(arg)
         if extra_kwargs:
             kwargs.update(extra_kwargs)
-        if not (args or kwargs or star_args or dstar_args or extra_kwargs):
+        if not (args or kwargs or dyn_args or dyn_kwargs):
             self.used_shortcuts.add('call_function_simple')
             return 'call_function_simple(%s, context)' % \
                    self.handle_node(node.node)
@@ -1065,8 +979,8 @@ class PythonTranslator(Translator):
             self.handle_node(node.node),
             self.to_tuple(args),
             ', '.join(['%r: %s' % i for i in kwargs.iteritems()]),
-            star_args,
-            dstar_args
+            dyn_args,
+            dyn_kwargs
         )
 
     def handle_add(self, node):
@@ -1087,6 +1001,16 @@ class PythonTranslator(Translator):
             self.handle_node(node.right)
         )
 
+    def handle_concat(self, node):
+        """
+        Convert some objects to unicode and concatenate them.
+        """
+        self.used_shortcuts.add('to_unicode')
+        return "u''.join(%s)" % self.to_tuple([
+            'to_unicode(%s)' % self.handle_node(arg)
+            for arg in node.args
+        ])
+
     def handle_div(self, node):
         """
         Divide two items.
@@ -1123,19 +1047,19 @@ class PythonTranslator(Translator):
             self.handle_node(node.right)
         )
 
-    def handle_unary_add(self, node):
+    def handle_pos(self, node):
         """
         One of the more or less unused nodes.
         """
-        return '(+%s)' % self.handle_node(node.expr)
+        return '(+%s)' % self.handle_node(node.node)
 
-    def handle_unary_sub(self, node):
+    def handle_neg(self, node):
         """
         Make a number negative.
         """
-        return '(-%s)' % self.handle_node(node.expr)
+        return '(-%s)' % self.handle_node(node.node)
 
-    def handle_power(self, node):
+    def handle_pow(self, node):
         """
         handle foo**bar
         """
@@ -1155,57 +1079,89 @@ class PythonTranslator(Translator):
             ) for key, value in node.items
         ])
 
+    def handle_set_expr(self, node):
+        """
+        Set constructor syntax.
+        """
+        self.need_set_import = True
+        return 'set([%s])' % ', '.join([self.handle_node(n)
+                                        for n in node.items])
+
     def handle_list(self, node):
         """
         We don't know tuples, tuples are lists for jinja.
         """
         return '[%s]' % ', '.join([
-            self.handle_node(n) for n in node.nodes
+            self.handle_node(n) for n in node.items
         ])
 
+    def handle_undefined(self, node):
+        """
+        Return the current undefined literal.
+        """
+        return 'undefined_singleton'
+
     def handle_and(self, node):
         """
         Handle foo and bar.
         """
-        return ' and '.join([
-            self.handle_node(n) for n in node.nodes
-        ])
+        return '(%s and %s)' % (
+            self.handle_node(node.left),
+            self.handle_node(node.right)
+        )
 
     def handle_or(self, node):
         """
         handle foo or bar.
         """
-        return ' or '.join([
-            self.handle_node(n) for n in node.nodes
-        ])
+        return '(%s or %s)' % (
+            self.handle_node(node.left),
+            self.handle_node(node.right)
+        )
 
     def handle_not(self, node):
         """
         handle not operator.
         """
-        return 'not %s' % self.handle_node(node.expr)
+        return '(not %s)' % self.handle_node(node.node)
 
-    def handle_slice(self, node):
+    def handle_slice(self, node, getslice_test=False):
         """
-        Slice access.
+        Slice access. Because of backwards compatibilty to python's
+        `__getslice__` this function takes a second parameter that lets this
+        method return a regular slice bracket call. If a regular slice bracket
+        call that is compatible to __getslice__ is not possible the return
+        value will be `None` so that a regular `get_attribute` wrapping can
+        happen.
         """
-        if node.lower is None:
-            lower = ''
+        if node.start is None:
+            start = not getslice_test and 'None' or ''
         else:
-            lower = self.handle_node(node.lower)
-        if node.upper is None:
-            upper = ''
+            start = self.handle_node(node.start)
+        if node.stop is None:
+            stop = not getslice_test and 'None' or ''
         else:
-            upper = self.handle_node(node.upper)
-        assert node.flags != 'OP_DELETE', 'wtf? shouldn\'t happen'
-        return '%s[%s:%s]' % (
-            self.handle_node(node.expr),
-            lower,
-            upper
-        )
+            stop = self.handle_node(node.stop)
+        if node.step is None:
+            step = 'None'
+        else:
+            if getslice_test:
+                return
+            step = self.handle_node(node.step)
+        if getslice_test:
+            return '[%s:%s]' % (start, stop)
+        return 'slice(%s, %s, %s)' % (start, stop, step)
 
-    def handle_sliceobj(self, node):
+    def handle_conditional_expr(self, node):
         """
-        Extended Slice access.
+        Handle conditional expressions.
         """
-        return '[%s]' % ':'.join([self.handle_node(n) for n in node.nodes])
+        if have_conditional_expr:
+            tmpl = '%(expr1)s if %(test)s else %(expr2)s'
+        else:
+            tmpl = '(%(test)s and (%(expr1)s,) or (%(expr2)s,))[0]'
+        return tmpl % {
+            'test':     self.handle_node(node.test),
+            'expr1':    self.handle_node(node.expr1),
+            'expr2':    self.handle_node(node.expr2)
+        }
diff --git a/jinja/utils.py b/jinja/utils.py
index a461bd7..bdea9e6 100644
--- a/jinja/utils.py
+++ b/jinja/utils.py
@@ -15,8 +15,7 @@ import re
 import sys
 import string
 from types import MethodType, FunctionType
-from compiler.ast import CallFunc, Name, Const
-from jinja.nodes import Trans
+from jinja import nodes
 from jinja.exceptions import SecurityException, TemplateNotFound
 
 # the python2.4 version of deque is missing the remove method
@@ -71,13 +70,6 @@ except NameError:
             rv.reverse()
         return rv
 
-# if we have extended debugger support we should really use it
-try:
-    from jinja._tbtools import *
-    has_extended_debugger = True
-except ImportError:
-    has_extended_debugger = False
-
 # group by support
 try:
     from itertools import groupby
@@ -204,13 +196,19 @@ def get_attribute(obj, name):
         raise AttributeError(name)
     if name[:2] == name[-2:] == '__':
         raise SecurityException('not allowed to access internal attributes')
-    if obj.__class__ in callable_types and name.startswith('func_') or \
-       name.startswith('im_'):
+    if getattr(obj, '__class__', None) in callable_types and \
+       name.startswith('func_') or name.startswith('im_'):
         raise SecurityException('not allowed to access function attributes')
     r = _getattr(obj, 'jinja_allowed_attributes', None)
     if r is not None and name not in r:
         raise SecurityException('disallowed attribute accessed')
-    return _getattr(obj, name)
+
+    # attribute lookups convert unicode strings to ascii bytestrings.
+    # this process could raise an UnicodeEncodeError.
+    try:
+        return _getattr(obj, name)
+    except UnicodeError:
+        raise AttributeError(name)
 
 
 def safe_range(start, stop=None, step=None):
@@ -352,7 +350,10 @@ def buffereater(f):
     Used by the python translator to capture output of substreams.
     (macros, filter sections etc)
     """
-    return lambda *a, **kw: capture_generator(f(*a, **kw))
+    def wrapped(*a, **kw):
+        __traceback_hide__ = True
+        return capture_generator(f(*a, **kw))
+    return wrapped
 
 
 def empty_block(context):
@@ -374,14 +375,16 @@ def collect_translations(ast):
     result = []
     while todo:
         node = todo.pop()
-        if node.__class__ is Trans:
+        if node.__class__ is nodes.Trans:
             result.append((node.lineno, node.singular, node.plural))
-        elif node.__class__ is CallFunc and \
-             node.node.__class__ is Name and \
+        elif node.__class__ is nodes.CallExpression and \
+             node.node.__class__ is nodes.NameExpression and \
              node.node.name == '_':
-            if len(node.args) == 1 and node.args[0].__class__ is Const:
+            if len(node.args) == 1 and not node.kwargs and not node.dyn_args \
+               and not node.dyn_kwargs and \
+               node.args[0].__class__ is nodes.ConstantExpression:
                 result.append((node.lineno, node.args[0].value, None))
-        todo.extend(node.getChildNodes())
+        todo.extend(node.get_child_nodes())
     result.sort(lambda a, b: cmp(a[0], b[0]))
     return result
 
@@ -609,3 +612,12 @@ class CacheDict(object):
         rv._mapping = deepcopy(self._mapping)
         rv._queue = deepcopy(self._queue)
         return rv
+
+
+NAMESPACE = {
+    'range':            safe_range,
+    'debug':            debug_helper,
+    'lipsum':           generate_lorem_ipsum,
+    'watchchanges':     watch_changes,
+    'rendertemplate':   render_included
+}
diff --git a/tests/runtime/bigtable.py b/tests/runtime/bigtable.py
index b1576df..ce853d9 100644
--- a/tests/runtime/bigtable.py
+++ b/tests/runtime/bigtable.py
@@ -7,12 +7,16 @@
 # Author: Jonas Borgström <jonas@edgewall.com>
 # Author: Armin Ronacher <armin.ronacher@active-4.com>
 
-import cgi
+import os
 import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+import cgi
 import timeit
 import jdebug
 from StringIO import StringIO
 
+
 try:
     from genshi.builder import tag
     from genshi.template import MarkupTemplate
diff --git a/tests/runtime/exception.py b/tests/runtime/exception.py
index 0af7955..9dffaeb 100644
--- a/tests/runtime/exception.py
+++ b/tests/runtime/exception.py
@@ -1,5 +1,8 @@
-import jdebug
+import os
 import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+
+import jdebug
 from jinja import Environment, DictLoader
 from jinja.exceptions import TemplateNotFound
 from wsgiref.simple_server import make_server
@@ -35,6 +38,7 @@ e = Environment(loader=DictLoader({
       <li><a href="code_runtime_error">runtime error in code</a></li>
       <li><a href="syntax_from_string">a syntax error from string</a></li>
       <li><a href="runtime_from_string">runtime error from a string</a></li>
+      <li><a href="multiple_templates">multiple templates</a></li>
     </ul>
   </body>
 </html>
@@ -53,17 +57,31 @@ e = Environment(loader=DictLoader({
     '/nested_syntax_error': u'''
 {% include 'syntax_broken' %}
     ''',
-
     '/code_runtime_error': u'''We have a runtime error here:
     {{ broken() }}''',
+    '/multiple_templates': '''\
+{{ fire_multiple_broken() }}
+''',
 
     'runtime_broken': '''\
 This is an included template
 {% set a = 1 / 0 %}''',
     'syntax_broken': '''\
 This is an included template
-{% raw %}just some foo'''
+{% raw %}just some foo''',
+    'multiple_broken': '''\
+Just some context:
+{% include 'macro_broken' %}
+{{ broken() }}
+''',
+    'macro_broken': '''\
+{% macro broken %}
+    {{ 1 / 0 }}
+{% endmacro %}
+'''
 }))
+e.globals['fire_multiple_broken'] = lambda: \
+    e.get_template('multiple_broken').render()
 
 FAILING_STRING_TEMPLATE = '{{ 1 / 0 }}'
 BROKEN_STRING_TEMPLATE = '{% if foo %}...{% endfor %}'
@@ -91,5 +109,5 @@ def test(environ, start_response):
 
 if __name__ == '__main__':
     from werkzeug.debug import DebuggedApplication
-    app = DebuggedApplication(test)
+    app = DebuggedApplication(test, True)
     make_server("localhost", 7000, app).serve_forever()
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
index 6ac49d4..cc85b19 100644
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -14,6 +14,7 @@ COMMENTS = '''\
   <li>{item}</li>
 <!--- endfor -->
 </ul>'''
+BYTEFALLBACK = u'''{{ 'foo'|pprint }}|{{ 'bär'|pprint }}'''
 
 
 def test_balancing():
@@ -29,3 +30,25 @@ def test_comments():
     tmpl = env.from_string(COMMENTS)
     assert tmpl.render(seq=range(3)) == ("<ul>\n  <li>0</li>\n  "
                                          "<li>1</li>\n  <li>2</li>\n</ul>")
+
+
+def test_string_escapes(env):
+    for char in u'\0', u'\u2668', u'\xe4', u'\t', u'\r', u'\n':
+        tmpl = env.from_string('{{ %s }}' % repr(char)[1:])
+        assert tmpl.render() == char
+    assert env.from_string('{{ "\N{HOT SPRINGS}" }}').render() == u'\u2668'
+
+
+def test_bytefallback(env):
+    tmpl = env.from_string(BYTEFALLBACK)
+    assert tmpl.render() == u"'foo'|u'b\\xe4r'"
+
+
+def test_operators(env):
+    from jinja.lexer import operators
+    for test, expect in operators.iteritems():
+        if test in '([{}])':
+            continue
+        stream = env.lexer.tokenize('{{ %s }}' % test)
+        stream.next()
+        assert stream.current.type == expect
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index 2335b9c..a1d21ad 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -46,14 +46,19 @@ def test_dict_loader():
 
 def test_package_loader():
     env = Environment(loader=package_loader)
-    tmpl = env.get_template('test.html')
-    assert tmpl.render().strip() == 'BAR'
-    try:
-        env.get_template('missing.html')
-    except TemplateNotFound:
-        pass
-    else:
-        raise AssertionError('expected template exception')
+    for x in xrange(2):
+        tmpl = env.get_template('test.html')
+        assert tmpl.render().strip() == 'BAR'
+        try:
+            env.get_template('missing.html')
+        except TemplateNotFound:
+            pass
+        else:
+            raise AssertionError('expected template exception')
+
+        # second run in native mode (no pkg_resources)
+        package_loader.force_native = True
+        del package_loader._load_func
 
 
 def test_filesystem_loader():
diff --git a/tests/test_macros.py b/tests/test_macros.py
index cf84f95..7e6d54a 100644
--- a/tests/test_macros.py
+++ b/tests/test_macros.py
@@ -55,6 +55,8 @@ CALLERUNDEFINED = '''\
 {{ test() }}\
 '''
 
+INCLUDETEMPLATE = '''{% macro test(foo) %}[{{ foo }}]{% endmacro %}'''
+
 
 def test_simple(env):
     tmpl = env.from_string(SIMPLE)
@@ -105,3 +107,8 @@ def test_complex_call(env):
 def test_caller_undefined(env):
     tmpl = env.from_string(CALLERUNDEFINED)
     assert tmpl.render() == 'True'
+
+
+def test_include(env):
+    tmpl = env.from_string('{% include "include" %}{{ test("foo") }}')
+    assert tmpl.render() == '[foo]'
diff --git a/tests/test_security.py b/tests/test_security.py
index 95d3371..78a1e0a 100644
--- a/tests/test_security.py
+++ b/tests/test_security.py
@@ -43,9 +43,9 @@ test_restricted = '''
 >>> env.from_string("{% for item.attribute in seq %}...{% endfor %}")
 Traceback (most recent call last):
     ...
-TemplateSyntaxError: can't assign to expression. (line 1)
+TemplateSyntaxError: cannot assign to expression (line 1)
 >>> env.from_string("{% for foo, bar.baz in seq %}...{% endfor %}")
 Traceback (most recent call last):
     ...
-TemplateSyntaxError: can't assign to expression. (line 1)
+TemplateSyntaxError: cannot assign to expression (line 1)
 '''
diff --git a/tests/test_syntax.py b/tests/test_syntax.py
index e6a8714..abcd058 100644
--- a/tests/test_syntax.py
+++ b/tests/test_syntax.py
@@ -6,20 +6,43 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+from jinja import Environment, DictLoader
+from jinja.exceptions import TemplateSyntaxError
+
 
 CALL = '''{{ foo('a', c='d', e='f', *['b'], **{'g': 'h'}) }}'''
 SLICING = '''{{ [1, 2, 3][:] }}|{{ [1, 2, 3][::-1] }}'''
 ATTR = '''{{ foo.bar }}|{{ foo['bar'] }}'''
 SUBSCRIPT = '''{{ foo[0] }}|{{ foo[-1] }}'''
 KEYATTR = '''{{ {'items': 'foo'}.items }}|{{ {}.items() }}'''
-TUPLE = '''{{ () }}'''
+TUPLE = '''{{ () }}|{{ (1,) }}|{{ (1, 2) }}'''
 MATH = '''{{ (1 + 1 * 2) - 3 / 2 }}|{{ 2**3 }}'''
 DIV = '''{{ 3 // 2 }}|{{ 3 / 2 }}|{{ 3 % 2 }}'''
 UNARY = '''{{ +3 }}|{{ -3 }}'''
+CONCAT = '''{{ [1, 2] ~ 'foo' }}'''
 COMPARE = '''{{ 1 > 0 }}|{{ 1 >= 1 }}|{{ 2 < 3 }}|{{ 2 == 2 }}|{{ 1 <= 1 }}'''
-LITERALS = '''{{ [] }}|{{ {} }}|{{ '' }}'''
+INOP = '''{{ 1 in [1, 2, 3] }}|{{ 1 not in [1, 2, 3] }}'''
+LITERALS = '''{{ [] }}|{{ {} }}|{{ () }}|{{ '' }}|{{ @() }}'''
 BOOL = '''{{ true and false }}|{{ false or true }}|{{ not false }}'''
 GROUPING = '''{{ (true and false) or (false and true) and not false }}'''
+CONDEXPR = '''{{ 0 if true else 1 }}'''
+DJANGOATTR = '''{{ [1, 2, 3].0 }}'''
+FILTERPRIORITY = '''{{ "foo"|upper + "bar"|upper }}'''
+REGEX = '''{{ @/\S+/.findall('foo bar baz') }}'''
+TUPLETEMPLATES = [
+    '{{ () }}',
+    '{{ (1, 2) }}',
+    '{{ (1, 2,) }}',
+    '{{ 1, }}',
+    '{{ 1, 2 }}',
+    '{% for foo, bar in seq %}...{% endfor %}',
+    '{% for x in foo, bar %}...{% endfor %}',
+    '{% for x in foo, %}...{% endfor %}',
+    '{% for x in foo, recursive %}...{% endfor %}',
+    '{% for x in foo, bar recursive %}...{% endfor %}',
+    '{% for x, in foo, recursive %}...{% endfor %}'
+]
+TRAILINGCOMMA = '''{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}|{{ @(1, 2,) }}'''
 
 
 def test_call():
@@ -52,7 +75,7 @@ def test_keyattr(env):
 
 def test_tuple(env):
     tmpl = env.from_string(TUPLE)
-    assert tmpl.render() == '[]'
+    assert tmpl.render() == '()|(1,)|(1, 2)'
 
 
 def test_math(env):
@@ -70,14 +93,24 @@ def test_unary(env):
     assert tmpl.render() == '3|-3'
 
 
+def test_concat(env):
+    tmpl = env.from_string(CONCAT)
+    assert tmpl.render() == '[1, 2]foo'
+
+
 def test_compare(env):
     tmpl = env.from_string(COMPARE)
     assert tmpl.render() == 'True|True|True|True|True'
 
 
+def test_inop(env):
+    tmpl = env.from_string(INOP)
+    assert tmpl.render() == 'True|False'
+
+
 def test_literals(env):
     tmpl = env.from_string(LITERALS)
-    assert tmpl.render() == '[]|{}|'
+    assert tmpl.render() == '[]|{}|()||set([])'
 
 
 def test_bool(env):
@@ -88,3 +121,78 @@ def test_bool(env):
 def test_grouping(env):
     tmpl = env.from_string(GROUPING)
     assert tmpl.render() == 'False'
+
+
+def test_django_attr(env):
+    tmpl = env.from_string(DJANGOATTR)
+    assert tmpl.render() == '1'
+
+
+def test_conditional_expression(env):
+    tmpl = env.from_string(CONDEXPR)
+    assert tmpl.render() == '0'
+
+
+def test_filter_priority(env):
+    tmpl = env.from_string(FILTERPRIORITY)
+    assert tmpl.render() == 'FOOBAR'
+
+
+def test_function_calls(env):
+    tests = [
+        (True, '*foo, bar'),
+        (True, '*foo, *bar'),
+        (True, '*foo, bar=42'),
+        (True, '**foo, *bar'),
+        (True, '**foo, bar'),
+        (False, 'foo, bar'),
+        (False, 'foo, bar=42'),
+        (False, 'foo, bar=23, *args'),
+        (False, 'a, b=c, *d, **e'),
+        (False, '*foo, **bar')
+    ]
+    for should_fail, sig in tests:
+        if should_fail:
+            try:
+                print env.from_string('{{ foo(%s) }}' % sig)
+            except TemplateSyntaxError:
+                continue
+            assert False, 'expected syntax error'
+        else:
+            env.from_string('foo(%s)' % sig)
+
+
+def test_regex(env):
+    tmpl = env.from_string(REGEX)
+    assert tmpl.render() == "['foo', 'bar', 'baz']"
+
+
+def test_tuple_expr(env):
+    for tmpl in TUPLETEMPLATES:
+        assert env.from_string(tmpl)
+
+
+def test_trailing_comma(env):
+    tmpl = env.from_string(TRAILINGCOMMA)
+    assert tmpl.render() == '(1, 2)|[1, 2]|{1: 2}|set([1, 2])'
+
+
+def test_extends_position():
+    env = Environment(loader=DictLoader({
+        'empty': '[{% block empty %}{% endblock %}]'
+    }))
+    tests = [
+        ('{% extends "empty" %}', '[!]'),
+        ('  {% extends "empty" %}', '[!]'),
+        ('  !\n', '  !\n!'),
+        ('{# foo #}  {% extends "empty" %}', '[!]'),
+        ('{% set foo = "blub" %}{% extends "empty" %}', None)
+    ]
+
+    for tmpl, expected_output in tests:
+        try:
+            tmpl = env.from_string(tmpl + '{% block empty %}!{% endblock %}')
+        except TemplateSyntaxError:
+            assert expected_output is None, 'got syntax error'
+        else:
+            assert expected_output == tmpl.render()
diff --git a/tests/test_tests.py b/tests/test_tests.py
index c28cd90..9d9a33f 100644
--- a/tests/test_tests.py
+++ b/tests/test_tests.py
@@ -11,7 +11,8 @@ DEFINED = '''{{ missing is defined }}|{{ true is defined }}'''
 EVEN = '''{{ 1 is even }}|{{ 2 is even }}'''
 LOWER = '''{{ "foo" is lower }}|{{ "FOO" is lower }}'''
 MATCHING = '''{{ "42" is matching('^\\d+$') }}|\
-{{ "foo" is matching('^\\d+$') }}'''
+{{ "foo" is matching('^\\d+$') }}|\
+{{ "foo bar" is matching @/^foo\\s+BAR$/i }}'''
 NUMERIC = '''{{ "43" is numeric }}|{{ "foo" is numeric }}|\
 {{ 42 is numeric }}'''
 ODD = '''{{ 1 is odd }}|{{ 2 is odd }}'''
@@ -20,6 +21,7 @@ SEQUENCE = '''{{ [1, 2, 3] is sequence }}|\
 {{ 42 is sequence }}'''
 UPPER = '''{{ "FOO" is upper }}|{{ "foo" is upper }}'''
 SAMEAS = '''{{ foo is sameas(false) }}|{{ 0 is sameas(false) }}'''
+NOPARENFORARG1 = '''{{ foo is sameas none }}'''
 
 
 def test_defined(env):
@@ -39,7 +41,7 @@ def test_lower(env):
 
 def test_matching(env):
     tmpl = env.from_string(MATCHING)
-    assert tmpl.render() == 'True|False'
+    assert tmpl.render() == 'True|False|True'
 
 
 def test_numeric(env):
@@ -65,3 +67,8 @@ def test_upper(env):
 def test_sameas(env):
     tmpl = env.from_string(SAMEAS)
     assert tmpl.render(foo=False) == 'True|False'
+
+
+def test_no_paren_for_arg1(env):
+    tmpl = env.from_string(NOPARENFORARG1)
+    assert tmpl.render(foo=None) == 'True'
diff --git a/tests/test_various.py b/tests/test_various.py
index cea56b8..9fb483f 100644
--- a/tests/test_various.py
+++ b/tests/test_various.py
@@ -29,11 +29,16 @@ KEYWORDS = '''\
 {{ while }}
 {{ pass }}
 {{ finally }}'''
-LIGHTKW = '''{{ call }}'''
 UNPACKING = '''{% for a, b, c in [[1, 2, 3]] %}{{ a }}|{{ b }}|{{ c }}{% endfor %}'''
 RAW = '''{% raw %}{{ FOO }} and {% BAR %}{% endraw %}'''
 CONST = '''{{ true }}|{{ false }}|{{ none }}|{{ undefined }}|\
 {{ none is defined }}|{{ undefined is defined }}'''
+LOCALSET = '''{% set foo = 0 %}\
+{% for item in [1, 2] %}{% set foo = 1 %}{% endfor %}\
+{{ foo }}'''
+NONLOCALSET = '''{% set foo = 0 %}\
+{% for item in [1, 2] %}{% set foo = 1! %}{% endfor %}\
+{{ foo }}'''
 CONSTASS1 = '''{% set true = 42 %}'''
 CONSTASS2 = '''{% for undefined in seq %}{% endfor %}'''
 
@@ -42,10 +47,6 @@ def test_keywords(env):
     env.from_string(KEYWORDS)
 
 
-def test_lightkw(env):
-    env.from_string(LIGHTKW)
-
-
 def test_unpacking(env):
     tmpl = env.from_string(UNPACKING)
     assert tmpl.render() == '1|2|3'
@@ -100,3 +101,13 @@ def test_const_assign(env):
             pass
         else:
             raise AssertionError('expected syntax error')
+
+
+def test_localset(env):
+    tmpl = env.from_string(LOCALSET)
+    assert tmpl.render() == '0'
+
+
+def test_nonlocalset(env):
+    tmpl = env.from_string(NONLOCALSET)
+    assert tmpl.render() == '1'
-- 
2.26.2