Added unittests for stream filtering and preprocessing features of extensions.

--HG--
branch : trunk

diff --git a/jinja2/environment.py b/jinja2/environment.py
index e24e78e4a03e95a752a40a5f3b867f734591e2a8..94d0679e50549a4d150b42545b0007ded5de2d61 100644 (file)
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -358,15 +358,12 @@ class Environment(object):
         """Called by the parser to do the preprocessing and filtering
         for all the extensions.  Returns a :class:`~jinja2.lexer.TokenStream`.
         """
-        def _stream(iterable):
-            if not isinstance(iterable, TokenStream):
-                iterable = TokenStream(iterable, name, filename)
-            return iterable
         source = self.preprocess(source, name, filename)
-        tokeniter = self.lexer.tokeniter(source, name, filename)
-        stream = _stream(self.lexer.wrap(tokeniter, name, filename))
+        stream = self.lexer.tokenize(source, name, filename)
         for ext in self.extensions.itervalues():
-            stream = _stream(ext.filter_stream(stream))
+            stream = ext.filter_stream(stream)
+            if not isinstance(stream, TokenStream):
+                stream = TokenStream(stream, name, filename)
         return stream
 
     def compile(self, source, name=None, filename=None, raw=False):

diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index 1f22ed7b89fa0dda1db8e79d43bc386a76889553..108e49ccf974b0970949397be637d0c7ff912457 100644 (file)
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -377,7 +377,6 @@ class Lexer(object):
 
     def tokenize(self, source, name=None, filename=None):
         """Calls tokeniter + tokenize and wraps it in a token stream.
-        This is currently only used for unittests.
         """
         stream = self.tokeniter(source, name, filename)
         return TokenStream(self.wrap(stream, name, filename), name, filename)

diff --git a/tests/test_ext.py b/tests/test_ext.py
index d3364bb61a4b7fedcf11c0230cf9b8c0ed9a77ee..8002789bfebdfb6d0dcc0099d415ffae5282bc3c 100644 (file)
--- a/tests/test_ext.py
+++ b/tests/test_ext.py
@@ -6,13 +6,18 @@
     :copyright: 2008 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+import re
 from jinja2 import Environment, nodes
-from jinja2.ext import Extension
+from jinja2.ext import Extension, Token
 
 
 importable_object = 23
 
 
+_line_re = re.compile(r'(\r\n|\r|\n)')
+_gettext_re = re.compile(r'_\((([^)\\]*(?:\\.[^)\\]*)*))\)(?s)')
+
+
 class TestExtension(Extension):
     tags = set(['test'])
     ext_attr = 42
@@ -34,6 +39,48 @@ class TestExtension(Extension):
         )
 
 
+class PreprocessorExtension(Extension):
+
+    def preprocess(self, source, name, filename=None):
+        return source.replace('[[TEST]]', '({{ foo }})')
+
+
+class StreamFilterExtension(Extension):
+
+    def filter_stream(self, stream):
+        for token in stream:
+            if token.type is 'data':
+                for t in self.interpolate(token):
+                    yield t
+            else:
+                yield token
+
+    def count_lines(self, value):
+        return len(_line_re.findall(value))
+
+    def interpolate(self, token):
+        pos = 0
+        end = len(token.value)
+        lineno = token.lineno
+        while 1:
+            match = _gettext_re.search(token.value, pos)
+            if match is None:
+                break
+            value = token.value[pos:match.start()]
+            if value:
+                yield Token(lineno, 'data', value)
+            lineno += self.count_lines(token.value)
+            yield Token(lineno, 'variable_begin', None)
+            yield Token(lineno, 'name', 'gettext')
+            yield Token(lineno, 'lparen', None)
+            yield Token(lineno, 'string', match.group(1))
+            yield Token(lineno, 'rparen', None)
+            yield Token(lineno, 'variable_end', None)
+            pos = match.end()
+        if pos < end:
+            yield Token(lineno, 'data', token.value[pos:])
+
+
 def test_loop_controls():
     env = Environment(extensions=['jinja2.ext.loopcontrols'])
 
@@ -78,3 +125,16 @@ def test_rebinding():
     for env in original, overlay:
         for ext in env.extensions.itervalues():
             assert ext.environment is env
+
+
+def test_preprocessor_extension():
+    env = Environment(extensions=[PreprocessorExtension])
+    tmpl = env.from_string('{[[TEST]]}')
+    assert tmpl.render(foo=42) == '{(42)}'
+
+
+def test_streamfilter_extension():
+    env = Environment(extensions=[StreamFilterExtension])
+    env.globals['gettext'] = lambda x: x.title()
+    tmpl = env.from_string('Foo _(bar) Baz')
+    assert tmpl.render() == 'Foo Bar Baz'