Added example extension that uses the stream filtering and added unittests.

--HG--
branch : trunk

diff --git a/docs/extensions.rst b/docs/extensions.rst
index 215a6845a3cf32f9a4b35a660b8de588cd7f6f05..1d25a2b9852e8ef9ba3e10f2134c286e753f0716 100644 (file)
--- a/docs/extensions.rst
+++ b/docs/extensions.rst
@@ -229,6 +229,11 @@ extensions:
 
         The value of the token.
 
+There is also a utility function in the lexer module that can count newline
+characters in strings:
+
+.. autofunction:: jinja2.lexer.count_newlines
+
 AST
 ~~~
 

diff --git a/ext/inlinegettext.py b/ext/inlinegettext.py
new file mode 100644 (file)
index 0000000..049ab3b
--- /dev/null
+++ b/ext/inlinegettext.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+    Inline Gettext
+    ~~~~~~~~~~~~~~
+
+    An example extension for Jinja2 that supports inline gettext calls.
+    Requires the i18n extension to be loaded.
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: BSD.
+"""
+import re
+from jinja2.ext import Extension
+from jinja2.lexer import Token, count_newlines
+from jinja2.exceptions import TemplateSyntaxError
+
+
+_outside_re = re.compile(r'\\?(gettext|_)\(')
+_inside_re = re.compile(r'\\?[()]')
+
+
+class InlineGettext(Extension):
+    """This extension implements support for inline gettext blocks::
+
+        <h1>_(Welcome)</h1>
+        <p>_(This is a paragraph)</p>
+
+    Requires the i18n extension to be loaded and configured.
+    """
+
+    def filter_stream(self, stream):
+        paren_stack = 0
+
+        for token in stream:
+            if token.type is not 'data':
+                yield token
+                continue
+
+            pos = 0
+            lineno = token.lineno
+
+            while 1:
+                if not paren_stack:
+                    match = _outside_re.search(token.value, pos)
+                else:
+                    match = _inside_re.search(token.value, pos)
+                if match is None:
+                    break
+                new_pos = match.start()
+                if new_pos > pos:
+                    preval = token.value[pos:new_pos]
+                    yield Token(lineno, 'data', preval)
+                    lineno += count_newlines(preval)
+                gtok = match.group()
+                if gtok[0] == '\\':
+                    yield Token(lineno, 'data', gtok[1:])
+                elif not paren_stack:
+                    yield Token(lineno, 'block_begin', None)
+                    yield Token(lineno, 'name', 'trans')
+                    yield Token(lineno, 'block_end', None)
+                    paren_stack = 1
+                else:
+                    if gtok == '(' or paren_stack > 1:
+                        yield Token(lineno, 'data', gtok)
+                    paren_stack += gtok == ')' and -1 or 1
+                    if not paren_stack:
+                        yield Token(lineno, 'block_begin', None)
+                        yield Token(lineno, 'name', 'endtrans')
+                        yield Token(lineno, 'block_end', None)
+                pos = match.end()
+
+            if pos < len(token.value):
+                yield Token(lineno, 'data', token.value[pos:])
+
+        if paren_stack:
+            raise TemplateSyntaxError('unclosed gettext expression',
+                                      token.lineno, stream.name,
+                                      stream.filename)

diff --git a/jinja2/ext.py b/jinja2/ext.py
index 9dfa87cc0c2df7af8a7d2088559ade310150b080..f60aade429a4a46dea2605f3e27324f04979dfdd 100644 (file)
--- a/jinja2/ext.py
+++ b/jinja2/ext.py
@@ -16,7 +16,6 @@ from jinja2.defaults import *
 from jinja2.environment import get_spontaneous_environment
 from jinja2.runtime import Undefined, concat
 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
-from jinja2.lexer import Token
 from jinja2.utils import contextfunction, import_string, Markup
 
 
@@ -80,6 +79,10 @@ class Extension(object):
         to filter tokens returned.  This method has to return an iterable of
         :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a
         :class:`~jinja2.lexer.TokenStream`.
+
+        In the `ext` folder of the Jinja2 source distribution there is a file
+        called `inlinegettext.py` which implements a filter that utilizes this
+        method.
         """
         return stream
 
@@ -261,6 +264,8 @@ class InternationalizationExtension(Extension):
                                 'pluralize section')
                 parser.fail('control structures in translatable sections are '
                             'not allowed')
+            elif parser.stream.eos:
+                parser.fail('unclosed translation block')
             else:
                 assert False, 'internal parser error'
 

diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index 108e49ccf974b0970949397be637d0c7ff912457..0597b7a80b46f63c4067f6190c27fd1b3367e2ed 100644 (file)
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -71,6 +71,13 @@ operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
                          sorted(operators, key=lambda x: -len(x))))
 
 
+def count_newlines(value):
+    """Count the number of newline characters in the string.  This is
+    useful for extensions that filter a stream.
+    """
+    return len(newline_re.findall(value))
+
+
 class Failure(object):
     """Class that raises a `TemplateSyntaxError` if called.
     Used by the `Lexer` to specify known errors.

diff --git a/tests/test_ext.py b/tests/test_ext.py
index 8002789bfebdfb6d0dcc0099d415ffae5282bc3c..b425e18d32dc112c3d5520f654b54bf7fc427f24 100644 (file)
--- a/tests/test_ext.py
+++ b/tests/test_ext.py
@@ -8,14 +8,14 @@
 """
 import re
 from jinja2 import Environment, nodes
-from jinja2.ext import Extension, Token
+from jinja2.ext import Extension
+from jinja2.lexer import Token, count_newlines
 
 
 importable_object = 23
 
 
-_line_re = re.compile(r'(\r\n|\r|\n)')
-_gettext_re = re.compile(r'_\((([^)\\]*(?:\\.[^)\\]*)*))\)(?s)')
+_gettext_re = re.compile(r'_\((.*?)\)')
 
 
 class TestExtension(Extension):
@@ -55,9 +55,6 @@ class StreamFilterExtension(Extension):
             else:
                 yield token
 
-    def count_lines(self, value):
-        return len(_line_re.findall(value))
-
     def interpolate(self, token):
         pos = 0
         end = len(token.value)
@@ -69,7 +66,7 @@ class StreamFilterExtension(Extension):
             value = token.value[pos:match.start()]
             if value:
                 yield Token(lineno, 'data', value)
-            lineno += self.count_lines(token.value)
+            lineno += count_newlines(token.value)
             yield Token(lineno, 'variable_begin', None)
             yield Token(lineno, 'name', 'gettext')
             yield Token(lineno, 'lparen', None)