From d02fc7db91a3f67f04489ddade7d9387d90eeab1 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 14 Jun 2008 14:19:47 +0200 Subject: [PATCH] Added example extension that uses the stream filtering and added unittests. --HG-- branch : trunk --- docs/extensions.rst | 5 +++ ext/inlinegettext.py | 78 ++++++++++++++++++++++++++++++++++++++++++++ jinja2/ext.py | 7 +++- jinja2/lexer.py | 7 ++++ tests/test_ext.py | 11 +++---- 5 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 ext/inlinegettext.py diff --git a/docs/extensions.rst b/docs/extensions.rst index 215a684..1d25a2b 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -229,6 +229,11 @@ extensions: The value of the token. +There is also a utility function in the lexer module that can count newline +characters in strings: + +.. autofunction:: jinja2.lexer.count_newlines + AST ~~~ diff --git a/ext/inlinegettext.py b/ext/inlinegettext.py new file mode 100644 index 0000000..049ab3b --- /dev/null +++ b/ext/inlinegettext.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" + Inline Gettext + ~~~~~~~~~~~~~~ + + An example extension for Jinja2 that supports inline gettext calls. + Requires the i18n extension to be loaded. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +import re +from jinja2.ext import Extension +from jinja2.lexer import Token, count_newlines +from jinja2.exceptions import TemplateSyntaxError + + +_outside_re = re.compile(r'\\?(gettext|_)\(') +_inside_re = re.compile(r'\\?[()]') + + +class InlineGettext(Extension): + """This extension implements support for inline gettext blocks:: + +

_(Welcome)

+

_(This is a paragraph)

+ + Requires the i18n extension to be loaded and configured. + """ + + def filter_stream(self, stream): + paren_stack = 0 + + for token in stream: + if token.type is not 'data': + yield token + continue + + pos = 0 + lineno = token.lineno + + while 1: + if not paren_stack: + match = _outside_re.search(token.value, pos) + else: + match = _inside_re.search(token.value, pos) + if match is None: + break + new_pos = match.start() + if new_pos > pos: + preval = token.value[pos:new_pos] + yield Token(lineno, 'data', preval) + lineno += count_newlines(preval) + gtok = match.group() + if gtok[0] == '\\': + yield Token(lineno, 'data', gtok[1:]) + elif not paren_stack: + yield Token(lineno, 'block_begin', None) + yield Token(lineno, 'name', 'trans') + yield Token(lineno, 'block_end', None) + paren_stack = 1 + else: + if gtok == '(' or paren_stack > 1: + yield Token(lineno, 'data', gtok) + paren_stack += gtok == ')' and -1 or 1 + if not paren_stack: + yield Token(lineno, 'block_begin', None) + yield Token(lineno, 'name', 'endtrans') + yield Token(lineno, 'block_end', None) + pos = match.end() + + if pos < len(token.value): + yield Token(lineno, 'data', token.value[pos:]) + + if paren_stack: + raise TemplateSyntaxError('unclosed gettext expression', + token.lineno, stream.name, + stream.filename) diff --git a/jinja2/ext.py b/jinja2/ext.py index 9dfa87c..f60aade 100644 --- a/jinja2/ext.py +++ b/jinja2/ext.py @@ -16,7 +16,6 @@ from jinja2.defaults import * from jinja2.environment import get_spontaneous_environment from jinja2.runtime import Undefined, concat from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.lexer import Token from jinja2.utils import contextfunction, import_string, Markup @@ -80,6 +79,10 @@ class Extension(object): to filter tokens returned. This method has to return an iterable of :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a :class:`~jinja2.lexer.TokenStream`. + + In the `ext` folder of the Jinja2 source distribution there is a file + called `inlinegettext.py` which implements a filter that utilizes this + method. """ return stream @@ -261,6 +264,8 @@ class InternationalizationExtension(Extension): 'pluralize section') parser.fail('control structures in translatable sections are ' 'not allowed') + elif parser.stream.eos: + parser.fail('unclosed translation block') else: assert False, 'internal parser error' diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 108e49c..0597b7a 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -71,6 +71,13 @@ operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))) +def count_newlines(value): + """Count the number of newline characters in the string. This is + useful for extensions that filter a stream. + """ + return len(newline_re.findall(value)) + + class Failure(object): """Class that raises a `TemplateSyntaxError` if called. Used by the `Lexer` to specify known errors. diff --git a/tests/test_ext.py b/tests/test_ext.py index 8002789..b425e18 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -8,14 +8,14 @@ """ import re from jinja2 import Environment, nodes -from jinja2.ext import Extension, Token +from jinja2.ext import Extension +from jinja2.lexer import Token, count_newlines importable_object = 23 -_line_re = re.compile(r'(\r\n|\r|\n)') -_gettext_re = re.compile(r'_\((([^)\\]*(?:\\.[^)\\]*)*))\)(?s)') +_gettext_re = re.compile(r'_\((.*?)\)') class TestExtension(Extension): @@ -55,9 +55,6 @@ class StreamFilterExtension(Extension): else: yield token - def count_lines(self, value): - return len(_line_re.findall(value)) - def interpolate(self, token): pos = 0 end = len(token.value) @@ -69,7 +66,7 @@ class StreamFilterExtension(Extension): value = token.value[pos:match.start()] if value: yield Token(lineno, 'data', value) - lineno += self.count_lines(token.value) + lineno += count_newlines(token.value) yield Token(lineno, 'variable_begin', None) yield Token(lineno, 'name', 'gettext') yield Token(lineno, 'lparen', None) -- 2.26.2