From 2e9396ba8f6cfa9007d29724c6a14d3905078bcf Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 16 Apr 2008 14:21:57 +0200 Subject: [PATCH] reimplemented {% trans %} --HG-- branch : trunk --- examples/translate.py | 6 ++ jinja2/compiler.py | 5 +- jinja2/environment.py | 22 +++--- jinja2/i18n.py | 155 ++++++++++++++++++++++++++++++++++++++++++ jinja2/lexer.py | 45 ++++-------- jinja2/nodes.py | 13 +++- jinja2/optimizer.py | 12 +++- jinja2/parser.py | 10 ++- jinja2/runtime.py | 41 ++++++----- 9 files changed, 245 insertions(+), 64 deletions(-) create mode 100644 examples/translate.py create mode 100644 jinja2/i18n.py diff --git a/examples/translate.py b/examples/translate.py new file mode 100644 index 0000000..8b38d02 --- /dev/null +++ b/examples/translate.py @@ -0,0 +1,6 @@ +from jinja2 import Environment + +print Environment().from_string("""\ +{% trans %}Hello {{ user }}!{% endtrans %} +{% trans count=users|count %}{{ count }} user{% pluralize %}{{ count }} users{% endtrans %} +""").render() diff --git a/jinja2/compiler.py b/jinja2/compiler.py index 9ab7adf..abaf861 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -50,7 +50,7 @@ def has_safe_repr(value): if value is None or value is NotImplemented or value is Ellipsis: return True if isinstance(value, (bool, int, long, float, complex, basestring, - StaticLoopContext)): + xrange, StaticLoopContext)): return True if isinstance(value, (tuple, list, set, frozenset)): for item in value: @@ -1030,6 +1030,5 @@ class CodeGenerator(NodeVisitor): self.write(')') def visit_Keyword(self, node, frame): - self.visit(node.key, frame) - self.write('=') + self.write(node.key + '=') self.visit(node.value, frame) diff --git a/jinja2/environment.py b/jinja2/environment.py index 33e8f76..2ed660c 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -67,8 +67,13 @@ class Environment(object): `loader` the loader which should be used. ========================= ============================================ """ + + # santity checks assert issubclass(undefined, Undefined), 'undefined must be ' \ 'a subclass of undefined because filters depend on it.' + assert block_start_string != variable_start_string != \ + comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' # lexer / parser information self.block_start_string = block_start_string @@ -136,7 +141,9 @@ class Environment(object): source = generate(node, self, filename) if raw: return source - if isinstance(filename, unicode): + if filename is None: + filename = '' + elif isinstance(filename, unicode): filename = filename.encode('utf-8') return compile(source, filename, 'exec') @@ -158,7 +165,8 @@ class Environment(object): def from_string(self, source, filename='', globals=None): """Load a template from a string.""" globals = self.make_globals(globals) - return Template(self, self.compile(source, filename), globals) + return Template(self, self.compile(source, filename, globals=globals), + globals) def make_globals(self, d): """Return a dict for the globals.""" @@ -187,16 +195,14 @@ class Template(object): def generate(self, *args, **kwargs): # assemble the context - local_context = dict(*args, **kwargs) - context = self.globals.copy() - context.update(local_context) + context = dict(*args, **kwargs) # if the environment is using the optimizer locals may never # override globals as optimizations might have happened # depending on values of certain globals. This assertion goes # away if the python interpreter is started with -O if __debug__ and self.environment.optimized: - overrides = set(local_context) & set(self.globals) + overrides = set(context) & set(self.globals) if overrides: plural = len(overrides) != 1 and 's' or '' raise AssertionError('the per template variable%s %s ' @@ -204,8 +210,8 @@ class Template(object): 'With an enabled optimizer this ' 'will lead to unexpected results.' % (plural, ', '.join(overrides), plural or ' a', plural)) - gen = self.root_render_func(context) - # skip the first item which is a reference to the stream + gen = self.root_render_func(dict(self.globals, **context)) + # skip the first item which is a reference to the context gen.next() return gen diff --git a/jinja2/i18n.py b/jinja2/i18n.py new file mode 100644 index 0000000..1b19321 --- /dev/null +++ b/jinja2/i18n.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" + jinja2.i18n + ~~~~~~~~~~~ + + i18n support for Jinja. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +from jinja2 import nodes +from jinja2.parser import _statement_end_tokens +from jinja2.exceptions import TemplateAssertionError + + +def parse_trans(parser): + """Parse a translatable tag.""" + lineno = parser.stream.expect('trans').lineno + + # skip colon for python compatibility + if parser.stream.current.type is 'colon': + parser.stream.next() + + # find all the variables referenced. Additionally a variable can be + # defined in the body of the trans block too, but this is checked at + # a later state. + plural_expr = None + variables = {} + while parser.stream.current.type is not 'block_end': + if variables: + parser.stream.expect('comma') + name = parser.stream.expect('name') + if name.value in variables: + raise TemplateAssertionError('translatable variable %r defined ' + 'twice.' % name.value, name.lineno, + parser.filename) + + # expressions + if parser.stream.current.type is 'assign': + parser.stream.next() + variables[name.value] = var = parser.parse_expression() + else: + variables[name.value] = var = nodes.Name(name.value, 'load') + if plural_expr is None: + plural_expr = var + parser.stream.expect('block_end') + + plural = plural_names = None + have_plural = False + referenced = set() + + # now parse until endtrans or pluralize + singular_names, singular = _parse_block(parser, True) + if singular_names: + referenced.update(singular_names) + if plural_expr is None: + plural_expr = nodes.Name(singular_names[0], 'load') + + # if we have a pluralize block, we parse that too + if parser.stream.current.type is 'pluralize': + have_plural = True + parser.stream.next() + if parser.stream.current.type is not 'block_end': + plural_expr = parser.parse_expression() + parser.stream.expect('block_end') + plural_names, plural = _parse_block(parser, False) + parser.stream.next() + referenced.update(plural_names) + else: + parser.stream.next() + parser.end_statement() + + # register free names as simple name expressions + for var in referenced: + if var not in variables: + variables[var] = nodes.Name(var, 'load') + + # no variables referenced? no need to escape + if not referenced: + singular = singular.replace('%%', '%') + if plural: + plural = plural.replace('%%', '%') + + if not have_plural: + if plural_expr is None: + raise TemplateAssertionError('pluralize without variables', + lineno, parser.filename) + plural_expr = None + + if variables: + variables = nodes.Dict([nodes.Pair(nodes.Const(x, lineno=lineno), y) + for x, y in variables.items()]) + else: + vairables = None + + node = _make_node(singular, plural, variables, plural_expr) + node.set_lineno(lineno) + return node + + +def _parse_block(parser, allow_pluralize): + """Parse until the next block tag with a given name.""" + referenced = [] + buf = [] + while 1: + if parser.stream.current.type is 'data': + buf.append(parser.stream.current.value.replace('%', '%%')) + parser.stream.next() + elif parser.stream.current.type is 'variable_begin': + parser.stream.next() + referenced.append(parser.stream.expect('name').value) + buf.append('%s') + parser.stream.expect('variable_end') + elif parser.stream.current.type is 'block_begin': + parser.stream.next() + if parser.stream.current.type is 'endtrans': + break + elif parser.stream.current.type is 'pluralize': + if allow_pluralize: + break + raise TemplateSyntaxError('a translatable section can have ' + 'only one pluralize section', + parser.stream.current.lineno, + parser.filename) + raise TemplateSyntaxError('control structures in translatable ' + 'sections are not allowed.', + parser.stream.current.lineno, + parser.filename) + else: + assert False, 'internal parser error' + + return referenced, u''.join(buf) + + +def _make_node(singular, plural, variables, plural_expr): + """Generates a useful node from the data provided.""" + # singular only: + if plural_expr is None: + gettext = nodes.Name('gettext', 'load') + node = nodes.Call(gettext, [nodes.Const(singular)], + [], None, None) + if variables: + node = nodes.Mod(node, variables) + + # singular and plural + else: + ngettext = nodes.Name('ngettext', 'load') + node = nodes.Call(ngettext, [ + nodes.Const(singular), + nodes.Const(plural), + plural_expr + ], [], None, None) + if variables: + node = nodes.Mod(node, variables) + return nodes.Output([node]) diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 1f033d7..931d7c1 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -236,18 +236,6 @@ class Lexer(object): (operator_re, 'operator', None) ] - #: if variables and blocks have the same delimiters we won't - #: receive any variable blocks in the parser. This variable is `True` - #: if we need that. - self.no_variable_block = ( - (environment.variable_start_string is - environment.variable_end_string is None) or - (environment.variable_start_string == - environment.block_start_string and - environment.variable_end_string == - environment.block_end_string) - ) - # assamble the root lexing rule. because "|" is ungreedy # we have to sort by length so that the lexer continues working # as expected when we have parsing rules like <% for block and @@ -256,11 +244,9 @@ class Lexer(object): # is required. root_tag_rules = [ ('comment', environment.comment_start_string), - ('block', environment.block_start_string) + ('block', environment.block_start_string), + ('variable', environment.variable_start_string) ] - if not self.no_variable_block: - root_tag_rules.append(('variable', - environment.variable_start_string)) root_tag_rules.sort(key=lambda x: len(x[1])) # now escape the rules. This is done here so that the escape @@ -309,6 +295,13 @@ class Lexer(object): block_suffix_re )), 'block_end', '#pop'), ] + tag_rules, + # variables + 'variable_begin': [ + (c('\-%s\s*|%s' % ( + e(environment.variable_end_string), + e(environment.variable_end_string) + )), 'variable_end', '#pop') + ] + tag_rules, # raw block 'raw_begin': [ (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( @@ -319,24 +312,12 @@ class Lexer(object): block_suffix_re )), ('data', 'raw_end'), '#pop'), (c('(.)'), (Failure('Missing end of raw directive'),), None) - ] - } - - # only add the variable rules to the list if we process variables - # the variable_end_string variable could be None and break things. - if not self.no_variable_block: - self.rules['variable_begin'] = [ - (c('\-%s\s*|%s' % ( - e(environment.variable_end_string), - e(environment.variable_end_string) - )), 'variable_end', '#pop') - ] + tag_rules - - # the same goes for the line_statement_prefix - if environment.line_statement_prefix is not None: - self.rules['linestatement_begin'] = [ + ], + # line statements + 'linestatement_begin': [ (c(r'\s*(\n|$)'), 'linestatement_end', '#pop') ] + tag_rules + } def tokenize(self, source, filename=None): """Works like `tokeniter` but returns a tokenstream of tokens and not diff --git a/jinja2/nodes.py b/jinja2/nodes.py index 40aae3c..51c3039 100644 --- a/jinja2/nodes.py +++ b/jinja2/nodes.py @@ -159,6 +159,16 @@ class Node(object): node.ctx = ctx todo.extend(node.iter_child_nodes()) + def set_lineno(self, lineno, override=False): + """Set the line numbers of the node and children.""" + todo = deque([self]) + while todo: + node = todo.popleft() + if 'lineno' in node.attributes: + if node.lineno is None or override: + node.lineno = lineno + todo.extend(node.iter_child_nodes()) + def set_environment(self, environment): """Set the environment for all nodes.""" todo = deque([self]) @@ -333,7 +343,8 @@ class Const(Literal): def from_untrusted(cls, value, lineno=None, environment=None): """Return a const object if the value is representable as constant value in the generated code, otherwise it will raise - an `Impossible` exception.""" + an `Impossible` exception. + """ from compiler import has_safe_repr if not has_safe_repr(value): raise Impossible() diff --git a/jinja2/optimizer.py b/jinja2/optimizer.py index bc23fcb..fd5922c 100644 --- a/jinja2/optimizer.py +++ b/jinja2/optimizer.py @@ -16,7 +16,7 @@ prerender a template, this module might speed up your templates a bit if you are using a lot of constants. - :copyright: Copyright 2008 by Christoph Hack. + :copyright: Copyright 2008 by Christoph Hack, Armin Ronacher. :license: GNU GPL. """ from jinja2 import nodes @@ -24,6 +24,16 @@ from jinja2.visitor import NodeVisitor, NodeTransformer from jinja2.runtime import LoopContext +# TODO +# - function calls to contant objects are not properly evaluated if the +# function is not representable at constant type. eg: +# {% for item in range(10) %} doesn't become +# for l_item in xrange(10: even though it would be possible +# - multiple Output() nodes should be concatenated into one node. +# for example the i18n system could output such nodes: +# "foo{% trans %}bar{% endtrans %}blah" + + def optimize(node, environment, context_hint=None): """The context hint can be used to perform an static optimization based on the context given.""" diff --git a/jinja2/parser.py b/jinja2/parser.py index 5185df3..fd43af7 100644 --- a/jinja2/parser.py +++ b/jinja2/parser.py @@ -13,7 +13,7 @@ from jinja2.exceptions import TemplateSyntaxError _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print', - 'macro', 'include']) + 'macro', 'include', 'trans']) _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq', 'in']) _statement_end_tokens = set(['elif', 'else', 'endblock', 'endfilter', 'endfor', 'endif', 'endmacro', 'variable_end', @@ -33,7 +33,6 @@ class Parser(object): self.source = unicode(source) self.filename = filename self.closed = False - self.no_variable_block = self.environment.lexer.no_variable_block self.stream = environment.lexer.tokenize(source, filename) def end_statement(self): @@ -235,6 +234,13 @@ class Parser(object): self.end_statement() return node + def parse_trans(self): + """Parse a translatable section.""" + # lazily imported because we don't want the i18n overhead + # if it's not used. (Even though the overhead is low) + from jinja2.i18n import parse_trans + return parse_trans(self) + def parse_expression(self, no_condexpr=False): """Parse an expression.""" if no_condexpr: diff --git a/jinja2/runtime.py b/jinja2/runtime.py index 70ef9d8..3e3721d 100644 --- a/jinja2/runtime.py +++ b/jinja2/runtime.py @@ -137,12 +137,20 @@ class LoopContextBase(object): self.index0 = 0 self.parent = parent + def cycle(self, *args): + """A replacement for the old ``{% cycle %}`` tag.""" + if not args: + raise TypeError('no items for cycling given') + return args[self.index0 % len(args)] + first = property(lambda x: x.index0 == 0) last = property(lambda x: x.revindex0 == 0) index = property(lambda x: x.index0 + 1) revindex = property(lambda x: x.length) revindex0 = property(lambda x: x.length - 1) - length = property(lambda x: len(x)) + + def __len__(self): + return self.length class LoopContext(LoopContextBase): @@ -171,7 +179,8 @@ class LoopContext(LoopContextBase): self.index0 += 1 return self._next(), self - def __len__(self): + @property + def length(self): if self._length is None: try: length = len(self._iterable) @@ -182,6 +191,9 @@ class LoopContext(LoopContextBase): self._length = length return self._length + def __repr__(self): + return 'LoopContext(%r)' % self.index0 + class StaticLoopContext(LoopContextBase): """The static loop context is used in the optimizer to "freeze" the @@ -192,19 +204,16 @@ class StaticLoopContext(LoopContextBase): def __init__(self, index0, length, parent): self.index0 = index0 self.parent = parent - self._length = length + self.length = length def __repr__(self): """The repr is used by the optimizer to dump the object.""" return 'StaticLoopContext(%r, %r, %r)' % ( self.index0, - self._length, + self.length, self.parent ) - def __len__(self): - return self._length - def make_static(self): return self @@ -267,19 +276,20 @@ class Undefined(object): def __init__(self, name=None, attr=None, extra=None): if attr is None: self._undefined_hint = '%r is undefined' % name + self._error_class = NameError else: - self._undefined_hint = 'attribute %r of %r is undefined' \ - % (attr, name) + self._undefined_hint = '%r has no attribute named %r' \ + % (name, attr) + self._error_class = AttributeError if extra is not None: self._undefined_hint += ' (' + extra + ')' - def fail_with_error(self, *args, **kwargs): - raise NameError(self._undefined_hint) + def _fail_with_error(self, *args, **kwargs): + raise self._error_class(self._undefined_hint) __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ __realdiv__ = __rrealdiv__ = __floordiv__ = __rfloordiv__ = \ __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getattr__ = __getitem__ = fail_with_error - del fail_with_error + __getattr__ = __getitem__ = _fail_with_error def __unicode__(self): return u'' @@ -311,7 +321,4 @@ class DebugUndefined(Undefined): class StrictUndefined(Undefined): """An undefined that barks on print and iteration.""" - def fail_with_error(self, *args, **kwargs): - raise NameError(self._undefined_hint) - __iter__ = __unicode__ = __len__ = fail_with_error - del fail_with_error + __iter__ = __unicode__ = __len__ = Undefined._fail_with_error -- 2.26.2