From 92f572f8489036f443fbe8a080868353d8718b65 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 26 Feb 2007 22:17:32 +0100 Subject: [PATCH 1/1] [svn] added new jinja trunk --HG-- branch : trunk --- jinja/__init__.py | 8 + jinja/ast.py | 108 +++++++++++ jinja/datastructure.py | 175 +++++++++++++++++ jinja/defaults.py | 15 ++ jinja/environment.py | 86 +++++++++ jinja/exceptions.py | 56 ++++++ jinja/filters.py | 203 ++++++++++++++++++++ jinja/lexer.py | 191 ++++++++++++++++++ jinja/nodes.py | 90 +++++++++ jinja/parser.py | 219 +++++++++++++++++++++ jinja/translators/__init__.py | 11 ++ jinja/translators/python.py | 351 ++++++++++++++++++++++++++++++++++ syntax.html | 61 ++++++ syntax.txt | 22 +++ test.html | 53 +++++ test.py | 7 + tests/mockup.txt | 153 +++++++++++++++ tests/run.py | 15 ++ 18 files changed, 1824 insertions(+) create mode 100644 jinja/__init__.py create mode 100644 jinja/ast.py create mode 100644 jinja/datastructure.py create mode 100644 jinja/defaults.py create mode 100644 jinja/environment.py create mode 100644 jinja/exceptions.py create mode 100644 jinja/filters.py create mode 100644 jinja/lexer.py create mode 100644 jinja/nodes.py create mode 100644 jinja/parser.py create mode 100644 jinja/translators/__init__.py create mode 100644 jinja/translators/python.py create mode 100644 syntax.html create mode 100644 syntax.txt create mode 100644 test.html create mode 100644 test.py create mode 100644 tests/mockup.txt create mode 100644 tests/run.py diff --git a/jinja/__init__.py b/jinja/__init__.py new file mode 100644 index 0000000..edf3439 --- /dev/null +++ b/jinja/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" + Jinja Sandboxed Template Engine + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" diff --git a/jinja/ast.py b/jinja/ast.py new file mode 100644 index 0000000..88f18a2 --- /dev/null +++ b/jinja/ast.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +""" + jinja.ast + ~~~~~~~~~ + + Advance Syntax Tree for jinja. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + +class Node(object): + """ + Baseclass of all nodes. For instance checking. + """ + __slots__ = () + + def __init__(self): + raise TypeError('cannot create %r instances' % + self.__class__.__name__) + + +class Expression(list, Node): + """ + Node that helds childnodes. Normally just used temporary. + """ + __slots__ = () + + def __init__(self, *args, **kwargs): + super(Expression, self).__init__(*args, **kwargs) + + def __repr__(self): + return 'Expression(%s)' % list.__repr__(self) + + +class Comment(Node): + """ + Node that helds a comment. We keep comments in the data + if some translator wants to forward it into the generated + code (for example the python translator). + """ + __slots__ = ('pos', 'comment',) + + def __init__(self, pos, comment): + self.pos = pos + self.comment = comment + + def __repr__(self): + return 'Comment(%r, %r)' % (self.pos, self.comment) + + +class Page(Node): + """ + Node that helds all root nodes. + """ + __slots__ = ('filename', 'nodes') + + def __init__(self, filename, nodes): + self.filename = filename + self.nodes = nodes + + def __repr__(self): + return 'Page(%r, %r)' % ( + self.filename, + self.nodes + ) + + +class Variable(Node): + """ + Node for variables + """ + __slots__ = ('pos', 'expression') + + def __init__(self, pos, expression): + self.pos = pos + self.expression = expression + + def __repr__(self): + return 'Variable(%r)' % self.expression + + +class Data(Node): + """ + Node for data outside of tags. + """ + __slots__ = ('pos', 'data') + + def __init__(self, pos, data): + self.pos = pos + self.data = data + + def __repr__(self): + return 'Data(%d, %r)' % (self.pos, self.data) + + +class Name(Node): + """ + Node for names. + """ + __slots__ = ('pos', 'data') + + def __init__(self, pos, data): + self.pos = pos + self.data = data + + def __repr__(self): + return 'Name(%d, %r)' % (self.pos, self.data) diff --git a/jinja/datastructure.py b/jinja/datastructure.py new file mode 100644 index 0000000..868397c --- /dev/null +++ b/jinja/datastructure.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +""" + jinja.datastructure + ~~~~~~~~~~~~~~~~~~~ + + Module that helds several data types used in the template engine. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + +# python2.3 compatibility. do not use this method for anything else +# then context reversing. +try: + _reversed = reversed +except NameError: + def _reversed(c): + return c[::-1] + + +class UndefinedType(object): + """ + An object that does not exist. + """ + + def __init__(self): + try: + Undefined + except NameError: + pass + else: + raise TypeError('cannot create %r instances' % + self.__class__.__name__) + + def __getitem__(self, arg): + return self + + def __iter__(self): + return iter(int, 0) + + def __getattr__(self, arg): + return self + + def __nonzero__(self): + return False + + def __len__(self): + return 0 + + def __str__(self): + return '' + + def __unicode__(self): + return u'' + + def __int__(self): + return 0 + + def __float__(self): + return 1 + + +Undefined = UndefinedType() + + +class Context(object): + """ + Dict like object. + """ + + __slots__ = ('stack') + + def __init__(*args, **kwargs): + try: + self = args[0] + self.environment = args[1] + initial = dict(*args[2:], **kwargs) + except: + raise TypeError('%r requires environment as first argument. ' + 'The rest of the arguments are forwarded to ' + 'the default dict constructor.') + initial.update( + false=False, + true=True, + none=None + ) + self._stack = [initial, {}] + + def pop(self): + if len(self._stack) <= 2: + raise ValueError('cannot pop initial layer') + return self._stack.pop() + + def push(self, data=None): + self._stack.append(data or {}) + + def __getitem__(self, name): + for d in _reversed(self._stack): + if name in d: + return d[name] + return Undefined + + def __setitem__(self, name, value): + self._stack[-1][name] = value + + def __delitem__(self, name): + if name in self._stack[-1]: + del self._stack[-1][name] + + def __repr__(self): + tmp = {} + for d in self._stack: + for key, value in d.iteritems(): + tmp[key] = value + return 'Context(%s)' % repr(tmp) + + +class TokenStream(object): + """ + A token stream works like a normal generator just that + it supports pushing tokens back to the stream. + """ + + def __init__(self, generator): + self._generator = generator + self._pushed = [] + self.last = (0, 'initial', '') + + def __iter__(self): + return self + + def __nonzero__(self): + """Are we at the end of the tokenstream?""" + if self._pushed: + return True + try: + self.push(self.next()) + except StopIteration: + return False + return True + + eos = property(lambda x: not x.__nonzero__(), doc=__nonzero__.__doc__) + + def next(self): + """Return the next token from the stream.""" + if self._pushed: + rv = self._pushed.pop() + else: + rv = self._generator.next() + self.last = rv + return rv + + def look(self): + """Pop and push a token, return it.""" + token = self.next() + self.push(*token) + return token + + def fetch_until(self, test, drop_needle=False): + """Fetch tokens until a function matches.""" + try: + while True: + token = self.next() + if test(*token): + if not drop_needle: + self.push(*token) + return + else: + yield token + except StopIteration: + raise IndexError('end of stream reached') + + def push(self, pos, token, data): + """Push an yielded token back to the stream.""" + self._pushed.append((pos, token, data)) diff --git a/jinja/defaults.py b/jinja/defaults.py new file mode 100644 index 0000000..4a2d6bb --- /dev/null +++ b/jinja/defaults.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +""" + jinja.defaults + ~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + + +DEFAULT_TAGS = {} + +DEFAULT_FILTERS = {} diff --git a/jinja/environment.py b/jinja/environment.py new file mode 100644 index 0000000..1f1a62c --- /dev/null +++ b/jinja/environment.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" + jinja.environment + ~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from jinja.lexer import Lexer +from jinja.parser import Parser +from jinja.exceptions import TagNotFound, FilterNotFound +from jinja.defaults import DEFAULT_TAGS, DEFAULT_FILTERS + + +class Environment(object): + """ + The jinja environment. + """ + + def __init__(self, + block_start_string='{%', + block_end_string='%}', + variable_start_string='{{', + variable_end_string='}}', + comment_start_string='{#', + comment_end_string='#}', + template_charset='utf-8', + charset='utf-8', + loader=None, + tags=None, + filters=None): + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + + # other stuff + self.template_charset = template_charset + self.charset = charset + self.loader = loader + self.tags = tags or DEFAULT_TAGS.copy() + self.filters = filters or DEFAULT_FILTERS.copy() + + # create lexer + self.lexer = Lexer(self) + + def parse(self, source): + """Function that creates a new parser and parses the source.""" + parser = Parser(self, source) + return parser.parse_page() + + def get_tag(self, name): + """ + Return the tag for a specific name. Raise a `TagNotFound` exception + if a tag with this name is not registered. + """ + if name not in self._tags: + raise TagNotFound(name) + return self._tags[name] + + def get_filter(self, name): + """ + Return the filter for a given name. Raise a `FilterNotFound` exception + if the requested filter is not registered. + """ + if name not in self._filters: + raise FilterNotFound(name) + return self._filters[name] + + def to_unicode(self, value): + """ + Convert a value to unicode with the rules defined on the environment. + """ + if isinstance(value, unicode): + return value + else: + try: + return unicode(value) + except UnicodeError: + return str(value).decode(self.charset, 'ignore') diff --git a/jinja/exceptions.py b/jinja/exceptions.py new file mode 100644 index 0000000..6156041 --- /dev/null +++ b/jinja/exceptions.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" + jinja.exceptions + ~~~~~~~~~~~~~~~~ + + Jinja exceptions. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + + +class TemplateError(RuntimeError): + pass + + +class TagNotFound(KeyError, TemplateError): + """ + The parser looked for a specific tag in the tag library but was unable to find + one. + """ + + def __init__(self, tagname): + super(TagNotFound, self).__init__('The tag %r does not exist.' % tagname) + self.tagname = tagname + + +class FilterNotFound(KeyError, TemplateError): + """ + The template engine looked for a filter but was unable to find it. + """ + + def __init__(self, filtername): + super(FilterNotFound, self).__init__('The filter %r does not exist.' % filtername) + self.filtername = filtername + + +class TemplateSyntaxError(SyntaxError, TemplateError): + """ + Raised to tell the user that there is a problem with the template. + """ + + def __init__(self, message, pos): + super(TemplateSyntaxError, self).__init__(message) + self.pos = pos + + +class TemplateRuntimeError(TemplateError): + """ + Raised by the template engine if a tag encountered an error when + rendering. + """ + + def __init__(self, message, pos): + super(TemplateRuntimeError, self).__init__(message) + self.pos = pos diff --git a/jinja/filters.py b/jinja/filters.py new file mode 100644 index 0000000..93ef90f --- /dev/null +++ b/jinja/filters.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +""" + jinja.filters + ~~~~~~~~~~~~~ + + Bundled jinja filters. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + + +def stringfilter(f): + """ + Decorator for filters that just work on unicode objects. + """ + def decorator(*args): + def wrapped(env, context, value): + args = list(args) + for idx, var in enumerate(args): + if isinstance(var, str): + args[idx] = env.to_unicode(var) + return f(env.to_unicode(value), *args) + return wrapped + return decorator + + +def do_replace(s, old, new, count=None): + """ + {{ s|replace(old, new, count=None) }} + + Return a copy of s with all occurrences of substring + old replaced by new. If the optional argument count is + given, only the first count occurrences are replaced. + """ + if count is None: + return s.replace(old, new) + return s.replace(old, new, count) +do_replace = stringfilter(do_replace) + + +def do_upper(s): + """ + {{ s|upper }} + + Return a copy of s converted to uppercase. + """ + return s.upper() +do_upper = stringfilter(do_upper) + + +def do_lower(s): + """ + {{ s|lower }} + + Return a copy of s converted to lowercase. + """ + return s.lower() +do_lower = stringfilter(do_lower) + + +def do_escape(s, attribute=False): + """ + {{ s|escape(attribute) }} + + XML escape &, <, and > in a string of data. If attribute is + True it also converts ``"`` to ``"`` + """ + s = s.replace("&", "&").replace("<", "<").replace(">", ">") + if attribute: + s = s.replace('"', """) + return s +do_escape = stringfilter(do_escape) + + +def do_addslashes(s): + """ + {{ s|addslashes }} + + Adds slashes to s. + """ + return s.encode('utf-8').encode('string-escape').decode('utf-8') +do_addslashes = stringfilter(do_addslashes) + + +def do_capitalize(s): + """ + {{ s|capitalize }} + + Return a copy of the string s with only its first character + capitalized. + """ + return s.capitalize() +do_capitalize = stringfilter(do_capitalize) + + +def do_title(s): + """ + {{ s|title }} + + Return a titlecased version of s, i.e. words start with uppercase + characters, all remaining cased characters have lowercase. + """ + return s.title() +do_title = stringfilter(do_title) + + +def do_default(default_value=u''): + """ + {{ s|default(default_value='') }} + + In case of s isn't set or True default will return default_value + which is '' per default. + """ + return lambda e, c, v: v or default_value + + +def do_join(d=u''): + """ + {{ sequence|join(d='') }} + + Return a string which is the concatenation of the strings in the + sequence. The separator between elements is d which is an empty + string per default. + """ + def wrapped(env, context, value): + d = env.to_unicode(d) + return d.join([env.to_unicode(x) for x in value]) + return wrapped + + +def do_count(): + """ + {{ var|count }} + + Return the length of var. In case if getting an integer or float + it will convert it into a string an return the length of the new + string. + If the object doesn't provide a __len__ function it will return + zero.st(value) + l.reverse() + return + """ + def wrapped(env, context, value): + try: + if type(value) in (int, float, long): + return len(str(var)) + return len(var) + except TypeError: + return 0 + return wrapped + + +def do_odd(): + """ + {{ var|odd }} + + Return true if the variable is odd. + """ + return lambda e, c, v: v % 2 == 1 + + +def do_even(): + """ + {{ var|even }} + + Return true of the variable is even. + """ + return lambda e, c, v: v % 2 == 0 + + +def do_reversed(): + """ + {{ var|reversed }} + + Return a reversed list of the iterable filtered. + """ + def wrapped(env, context, value): + try: + return value[::-1] + except: + l = list(value) + l.reverse() + return l + return wrapped + + +FILTERS = { + 'replace': do_replace, + 'upper': do_upper, + 'lower': do_lower, + 'escape': do_escape, + 'e': do_escape, + 'addslashes': do_addslashes, + 'capitalize': do_capitalize, + 'title': do_title, + 'default': do_default, + 'join': do_join, + 'count': do_count, + 'odd': do_odd, + 'even': do_even, + 'reversed': do_reversed +} diff --git a/jinja/lexer.py b/jinja/lexer.py new file mode 100644 index 0000000..ce2541d --- /dev/null +++ b/jinja/lexer.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +""" + jinja.lexer + ~~~~~~~~~~~ +""" +import re +from jinja.datastructure import TokenStream +from jinja.exceptions import TemplateSyntaxError + + +# static regular expressions +whitespace_re = re.compile(r'\s+(?m)') +name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*[!?]?') +string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" + r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)') +number_re = re.compile(r'\d+(\.\d+)*') + +operator_re = re.compile('(%s)' % '|'.join( + isinstance(x, unicode) and str(x) or re.escape(x) for x in [ + # math operators + '+', '-', '*', '/', '%', + # braces and parenthesis + '[', ']', '(', ')', '{', '}', + # attribute access and comparison / logical operators + '.', ',', '|', '==', '<', '>', '<=', '>=', '!=', + ur'or\b', ur'and\b', ur'not\b' +])) + + +class Failure(object): + """ + Class that raises a `TemplateSyntaxError` if called. + Used by the `Lexer` to specify known errors. + """ + + def __init__(self, message, cls=TemplateSyntaxError): + self.message = message + self.error_class = cls + + def __call__(self, position): + raise self.error_class(self.message, position) + + +class Lexer(object): + """ + Class that implements a lexer for a given environment. Automatically + created by the environment class, usually you don't have to do that. + """ + + def __init__(self, environment): + # shortcuts + c = lambda x: re.compile(x, re.M | re.S) + e = re.escape + + # parsing rules for tags + tag_rules = [ + (whitespace_re, None, None), + (number_re, 'number', None), + (operator_re, 'operator', None), + (name_re, 'name', None), + (string_re, 'string', None) + ] + + # global parsing rules + self.rules = { + 'root': [ + (c('(.*?)(?:(?P' + + e(environment.comment_start_string) + + ')|(?P' + + e(environment.block_start_string) + + ')|(?P' + + e(environment.variable_start_string) + + '))'), ('data', '#bygroup'), '#bygroup'), + (c('.+'), 'data', None) + ], + 'comment_begin': [ + (c(r'(.*?)(%s)' % e(environment.comment_end_string)), + ('comment', 'comment_end'), '#pop'), + (c('(.)'), (Failure('Missing end of comment tag'),), None) + ], + 'block_begin': [ + (c(e(environment.block_end_string)), 'block_end', '#pop') + ] + tag_rules, + 'variable_begin': [ + (c(e(environment.variable_end_string)), 'variable_end', + '#pop') + ] + tag_rules + } + + def tokenize(self, source): + """ + Simple tokenize function that yields ``(position, type, contents)`` + tuples. Wrap the generator returned by this function in a + `TokenStream` to get real token instances and be able to push tokens + back to the stream. That's for example done by the parser. + """ + return TokenStream(self.tokeniter(source)) + + def tokeniter(self, source): + """ + This method tokenizes the text and returns the tokens in a generator. + Normally it's a better idea to use the `tokenize` function which + returns a `TokenStream` but in some situations it can be useful + to use this function since it can be marginally faster. + """ + pos = 0 + stack = ['root'] + statetokens = self.rules['root'] + source_length = len(source) + + while True: + # tokenizer loop + for regex, tokens, new_state in statetokens: + m = regex.match(source, pos) + if m: + # tuples support more options + if isinstance(tokens, tuple): + for idx, token in enumerate(tokens): + # hidden group + if token is None: + continue + # failure group + elif isinstance(token, Failure): + raise token(m.start(idx + 1)) + # bygroup is a bit more complex, in that case we + # yield for the current token the first named + # group that matched + elif token == '#bygroup': + for key, value in m.groupdict().iteritems(): + if value is not None: + yield m.start(key), key, value + break + else: + raise RuntimeError('%r wanted to resolve ' + 'the token dynamically' + ' but no group matched' + % regex) + # normal group + else: + data = m.group(idx + 1) + if data: + yield m.start(idx + 1), token, data + # strings as token just are yielded as it, but just + # if the data is not empty + else: + data = m.group() + if tokens is not None: + if data: + yield pos, tokens, data + # fetch new position into new variable so that we can check + # if there is a internal parsing error which would result + # in an infinite loop + pos2 = m.end() + # handle state changes + if new_state is not None: + # remove the uppermost state + if new_state == '#pop': + stack.pop() + # resolve the new state by group checking + elif new_state == '#bygroup': + for key, value in m.groupdict().iteritems(): + if value is not None: + stack.append(key) + break + else: + raise RuntimeError('%r wanted to resolve the ' + 'new state dynamically but' + ' no group matched' % + regex) + # direct state name given + else: + stack.append(new_state) + statetokens = self.rules[stack[-1]] + # we are still at the same position and no stack change. + # this means a loop without break condition, avoid that and + # raise error + elif pos2 == pos: + raise RuntimeError('%r yielded empty string without ' + 'stack change' % regex) + # publish new function and start again + pos = pos2 + break + # if loop terminated without break we havn't found a single match + # either we are at the end of the file or we have a problem + else: + # end of text + if pos >= source_length: + return + # something went wrong + raise TemplateSyntaxError('unexpected char %r at %d' % + (source[pos], pos), pos) diff --git a/jinja/nodes.py b/jinja/nodes.py new file mode 100644 index 0000000..580310a --- /dev/null +++ b/jinja/nodes.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +""" + jinja.nodes + ~~~~~~~~~~~ + + Additional nodes for jinja. Look like nodes from the ast. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from compiler.ast import Node + + +class Text(Node): + """ + Node that represents normal text. + """ + + def __init__(self, pos, text): + self.pos = pos + self.text = text + + def __repr__(self): + return 'Text(%r)' % (self.text,) + + +class NodeList(list, Node): + """ + A node that stores multiple childnodes. + """ + + def __init__(self, pos, data=None): + self.pos = pos + list.__init__(self, data or ()) + + def __repr__(self): + return 'NodeList(%s)' % list.__repr__(self) + + +class ForLoop(Node): + """ + A node that represents a for loop + """ + + def __init__(self, pos, item, seq, body, else_): + self.pos = pos + self.item = item + self.seq = seq + self.body = body + self.else_ = else_ + + def __repr__(self): + return 'ForLoop(%r, %r, %r, %r)' % ( + self.item, + self.seq, + self.body, + self.else_ + ) + + +class IfCondition(Node): + """ + A node that represents an if condition. + """ + + def __init__(self, pos, test, body, else_): + self.pos = pos + self.test = test + self.body = body + self.else_ = else_ + + def __repr__(self): + return 'IfCondition(%r, %r, %r)' % ( + self.test, + self.body, + self.else_ + ) + + +class Print(Node): + """ + A node that represents variable tags and print calls + """ + + def __init__(self, pos, variable): + self.pos = pos + self.variable = variable + + def __repr__(self): + return 'Print(%r)' % (self.variable,) diff --git a/jinja/parser.py b/jinja/parser.py new file mode 100644 index 0000000..7ab0fe8 --- /dev/null +++ b/jinja/parser.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +""" + jinja.parser + ~~~~~~~~~~~~ + + Implements the template parser. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +import re +from compiler import ast, parse +from jinja import nodes +from jinja.datastructure import TokenStream +from jinja.exceptions import TemplateSyntaxError + + +# callback functions for the subparse method +end_of_block = lambda p, t, d: t == 'block_end' +end_of_variable = lambda p, t, d: t == 'variable_end' +switch_for = lambda p, t, d: t == 'name' and d in ('else', 'endfor') +end_of_for = lambda p, t, d: t == 'name' and d == 'endfor' +switch_if = lambda p, t, d: t == 'name' and d in ('else', 'endif') +end_of_if = lambda p, t, d: t == 'name' and d == 'endif' + + +class Parser(object): + """ + The template parser class. + + Transforms sourcecode into an abstract syntax tree:: + + >>> parse("{% for item in seq|reversed %}{{ item }}{% endfor %}") + Document(ForLoop(AssignName('item'), Filter(Name('seq'), Name('reversed')), + Print('item'), None)) + >>> parse("{% if true %}foo{% else %}bar{% endif %}") + Document(IfCondition(Name('true'), Data('foo'), Data('bar'))) + >>> parse("{% if false %}...{% elif 0 > 1 %}...{% else %}...{% endif %}") + Document(IfCondition(Name('false'), Data('...'), + IfCondition(Compare('>', Const(0), Const(1)), + Data('...'), Data('...')))) + """ + + def __init__(self, environment, source, filename=None): + self.environment = environment + if isinstance(source, str): + source = source.decode(environment.template_charset, 'ignore') + self.source = source + self.filename = filename + self.tokenstream = environment.lexer.tokenize(source) + self._parsed = False + + self.directives = { + 'for': self.handle_for_directive, + 'if': self.handle_if_directive, + 'print': self.handle_print_directive + } + + def handle_for_directive(self, pos, gen): + """ + Handle a for directive and return a ForLoop node + """ + ast = self.parse_python(pos, gen, 'for %s:pass\nelse:pass') + body = self.subparse(switch_for) + + # do we have an else section? + if self.tokenstream.next()[2] == 'else': + self.close_remaining_block() + else_ = self.subparse(end_of_for, True) + else: + else_ = None + self.close_remaining_block() + + return nodes.ForLoop(pos, ast.assign, ast.list, body, else_) + + def handle_if_directive(self, pos, gen): + """ + Handle if/else blocks. elif is not supported by now. + """ + ast = self.parse_python(pos, gen, 'if %s:pass\nelse:pass') + body = self.subparse(switch_if) + + # do we have an else section? + if self.tokenstream.next()[2] == 'else': + self.close_remaining_block() + else_ = self.subparse(end_of_if, True) + else: + else_ = None + self.close_remaining_block() + + return nodes.IfCondition(pos, ast.tests[0][0], body, else_) + + def handle_print_directive(self, pos, gen): + """ + Handle {{ foo }} and {% print foo %}. + """ + ast = self.parse_python(pos, gen, 'print_(%s)') + # ast is something like Discard(CallFunc(Name('print_'), ...)) + # so just use the args + arguments = ast.expr.args + # we only accept one argument + if len(arguments) != 1: + raise TemplateSyntaxError('invalid argument count for print; ' + 'print requires exactly one argument, ' + 'got %d.' % len(arguments), pos) + return nodes.Print(pos, arguments[0]) + + def parse_python(self, pos, gen, template='%s'): + """ + Convert the passed generator into a flat string representing + python sourcecode and return an ast node or raise a + TemplateSyntaxError. + """ + tokens = [] + for t_pos, t_token, t_data in gen: + if t_token == 'string': + tokens.append('u' + t_data) + else: + tokens.append(t_data) + source = '\xef\xbb\xbf' + (template % (u' '.join(tokens)).encode('utf-8')) + try: + ast = parse(source, 'exec') + except SyntaxError, e: + raise TemplateSyntaxError(str(e), pos + e.offset - 1) + assert len(ast.node.nodes) == 1, 'get %d nodes, 1 expected' % len(ast.node.nodes) + return ast.node.nodes[0] + + def parse(self): + """ + Parse the template and return a nodelist. + """ + return self.subparse(None) + + def subparse(self, test, drop_needle=False): + """ + Helper function used to parse the sourcecode until the test + function which is passed a tuple in the form (pos, token, data) + returns True. In that case the current token is pushed back to + the tokenstream and the generator ends. + + The test function is only called for the first token after a + block tag. Variable tags are *not* aliases for {% print %} in + that case. + + If drop_needle is True the needle_token is removed from the tokenstream. + """ + def finish(): + """Helper function to remove unused nodelists.""" + if len(result) == 1: + return result[0] + return result + + pos = self.tokenstream.last[0] + result = nodes.NodeList(pos) + for pos, token, data in self.tokenstream: + # this token marks the begin or a variable section. + # parse everything till the end of it. + if token == 'variable_begin': + gen = self.tokenstream.fetch_until(end_of_variable, True) + result.append(self.directives['print'](pos, gen)) + + # this token marks the start of a block. like for variables + # just parse everything until the end of the block + elif token == 'block_begin': + gen = self.tokenstream.fetch_until(end_of_block, True) + try: + pos, token, data = gen.next() + except StopIteration: + raise TemplateSyntaxError('unexpected end of block', pos) + + # first token *must* be a name token + if token != 'name': + raise TemplateSyntaxError('unexpected %r token' % token, pos) + + # if a test function is passed to subparse we check if we + # reached the end of such a requested block. + if test is not None and test(pos, token, data): + if not drop_needle: + self.tokenstream.push(pos, token, data) + return finish() + + # the first token tells us which directive we want to call. + # if if doesn't match any existing directive it's like a + # template syntax error. + if data in self.directives: + node = self.directives[data](pos, gen) + else: + raise TemplateSyntaxError('unknown directive %r' % data, pos) + result.append(node) + + # here the only token we should get is "data". all other + # tokens just exist in block or variable sections. (if the + # tokenizer is not brocken) + elif token == 'data': + result.append(nodes.Text(pos, data)) + + # so this should be unreachable code + else: + raise AssertionError('unexpected token %r' % token) + + # still here and a test function is provided? raise and error + if test is not None: + raise TemplateSyntaxError('unexpected end of template', pos) + return finish() + + def close_remaining_block(self): + """ + If we opened a block tag because one of our tags requires an end + tag we can use this method to drop the rest of the block from + the stream. If the next token isn't the block end we throw an + error. + """ + pos = self.tokenstream.last[0] + try: + pos, token, data = self.tokenstream.next() + except StopIteration: + raise TemplateSyntaxError('missing closing tag', pos) + if token != 'block_end': + raise TemplateSyntaxError('expected close tag, found %r' % token, pos) diff --git a/jinja/translators/__init__.py b/jinja/translators/__init__.py new file mode 100644 index 0000000..b92b79e --- /dev/null +++ b/jinja/translators/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +""" + jinja.translators + ~~~~~~~~~~~~~~~~~ + + The submodules of this module provide translators for the jinja ast + which basically just is the python ast with a few more nodes. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" diff --git a/jinja/translators/python.py b/jinja/translators/python.py new file mode 100644 index 0000000..927f892 --- /dev/null +++ b/jinja/translators/python.py @@ -0,0 +1,351 @@ +# -*- coding: utf-8 -*- +""" + jinja.translators.python + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This module translates a jinja ast into python code. + + :copyright: 2006 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from compiler import ast +from jinja import nodes + + +class PythonTranslator(object): + """ + Pass this translator a ast tree to get valid python code. + """ + + def __init__(self, environment, node): + self.environment = environment + self.node = node + self.indention = 0 + self.last_pos = 0 + + self.handlers = { + # jinja nodes + nodes.Text: self.handle_template_text, + nodes.NodeList: self.handle_node_list, + nodes.ForLoop: self.handle_for_loop, + nodes.IfCondition: self.handle_if_condition, + nodes.Print: self.handle_print, + # used python nodes + ast.Name: self.handle_name, + ast.AssName: self.handle_name, + ast.Compare: self.handle_compare, + ast.Const: self.handle_const, + ast.Subscript: self.handle_subscript, + ast.Getattr: self.handle_getattr, + ast.AssTuple: self.handle_ass_tuple, + ast.Bitor: self.handle_bitor, + ast.CallFunc: self.handle_call_func, + ast.Add: self.handle_add, + ast.Sub: self.handle_sub, + ast.Div: self.handle_div, + ast.Mul: self.handle_mul, + ast.Mod: self.handle_mod, + ast.UnarySub: self.handle_unary_sub, + ast.Power: self.handle_power, + ast.Dict: self.handle_dict, + ast.List: self.handle_list, + ast.Tuple: self.handle_list, + ast.And: self.handle_and, + ast.Or: self.handle_or, + ast.Not: self.handle_not + } + + def indent(self, text): + """ + Indent the current text. + """ + return (' ' * (self.indention * 4)) + text + + def handle_node(self, node): + """ + Handle one node + """ + if node.__class__ in self.handlers: + out = self.handlers[node.__class__](node) + else: + raise AssertionError('unhandled node %r' % node.__class__) + return out + + # -- jinja nodes + + def handle_node_list(self, node): + """ + In some situations we might have a node list. It's just + a collection of multiple statements. + """ + self.last_pos = node.pos + buf = [] + for n in node: + buf.append(self.handle_node(n)) + return '\n'.join(buf) + + def handle_for_loop(self, node): + """ + Handle a for loop. Pretty basic, just that we give the else + clause a different behavior. + """ + self.last_pos = node.pos + buf = [] + write = lambda x: buf.append(self.indent(x)) + write('context.push()') + write('parent_loop = context[\'loop\']') + write('loop_data = None') + write('for (loop_data, %s) in environment.iterate(%s):' % ( + self.handle_node(node.item), + self.handle_node(node.seq) + )) + self.indention += 1 + write('loop_data.parent = parent_loop') + write('context[\'loop\'] = loop_data') + buf.append(self.handle_node(node.body)) + self.indention -= 1 + if node.else_ is not None: + write('if loop_data is None:') + self.indention += 1 + buf.append(self.handle_node(node.else_)) + self.indention -= 1 + write('context.pop()') + return '\n'.join(buf) + + def handle_if_condition(self, node): + """ + Handle an if condition node. + """ + self.last_pos = node.pos + buf = [] + write = lambda x: buf.append(self.indent(x)) + write('if %s:' % self.handle_node(node.test)) + self.indention += 1 + buf.append(self.handle_node(node.body)) + self.indention -= 1 + if node.else_ is not None: + write('else:') + self.indention += 1 + buf.append(self.handle_node(node.else_)) + self.indention -= 1 + return '\n'.join(buf) + + def handle_print(self, node): + """ + Handle a print statement. + """ + self.last_pos = node.pos + return self.indent('write_var(%s)' % self.handle_node(node.variable)) + + def handle_template_text(self, node): + """ + Handle data around nodes. + """ + self.last_pos = node.pos + return self.indent('write(%r)' % node.text) + + # -- python nodes + + def handle_name(self, node): + """ + Handle name assignments and name retreivement. + """ + return 'context[%r]' % node.name + + def handle_compare(self, node): + """ + Any sort of comparison + """ + buf = [] + buf.append(self.handle_node(node.expr)) + for op, n in node.ops: + buf.append(op) + buf.append(self.handle_node(n)) + return ' '.join(buf) + + def handle_const(self, node): + """ + Constant values in expressions. + """ + return repr(node.value) + + def handle_subscript(self, node): + """ + Handle variable based attribute access foo['bar']. + """ + if len(node.subs) != 1: + raise TemplateSyntaxError('attribute access requires one argument', + self.last_pos) + assert node.flags != 'OP_DELETE', 'wtf? do we support that?' + return 'get_attribute(%s, %s)' % ( + self.handle_node(node.expr), + self.handle_node(node.subs[0]) + ) + + def handle_getattr(self, node): + """ + Handle hardcoded attribute access. foo.bar + """ + return 'get_attribute(%s, %r)' % ( + self.handle_node(node.expr), + node.attrname + ) + + def handle_ass_tuple(self, node): + """ + Tuple unpacking loops. + """ + return '(%s)' % ', '.join([self.handle_node(n) for n in node.nodes]) + + def handle_bitor(self, node): + """ + We use the pipe operator for filtering. + """ + return 'environment.apply_filters(%s, %r)' % ( + self.handle_node(node.nodes[0]), + [self.handle_node(n) for n in node.nodes[1:]] + ) + + def handle_call_func(self, node): + """ + Handle function calls. + """ + args = [] + kwargs = {} + star_args = dstar_args = None + if node.star_args is not None: + star_args = self.handle_node(node.star_args) + if node.dstar_args is not None: + dstar_args = self.handle_node(node.dstar_args) + for arg in node.args: + if arg.__class__ is ast.Keyword: + kwargs[arg.name] = self.handle_node(arg.expr) + else: + args.append(self.handle_node(arg)) + return 'environment.call_function(%s, [%s], {%s}, %s, %s)' % ( + self.handle_node(node.node), + ', '.join(args), + ', '.join(['%r: %s' % i for i in kwargs.iteritems()]), + star_args, + dstar_args + ) + + def handle_add(self, node): + """ + Add two items. + """ + return '(%s + %s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_sub(self, node): + """ + Sub two items. + """ + return '(%s - %s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_div(self, node): + """ + Divide two items. + """ + return '(%s / %s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_mul(self, node): + """ + Multiply two items. + """ + return '(%s * %s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_mod(self, node): + """ + Apply modulo. + """ + return '(%s %% %s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_unary_sub(self, node): + """ + Make a number negative. + """ + return '(-%s)' % self.handle_node(node.expr) + + def handle_power(self, node): + """ + handle foo**bar + """ + return '(%s**%s)' % ( + self.handle_node(node.left), + self.handle_node(node.right) + ) + + def handle_dict(self, node): + """ + Dict constructor syntax. + """ + return '{%s}' % ', '.join([ + '%s: %s' % ( + self.handle_node(key), + self.handle_node(value) + ) for key, value in node.items + ]) + + def handle_list(self, node): + """ + We don't know tuples, tuples are lists for jinja. + """ + return '[%s]' % ', '.join([ + self.handle_node(n) for n in nodes + ]) + + def handle_and(self, node): + """ + Handle foo and bar. + """ + return ' and '.join([ + self.handle_node(n) for n in node.nodes + ]) + + def handle_or(self, node): + """ + handle foo or bar. + """ + return ' or '.join([ + self.handle_node(n) for n in self.nodse + ]) + + def handle_not(self, node): + """ + handle not operator. + """ + return 'not %s' % self.handle_node(node.expr) + + def translate(self): + self.indention = 1 + lines = [ + 'def generate(environment, context, write, write_var=None):', + ' """This function was automatically generated by', + ' the jinja python translator. do not edit."""', + ' if write_var is None:', + ' write_var = write' + ] + lines.append(self.handle_node(self.node)) + return '\n'.join(lines) + + +def translate(environment, node): + """ + Do the translation to python. + """ + return PythonTranslator(environment, node).translate() diff --git a/syntax.html b/syntax.html new file mode 100644 index 0000000..bc10e08 --- /dev/null +++ b/syntax.html @@ -0,0 +1,61 @@ +Jinja 1.0 Syntax +================ + +
    +{% for char in my_string|upper|replace(" ", "") %} +
  • {{ loop.index }} - {{ char|e }}
    • +{% endfor %} +
+ +{{ variable|strip(" ")|escape|replace("a", "b") }} + +{% if item == 42 %} + ... +{% endif %} + +{% if item|odd? %} + applies the odd? filter which returns true if the + item in is odd. +{% endif %} + +{{ item|e(true) }} -- escape the variable for attributes + +
    +{% for item in seq %} +
  • {{ item.current|e }} + {% if item.items %}
      {% recurse item.items %}
    {% endif %} +
    • +{% endfor %} +
+ +How a Filter Looks Like +======================= + +def replace(search, repl): + def wrapped(env, value): + return env.to_unicode(value).replace(search, repl) + return wrapped + + +def escape(attr=False): + def wrapped(env, value): + return cgi.escape(env.to_unicode(value), attr) + return wrapped + + +def odd(): + return lambda env, value: value % 2 == 1 +odd.__name__ = 'odd?' + + +@stringfilter +def replace(value, search, repl): + return value.replace(search, repl) + + +def stringfilter(f): + def decorator(*args): + def wrapped(env, value): + return f(env.to_unicode(value), *args) + return wrapped + return decorator diff --git a/syntax.txt b/syntax.txt new file mode 100644 index 0000000..381ec74 --- /dev/null +++ b/syntax.txt @@ -0,0 +1,22 @@ +For Loops +--------- + +Simple Syntax:: + + {% for in %} + + {% endfor %} + +Item Unpacking:: + + {% for , in %} + + {% endfor %} + +With else block:: + + {% for in %} + + {% else %} + + {% endfor %} diff --git a/test.html b/test.html new file mode 100644 index 0000000..f846b06 --- /dev/null +++ b/test.html @@ -0,0 +1,53 @@ + + + + {% block "page_title" %}Untitled{% endblock %} | My Webpage + + + + {% block html_head %}{% endblock %} + + + +
+ {% block body %} + content goes here. + {% endblock %} +
+ + + + diff --git a/test.py b/test.py new file mode 100644 index 0000000..e090dac --- /dev/null +++ b/test.py @@ -0,0 +1,7 @@ +from jinja.environment import Environment + +e = Environment() + +def test(x): + for pos, token, data in e.lexer.tokenize(x): + print '%-8d%-30r%-40r' % (pos, token, data) diff --git a/tests/mockup.txt b/tests/mockup.txt new file mode 100644 index 0000000..ff7e5b2 --- /dev/null +++ b/tests/mockup.txt @@ -0,0 +1,153 @@ +source index.html:: + + {% page extends='layout.html', defaultfilter='autoescape', + charset='utf-8' %} + + {% def title %}Index | {{ super.title }}{% enddef %} + {% def body %} +

Index

+

+ This is just a mockup template so that you can see how the new + jinja syntax looks like. It also shows the python output of this + template when compiled. +

+
    + {% for user in users %} +
  • {{ user.username }}
    • + {% endfor %} +
+ {% enddef %} + +source layout.html:: + + {% page defaultfilter='autoescape', charset='utf-8' %} + + + {% block title %}My Webpage{% endblock %} + + +
+

My Webpage

+

{{ self.title }}

+
+
+ {{ self.body }} +
+
+ {% block footer %} + Copyright 2007 by the Pocoo Team. + {% endblock %} +
+ + + +generated python code:: + + from jinja.runtime import Template, Markup + + class Layer1Template(Template): + + defaultfilter = 'autoescape' + + def p_title(self): + yield Markup(u'My Webpage') + + def p_footer(self): + yield Markup(u'\n Copyright 2007 by the Pocoo Team.\n ') + + def p_outer_body(self): + yield Markup(u'\n \n \n') + for i in self.resolve('self.title'): + yield i + yield Markup(u'\n \n \n
\n' + u'

My Webpage

\n

') + for i in self.resolve('self.title'): + yield i + yield Markup(u'

\n
\n
\n ') + for i in self.resolve('self.body'): + yield i + yield Markup(u'\n
\n
\n ') + for i in self.resolve('self.footer'): + yield i + yield Markup(u'\n
\n \nIndex\n

\n This is just a mockup ' + u'template so that you can see how the new jinja syntax ' + u'looks like. It also shows the python output of this ' + u'template when compiled.\n

\n
    \n ') + iteration_1 = self.start_iteration(['user'], 'users') + for _ in iteration_1: + yield Markup(u'\n
  • ') + for i in self.resolve('user.username'): + yield i + yield Markup(u'
    • \n ') + iteration_1.close() + + def get_template(context): + return Layer2Template(context) + +generated javascript code:: + + (function(c) { + var m = function(x) { return new Jinja.Markup(x) }; + + var l1 = function() {}; + l1.defaultfilter = 'autoescape'; + l1.prototype.p_title = function() { + this.yield(m('My Webpage')); + }; + l1.prototype.p_footer = function() { + this.yield(m('\n Copyright 2007 by the Pocoo Team.\n ')); + }; + l1.prototype.p_outer_body = function() { + this.yield(m('\n \n \n')); + this.resolve('self.title'); + this.yield(m('\n \n \n
    \n' + '

    My Webpage

    \n

    ')); + this.resolve('self.title'); + this.yield(m('

    \n
    \n
    \n ')); + this.resolve('self.body'); + this.yield(m('\n
    \n
    \n ')); + this.resolve('self.footer'); + this.yield(m('\n
    \n \nIndex\n

    \n This is just a mockup ' + 'template so that you can see how the new jinja syntax ' + 'looks like. It also shows the python output of this ' + 'template when compiled.\n

    \n
      \n ')); + var iteration_1 = this.startIteration(['user'], 'users'); + while (iteration_1.next()) { + this.yield(m('\n
    • ')); + this.resolve('user.username'); + this.yield('
      • \n '); + }; + iteration_1.close(); + }; + + return function(context) { + return new Jinja.Template(context, l1, l2); + }; + })(); diff --git a/tests/run.py b/tests/run.py new file mode 100644 index 0000000..b323b64 --- /dev/null +++ b/tests/run.py @@ -0,0 +1,15 @@ +from jinja import Environment, FileSystemLoader + +env = Environment( + loader=FileSystemLoader(['.'], suffix='') +) +env.filters['odd?'] = lambda c, x: x % 2 == 1 +env.filters['even?'] = lambda x, x: x % 2 == 0 + +t = env.get_template('index.html') +print t.render( + users=[ + dict(username='foo', user_id=1), + dict(username='bar', user_id=2) + ] +) -- 2.26.2