From 7f15ef846f86647eeb0f2cc14a06e48862423d63 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Fri, 16 May 2008 09:11:39 +0200 Subject: [PATCH] improved exception system. now both name (load name) and filename are passed. --HG-- branch : trunk --- docs/extensions.rst | 2 +- jinja2/compiler.py | 13 +++++---- jinja2/environment.py | 10 +++---- jinja2/exceptions.py | 9 ++++-- jinja2/ext.py | 22 ++++++--------- jinja2/lexer.py | 65 +++++++++++++++---------------------------- jinja2/parser.py | 48 ++++++++++++++++---------------- 7 files changed, 76 insertions(+), 93 deletions(-) diff --git a/docs/extensions.rst b/docs/extensions.rst index 4f9aa22..fd71922 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -167,7 +167,7 @@ extensions: .. autoclass:: jinja2.parser.Parser :members: parse_expression, parse_tuple, parse_assign_target, - parse_statements, free_identifier + parse_statements, free_identifier, fail .. attribute:: filename diff --git a/jinja2/compiler.py b/jinja2/compiler.py index 9f99045..52a85cd 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -597,7 +597,8 @@ class CodeGenerator(NodeVisitor): raise TemplateAssertionError('It\'s not possible to set and ' 'access variables derived from ' 'an outer scope! (affects: %s' % - vars, node.lineno, self.filename) + vars, node.lineno, self.name, + self.filename) # remove variables from a closure from the frame's undeclared # identifiers. @@ -648,7 +649,7 @@ class CodeGenerator(NodeVisitor): if block.name in self.blocks: raise TemplateAssertionError('block %r defined twice' % block.name, block.lineno, - self.filename) + self.name, self.filename) self.blocks[block.name] = block # find all imports and import them @@ -751,7 +752,7 @@ class CodeGenerator(NodeVisitor): if not frame.toplevel: raise TemplateAssertionError('cannot use extend from a non ' 'top-level scope', node.lineno, - self.filename) + self.name, self.filename) # if the number of extends statements in general is zero so # far, we don't have to add a check if something extended @@ -1392,7 +1393,8 @@ class CodeGenerator(NodeVisitor): func = self.environment.filters.get(node.name) if func is None: raise TemplateAssertionError('no filter named %r' % node.name, - node.lineno, self.filename) + node.lineno, self.name, + self.filename) if getattr(func, 'contextfilter', False): self.write('context, ') elif getattr(func, 'environmentfilter', False): @@ -1410,7 +1412,8 @@ class CodeGenerator(NodeVisitor): self.write(self.tests[node.name] + '(') if node.name not in self.environment.tests: raise TemplateAssertionError('no test named %r' % node.name, - node.lineno, self.filename) + node.lineno, self.name, + self.filename) self.visit(node.node, frame) self.signature(node, frame) self.write(')') diff --git a/jinja2/environment.py b/jinja2/environment.py index f64b150..3f6855a 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -288,7 +288,7 @@ class Environment(object): except (TypeError, LookupError): return self.undefined(obj=obj, name=argument) - def parse(self, source, filename=None): + def parse(self, source, name=None, filename=None): """Parse the sourcecode and return the abstract syntax tree. This tree of nodes is used by the compiler to convert the template into executable source- or bytecode. This is useful for debugging or to @@ -298,19 +298,19 @@ class Environment(object): this gives you a good overview of the node tree generated. """ try: - return Parser(self, source, filename).parse() + return Parser(self, source, name, filename).parse() except TemplateSyntaxError, e: from jinja2.debug import translate_syntax_error exc_type, exc_value, tb = translate_syntax_error(e) raise exc_type, exc_value, tb - def lex(self, source, filename=None): + def lex(self, source, name=None, filename=None): """Lex the given sourcecode and return a generator that yields tokens as tuples in the form ``(lineno, token_type, value)``. This can be useful for :ref:`extension development ` and debugging templates. """ - return self.lexer.tokeniter(source, filename) + return self.lexer.tokeniter(source, name, filename) def compile(self, source, name=None, filename=None, raw=False): """Compile a node or template source code. The `name` parameter is @@ -326,7 +326,7 @@ class Environment(object): mainly used internally. """ if isinstance(source, basestring): - source = self.parse(source, filename) + source = self.parse(source, name, filename) if self.optimized: node = optimize(source, self) source = generate(node, self, name, filename) diff --git a/jinja2/exceptions.py b/jinja2/exceptions.py index 2653952..147ddb9 100644 --- a/jinja2/exceptions.py +++ b/jinja2/exceptions.py @@ -35,10 +35,15 @@ class TemplateNotFound(IOError, LookupError, TemplateError): class TemplateSyntaxError(TemplateError): """Raised to tell the user that there is a problem with the template.""" - def __init__(self, message, lineno, filename): - TemplateError.__init__(self, '%s (line %s)' % (message, lineno)) + def __init__(self, message, lineno, name=None, filename=None): + if name is not None: + extra = '%s, line %d' % (name, lineno) + else: + extra = 'line %d' % lineno + TemplateError.__init__(self, '%s (%s)' % (message, extra)) self.message = message self.lineno = lineno + self.name = name self.filename = filename diff --git a/jinja2/ext.py b/jinja2/ext.py index bd64cce..a489f97 100644 --- a/jinja2/ext.py +++ b/jinja2/ext.py @@ -149,9 +149,9 @@ class InternationalizationExtension(Extension): name = parser.stream.expect('name') if name.value in variables: - raise TemplateAssertionError('translatable variable %r defined ' - 'twice.' % name.value, name.lineno, - parser.filename) + parser.fail('translatable variable %r defined twice.' % + name.value, name.lineno, + exc=TemplateAssertionError) # expressions if parser.stream.current.type is 'assign': @@ -202,8 +202,7 @@ class InternationalizationExtension(Extension): if not have_plural: plural_expr = None elif plural_expr is None: - raise TemplateAssertionError('pluralize without variables', - lineno, parser.filename) + parser.fail('pluralize without variables', lineno) if variables: variables = nodes.Dict([nodes.Pair(nodes.Const(x, lineno=lineno), y) @@ -236,15 +235,10 @@ class InternationalizationExtension(Extension): elif parser.stream.current.test('name:pluralize'): if allow_pluralize: break - raise TemplateSyntaxError('a translatable section can ' - 'have only one pluralize ' - 'section', - parser.stream.current.lineno, - parser.filename) - raise TemplateSyntaxError('control structures in translatable' - ' sections are not allowed.', - parser.stream.current.lineno, - parser.filename) + parser.fail('a translatable section can have only one ' + 'pluralize section') + parser.fail('control structures in translatable sections are ' + 'not allowed') else: assert False, 'internal parser error' diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 639b285..2719dcc 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -69,37 +69,6 @@ assert len(operators) == len(reverse_operators), 'operators dropped' operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))) -simple_escapes = { - 'a': '\a', - 'n': '\n', - 'r': '\r', - 'f': '\f', - 't': '\t', - 'v': '\v', - '\\': '\\', - '"': '"', - "'": "'", - '0': '\x00' -} -unicode_escapes = { - 'x': 2, - 'u': 4, - 'U': 8 -} - - -def unescape_string(lineno, filename, s): - r"""Unescape a string. Supported escapes: - \a, \n, \r\, \f, \v, \\, \", \', \0 - - \x00, \u0000, \U00000000, \N{...} - """ - try: - return s.encode('ascii', 'backslashreplace').decode('unicode-escape') - except UnicodeError, e: - msg = str(e).split(':')[-1].strip() - raise TemplateSyntaxError(msg, lineno, filename) - class Failure(object): """Class that raises a `TemplateSyntaxError` if called. @@ -186,10 +155,11 @@ class TokenStream(object): one token ahead. The current active token is stored as :attr:`current`. """ - def __init__(self, generator, filename): + def __init__(self, generator, name, filename): self._next = generator.next self._pushed = deque() self.current = Token(1, 'initial', '') + self.name = name self.filename = filename self.next() @@ -258,11 +228,11 @@ class TokenStream(object): raise TemplateSyntaxError('unexpected end of template, ' 'expected %r.' % expr, self.current.lineno, - self.filename) + self.name, self.filename) raise TemplateSyntaxError("expected token %r, got %r" % (expr, str(self.current)), self.current.lineno, - self.filename) + self.name, self.filename) try: return self.current finally: @@ -398,7 +368,7 @@ class Lexer(object): ] + tag_rules } - def tokenize(self, source, filename=None): + def tokenize(self, source, name=None, filename=None): """Works like `tokeniter` but returns a tokenstream of tokens and not a generator or token tuples. Additionally all token values are already converted into types and postprocessed. For example comments are removed, @@ -406,7 +376,7 @@ class Lexer(object): """ source = unicode(source) def generate(): - for lineno, token, value in self.tokeniter(source, filename): + for lineno, token, value in self.tokeniter(source, name, filename): if token in ('comment_begin', 'comment', 'comment_end'): continue elif token == 'linestatement_begin': @@ -426,7 +396,17 @@ class Lexer(object): elif token == 'name': value = str(value) elif token == 'string': - value = unescape_string(lineno, filename, value[1:-1]) + # try to unescape string + try: + value = value[1:-1] \ + .encode('ascii', 'backslashreplace') \ + .decode('unicode-escape') + except Exception, e: + msg = str(e).split(':')[-1].strip() + raise TemplateSyntaxError(msg, lineno, name, filename) + # if we can express it as bytestring (ascii only) + # we do that for support of semi broken APIs + # as datetime.datetime.strftime try: value = str(value) except UnicodeError: @@ -438,9 +418,9 @@ class Lexer(object): elif token == 'operator': token = operators[value] yield Token(lineno, token, value) - return TokenStream(generate(), filename) + return TokenStream(generate(), name, filename) - def tokeniter(self, source, filename=None): + def tokeniter(self, source, name, filename=None): """This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template. The output you get is not compatible with the input the jinja parser @@ -520,14 +500,15 @@ class Lexer(object): elif data in ('}', ')', ']'): if not balancing_stack: raise TemplateSyntaxError('unexpected "%s"' % - data, lineno, + data, lineno, name, filename) expected_op = balancing_stack.pop() if expected_op != data: raise TemplateSyntaxError('unexpected "%s", ' 'expected "%s"' % (data, expected_op), - lineno, filename) + lineno, name, + filename) # yield items if tokens is not None: yield lineno, tokens, data @@ -576,4 +557,4 @@ class Lexer(object): # something went wrong raise TemplateSyntaxError('unexpected char %r at %d' % (source[pos], pos), lineno, - filename) + name, filename) diff --git a/jinja2/parser.py b/jinja2/parser.py index d9bf492..70c1cdc 100644 --- a/jinja2/parser.py +++ b/jinja2/parser.py @@ -23,11 +23,12 @@ class Parser(object): extensions and can be used to parse expressions or statements. """ - def __init__(self, environment, source, filename=None): + def __init__(self, environment, source, name=None, filename=None): self.environment = environment if isinstance(filename, unicode): filename = filename.encode('utf-8') self.source = unicode(source) + self.name = name self.filename = filename self.closed = False self.stream = environment.lexer.tokenize(self.source, filename) @@ -37,6 +38,15 @@ class Parser(object): self.extensions[tag] = extension.parse self._last_identifier = 0 + def fail(self, msg, lineno=None, exc=TemplateSyntaxError): + """Convenience method that raises `exc` with the message, passed + line number or last line number as well as the current name and + filename. + """ + if lineno is None: + lineno = self.stream.current.lineno + raise TemplateSyntaxError(msg, lineno, self.name, self.filename) + def is_tuple_end(self, extra_end_rules=None): """Are we at the end of a tuple?""" if self.stream.current.type in ('variable_end', 'block_end', 'rparen'): @@ -56,8 +66,7 @@ class Parser(object): """Parse a single statement.""" token = self.stream.current if token.type is not 'name': - raise TemplateSyntaxError('tag name expected', token.lineno, - self.filename) + self.fail('tag name expected', token.lineno) if token.value in _statement_keywords: return getattr(self, 'parse_' + self.stream.current.value)() if token.value == 'call': @@ -67,8 +76,7 @@ class Parser(object): ext = self.extensions.get(token.value) if ext is not None: return ext(self) - raise TemplateSyntaxError('unknown tag %r' % token.value, - token.lineno, self.filename) + self.fail('unknown tag %r' % token.value, token.lineno) def parse_statements(self, end_tokens, drop_needle=False): """Parse multiple statements into a list until one of the end tokens @@ -194,10 +202,9 @@ class Parser(object): break target = self.parse_assign_target(name_only=True) if target.name.startswith('__'): - raise TemplateAssertionError('names starting with two ' - 'underscores can not be ' - 'imported', target.lineno, - self.filename) + self.fail('names starting with two underscores can not ' + 'be imported', target.lineno, + exc=TemplateAssertionError) if self.stream.skip_if('name:as'): alias = self.parse_assign_target(name_only=True) node.names.append((target.name, alias.name)) @@ -236,8 +243,7 @@ class Parser(object): node.call = self.parse_expression() if not isinstance(node.call, nodes.Call): - raise TemplateSyntaxError('expected call', node.lineno, - self.filename) + self.fail('expected call', node.lineno) node.body = self.parse_statements(('name:endcall',), drop_needle=True) return node @@ -285,9 +291,8 @@ class Parser(object): target = self.parse_primary(with_postfix=False) target.set_ctx('store') if not target.can_assign(): - raise TemplateSyntaxError('can\'t assign to %r' % - target.__class__.__name__.lower(), - target.lineno, self.filename) + self.fail('can\'t assign to %r' % target.__class__. + __name__.lower(), target.lineno) return target def parse_expression(self, with_condexpr=True): @@ -469,9 +474,7 @@ class Parser(object): elif token.type is 'lbrace': node = self.parse_dict() else: - raise TemplateSyntaxError("unexpected token '%s'" % - (token,), token.lineno, - self.filename) + self.fail("unexpected token '%s'" % (token,), token.lineno) if with_postfix: node = self.parse_postfix(node) return node @@ -563,8 +566,7 @@ class Parser(object): if token.type is 'dot': attr_token = self.stream.current if attr_token.type not in ('name', 'integer'): - raise TemplateSyntaxError('expected name or number', - attr_token.lineno, self.filename) + self.fail('expected name or number', attr_token.lineno) arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) self.stream.next() elif token.type is 'lbracket': @@ -579,8 +581,7 @@ class Parser(object): else: arg = nodes.Tuple(args, self.lineno, self.filename) else: - raise TemplateSyntaxError('expected subscript expression', - self.lineno, self.filename) + self.fail('expected subscript expression', self.lineno) return nodes.Subscript(node, arg, 'load', lineno=token.lineno) def parse_subscribed(self): @@ -623,9 +624,8 @@ class Parser(object): def ensure(expr): if not expr: - raise TemplateSyntaxError('invalid syntax for function ' - 'call expression', token.lineno, - self.filename) + self.fail('invalid syntax for function call expression', + token.lineno) while self.stream.current.type is not 'rparen': if require_comma: -- 2.26.2