jinja2/parser.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.parser
   4     ~~~~~~~~~~~~~
   5
   6     Implements the template parser.
   7
   8     :copyright: (c) 2010 by the Jinja Team.
   9     :license: BSD, see LICENSE for more details.
  10 """
  11 from jinja2 import nodes
  12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
  13 from jinja2.utils import next
  14 from jinja2.lexer import describe_token, describe_token_expr
  15
  16
  17 #: statements that callinto
  18 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
  19                                  'macro', 'include', 'from', 'import',
  20                                  'set'])
  21 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
  22
  23
  24 class Parser(object):
  25     """This is the central parsing class Jinja2 uses.  It's passed to
  26     extensions and can be used to parse expressions or statements.
  27     """
  28
  29     def __init__(self, environment, source, name=None, filename=None,
  30                  state=None):
  31         self.environment = environment
  32         self.stream = environment._tokenize(source, name, filename, state)
  33         self.name = name
  34         self.filename = filename
  35         self.closed = False
  36         self.extensions = {}
  37         for extension in environment.iter_extensions():
  38             for tag in extension.tags:
  39                 self.extensions[tag] = extension.parse
  40         self._last_identifier = 0
  41         self._tag_stack = []
  42         self._end_token_stack = []
  43
  44     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  45         """Convenience method that raises `exc` with the message, passed
  46         line number or last line number as well as the current name and
  47         filename.
  48         """
  49         if lineno is None:
  50             lineno = self.stream.current.lineno
  51         raise exc(msg, lineno, self.name, self.filename)
  52
  53     def _fail_ut_eof(self, name, end_token_stack, lineno):
  54         expected = []
  55         for exprs in end_token_stack:
  56             expected.extend(map(describe_token_expr, exprs))
  57         if end_token_stack:
  58             currently_looking = ' or '.join(
  59                 "'%s'" % describe_token_expr(expr)
  60                 for expr in end_token_stack[-1])
  61         else:
  62             currently_looking = None
  63
  64         if name is None:
  65             message = ['Unexpected end of template.']
  66         else:
  67             message = ['Encountered unknown tag \'%s\'.' % name]
  68
  69         if currently_looking:
  70             if name is not None and name in expected:
  71                 message.append('You probably made a nesting mistake. Jinja '
  72                                'is expecting this tag, but currently looking '
  73                                'for %s.' % currently_looking)
  74             else:
  75                 message.append('Jinja was looking for the following tags: '
  76                                '%s.' % currently_looking)
  77
  78         if self._tag_stack:
  79             message.append('The innermost block that needs to be '
  80                            'closed is \'%s\'.' % self._tag_stack[-1])
  81
  82         self.fail(' '.join(message), lineno)
  83
  84     def fail_unknown_tag(self, name, lineno=None):
  85         """Called if the parser encounters an unknown tag.  Tries to fail
  86         with a human readable error message that could help to identify
  87         the problem.
  88         """
  89         return self._fail_ut_eof(name, self._end_token_stack, lineno)
  90
  91     def fail_eof(self, end_tokens=None, lineno=None):
  92         """Like fail_unknown_tag but for end of template situations."""
  93         stack = list(self._end_token_stack)
  94         if end_tokens is not None:
  95             stack.append(end_tokens)
  96         return self._fail_ut_eof(None, stack, lineno)
  97
  98     def is_tuple_end(self, extra_end_rules=None):
  99         """Are we at the end of a tuple?"""
 100         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
 101             return True
 102         elif extra_end_rules is not None:
 103             return self.stream.current.test_any(extra_end_rules)
 104         return False
 105
 106     def free_identifier(self, lineno=None):
 107         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 108         self._last_identifier += 1
 109         rv = object.__new__(nodes.InternalName)
 110         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
 111         return rv
 112
 113     def parse_statement(self):
 114         """Parse a single statement."""
 115         token = self.stream.current
 116         if token.type != 'name':
 117             self.fail('tag name expected', token.lineno)
 118         self._tag_stack.append(token.value)
 119         pop_tag = True
 120         try:
 121             if token.value in _statement_keywords:
 122                 return getattr(self, 'parse_' + self.stream.current.value)()
 123             if token.value == 'call':
 124                 return self.parse_call_block()
 125             if token.value == 'filter':
 126                 return self.parse_filter_block()
 127             ext = self.extensions.get(token.value)
 128             if ext is not None:
 129                 return ext(self)
 130
 131             # did not work out, remove the token we pushed by accident
 132             # from the stack so that the unknown tag fail function can
 133             # produce a proper error message.
 134             self._tag_stack.pop()
 135             pop_tag = False
 136             self.fail_unknown_tag(token.value, token.lineno)
 137         finally:
 138             if pop_tag:
 139                 self._tag_stack.pop()
 140
 141     def parse_statements(self, end_tokens, drop_needle=False):
 142         """Parse multiple statements into a list until one of the end tokens
 143         is reached.  This is used to parse the body of statements as it also
 144         parses template data if appropriate.  The parser checks first if the
 145         current token is a colon and skips it if there is one.  Then it checks
 146         for the block end and parses until if one of the `end_tokens` is
 147         reached.  Per default the active token in the stream at the end of
 148         the call is the matched end token.  If this is not wanted `drop_needle`
 149         can be set to `True` and the end token is removed.
 150         """
 151         # the first token may be a colon for python compatibility
 152         self.stream.skip_if('colon')
 153
 154         # in the future it would be possible to add whole code sections
 155         # by adding some sort of end of statement token and parsing those here.
 156         self.stream.expect('block_end')
 157         result = self.subparse(end_tokens)
 158
 159         # we reached the end of the template too early, the subparser
 160         # does not check for this, so we do that now
 161         if self.stream.current.type == 'eof':
 162             self.fail_eof(end_tokens)
 163
 164         if drop_needle:
 165             next(self.stream)
 166         return result
 167
 168     def parse_set(self):
 169         """Parse an assign statement."""
 170         lineno = next(self.stream).lineno
 171         target = self.parse_assign_target()
 172         self.stream.expect('assign')
 173         expr = self.parse_tuple()
 174         return nodes.Assign(target, expr, lineno=lineno)
 175
 176     def parse_for(self):
 177         """Parse a for loop."""
 178         lineno = self.stream.expect('name:for').lineno
 179         target = self.parse_assign_target(extra_end_rules=('name:in',))
 180         self.stream.expect('name:in')
 181         iter = self.parse_tuple(with_condexpr=False,
 182                                 extra_end_rules=('name:recursive',))
 183         test = None
 184         if self.stream.skip_if('name:if'):
 185             test = self.parse_expression()
 186         recursive = self.stream.skip_if('name:recursive')
 187         body = self.parse_statements(('name:endfor', 'name:else'))
 188         if next(self.stream).value == 'endfor':
 189             else_ = []
 190         else:
 191             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
 192         return nodes.For(target, iter, body, else_, test,
 193                          recursive, lineno=lineno)
 194
 195     def parse_if(self):
 196         """Parse an if construct."""
 197         node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
 198         while 1:
 199             node.test = self.parse_tuple(with_condexpr=False)
 200             node.body = self.parse_statements(('name:elif', 'name:else',
 201                                                'name:endif'))
 202             token = next(self.stream)
 203             if token.test('name:elif'):
 204                 new_node = nodes.If(lineno=self.stream.current.lineno)
 205                 node.else_ = [new_node]
 206                 node = new_node
 207                 continue
 208             elif token.test('name:else'):
 209                 node.else_ = self.parse_statements(('name:endif',),
 210                                                    drop_needle=True)
 211             else:
 212                 node.else_ = []
 213             break
 214         return result
 215
 216     def parse_block(self):
 217         node = nodes.Block(lineno=next(self.stream).lineno)
 218         node.name = self.stream.expect('name').value
 219         node.scoped = self.stream.skip_if('name:scoped')
 220
 221         # common problem people encounter when switching from django
 222         # to jinja.  we do not support hyphens in block names, so let's
 223         # raise a nicer error message in that case.
 224         if self.stream.current.type == 'sub':
 225             self.fail('Block names in Jinja have to be valid Python '
 226                       'identifiers and may not contain hypens, use an '
 227                       'underscore instead.')
 228
 229         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
 230         self.stream.skip_if('name:' + node.name)
 231         return node
 232
 233     def parse_extends(self):
 234         node = nodes.Extends(lineno=next(self.stream).lineno)
 235         node.template = self.parse_expression()
 236         return node
 237
 238     def parse_import_context(self, node, default):
 239         if self.stream.current.test_any('name:with', 'name:without') and \
 240            self.stream.look().test('name:context'):
 241             node.with_context = next(self.stream).value == 'with'
 242             self.stream.skip()
 243         else:
 244             node.with_context = default
 245         return node
 246
 247     def parse_include(self):
 248         node = nodes.Include(lineno=next(self.stream).lineno)
 249         node.template = self.parse_expression()
 250         if self.stream.current.test('name:ignore') and \
 251            self.stream.look().test('name:missing'):
 252             node.ignore_missing = True
 253             self.stream.skip(2)
 254         else:
 255             node.ignore_missing = False
 256         return self.parse_import_context(node, True)
 257
 258     def parse_import(self):
 259         node = nodes.Import(lineno=next(self.stream).lineno)
 260         node.template = self.parse_expression()
 261         self.stream.expect('name:as')
 262         node.target = self.parse_assign_target(name_only=True).name
 263         return self.parse_import_context(node, False)
 264
 265     def parse_from(self):
 266         node = nodes.FromImport(lineno=next(self.stream).lineno)
 267         node.template = self.parse_expression()
 268         self.stream.expect('name:import')
 269         node.names = []
 270
 271         def parse_context():
 272             if self.stream.current.value in ('with', 'without') and \
 273                self.stream.look().test('name:context'):
 274                 node.with_context = next(self.stream).value == 'with'
 275                 self.stream.skip()
 276                 return True
 277             return False
 278
 279         while 1:
 280             if node.names:
 281                 self.stream.expect('comma')
 282             if self.stream.current.type == 'name':
 283                 if parse_context():
 284                     break
 285                 target = self.parse_assign_target(name_only=True)
 286                 if target.name.startswith('_'):
 287                     self.fail('names starting with an underline can not '
 288                               'be imported', target.lineno,
 289                               exc=TemplateAssertionError)
 290                 if self.stream.skip_if('name:as'):
 291                     alias = self.parse_assign_target(name_only=True)
 292                     node.names.append((target.name, alias.name))
 293                 else:
 294                     node.names.append(target.name)
 295                 if parse_context() or self.stream.current.type != 'comma':
 296                     break
 297             else:
 298                 break
 299         if not hasattr(node, 'with_context'):
 300             node.with_context = False
 301             self.stream.skip_if('comma')
 302         return node
 303
 304     def parse_signature(self, node):
 305         node.args = args = []
 306         node.defaults = defaults = []
 307         self.stream.expect('lparen')
 308         while self.stream.current.type != 'rparen':
 309             if args:
 310                 self.stream.expect('comma')
 311             arg = self.parse_assign_target(name_only=True)
 312             arg.set_ctx('param')
 313             if self.stream.skip_if('assign'):
 314                 defaults.append(self.parse_expression())
 315             args.append(arg)
 316         self.stream.expect('rparen')
 317
 318     def parse_call_block(self):
 319         node = nodes.CallBlock(lineno=next(self.stream).lineno)
 320         if self.stream.current.type == 'lparen':
 321             self.parse_signature(node)
 322         else:
 323             node.args = []
 324             node.defaults = []
 325
 326         node.call = self.parse_expression()
 327         if not isinstance(node.call, nodes.Call):
 328             self.fail('expected call', node.lineno)
 329         node.body = self.parse_statements(('name:endcall',), drop_needle=True)
 330         return node
 331
 332     def parse_filter_block(self):
 333         node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 334         node.filter = self.parse_filter(None, start_inline=True)
 335         node.body = self.parse_statements(('name:endfilter',),
 336                                           drop_needle=True)
 337         return node
 338
 339     def parse_macro(self):
 340         node = nodes.Macro(lineno=next(self.stream).lineno)
 341         node.name = self.parse_assign_target(name_only=True).name
 342         self.parse_signature(node)
 343         node.body = self.parse_statements(('name:endmacro',),
 344                                           drop_needle=True)
 345         return node
 346
 347     def parse_print(self):
 348         node = nodes.Output(lineno=next(self.stream).lineno)
 349         node.nodes = []
 350         while self.stream.current.type != 'block_end':
 351             if node.nodes:
 352                 self.stream.expect('comma')
 353             node.nodes.append(self.parse_expression())
 354         return node
 355
 356     def parse_assign_target(self, with_tuple=True, name_only=False,
 357                             extra_end_rules=None):
 358         """Parse an assignment target.  As Jinja2 allows assignments to
 359         tuples, this function can parse all allowed assignment targets.  Per
 360         default assignments to tuples are parsed, that can be disable however
 361         by setting `with_tuple` to `False`.  If only assignments to names are
 362         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 363         parameter is forwarded to the tuple parsing function.
 364         """
 365         if name_only:
 366             token = self.stream.expect('name')
 367             target = nodes.Name(token.value, 'store', lineno=token.lineno)
 368         else:
 369             if with_tuple:
 370                 target = self.parse_tuple(simplified=True,
 371                                           extra_end_rules=extra_end_rules)
 372             else:
 373                 target = self.parse_primary()
 374             target.set_ctx('store')
 375         if not target.can_assign():
 376             self.fail('can\'t assign to %r' % target.__class__.
 377                       __name__.lower(), target.lineno)
 378         return target
 379
 380     def parse_expression(self, with_condexpr=True):
 381         """Parse an expression.  Per default all expressions are parsed, if
 382         the optional `with_condexpr` parameter is set to `False` conditional
 383         expressions are not parsed.
 384         """
 385         if with_condexpr:
 386             return self.parse_condexpr()
 387         return self.parse_or()
 388
 389     def parse_condexpr(self):
 390         lineno = self.stream.current.lineno
 391         expr1 = self.parse_or()
 392         while self.stream.skip_if('name:if'):
 393             expr2 = self.parse_or()
 394             if self.stream.skip_if('name:else'):
 395                 expr3 = self.parse_condexpr()
 396             else:
 397                 expr3 = None
 398             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 399             lineno = self.stream.current.lineno
 400         return expr1
 401
 402     def parse_or(self):
 403         lineno = self.stream.current.lineno
 404         left = self.parse_and()
 405         while self.stream.skip_if('name:or'):
 406             right = self.parse_and()
 407             left = nodes.Or(left, right, lineno=lineno)
 408             lineno = self.stream.current.lineno
 409         return left
 410
 411     def parse_and(self):
 412         lineno = self.stream.current.lineno
 413         left = self.parse_not()
 414         while self.stream.skip_if('name:and'):
 415             right = self.parse_not()
 416             left = nodes.And(left, right, lineno=lineno)
 417             lineno = self.stream.current.lineno
 418         return left
 419
 420     def parse_not(self):
 421         if self.stream.current.test('name:not'):
 422             lineno = next(self.stream).lineno
 423             return nodes.Not(self.parse_not(), lineno=lineno)
 424         return self.parse_compare()
 425
 426     def parse_compare(self):
 427         lineno = self.stream.current.lineno
 428         expr = self.parse_add()
 429         ops = []
 430         while 1:
 431             token_type = self.stream.current.type
 432             if token_type in _compare_operators:
 433                 next(self.stream)
 434                 ops.append(nodes.Operand(token_type, self.parse_add()))
 435             elif self.stream.skip_if('name:in'):
 436                 ops.append(nodes.Operand('in', self.parse_add()))
 437             elif self.stream.current.test('name:not') and \
 438                  self.stream.look().test('name:in'):
 439                 self.stream.skip(2)
 440                 ops.append(nodes.Operand('notin', self.parse_add()))
 441             else:
 442                 break
 443             lineno = self.stream.current.lineno
 444         if not ops:
 445             return expr
 446         return nodes.Compare(expr, ops, lineno=lineno)
 447
 448     def parse_add(self):
 449         lineno = self.stream.current.lineno
 450         left = self.parse_sub()
 451         while self.stream.current.type == 'add':
 452             next(self.stream)
 453             right = self.parse_sub()
 454             left = nodes.Add(left, right, lineno=lineno)
 455             lineno = self.stream.current.lineno
 456         return left
 457
 458     def parse_sub(self):
 459         lineno = self.stream.current.lineno
 460         left = self.parse_concat()
 461         while self.stream.current.type == 'sub':
 462             next(self.stream)
 463             right = self.parse_concat()
 464             left = nodes.Sub(left, right, lineno=lineno)
 465             lineno = self.stream.current.lineno
 466         return left
 467
 468     def parse_concat(self):
 469         lineno = self.stream.current.lineno
 470         args = [self.parse_mul()]
 471         while self.stream.current.type == 'tilde':
 472             next(self.stream)
 473             args.append(self.parse_mul())
 474         if len(args) == 1:
 475             return args[0]
 476         return nodes.Concat(args, lineno=lineno)
 477
 478     def parse_mul(self):
 479         lineno = self.stream.current.lineno
 480         left = self.parse_div()
 481         while self.stream.current.type == 'mul':
 482             next(self.stream)
 483             right = self.parse_div()
 484             left = nodes.Mul(left, right, lineno=lineno)
 485             lineno = self.stream.current.lineno
 486         return left
 487
 488     def parse_div(self):
 489         lineno = self.stream.current.lineno
 490         left = self.parse_floordiv()
 491         while self.stream.current.type == 'div':
 492             next(self.stream)
 493             right = self.parse_floordiv()
 494             left = nodes.Div(left, right, lineno=lineno)
 495             lineno = self.stream.current.lineno
 496         return left
 497
 498     def parse_floordiv(self):
 499         lineno = self.stream.current.lineno
 500         left = self.parse_mod()
 501         while self.stream.current.type == 'floordiv':
 502             next(self.stream)
 503             right = self.parse_mod()
 504             left = nodes.FloorDiv(left, right, lineno=lineno)
 505             lineno = self.stream.current.lineno
 506         return left
 507
 508     def parse_mod(self):
 509         lineno = self.stream.current.lineno
 510         left = self.parse_pow()
 511         while self.stream.current.type == 'mod':
 512             next(self.stream)
 513             right = self.parse_pow()
 514             left = nodes.Mod(left, right, lineno=lineno)
 515             lineno = self.stream.current.lineno
 516         return left
 517
 518     def parse_pow(self):
 519         lineno = self.stream.current.lineno
 520         left = self.parse_unary()
 521         while self.stream.current.type == 'pow':
 522             next(self.stream)
 523             right = self.parse_unary()
 524             left = nodes.Pow(left, right, lineno=lineno)
 525             lineno = self.stream.current.lineno
 526         return left
 527
 528     def parse_unary(self, with_filter=True):
 529         token_type = self.stream.current.type
 530         lineno = self.stream.current.lineno
 531         if token_type == 'sub':
 532             next(self.stream)
 533             node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 534         elif token_type == 'add':
 535             next(self.stream)
 536             node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 537         else:
 538             node = self.parse_primary()
 539         node = self.parse_postfix(node)
 540         if with_filter:
 541             node = self.parse_filter_expr(node)
 542         return node
 543
 544     def parse_primary(self):
 545         token = self.stream.current
 546         if token.type == 'name':
 547             if token.value in ('true', 'false', 'True', 'False'):
 548                 node = nodes.Const(token.value in ('true', 'True'),
 549                                    lineno=token.lineno)
 550             elif token.value in ('none', 'None'):
 551                 node = nodes.Const(None, lineno=token.lineno)
 552             else:
 553                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
 554             next(self.stream)
 555         elif token.type == 'string':
 556             next(self.stream)
 557             buf = [token.value]
 558             lineno = token.lineno
 559             while self.stream.current.type == 'string':
 560                 buf.append(self.stream.current.value)
 561                 next(self.stream)
 562             node = nodes.Const(''.join(buf), lineno=lineno)
 563         elif token.type in ('integer', 'float'):
 564             next(self.stream)
 565             node = nodes.Const(token.value, lineno=token.lineno)
 566         elif token.type == 'lparen':
 567             next(self.stream)
 568             node = self.parse_tuple(explicit_parentheses=True)
 569             self.stream.expect('rparen')
 570         elif token.type == 'lbracket':
 571             node = self.parse_list()
 572         elif token.type == 'lbrace':
 573             node = self.parse_dict()
 574         else:
 575             self.fail("unexpected '%s'" % describe_token(token), token.lineno)
 576         return node
 577
 578     def parse_tuple(self, simplified=False, with_condexpr=True,
 579                     extra_end_rules=None, explicit_parentheses=False):
 580         """Works like `parse_expression` but if multiple expressions are
 581         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 582         This method could also return a regular expression instead of a tuple
 583         if no commas where found.
 584
 585         The default parsing mode is a full tuple.  If `simplified` is `True`
 586         only names and literals are parsed.  The `no_condexpr` parameter is
 587         forwarded to :meth:`parse_expression`.
 588
 589         Because tuples do not require delimiters and may end in a bogus comma
 590         an extra hint is needed that marks the end of a tuple.  For example
 591         for loops support tuples between `for` and `in`.  In that case the
 592         `extra_end_rules` is set to ``['name:in']``.
 593
 594         `explicit_parentheses` is true if the parsing was triggered by an
 595         expression in parentheses.  This is used to figure out if an empty
 596         tuple is a valid expression or not.
 597         """
 598         lineno = self.stream.current.lineno
 599         if simplified:
 600             parse = self.parse_primary
 601         elif with_condexpr:
 602             parse = self.parse_expression
 603         else:
 604             parse = lambda: self.parse_expression(with_condexpr=False)
 605         args = []
 606         is_tuple = False
 607         while 1:
 608             if args:
 609                 self.stream.expect('comma')
 610             if self.is_tuple_end(extra_end_rules):
 611                 break
 612             args.append(parse())
 613             if self.stream.current.type == 'comma':
 614                 is_tuple = True
 615             else:
 616                 break
 617             lineno = self.stream.current.lineno
 618
 619         if not is_tuple:
 620             if args:
 621                 return args[0]
 622
 623             # if we don't have explicit parentheses, an empty tuple is
 624             # not a valid expression.  This would mean nothing (literally
 625             # nothing) in the spot of an expression would be an empty
 626             # tuple.
 627             if not explicit_parentheses:
 628                 self.fail('Expected an expression, got \'%s\'' %
 629                           describe_token(self.stream.current))
 630
 631         return nodes.Tuple(args, 'load', lineno=lineno)
 632
 633     def parse_list(self):
 634         token = self.stream.expect('lbracket')
 635         items = []
 636         while self.stream.current.type != 'rbracket':
 637             if items:
 638                 self.stream.expect('comma')
 639             if self.stream.current.type == 'rbracket':
 640                 break
 641             items.append(self.parse_expression())
 642         self.stream.expect('rbracket')
 643         return nodes.List(items, lineno=token.lineno)
 644
 645     def parse_dict(self):
 646         token = self.stream.expect('lbrace')
 647         items = []
 648         while self.stream.current.type != 'rbrace':
 649             if items:
 650                 self.stream.expect('comma')
 651             if self.stream.current.type == 'rbrace':
 652                 break
 653             key = self.parse_expression()
 654             self.stream.expect('colon')
 655             value = self.parse_expression()
 656             items.append(nodes.Pair(key, value, lineno=key.lineno))
 657         self.stream.expect('rbrace')
 658         return nodes.Dict(items, lineno=token.lineno)
 659
 660     def parse_postfix(self, node):
 661         while 1:
 662             token_type = self.stream.current.type
 663             if token_type == 'dot' or token_type == 'lbracket':
 664                 node = self.parse_subscript(node)
 665             # calls are valid both after postfix expressions (getattr
 666             # and getitem) as well as filters and tests
 667             elif token_type == 'lparen':
 668                 node = self.parse_call(node)
 669             else:
 670                 break
 671         return node
 672
 673     def parse_filter_expr(self, node):
 674         while 1:
 675             token_type = self.stream.current.type
 676             if token_type == 'pipe':
 677                 node = self.parse_filter(node)
 678             elif token_type == 'name' and self.stream.current.value == 'is':
 679                 node = self.parse_test(node)
 680             # calls are valid both after postfix expressions (getattr
 681             # and getitem) as well as filters and tests
 682             elif token_type == 'lparen':
 683                 node = self.parse_call(node)
 684             else:
 685                 break
 686         return node
 687
 688     def parse_subscript(self, node):
 689         token = next(self.stream)
 690         if token.type == 'dot':
 691             attr_token = self.stream.current
 692             next(self.stream)
 693             if attr_token.type == 'name':
 694                 return nodes.Getattr(node, attr_token.value, 'load',
 695                                      lineno=token.lineno)
 696             elif attr_token.type != 'integer':
 697                 self.fail('expected name or number', attr_token.lineno)
 698             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 699             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 700         if token.type == 'lbracket':
 701             priority_on_attribute = False
 702             args = []
 703             while self.stream.current.type != 'rbracket':
 704                 if args:
 705                     self.stream.expect('comma')
 706                 args.append(self.parse_subscribed())
 707             self.stream.expect('rbracket')
 708             if len(args) == 1:
 709                 arg = args[0]
 710             else:
 711                 arg = nodes.Tuple(args, 'load', lineno=token.lineno)
 712             return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
 713         self.fail('expected subscript expression', self.lineno)
 714
 715     def parse_subscribed(self):
 716         lineno = self.stream.current.lineno
 717
 718         if self.stream.current.type == 'colon':
 719             next(self.stream)
 720             args = [None]
 721         else:
 722             node = self.parse_expression()
 723             if self.stream.current.type != 'colon':
 724                 return node
 725             next(self.stream)
 726             args = [node]
 727
 728         if self.stream.current.type == 'colon':
 729             args.append(None)
 730         elif self.stream.current.type not in ('rbracket', 'comma'):
 731             args.append(self.parse_expression())
 732         else:
 733             args.append(None)
 734
 735         if self.stream.current.type == 'colon':
 736             next(self.stream)
 737             if self.stream.current.type not in ('rbracket', 'comma'):
 738                 args.append(self.parse_expression())
 739             else:
 740                 args.append(None)
 741         else:
 742             args.append(None)
 743
 744         return nodes.Slice(lineno=lineno, *args)
 745
 746     def parse_call(self, node):
 747         token = self.stream.expect('lparen')
 748         args = []
 749         kwargs = []
 750         dyn_args = dyn_kwargs = None
 751         require_comma = False
 752
 753         def ensure(expr):
 754             if not expr:
 755                 self.fail('invalid syntax for function call expression',
 756                           token.lineno)
 757
 758         while self.stream.current.type != 'rparen':
 759             if require_comma:
 760                 self.stream.expect('comma')
 761                 # support for trailing comma
 762                 if self.stream.current.type == 'rparen':
 763                     break
 764             if self.stream.current.type == 'mul':
 765                 ensure(dyn_args is None and dyn_kwargs is None)
 766                 next(self.stream)
 767                 dyn_args = self.parse_expression()
 768             elif self.stream.current.type == 'pow':
 769                 ensure(dyn_kwargs is None)
 770                 next(self.stream)
 771                 dyn_kwargs = self.parse_expression()
 772             else:
 773                 ensure(dyn_args is None and dyn_kwargs is None)
 774                 if self.stream.current.type == 'name' and \
 775                     self.stream.look().type == 'assign':
 776                     key = self.stream.current.value
 777                     self.stream.skip(2)
 778                     value = self.parse_expression()
 779                     kwargs.append(nodes.Keyword(key, value,
 780                                                 lineno=value.lineno))
 781                 else:
 782                     ensure(not kwargs)
 783                     args.append(self.parse_expression())
 784
 785             require_comma = True
 786         self.stream.expect('rparen')
 787
 788         if node is None:
 789             return args, kwargs, dyn_args, dyn_kwargs
 790         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
 791                           lineno=token.lineno)
 792
 793     def parse_filter(self, node, start_inline=False):
 794         while self.stream.current.type == 'pipe' or start_inline:
 795             if not start_inline:
 796                 next(self.stream)
 797             token = self.stream.expect('name')
 798             name = token.value
 799             while self.stream.current.type == 'dot':
 800                 next(self.stream)
 801                 name += '.' + self.stream.expect('name').value
 802             if self.stream.current.type == 'lparen':
 803                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 804             else:
 805                 args = []
 806                 kwargs = []
 807                 dyn_args = dyn_kwargs = None
 808             node = nodes.Filter(node, name, args, kwargs, dyn_args,
 809                                 dyn_kwargs, lineno=token.lineno)
 810             start_inline = False
 811         return node
 812
 813     def parse_test(self, node):
 814         token = next(self.stream)
 815         if self.stream.current.test('name:not'):
 816             next(self.stream)
 817             negated = True
 818         else:
 819             negated = False
 820         name = self.stream.expect('name').value
 821         while self.stream.current.type == 'dot':
 822             next(self.stream)
 823             name += '.' + self.stream.expect('name').value
 824         dyn_args = dyn_kwargs = None
 825         kwargs = []
 826         if self.stream.current.type == 'lparen':
 827             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 828         elif self.stream.current.type in ('name', 'string', 'integer',
 829                                           'float', 'lparen', 'lbracket',
 830                                           'lbrace') and not \
 831              self.stream.current.test_any('name:else', 'name:or',
 832                                           'name:and'):
 833             if self.stream.current.test('name:is'):
 834                 self.fail('You cannot chain multiple tests with is')
 835             args = [self.parse_expression()]
 836         else:
 837             args = []
 838         node = nodes.Test(node, name, args, kwargs, dyn_args,
 839                           dyn_kwargs, lineno=token.lineno)
 840         if negated:
 841             node = nodes.Not(node, lineno=token.lineno)
 842         return node
 843
 844     def subparse(self, end_tokens=None):
 845         body = []
 846         data_buffer = []
 847         add_data = data_buffer.append
 848
 849         if end_tokens is not None:
 850             self._end_token_stack.append(end_tokens)
 851
 852         def flush_data():
 853             if data_buffer:
 854                 lineno = data_buffer[0].lineno
 855                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 856                 del data_buffer[:]
 857
 858         try:
 859             while self.stream:
 860                 token = self.stream.current
 861                 if token.type == 'data':
 862                     if token.value:
 863                         add_data(nodes.TemplateData(token.value,
 864                                                     lineno=token.lineno))
 865                     next(self.stream)
 866                 elif token.type == 'variable_begin':
 867                     next(self.stream)
 868                     add_data(self.parse_tuple(with_condexpr=True))
 869                     self.stream.expect('variable_end')
 870                 elif token.type == 'block_begin':
 871                     flush_data()
 872                     next(self.stream)
 873                     if end_tokens is not None and \
 874                        self.stream.current.test_any(*end_tokens):
 875                         return body
 876                     rv = self.parse_statement()
 877                     if isinstance(rv, list):
 878                         body.extend(rv)
 879                     else:
 880                         body.append(rv)
 881                     self.stream.expect('block_end')
 882                 else:
 883                     raise AssertionError('internal parsing error')
 884
 885             flush_data()
 886         finally:
 887             if end_tokens is not None:
 888                 self._end_token_stack.pop()
 889
 890         return body
 891
 892     def parse(self):
 893         """Parse the whole template into a `Template` node."""
 894         result = nodes.Template(self.subparse(), lineno=1)
 895         result.set_environment(self.environment)
 896         return result