jinja2/parser.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     jinja2.parser
   4     ~~~~~~~~~~~~~
   5
   6     Implements the template parser.
   7
   8     :copyright: 2008 by Armin Ronacher.
   9     :license: BSD, see LICENSE for more details.
  10 """
  11 from jinja2 import nodes
  12 from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
  13
  14
  15 _statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
  16                                  'macro', 'include', 'from', 'import',
  17                                  'set'])
  18 _compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
  19
  20
  21 class Parser(object):
  22     """This is the central parsing class Jinja2 uses.  It's passed to
  23     extensions and can be used to parse expressions or statements.
  24     """
  25
  26     def __init__(self, environment, source, name=None, filename=None):
  27         self.environment = environment
  28         self.source = unicode(source)
  29         self.name = name
  30         self.filename = filename
  31         self.closed = False
  32         self.stream = environment.lexer.tokenize(self.source, name, filename)
  33         self.extensions = {}
  34         for extension in environment.extensions.itervalues():
  35             for tag in extension.tags:
  36                 self.extensions[tag] = extension.parse
  37         self._last_identifier = 0
  38
  39     def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
  40         """Convenience method that raises `exc` with the message, passed
  41         line number or last line number as well as the current name and
  42         filename.
  43         """
  44         if lineno is None:
  45             lineno = self.stream.current.lineno
  46         raise exc(msg, lineno, self.name, self.filename)
  47
  48     def is_tuple_end(self, extra_end_rules=None):
  49         """Are we at the end of a tuple?"""
  50         if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
  51             return True
  52         elif extra_end_rules is not None:
  53             return self.stream.current.test_any(extra_end_rules)
  54         return False
  55
  56     def free_identifier(self, lineno=None):
  57         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
  58         self._last_identifier += 1
  59         rv = object.__new__(nodes.InternalName)
  60         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
  61         return rv
  62
  63     def parse_statement(self):
  64         """Parse a single statement."""
  65         token = self.stream.current
  66         if token.type is not 'name':
  67             self.fail('tag name expected', token.lineno)
  68         if token.value in _statement_keywords:
  69             return getattr(self, 'parse_' + self.stream.current.value)()
  70         if token.value == 'call':
  71             return self.parse_call_block()
  72         if token.value == 'filter':
  73             return self.parse_filter_block()
  74         ext = self.extensions.get(token.value)
  75         if ext is not None:
  76             return ext(self)
  77         self.fail('unknown tag %r' % token.value, token.lineno)
  78
  79     def parse_statements(self, end_tokens, drop_needle=False):
  80         """Parse multiple statements into a list until one of the end tokens
  81         is reached.  This is used to parse the body of statements as it also
  82         parses template data if appropriate.  The parser checks first if the
  83         current token is a colon and skips it if there is one.  Then it checks
  84         for the block end and parses until if one of the `end_tokens` is
  85         reached.  Per default the active token in the stream at the end of
  86         the call is the matched end token.  If this is not wanted `drop_needle`
  87         can be set to `True` and the end token is removed.
  88         """
  89         # the first token may be a colon for python compatibility
  90         self.stream.skip_if('colon')
  91
  92         # in the future it would be possible to add whole code sections
  93         # by adding some sort of end of statement token and parsing those here.
  94         self.stream.expect('block_end')
  95         result = self.subparse(end_tokens)
  96
  97         if drop_needle:
  98             self.stream.next()
  99         return result
 100
 101     def parse_set(self):
 102         """Parse an assign statement."""
 103         lineno = self.stream.next().lineno
 104         target = self.parse_assign_target()
 105         self.stream.expect('assign')
 106         expr = self.parse_tuple()
 107         return nodes.Assign(target, expr, lineno=lineno)
 108
 109     def parse_for(self):
 110         """Parse a for loop."""
 111         lineno = self.stream.expect('name:for').lineno
 112         target = self.parse_assign_target(extra_end_rules=('name:in',))
 113         self.stream.expect('name:in')
 114         iter = self.parse_tuple(with_condexpr=False,
 115                                 extra_end_rules=('name:recursive',))
 116         test = None
 117         if self.stream.skip_if('name:if'):
 118             test = self.parse_expression()
 119         recursive = self.stream.skip_if('name:recursive')
 120         body = self.parse_statements(('name:endfor', 'name:else'))
 121         if self.stream.next().value == 'endfor':
 122             else_ = []
 123         else:
 124             else_ = self.parse_statements(('name:endfor',), drop_needle=True)
 125         return nodes.For(target, iter, body, else_, test,
 126                          recursive, lineno=lineno)
 127
 128     def parse_if(self):
 129         """Parse an if construct."""
 130         node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
 131         while 1:
 132             node.test = self.parse_tuple(with_condexpr=False)
 133             node.body = self.parse_statements(('name:elif', 'name:else',
 134                                                'name:endif'))
 135             token = self.stream.next()
 136             if token.test('name:elif'):
 137                 new_node = nodes.If(lineno=self.stream.current.lineno)
 138                 node.else_ = [new_node]
 139                 node = new_node
 140                 continue
 141             elif token.test('name:else'):
 142                 node.else_ = self.parse_statements(('name:endif',),
 143                                                    drop_needle=True)
 144             else:
 145                 node.else_ = []
 146             break
 147         return result
 148
 149     def parse_block(self):
 150         node = nodes.Block(lineno=self.stream.next().lineno)
 151         node.name = self.stream.expect('name').value
 152         node.body = self.parse_statements(('name:endblock',), drop_needle=True)
 153         self.stream.skip_if('name:' + node.name)
 154         return node
 155
 156     def parse_extends(self):
 157         node = nodes.Extends(lineno=self.stream.next().lineno)
 158         node.template = self.parse_expression()
 159         return node
 160
 161     def parse_import_context(self, node, default):
 162         if self.stream.current.test_any('name:with', 'name:without') and \
 163            self.stream.look().test('name:context'):
 164             node.with_context = self.stream.next().value == 'with'
 165             self.stream.skip()
 166         else:
 167             node.with_context = default
 168         return node
 169
 170     def parse_include(self):
 171         node = nodes.Include(lineno=self.stream.next().lineno)
 172         node.template = self.parse_expression()
 173         return self.parse_import_context(node, True)
 174
 175     def parse_import(self):
 176         node = nodes.Import(lineno=self.stream.next().lineno)
 177         node.template = self.parse_expression()
 178         self.stream.expect('name:as')
 179         node.target = self.parse_assign_target(name_only=True).name
 180         return self.parse_import_context(node, False)
 181
 182     def parse_from(self):
 183         node = nodes.FromImport(lineno=self.stream.next().lineno)
 184         node.template = self.parse_expression()
 185         self.stream.expect('name:import')
 186         node.names = []
 187
 188         def parse_context():
 189             if self.stream.current.value in ('with', 'without') and \
 190                self.stream.look().test('name:context'):
 191                 node.with_context = self.stream.next().value == 'with'
 192                 self.stream.skip()
 193                 return True
 194             return False
 195
 196         while 1:
 197             if node.names:
 198                 self.stream.expect('comma')
 199             if self.stream.current.type is 'name':
 200                 if parse_context():
 201                     break
 202                 target = self.parse_assign_target(name_only=True)
 203                 if target.name.startswith('_'):
 204                     self.fail('names starting with an underline can not '
 205                               'be imported', target.lineno,
 206                               exc=TemplateAssertionError)
 207                 if self.stream.skip_if('name:as'):
 208                     alias = self.parse_assign_target(name_only=True)
 209                     node.names.append((target.name, alias.name))
 210                 else:
 211                     node.names.append(target.name)
 212                 if parse_context() or self.stream.current.type is not 'comma':
 213                     break
 214             else:
 215                 break
 216         if not hasattr(node, 'with_context'):
 217             node.with_context = False
 218             self.stream.skip_if('comma')
 219         return node
 220
 221     def parse_signature(self, node):
 222         node.args = args = []
 223         node.defaults = defaults = []
 224         self.stream.expect('lparen')
 225         while self.stream.current.type is not 'rparen':
 226             if args:
 227                 self.stream.expect('comma')
 228             arg = self.parse_assign_target(name_only=True)
 229             arg.set_ctx('param')
 230             if self.stream.skip_if('assign'):
 231                 defaults.append(self.parse_expression())
 232             args.append(arg)
 233         self.stream.expect('rparen')
 234
 235     def parse_call_block(self):
 236         node = nodes.CallBlock(lineno=self.stream.next().lineno)
 237         if self.stream.current.type is 'lparen':
 238             self.parse_signature(node)
 239         else:
 240             node.args = []
 241             node.defaults = []
 242
 243         node.call = self.parse_expression()
 244         if not isinstance(node.call, nodes.Call):
 245             self.fail('expected call', node.lineno)
 246         node.body = self.parse_statements(('name:endcall',), drop_needle=True)
 247         return node
 248
 249     def parse_filter_block(self):
 250         node = nodes.FilterBlock(lineno=self.stream.next().lineno)
 251         node.filter = self.parse_filter(None, start_inline=True)
 252         node.body = self.parse_statements(('name:endfilter',),
 253                                           drop_needle=True)
 254         return node
 255
 256     def parse_macro(self):
 257         node = nodes.Macro(lineno=self.stream.next().lineno)
 258         node.name = self.parse_assign_target(name_only=True).name
 259         self.parse_signature(node)
 260         node.body = self.parse_statements(('name:endmacro',),
 261                                           drop_needle=True)
 262         return node
 263
 264     def parse_print(self):
 265         node = nodes.Output(lineno=self.stream.next().lineno)
 266         node.nodes = []
 267         while self.stream.current.type is not 'block_end':
 268             if node.nodes:
 269                 self.stream.expect('comma')
 270             node.nodes.append(self.parse_expression())
 271         return node
 272
 273     def parse_assign_target(self, with_tuple=True, name_only=False,
 274                             extra_end_rules=None):
 275         """Parse an assignment target.  As Jinja2 allows assignments to
 276         tuples, this function can parse all allowed assignment targets.  Per
 277         default assignments to tuples are parsed, that can be disable however
 278         by setting `with_tuple` to `False`.  If only assignments to names are
 279         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 280         parameter is forwarded to the tuple parsing function.
 281         """
 282         if name_only:
 283             token = self.stream.expect('name')
 284             target = nodes.Name(token.value, 'store', lineno=token.lineno)
 285         else:
 286             if with_tuple:
 287                 target = self.parse_tuple(simplified=True,
 288                                           extra_end_rules=extra_end_rules)
 289             else:
 290                 target = self.parse_primary(with_postfix=False)
 291             target.set_ctx('store')
 292         if not target.can_assign():
 293             self.fail('can\'t assign to %r' % target.__class__.
 294                       __name__.lower(), target.lineno)
 295         return target
 296
 297     def parse_expression(self, with_condexpr=True):
 298         """Parse an expression.  Per default all expressions are parsed, if
 299         the optional `with_condexpr` parameter is set to `False` conditional
 300         expressions are not parsed.
 301         """
 302         if with_condexpr:
 303             return self.parse_condexpr()
 304         return self.parse_or()
 305
 306     def parse_condexpr(self):
 307         lineno = self.stream.current.lineno
 308         expr1 = self.parse_or()
 309         while self.stream.skip_if('name:if'):
 310             expr2 = self.parse_or()
 311             self.stream.expect('name:else')
 312             expr3 = self.parse_condexpr()
 313             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 314             lineno = self.stream.current.lineno
 315         return expr1
 316
 317     def parse_or(self):
 318         lineno = self.stream.current.lineno
 319         left = self.parse_and()
 320         while self.stream.skip_if('name:or'):
 321             right = self.parse_and()
 322             left = nodes.Or(left, right, lineno=lineno)
 323             lineno = self.stream.current.lineno
 324         return left
 325
 326     def parse_and(self):
 327         lineno = self.stream.current.lineno
 328         left = self.parse_compare()
 329         while self.stream.skip_if('name:and'):
 330             right = self.parse_compare()
 331             left = nodes.And(left, right, lineno=lineno)
 332             lineno = self.stream.current.lineno
 333         return left
 334
 335     def parse_compare(self):
 336         lineno = self.stream.current.lineno
 337         expr = self.parse_add()
 338         ops = []
 339         while 1:
 340             token_type = self.stream.current.type
 341             if token_type in _compare_operators:
 342                 self.stream.next()
 343                 ops.append(nodes.Operand(token_type, self.parse_add()))
 344             elif self.stream.skip_if('name:in'):
 345                 ops.append(nodes.Operand('in', self.parse_add()))
 346             elif self.stream.current.test('name:not') and \
 347                  self.stream.look().test('name:in'):
 348                 self.stream.skip(2)
 349                 ops.append(nodes.Operand('notin', self.parse_add()))
 350             else:
 351                 break
 352             lineno = self.stream.current.lineno
 353         if not ops:
 354             return expr
 355         return nodes.Compare(expr, ops, lineno=lineno)
 356
 357     def parse_add(self):
 358         lineno = self.stream.current.lineno
 359         left = self.parse_sub()
 360         while self.stream.current.type is 'add':
 361             self.stream.next()
 362             right = self.parse_sub()
 363             left = nodes.Add(left, right, lineno=lineno)
 364             lineno = self.stream.current.lineno
 365         return left
 366
 367     def parse_sub(self):
 368         lineno = self.stream.current.lineno
 369         left = self.parse_concat()
 370         while self.stream.current.type is 'sub':
 371             self.stream.next()
 372             right = self.parse_concat()
 373             left = nodes.Sub(left, right, lineno=lineno)
 374             lineno = self.stream.current.lineno
 375         return left
 376
 377     def parse_concat(self):
 378         lineno = self.stream.current.lineno
 379         args = [self.parse_mul()]
 380         while self.stream.current.type is 'tilde':
 381             self.stream.next()
 382             args.append(self.parse_mul())
 383         if len(args) == 1:
 384             return args[0]
 385         return nodes.Concat(args, lineno=lineno)
 386
 387     def parse_mul(self):
 388         lineno = self.stream.current.lineno
 389         left = self.parse_div()
 390         while self.stream.current.type is 'mul':
 391             self.stream.next()
 392             right = self.parse_div()
 393             left = nodes.Mul(left, right, lineno=lineno)
 394             lineno = self.stream.current.lineno
 395         return left
 396
 397     def parse_div(self):
 398         lineno = self.stream.current.lineno
 399         left = self.parse_floordiv()
 400         while self.stream.current.type is 'div':
 401             self.stream.next()
 402             right = self.parse_floordiv()
 403             left = nodes.Div(left, right, lineno=lineno)
 404             lineno = self.stream.current.lineno
 405         return left
 406
 407     def parse_floordiv(self):
 408         lineno = self.stream.current.lineno
 409         left = self.parse_mod()
 410         while self.stream.current.type is 'floordiv':
 411             self.stream.next()
 412             right = self.parse_mod()
 413             left = nodes.FloorDiv(left, right, lineno=lineno)
 414             lineno = self.stream.current.lineno
 415         return left
 416
 417     def parse_mod(self):
 418         lineno = self.stream.current.lineno
 419         left = self.parse_pow()
 420         while self.stream.current.type is 'mod':
 421             self.stream.next()
 422             right = self.parse_pow()
 423             left = nodes.Mod(left, right, lineno=lineno)
 424             lineno = self.stream.current.lineno
 425         return left
 426
 427     def parse_pow(self):
 428         lineno = self.stream.current.lineno
 429         left = self.parse_unary()
 430         while self.stream.current.type is 'pow':
 431             self.stream.next()
 432             right = self.parse_unary()
 433             left = nodes.Pow(left, right, lineno=lineno)
 434             lineno = self.stream.current.lineno
 435         return left
 436
 437     def parse_unary(self):
 438         token_type = self.stream.current.type
 439         lineno = self.stream.current.lineno
 440         if token_type is 'name' and self.stream.current.value == 'not':
 441             self.stream.next()
 442             node = self.parse_unary()
 443             return nodes.Not(node, lineno=lineno)
 444         if token_type is 'sub':
 445             self.stream.next()
 446             node = self.parse_unary()
 447             return nodes.Neg(node, lineno=lineno)
 448         if token_type is 'add':
 449             self.stream.next()
 450             node = self.parse_unary()
 451             return nodes.Pos(node, lineno=lineno)
 452         return self.parse_primary()
 453
 454     def parse_primary(self, with_postfix=True):
 455         token = self.stream.current
 456         if token.type is 'name':
 457             if token.value in ('true', 'false', 'True', 'False'):
 458                 node = nodes.Const(token.value in ('true', 'True'),
 459                                    lineno=token.lineno)
 460             elif token.value in ('none', 'None'):
 461                 node = nodes.Const(None, lineno=token.lineno)
 462             else:
 463                 node = nodes.Name(token.value, 'load', lineno=token.lineno)
 464             self.stream.next()
 465         elif token.type in ('integer', 'float', 'string'):
 466             self.stream.next()
 467             node = nodes.Const(token.value, lineno=token.lineno)
 468         elif token.type is 'lparen':
 469             self.stream.next()
 470             node = self.parse_tuple()
 471             self.stream.expect('rparen')
 472         elif token.type is 'lbracket':
 473             node = self.parse_list()
 474         elif token.type is 'lbrace':
 475             node = self.parse_dict()
 476         else:
 477             self.fail("unexpected token '%s'" % (token,), token.lineno)
 478         if with_postfix:
 479             node = self.parse_postfix(node)
 480         return node
 481
 482     def parse_tuple(self, simplified=False, with_condexpr=True,
 483                     extra_end_rules=None):
 484         """Works like `parse_expression` but if multiple expressions are
 485         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 486         This method could also return a regular expression instead of a tuple
 487         if no commas where found.
 488
 489         The default parsing mode is a full tuple.  If `simplified` is `True`
 490         only names and literals are parsed.  The `no_condexpr` parameter is
 491         forwarded to :meth:`parse_expression`.
 492
 493         Because tuples do not require delimiters and may end in a bogus comma
 494         an extra hint is needed that marks the end of a tuple.  For example
 495         for loops support tuples between `for` and `in`.  In that case the
 496         `extra_end_rules` is set to ``['name:in']``.
 497         """
 498         lineno = self.stream.current.lineno
 499         if simplified:
 500             parse = lambda: self.parse_primary(with_postfix=False)
 501         elif with_condexpr:
 502             parse = self.parse_expression
 503         else:
 504             parse = lambda: self.parse_expression(with_condexpr=False)
 505         args = []
 506         is_tuple = False
 507         while 1:
 508             if args:
 509                 self.stream.expect('comma')
 510             if self.is_tuple_end(extra_end_rules):
 511                 break
 512             args.append(parse())
 513             if self.stream.current.type is 'comma':
 514                 is_tuple = True
 515             else:
 516                 break
 517             lineno = self.stream.current.lineno
 518         if not is_tuple and args:
 519             return args[0]
 520         return nodes.Tuple(args, 'load', lineno=lineno)
 521
 522     def parse_list(self):
 523         token = self.stream.expect('lbracket')
 524         items = []
 525         while self.stream.current.type is not 'rbracket':
 526             if items:
 527                 self.stream.expect('comma')
 528             if self.stream.current.type == 'rbracket':
 529                 break
 530             items.append(self.parse_expression())
 531         self.stream.expect('rbracket')
 532         return nodes.List(items, lineno=token.lineno)
 533
 534     def parse_dict(self):
 535         token = self.stream.expect('lbrace')
 536         items = []
 537         while self.stream.current.type is not 'rbrace':
 538             if items:
 539                 self.stream.expect('comma')
 540             if self.stream.current.type == 'rbrace':
 541                 break
 542             key = self.parse_expression()
 543             self.stream.expect('colon')
 544             value = self.parse_expression()
 545             items.append(nodes.Pair(key, value, lineno=key.lineno))
 546         self.stream.expect('rbrace')
 547         return nodes.Dict(items, lineno=token.lineno)
 548
 549     def parse_postfix(self, node):
 550         while 1:
 551             token_type = self.stream.current.type
 552             if token_type is 'dot' or token_type is 'lbracket':
 553                 node = self.parse_subscript(node)
 554             elif token_type is 'lparen':
 555                 node = self.parse_call(node)
 556             elif token_type is 'pipe':
 557                 node = self.parse_filter(node)
 558             elif token_type is 'name' and self.stream.current.value == 'is':
 559                 node = self.parse_test(node)
 560             else:
 561                 break
 562         return node
 563
 564     def parse_subscript(self, node):
 565         token = self.stream.next()
 566         if token.type is 'dot':
 567             attr_token = self.stream.current
 568             if attr_token.type not in ('name', 'integer'):
 569                 self.fail('expected name or number', attr_token.lineno)
 570             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 571             self.stream.next()
 572         elif token.type is 'lbracket':
 573             args = []
 574             while self.stream.current.type is not 'rbracket':
 575                 if args:
 576                     self.stream.expect('comma')
 577                 args.append(self.parse_subscribed())
 578             self.stream.expect('rbracket')
 579             if len(args) == 1:
 580                 arg = args[0]
 581             else:
 582                 arg = nodes.Tuple(args, self.lineno, self.filename)
 583         else:
 584             self.fail('expected subscript expression', self.lineno)
 585         return nodes.Subscript(node, arg, 'load', lineno=token.lineno)
 586
 587     def parse_subscribed(self):
 588         lineno = self.stream.current.lineno
 589
 590         if self.stream.current.type is 'colon':
 591             self.stream.next()
 592             args = [None]
 593         else:
 594             node = self.parse_expression()
 595             if self.stream.current.type is not 'colon':
 596                 return node
 597             self.stream.next()
 598             args = [node]
 599
 600         if self.stream.current.type is 'colon':
 601             args.append(None)
 602         elif self.stream.current.type not in ('rbracket', 'comma'):
 603             args.append(self.parse_expression())
 604         else:
 605             args.append(None)
 606
 607         if self.stream.current.type is 'colon':
 608             self.stream.next()
 609             if self.stream.current.type not in ('rbracket', 'comma'):
 610                 args.append(self.parse_expression())
 611             else:
 612                 args.append(None)
 613         else:
 614             args.append(None)
 615
 616         return nodes.Slice(lineno=lineno, *args)
 617
 618     def parse_call(self, node):
 619         token = self.stream.expect('lparen')
 620         args = []
 621         kwargs = []
 622         dyn_args = dyn_kwargs = None
 623         require_comma = False
 624
 625         def ensure(expr):
 626             if not expr:
 627                 self.fail('invalid syntax for function call expression',
 628                           token.lineno)
 629
 630         while self.stream.current.type is not 'rparen':
 631             if require_comma:
 632                 self.stream.expect('comma')
 633                 # support for trailing comma
 634                 if self.stream.current.type is 'rparen':
 635                     break
 636             if self.stream.current.type is 'mul':
 637                 ensure(dyn_args is None and dyn_kwargs is None)
 638                 self.stream.next()
 639                 dyn_args = self.parse_expression()
 640             elif self.stream.current.type is 'pow':
 641                 ensure(dyn_kwargs is None)
 642                 self.stream.next()
 643                 dyn_kwargs = self.parse_expression()
 644             else:
 645                 ensure(dyn_args is None and dyn_kwargs is None)
 646                 if self.stream.current.type is 'name' and \
 647                     self.stream.look().type is 'assign':
 648                     key = self.stream.current.value
 649                     self.stream.skip(2)
 650                     value = self.parse_expression()
 651                     kwargs.append(nodes.Keyword(key, value,
 652                                                 lineno=value.lineno))
 653                 else:
 654                     ensure(not kwargs)
 655                     args.append(self.parse_expression())
 656
 657             require_comma = True
 658         self.stream.expect('rparen')
 659
 660         if node is None:
 661             return args, kwargs, dyn_args, dyn_kwargs
 662         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
 663                           lineno=token.lineno)
 664
 665     def parse_filter(self, node, start_inline=False):
 666         while self.stream.current.type == 'pipe' or start_inline:
 667             if not start_inline:
 668                 self.stream.next()
 669             token = self.stream.expect('name')
 670             name = token.value
 671             while self.stream.current.type is 'dot':
 672                 self.stream.next()
 673                 name += '.' + self.stream.expect('name').value
 674             if self.stream.current.type is 'lparen':
 675                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 676             else:
 677                 args = []
 678                 kwargs = []
 679                 dyn_args = dyn_kwargs = None
 680             node = nodes.Filter(node, name, args, kwargs, dyn_args,
 681                                 dyn_kwargs, lineno=token.lineno)
 682             start_inline = False
 683         return node
 684
 685     def parse_test(self, node):
 686         token = self.stream.next()
 687         if self.stream.current.test('name:not'):
 688             self.stream.next()
 689             negated = True
 690         else:
 691             negated = False
 692         name = self.stream.expect('name').value
 693         while self.stream.current.type is 'dot':
 694             self.stream.next()
 695             name += '.' + self.stream.expect('name').value
 696         dyn_args = dyn_kwargs = None
 697         kwargs = []
 698         if self.stream.current.type is 'lparen':
 699             args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
 700         elif self.stream.current.type in ('name', 'string', 'integer',
 701                                           'float', 'lparen', 'lbracket',
 702                                           'lbrace'):
 703             args = [self.parse_expression()]
 704         else:
 705             args = []
 706         node = nodes.Test(node, name, args, kwargs, dyn_args,
 707                           dyn_kwargs, lineno=token.lineno)
 708         if negated:
 709             node = nodes.Not(node, lineno=token.lineno)
 710         return node
 711
 712     def subparse(self, end_tokens=None):
 713         body = []
 714         data_buffer = []
 715         add_data = data_buffer.append
 716
 717         def flush_data():
 718             if data_buffer:
 719                 lineno = data_buffer[0].lineno
 720                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 721                 del data_buffer[:]
 722
 723         while self.stream:
 724             token = self.stream.current
 725             if token.type is 'data':
 726                 if token.value:
 727                     add_data(nodes.TemplateData(token.value,
 728                                                 lineno=token.lineno))
 729                 self.stream.next()
 730             elif token.type is 'variable_begin':
 731                 self.stream.next()
 732                 add_data(self.parse_tuple(with_condexpr=True))
 733                 self.stream.expect('variable_end')
 734             elif token.type is 'block_begin':
 735                 flush_data()
 736                 self.stream.next()
 737                 if end_tokens is not None and \
 738                    self.stream.current.test_any(*end_tokens):
 739                     return body
 740                 rv = self.parse_statement()
 741                 if isinstance(rv, list):
 742                     body.extend(rv)
 743                 else:
 744                     body.append(rv)
 745                 self.stream.expect('block_end')
 746             else:
 747                 raise AssertionError('internal parsing error')
 748
 749         flush_data()
 750         return body
 751
 752     def parse(self):
 753         """Parse the whole template into a `Template` node."""
 754         result = nodes.Template(self.subparse(), lineno=1)
 755         result.set_environment(self.environment)
 756         return result