added first code for parser extensions and moved some code in speedups around

[jinja2.git] / jinja2 / datastructure.py

diff --git a/jinja2/datastructure.py b/jinja2/datastructure.py
index 2777df5eb7a1de60abf6ce590bd9bc279ab4791b..9dab02b83099d4abe710ceb695359f9c09e7b673 100644 (file)
--- a/jinja2/datastructure.py
+++ b/jinja2/datastructure.py
@@ -13,9 +13,6 @@ from collections import deque
 from jinja2.exceptions import TemplateSyntaxError, TemplateRuntimeError
 
 
-_missing = object()
-
-
 class Token(tuple):
     """
     Token class.
@@ -32,7 +29,27 @@ class Token(tuple):
             return self.type
         elif self.type in reverse_operators:
             return reverse_operators[self.type]
-        return self.value
+        return '%s:%s' % (self.type, self.value)
+
+    def test(self, expr):
+        """Test a token against a token expression.  This can either be a
+        token type or 'token_type:token_value'.  This can only test against
+        string values!
+        """
+        # here we do a regular string equality check as test_many is usually
+        # passed an iterable of not interned strings.
+        if self.type == expr:
+            return True
+        elif ':' in expr:
+            return expr.split(':', 1) == [self.type, self.value]
+        return False
+
+    def test_many(self, iterable):
+        """Test against multiple token expressions."""
+        for expr in iterable:
+            if self.test(expr):
+                return True
+        return False
 
     def __repr__(self):
         return 'Token(%r, %r, %r)' % (
@@ -59,7 +76,7 @@ class TokenStreamIterator(object):
         if token.type == 'eof':
             self._stream.close()
             raise StopIteration()
-        self._stream.next()
+        self._stream.next(False)
         return token
 
 
@@ -96,43 +113,42 @@ class TokenStream(object):
 
     def look(self):
         """Look at the next token."""
-        old_token = self.current
-        next = self.next()
-        self.push(old_token)
-        self.push(next)
-        return next
+        old_token = self.next()
+        result = self.current
+        self.push(result)
+        self.current = old_token
+        return result
 
     def skip(self, n):
         """Got n tokens ahead."""
         for x in xrange(n):
             self.next()
 
-    def next(self):
-        """Go one token ahead."""
-        if self._pushed:
-            self.current = self._pushed.popleft()
-        elif self.current.type != 'eof':
-            try:
-                self.current = self._next()
-            except StopIteration:
-                self.close()
+    def next(self, skip_eol=True):
+        """Go one token ahead and return the old one"""
+        rv = self.current
+        while 1:
+            if self._pushed:
+                self.current = self._pushed.popleft()
+            elif self.current.type is not 'eof':
+                try:
+                    self.current = self._next()
+                except StopIteration:
+                    self.close()
+            if not skip_eol or self.current.type is not 'eol':
+                break
+        return rv
 
     def close(self):
         """Close the stream."""
         self.current = Token(self.current.lineno, 'eof', '')
         self._next = None
 
-    def expect(self, token_type, token_value=_missing):
+    def expect(self, expr):
         """Expect a given token type and return it"""
-        if self.current.type is not token_type:
+        if not self.current.test(expr):
             raise TemplateSyntaxError("expected token %r, got %r" %
-                                      (token_type, self.current.type),
-                                      self.current.lineno,
-                                      self.filename)
-        elif token_value is not _missing and \
-             self.current.value != token_value:
-            raise TemplateSyntaxError("expected %r, got %r" %
-                                      (token_value, self.current.value),
+                                      (expr, self.current),
                                       self.current.lineno,
                                       self.filename)
         try: