added support for line statement prefixes (cheetah/mako/erb like)

--HG--
branch : trunk

diff --git a/jinja2/environment.py b/jinja2/environment.py
index 8d5e9d458a9c1239d754f5bcd8872d0b1ba10f65..8458a2381dbebe13cd096386a46cf4c8e7d90e06 100644 (file)
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -30,8 +30,8 @@ class Environment(object):
                  variable_end_string='}}',
                  comment_start_string='{#',
                  comment_end_string='#}',
+                 line_statement_prefix=None,
                  trim_blocks=False,
-                 template_charset='utf-8',
                  loader=None):
         """Here the possible initialization parameters:
 
@@ -46,10 +46,12 @@ class Environment(object):
                                   comment. defaults to ``'{#'``.
         `comment_end_string`      the string marking the end of a comment.
                                   defaults to ``'#}'``.
+        `line_statement_prefix`   If given and a string, this will be used as
+                                  prefix for line based statements.  See the
+                                  documentation for more details.
         `trim_blocks`             If this is set to ``True`` the first newline
                                   after a block is removed (block, not
                                   variable tag!). Defaults to ``False``.
-        `template_charset`        the charset of the templates.
         `loader`                  the loader which should be used.
         ========================= ============================================
         """
@@ -61,8 +63,8 @@ class Environment(object):
         self.variable_end_string = variable_end_string
         self.comment_start_string = comment_start_string
         self.comment_end_string = comment_end_string
+        self.line_statement_prefix = line_statement_prefix
         self.trim_blocks = trim_blocks
-        self.template_charset = template_charset
 
         # defaults
         self.filters = DEFAULT_FILTERS.copy()

diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index b19e4caa54974c29f26c27322a3bd320422ffae7..5d8ab7b4cb6220a6dde4aeec1c65d4bf269969ca 100644 (file)
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -196,6 +196,7 @@ class LexerMeta(type):
                     environment.variable_end_string,
                     environment.comment_start_string,
                     environment.comment_end_string,
+                    environment.line_statement_prefix,
                     environment.trim_blocks))
 
         # use the cached lexer if possible
@@ -226,7 +227,6 @@ class Lexer(object):
 
         # lexing rules for tags
         tag_rules = [
-            (eol_re, 'eol', None),
             (whitespace_re, None, None),
             (float_re, 'float', None),
             (integer_re, 'integer', None),
@@ -262,7 +262,17 @@ class Lexer(object):
         if not self.no_variable_block:
             root_tag_rules.append(('variable',
                                    environment.variable_start_string))
-        root_tag_rules.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
+        root_tag_rules.sort(key=lambda x: len(x[1]))
+
+        # now escape the rules.  This is done here so that the escape
+        # signs don't count for the lengths of the tags.
+        root_tag_rules = [(a, e(b)) for a, b in root_tag_rules]
+
+        # if we have a line statement prefix we need an extra rule for
+        # that.  We add this rule *after* all the others.
+        if environment.line_statement_prefix is not None:
+            prefix = e(environment.line_statement_prefix)
+            root_tag_rules.insert(0, ('linestatement', '^\s*' + prefix))
 
         # block suffix if trimming is enabled
         block_suffix_re = environment.trim_blocks and '\\n?' or ''
@@ -277,7 +287,7 @@ class Lexer(object):
                         e(environment.block_start_string),
                         e(environment.block_end_string)
                     )] + [
-                        '(?P<%s_begin>\s*%s\-|%s)' % (n, e(r), e(r))
+                        '(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
                         for n, r in root_tag_rules
                     ])), ('data', '#bygroup'), '#bygroup'),
                 # data
@@ -323,6 +333,12 @@ class Lexer(object):
                 )), 'variable_end', '#pop')
             ] + tag_rules
 
+        # the same goes for the line_statement_prefix
+        if environment.line_statement_prefix is not None:
+            self.rules['linestatement_begin'] = [
+                (c(r'\s*(\n|$)'), 'linestatement_end', '#pop')
+            ] + tag_rules
+
     def tokenize(self, source, filename=None):
         """
         Works like `tokeniter` but returns a tokenstream of tokens and not a
@@ -331,10 +347,15 @@ class Lexer(object):
         already keyword tokens, not named tokens, comments are removed,
         integers and floats converted, strings unescaped etc.
         """
+        source = unicode(source)
         def generate():
             for lineno, token, value in self.tokeniter(source, filename):
                 if token in ('comment_begin', 'comment', 'comment_end'):
                     continue
+                elif token == 'linestatement_begin':
+                    token = 'block_begin'
+                elif token == 'linestatement_end':
+                    token = 'block_end'
                 elif token == 'data':
                     try:
                         value = str(value)

diff --git a/jinja2/parser.py b/jinja2/parser.py
index 400b9be35f06c76437a07598e681831e4f987839..74660f6c2189996073fdbf7d49ddc5a402abc5bd 100644 (file)
--- a/jinja2/parser.py
+++ b/jinja2/parser.py
@@ -28,11 +28,9 @@ class Parser(object):
 
     def __init__(self, environment, source, filename=None):
         self.environment = environment
-        if isinstance(source, str):
-            source = source.decode(environment.template_charset, 'ignore')
         if isinstance(filename, unicode):
             filename = filename.encode('utf-8')
-        self.source = source
+        self.source = unicode(source)
         self.filename = filename
         self.closed = False
         self.no_variable_block = self.environment.lexer.no_variable_block