From 59b6bd5d8480fcdc61936e691969e72fb4bb2e83 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 30 Mar 2009 21:00:16 +0200 Subject: [PATCH] Added support for line-based comments. --HG-- branch : trunk --- CHANGES | 1 + docs/templates.rst | 8 ++++++ jinja2/defaults.py | 1 + jinja2/environment.py | 23 +++++++++++------ jinja2/ext.py | 1 + jinja2/lexer.py | 57 ++++++++++++++++++++++++++++-------------- tests/test_old_bugs.py | 6 +++++ tests/test_parser.py | 28 ++++++++++++++++++++- 8 files changed, 98 insertions(+), 27 deletions(-) diff --git a/CHANGES b/CHANGES index 213cd72..b218e87 100644 --- a/CHANGES +++ b/CHANGES @@ -19,6 +19,7 @@ Version 2.2 variable and regular variable *after* the loop if that variable was unused *before* the loop. (#331) - Added support for optional `scoped` modifier to blocks. +- Added support for line-comments. Version 2.1.1 ------------- diff --git a/docs/templates.rst b/docs/templates.rst index 2187638..7714416 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -252,6 +252,14 @@ precedes it. For better readability statements that start a block (such as # endfor +Since Jinja 2.2 line-based comments are available as well. For example if +the line-comment prefix is configured to be ``##`` everything from ``##`` to +the end of the line is ignored (excluding the newline sign):: + + # for item in seq: +
  • {{ item }}
    • ## this comment is ignored + # endfor + .. _template-inheritance: diff --git a/jinja2/defaults.py b/jinja2/defaults.py index b1a2f6a..458485e 100644 --- a/jinja2/defaults.py +++ b/jinja2/defaults.py @@ -19,6 +19,7 @@ VARIABLE_END_STRING = '}}' COMMENT_START_STRING = '{#' COMMENT_END_STRING = '#}' LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None TRIM_BLOCKS = False NEWLINE_SEQUENCE = '\n' diff --git a/jinja2/environment.py b/jinja2/environment.py index bfbb662..23bf24b 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -118,6 +118,12 @@ class Environment(object): If given and a string, this will be used as prefix for line based statements. See also :ref:`line-statements`. + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + based comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + `trim_blocks` If this is set to ``True`` the first newline after a block is removed (block, not variable tag!). Defaults to `False`. @@ -198,6 +204,7 @@ class Environment(object): comment_start_string=COMMENT_START_STRING, comment_end_string=COMMENT_END_STRING, line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, extensions=(), @@ -228,6 +235,7 @@ class Environment(object): self.comment_start_string = comment_start_string self.comment_end_string = comment_end_string self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix self.trim_blocks = trim_blocks self.newline_sequence = newline_sequence @@ -266,10 +274,10 @@ class Environment(object): def overlay(self, block_start_string=missing, block_end_string=missing, variable_start_string=missing, variable_end_string=missing, comment_start_string=missing, comment_end_string=missing, - line_statement_prefix=missing, trim_blocks=missing, - extensions=missing, optimized=missing, undefined=missing, - finalize=missing, autoescape=missing, loader=missing, - cache_size=missing, auto_reload=missing, + line_statement_prefix=missing, line_comment_prefix=missing, + trim_blocks=missing, extensions=missing, optimized=missing, + undefined=missing, finalize=missing, autoescape=missing, + loader=missing, cache_size=missing, auto_reload=missing, bytecode_cache=missing): """Create a new overlay environment that shares all the data with the current environment except of cache and the overriden attributes. @@ -560,6 +568,7 @@ class Template(object): comment_start_string=COMMENT_START_STRING, comment_end_string=COMMENT_END_STRING, line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, extensions=(), @@ -570,9 +579,9 @@ class Template(object): env = get_spontaneous_environment( block_start_string, block_end_string, variable_start_string, variable_end_string, comment_start_string, comment_end_string, - line_statement_prefix, trim_blocks, newline_sequence, - frozenset(extensions), optimized, undefined, finalize, - autoescape, None, 0, False, None) + line_statement_prefix, line_comment_prefix, trim_blocks, + newline_sequence, frozenset(extensions), optimized, undefined, + finalize, autoescape, None, 0, False, None) return env.from_string(source, template_class=cls) @classmethod diff --git a/jinja2/ext.py b/jinja2/ext.py index ea3a894..bd61ebc 100644 --- a/jinja2/ext.py +++ b/jinja2/ext.py @@ -429,6 +429,7 @@ def babel_extract(fileobj, keywords, comment_tags, options): options.get('comment_start_string', COMMENT_START_STRING), options.get('comment_end_string', COMMENT_END_STRING), options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, + options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, str(options.get('trim_blocks', TRIM_BLOCKS)).lower() in \ ('1', 'on', 'yes', 'true'), NEWLINE_SEQUENCE, frozenset(extensions), diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 39aac9f..10cfd63 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -78,6 +78,9 @@ TOKEN_COMMENT_END = intern('comment_end') TOKEN_COMMENT = intern('comment') TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') TOKEN_LINESTATEMENT_END = intern('linestatement_end') +TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') +TOKEN_LINECOMMENT_END = intern('linecomment_end') +TOKEN_LINECOMMENT = intern('linecomment') TOKEN_DATA = intern('data') TOKEN_INITIAL = intern('initial') TOKEN_EOF = intern('eof') @@ -117,6 +120,11 @@ assert len(operators) == len(reverse_operators), 'operators dropped' operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))) +ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, + TOKEN_COMMENT_END, TOKEN_WHITESPACE, + TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN, + TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT]) + def count_newlines(value): """Count the number of newline characters in the string. This is @@ -125,6 +133,28 @@ def count_newlines(value): return len(newline_re.findall(value)) +def compile_rules(environment): + """Compiles all the rules from the environment into a list of rules.""" + e = re.escape + rules = [ + (len(environment.comment_start_string), 'comment', + e(environment.comment_start_string)), + (len(environment.block_start_string), 'block', + e(environment.block_start_string)), + (len(environment.variable_start_string), 'variable', + e(environment.variable_start_string)), + ] + + if environment.line_statement_prefix is not None: + rules.append((len(environment.line_statement_prefix), 'linestatement', + '^\\s*' + e(environment.line_statement_prefix))) + if environment.line_comment_prefix is not None: + rules.append((len(environment.line_comment_prefix), 'linecomment', + '\\s*' + e(environment.line_comment_prefix))) + + return [x[1:] for x in sorted(rules, reverse=True)] + + class Failure(object): """Class that raises a `TemplateSyntaxError` if called. Used by the `Lexer` to specify known errors. @@ -302,6 +332,7 @@ def get_lexer(environment): environment.comment_start_string, environment.comment_end_string, environment.line_statement_prefix, + environment.line_comment_prefix, environment.trim_blocks, environment.newline_sequence) lexer = _lexer_cache.get(key) @@ -340,22 +371,7 @@ class Lexer(object): # <%= for variables. (if someone wants asp like syntax) # variables are just part of the rules if variable processing # is required. - root_tag_rules = [ - ('comment', environment.comment_start_string), - ('block', environment.block_start_string), - ('variable', environment.variable_start_string) - ] - root_tag_rules.sort(key=lambda x: -len(x[1])) - - # now escape the rules. This is done here so that the escape - # signs don't count for the lengths of the tags. - root_tag_rules = [(a, e(b)) for a, b in root_tag_rules] - - # if we have a line statement prefix we need an extra rule for - # that. We add this rule *after* all the others. - if environment.line_statement_prefix is not None: - prefix = e(environment.line_statement_prefix) - root_tag_rules.insert(0, ('linestatement', '^\s*' + prefix)) + root_tag_rules = compile_rules(environment) # block suffix if trimming is enabled block_suffix_re = environment.trim_blocks and '\\n?' or '' @@ -416,7 +432,11 @@ class Lexer(object): # line statements TOKEN_LINESTATEMENT_BEGIN: [ (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') - ] + tag_rules + ] + tag_rules, + # line comments + TOKEN_LINECOMMENT_BEGIN: [ + (c(r'.*?(?=\n|$)'), TOKEN_LINECOMMENT_END, '#pop') + ] } def _normalize_newlines(self, value): @@ -434,8 +454,7 @@ class Lexer(object): every token in a :class:`Token` and converts the value. """ for lineno, token, value in stream: - if token in ('comment_begin', 'comment', 'comment_end', - 'whitespace'): + if token in ignored_tokens: continue elif token == 'linestatement_begin': token = 'block_begin' diff --git a/tests/test_old_bugs.py b/tests/test_old_bugs.py index f77e60c..59d6037 100644 --- a/tests/test_old_bugs.py +++ b/tests/test_old_bugs.py @@ -60,3 +60,9 @@ def test_weird_inline_comment(): env = Environment(line_statement_prefix='%') raises(TemplateSyntaxError, env.from_string, '% for item in seq {# missing #}\n...% endfor') + + +def test_old_macro_loop_scoping_bug(env): + tmpl = env.from_string('{% for i in (1, 2) %}{{ i }}{% endfor %}' + '{% macro i() %}3{% endmacro %}{{ i() }}') + assert tmpl.render() == '123' diff --git a/tests/test_parser.py b/tests/test_parser.py index fb338e1..640394e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -28,10 +28,17 @@ COMMENT_SYNTAX = '''\ ''' MAKO_SYNTAX = '''\ +<%# regular comment %> % for item in seq: ${item} % endfor''' +MAKO_SYNTAX_LINECOMMENTS = '''\ +<%# regular comment %> +% for item in seq: + ${item} ## the rest of the stuff +% endfor''' + BALANCING = '''{{{'foo':'bar'}.foo}}''' STARTCOMMENT = '''{# foo comment @@ -39,6 +46,14 @@ and bar comment #} {% macro blub() %}foo{% endmacro %} {{ blub() }}''' +LINE_SYNTAX_PRIORITY = '''\ +/* ignore me. + I'm a multiline comment */ +# for item in seq: +* ${item} ## this is just extra stuff +# endfor +''' + def test_php_syntax(): env = Environment('', '', '') @@ -72,4 +87,15 @@ def test_line_syntax(): env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%') tmpl = env.from_string(MAKO_SYNTAX) assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \ - range(5) + range(5) + + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##') + tmpl = env.from_string(MAKO_SYNTAX_LINECOMMENTS) + assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \ + range(5) + + +def test_line_syntax_priority(): + env = Environment('{%', '%}', '${', '}', '/*', '*/', '#', '##') + tmpl = env.from_string(LINE_SYNTAX_PRIORITY) + assert tmpl.render(seq=[1, 2]).strip() == '* 1\n* 2' -- 2.26.2