From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Wed, 10 Feb 2010 00:35:13 +0000 (+0100)
Subject: Down to 7 failures for Python 3. We're onto something.
X-Git-Tag: 2.3~7
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=0d242be185f34fcc3b4466545393e9ccbc86d199;p=jinja2.git

Down to 7 failures for Python 3. We're onto something.

--HG--
branch : trunk
---

diff --git a/custom_fixers/fix_broken_reraising.py b/custom_fixers/fix_broken_reraising.py
new file mode 100644
index 0000000..fd0ea68
--- /dev/null
+++ b/custom_fixers/fix_broken_reraising.py
@@ -0,0 +1,21 @@
+from lib2to3 import fixer_base, pytree
+from lib2to3.fixer_util import Name, BlankLine, Name, Attr, ArgList
+
+
+class FixBrokenReraising(fixer_base.BaseFix):
+    PATTERN = """
+    raise_stmt< 'raise' any ',' val=any ',' tb=any >
+    """
+
+    # run before the broken 2to3 checker with the same goal
+    # tries to rewrite it with a rule that does not work out for jinja
+    run_order = 1
+
+    def transform(self, node, results):
+        tb = results['tb'].clone()
+        tb.prefix = ''
+        with_tb = Attr(results['val'].clone(), Name('with_traceback')) + \
+                  [ArgList([tb])]
+        new = pytree.Node(self.syms.simple_stmt, [Name("raise")] + with_tb)
+        new.prefix = node.prefix
+        return new
diff --git a/jinja2/compiler.py b/jinja2/compiler.py
index 672d696..efe534b 100644
--- a/jinja2/compiler.py
+++ b/jinja2/compiler.py
@@ -36,6 +36,14 @@ else:
     have_condexpr = True
 
 
+# what method to iterate over items do we want to use for dict iteration
+# in generated code?  on 2.x let's go with iteritems, on 3.x with items
+if hasattr(dict, 'iteritems'):
+    dict_item_iter = 'iteritems'
+else:
+    dict_item_iter = 'items'
+
+
 def generate(node, environment, name, filename, stream=None):
     """Generate the python source for a node tree."""
     if not isinstance(node, nodes.Template):
@@ -867,7 +875,7 @@ class CodeGenerator(NodeVisitor):
         self.visit(node.template, frame)
         self.write(', %r)' % self.name)
         self.writeline('for name, parent_block in parent_template.'
-                       'blocks.iteritems():')
+                       'blocks.%s():' % dict_item_iter)
         self.indent()
         self.writeline('context.blocks.setdefault(name, []).'
                        'append(parent_block)')
diff --git a/jinja2/environment.py b/jinja2/environment.py
index fe7ed02..e3c64e3 100644
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -19,7 +19,7 @@ from jinja2.runtime import Undefined, new_context
 from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \
      TemplatesNotFound
 from jinja2.utils import import_string, LRUCache, Markup, missing, \
-     concat, consume, internalcode
+     concat, consume, internalcode, _encode_filename
 
 
 # for direct template usage we have up to ten living environments
@@ -375,9 +375,7 @@ class Environment(object):
 
     def _parse(self, source, name, filename):
         """Internal parsing function used by `parse` and `compile`."""
-        if isinstance(filename, unicode):
-            filename = filename.encode('utf-8')
-        return Parser(self, source, name, filename).parse()
+        return Parser(self, source, name, _encode_filename(filename)).parse()
 
     def lex(self, source, name=None, filename=None):
         """Lex the given sourcecode and return a generator that yields
@@ -442,8 +440,8 @@ class Environment(object):
                 return source
             if filename is None:
                 filename = '<template>'
-            elif isinstance(filename, unicode):
-                filename = filename.encode('utf-8')
+            else:
+                filename = _encode_filename(filename)
             return compile(source, filename, 'exec')
         except TemplateSyntaxError:
             exc_info = sys.exc_info()
diff --git a/jinja2/exceptions.py b/jinja2/exceptions.py
index 69a877b..771f6a8 100644
--- a/jinja2/exceptions.py
+++ b/jinja2/exceptions.py
@@ -92,7 +92,7 @@ class TemplateSyntaxError(TemplateError):
     def __unicode__(self):
         # for translated errors we only return the message
         if self.translated:
-            return self.message.encode('utf-8')
+            return self.message
 
         # otherwise attach some stuff
         location = 'line %d' % self.lineno
diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index b974199..98d85d9 100644
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -530,7 +530,8 @@ class Lexer(object):
                     raise TemplateSyntaxError(msg, lineno, name, filename)
                 # if we can express it as bytestring (ascii only)
                 # we do that for support of semi broken APIs
-                # as datetime.datetime.strftime
+                # as datetime.datetime.strftime.  On python 3 this
+                # call becomes a noop thanks to 2to3
                 try:
                     value = str(value)
                 except UnicodeError:
diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 5972139..424c1cd 100644
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -494,13 +494,18 @@ class Filter(Expr):
     def as_const(self, obj=None):
         if self.node is obj is None:
             raise Impossible()
-        filter = self.environment.filters.get(self.name)
-        if filter is None or getattr(filter, 'contextfilter', False):
+        # we have to be careful here because we call filter_ below.
+        # if this variable would be called filter, 2to3 would wrap the
+        # call in a list beause it is assuming we are talking about the
+        # builtin filter function here which no longer returns a list in
+        # python 3.  because of that, do not rename filter_ to filter!
+        filter_ = self.environment.filters.get(self.name)
+        if filter_ is None or getattr(filter_, 'contextfilter', False):
             raise Impossible()
         if obj is None:
             obj = self.node.as_const()
         args = [x.as_const() for x in self.args]
-        if getattr(filter, 'environmentfilter', False):
+        if getattr(filter_, 'environmentfilter', False):
             args.insert(0, self.environment)
         kwargs = dict(x.as_const() for x in self.kwargs)
         if self.dyn_args is not None:
@@ -514,7 +519,7 @@ class Filter(Expr):
             except:
                 raise Impossible()
         try:
-            return filter(obj, *args, **kwargs)
+            return filter_(obj, *args, **kwargs)
         except:
             raise Impossible()
 
diff --git a/jinja2/testsuite/ext.py b/jinja2/testsuite/ext.py
index 3bb5276..ddb81eb 100644
--- a/jinja2/testsuite/ext.py
+++ b/jinja2/testsuite/ext.py
@@ -17,6 +17,13 @@ from jinja2 import Environment, DictLoader, contextfunction, nodes
 from jinja2.exceptions import TemplateAssertionError
 from jinja2.ext import Extension
 from jinja2.lexer import Token, count_newlines
+from jinja2.utils import next
+
+# 2.x / 3.x
+try:
+    from io import BytesIO
+except ImportError:
+    from StringIO import StringIO as BytesIO
 
 
 importable_object = 23
@@ -81,7 +88,7 @@ class TestExtension(Extension):
             self.attr('ext_attr'),
             nodes.ImportedName(__name__ + '.importable_object'),
             nodes.ContextReference()
-        ])]).set_lineno(parser.stream.next().lineno)
+        ])]).set_lineno(next(parser.stream).lineno)
 
     def _dump(self, sandboxed, ext_attr, imported_object, context):
         return '%s|%s|%s|%s' % (
@@ -222,12 +229,11 @@ class InternationalizationTestCase(JinjaTestCase):
 
     def test_extract(self):
         from jinja2.ext import babel_extract
-        from StringIO import StringIO
-        source = StringIO('''
+        source = BytesIO('''
         {{ gettext('Hello World') }}
         {% trans %}Hello World{% endtrans %}
         {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
-        ''')
+        '''.encode('ascii')) # make python 3 happy
         assert list(babel_extract(source, ('gettext', 'ngettext', '_'), [], {})) == [
             (2, 'gettext', u'Hello World', []),
             (3, 'gettext', u'Hello World', []),
@@ -236,14 +242,13 @@ class InternationalizationTestCase(JinjaTestCase):
 
     def test_comment_extract(self):
         from jinja2.ext import babel_extract
-        from StringIO import StringIO
-        source = StringIO('''
+        source = BytesIO('''
         {# trans first #}
         {{ gettext('Hello World') }}
         {% trans %}Hello World{% endtrans %}{# trans second #}
         {#: third #}
         {% trans %}{{ users }} user{% pluralize %}{{ users }} users{% endtrans %}
-        ''')
+        '''.encode('utf-8')) # make python 3 happy
         assert list(babel_extract(source, ('gettext', 'ngettext', '_'), ['trans', ':'], {})) == [
             (3, 'gettext', u'Hello World', ['first']),
             (4, 'gettext', u'Hello World', ['second']),
diff --git a/jinja2/utils.py b/jinja2/utils.py
index 57fd3c5..0ba4645 100644
--- a/jinja2/utils.py
+++ b/jinja2/utils.py
@@ -72,23 +72,28 @@ except NameError:
         return x.next()
 
 
-# ironpython without stdlib doesn't have keyword
-try:
-    from keyword import iskeyword as is_python_keyword
-except ImportError:
-    _py_identifier_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9]*$')
-    def is_python_keyword(name):
-        if _py_identifier_re.search(name) is None:
-            return False
-        try:
-            exec name + " = 42"
-        except SyntaxError:
-            return False
-        return True
+# if this python version is unable to deal with unicode filenames
+# when passed to encode we let this function encode it properly.
+# This is used in a couple of places.  As far as Jinja is concerned
+# filenames are unicode *or* bytestrings in 2.x and unicode only in
+# 3.x because compile cannot handle bytes
+if sys.version_info < (3, 0):
+    def _encode_filename(filename):
+        if isinstance(filename, unicode):
+            return filename.encode('utf-8')
+        return filename
+else:
+    def _encode_filename(filename):
+        assert filename is None or isinstance(filename, str), \
+            'filenames must be strings'
+        return filename
+
+from keyword import iskeyword as is_python_keyword
 
 
 # common types.  These do exist in the special types module too which however
-# does not exist in IronPython out of the box.
+# does not exist in IronPython out of the box.  Also that way we don't have
+# to deal with implementation specific stuff here
 class _C(object):
     def method(self): pass
 def _func():