From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Fri, 23 May 2008 20:26:45 +0000 (+0200)
Subject: Environment.lex returns unicode tokens now, even if the input data was a bytestring.
X-Git-Tag: 2.0rc1~31
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=5c047ea2c3cd5e3a744e2298e032f41929aa2974;p=jinja2.git

Environment.lex returns unicode tokens now, even if the input data was a bytestring.

--HG--
branch : trunk
---

diff --git a/jinja2-debug.py b/jinja2-debug.py
index a250a62..2a870e8 100755
--- a/jinja2-debug.py
+++ b/jinja2-debug.py
@@ -13,7 +13,8 @@ import sys
 import jinja2
 from werkzeug import script
 
-env = jinja2.Environment(extensions=['jinja2.ext.i18n', 'jinja2.ext.do'])
+env = jinja2.Environment(extensions=['jinja2.ext.i18n', 'jinja2.ext.do',
+                                     'jinja2.ext.loopcontrols'])
 
 def shell_init_func():
     def _compile(x):
diff --git a/jinja2/ext.py b/jinja2/ext.py
index 63e8f5b..701f54d 100644
--- a/jinja2/ext.py
+++ b/jinja2/ext.py
@@ -25,7 +25,7 @@ GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
 
 
 class ExtensionRegistry(type):
-    """Gives the extension a unique identifier."""
+    """Gives the extension an unique identifier."""
 
     def __new__(cls, name, bases, d):
         rv = type.__new__(cls, name, bases, d)
@@ -95,13 +95,18 @@ class Extension(object):
                           dyn_args, dyn_kwargs, lineno=lineno)
 
 
+@contextfunction
+def _gettext_alias(context, string):
+    return context.resolve('gettext')(string)
+
+
 class InternationalizationExtension(Extension):
     """This extension adds gettext support to Jinja2."""
     tags = set(['trans'])
 
     def __init__(self, environment):
         Extension.__init__(self, environment)
-        environment.globals['_'] = contextfunction(lambda c, x: c['gettext'](x))
+        environment.globals['_'] = _gettext_alias
         environment.extend(
             install_gettext_translations=self._install,
             install_null_translations=self._install_null,
diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index 7f0b33f..92ff12e 100644
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -371,7 +371,6 @@ class Lexer(object):
         converted into types and postprocessed. For example comments are removed,
         integers and floats converted, strings unescaped etc.
         """
-        source = unicode(source)
         def generate():
             for lineno, token, value in self.tokeniter(source, name, filename):
                 if token in ('comment_begin', 'comment', 'comment_end',
@@ -425,7 +424,7 @@ class Lexer(object):
         wants.  The parser uses the `tokenize` function with returns a
         `TokenStream` and postprocessed tokens.
         """
-        source = '\n'.join(source.splitlines())
+        source = u'\n'.join(unicode(source).splitlines())
         pos = 0
         lineno = 1
         stack = ['root']