reduce code size of parser and speed it up a little by statically switching to unicod...

diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py
index fb9c001903334b3dd3c5977ab0e8ba31fa09d0ea..f31e5be53f4bf1dc7a2864104b0f0c651821b182 100644 (file)
--- a/Cython/Compiler/Lexicon.py
+++ b/Cython/Compiler/Lexicon.py
@@ -1,3 +1,4 @@
+# cython: language_level=3
 #
 #   Cython Scanner - Lexical Definitions
 #

diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd
index 486f758f049a570367c991e21816dd3b42d9b6dc..d9ed00eef2b583c3468d733bb58c605593a08d08 100644 (file)
--- a/Cython/Compiler/Parsing.pxd
+++ b/Cython/Compiler/Parsing.pxd
@@ -3,6 +3,8 @@
 cimport cython
 from Cython.Compiler.Scanning cimport PyrexScanner
 
+ctypedef object (*p_sub_expr_func)(object)
+
 # entry points
 
 cpdef p_module(PyrexScanner s, pxd, full_module_name)
@@ -13,8 +15,8 @@ cpdef p_code(PyrexScanner s, level= *)
 cdef p_ident(PyrexScanner s, message =*)
 cdef p_ident_list(PyrexScanner s)
 
-cdef p_binop_operator(PyrexScanner s)
-cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
+cdef tuple p_binop_operator(PyrexScanner s)
+cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_sub_expr)
 cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
 cdef p_lambdef_nocond(PyrexScanner s)
 cdef p_test(PyrexScanner s)
@@ -29,12 +31,13 @@ cdef p_starred_expr(PyrexScanner s)
 cdef p_cascaded_cmp(PyrexScanner s)
 cdef p_cmp_op(PyrexScanner s)
 cdef p_bit_expr(PyrexScanner s)
-cpdef p_xor_expr(PyrexScanner s)
-cpdef p_and_expr(PyrexScanner s)
-cpdef p_shift_expr(PyrexScanner s)
-cpdef p_arith_expr(PyrexScanner s)
-cpdef p_term(PyrexScanner s)
-cpdef p_factor(PyrexScanner s)
+cdef p_xor_expr(s)
+cdef p_and_expr(s)
+cdef p_shift_expr(s)
+cdef p_arith_expr(s)
+cdef p_term(s)
+cdef p_factor(s)
+cdef _p_factor(PyrexScanner s)
 cdef p_typecast(PyrexScanner s)
 cdef p_sizeof(PyrexScanner s)
 cdef p_yield_expression(PyrexScanner s)

diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py
index 09c7139ec5355c7d6168f4d9e1fb1efb7ed62e10..904d2fdafaf3fd37adddf96047fb90ac269ec921 100644 (file)
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
@@ -1,4 +1,4 @@
-# cython: auto_cpdef=True, infer_types=True
+# cython: auto_cpdef=True, infer_types=True, language_level=3
 #
 #   Pyrex Parser
 #
@@ -269,6 +269,10 @@ def p_term(s):
 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
 
 def p_factor(s):
+    # little indirection for C-ification purposes
+    return _p_factor(s)
+
+def _p_factor(s):
     sy = s.sy
     if sy in ('+', '-', '~'):
         op = s.sy
@@ -1701,7 +1705,6 @@ def p_statement(s, ctx, first_statement = 0):
         return p_IF_statement(s, ctx)
     elif s.sy == 'DECORATOR':
         if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):
-            print ctx.level
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)

diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py
index dab1d6b7764692119486914170dc91fd59d60d33..ff719ca68d891700102c4edc8608ef8bfeee8ce5 100644 (file)
--- a/Cython/Compiler/Scanning.py
+++ b/Cython/Compiler/Scanning.py
@@ -1,3 +1,4 @@
+# cython: infer_types=True, language_level=3
 #
 #   Cython Scanner
 #
@@ -7,7 +8,7 @@ import os
 import platform
 
 import cython
-cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object,
+cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=unicode,
                print_function=object)
 
 from Cython import Plex, Utils

diff --git a/setup.py b/setup.py
index 1db45e0fde568d31b2db3df87ecee228f97a8106..4325a2d3a01b905afb70549b3f082871aa3e279b 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -88,6 +88,7 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
     source_root = os.path.abspath(os.path.dirname(__file__))
     compiled_modules = ["Cython.Plex.Scanners",
                         "Cython.Plex.Actions",
+                        "Cython.Compiler.Lexicon",
                         "Cython.Compiler.Scanning",
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",