From 90cfb960eec7f4e112f202a97fda6094cf8480a4 Mon Sep 17 00:00:00 2001
From: Stefan Behnel <scoder@users.berlios.de>
Date: Wed, 8 Dec 2010 22:53:11 +0100
Subject: [PATCH] reduce code size of parser and speed it up a little by
 statically switching to unicode in more places

---
 Cython/Compiler/Lexicon.py  |  1 +
 Cython/Compiler/Parsing.pxd | 19 +++++++++++--------
 Cython/Compiler/Parsing.py  |  7 +++++--
 Cython/Compiler/Scanning.py |  3 ++-
 setup.py                    |  1 +
 5 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py
index fb9c0019..f31e5be5 100644
--- a/Cython/Compiler/Lexicon.py
+++ b/Cython/Compiler/Lexicon.py
@@ -1,3 +1,4 @@
+# cython: language_level=3
 #
 #   Cython Scanner - Lexical Definitions
 #
diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd
index 486f758f..d9ed00ee 100644
--- a/Cython/Compiler/Parsing.pxd
+++ b/Cython/Compiler/Parsing.pxd
@@ -3,6 +3,8 @@
 cimport cython
 from Cython.Compiler.Scanning cimport PyrexScanner
 
+ctypedef object (*p_sub_expr_func)(object)
+
 # entry points
 
 cpdef p_module(PyrexScanner s, pxd, full_module_name)
@@ -13,8 +15,8 @@ cpdef p_code(PyrexScanner s, level= *)
 cdef p_ident(PyrexScanner s, message =*)
 cdef p_ident_list(PyrexScanner s)
 
-cdef p_binop_operator(PyrexScanner s)
-cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
+cdef tuple p_binop_operator(PyrexScanner s)
+cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_sub_expr)
 cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
 cdef p_lambdef_nocond(PyrexScanner s)
 cdef p_test(PyrexScanner s)
@@ -29,12 +31,13 @@ cdef p_starred_expr(PyrexScanner s)
 cdef p_cascaded_cmp(PyrexScanner s)
 cdef p_cmp_op(PyrexScanner s)
 cdef p_bit_expr(PyrexScanner s)
-cpdef p_xor_expr(PyrexScanner s)
-cpdef p_and_expr(PyrexScanner s)
-cpdef p_shift_expr(PyrexScanner s)
-cpdef p_arith_expr(PyrexScanner s)
-cpdef p_term(PyrexScanner s)
-cpdef p_factor(PyrexScanner s)
+cdef p_xor_expr(s)
+cdef p_and_expr(s)
+cdef p_shift_expr(s)
+cdef p_arith_expr(s)
+cdef p_term(s)
+cdef p_factor(s)
+cdef _p_factor(PyrexScanner s)
 cdef p_typecast(PyrexScanner s)
 cdef p_sizeof(PyrexScanner s)
 cdef p_yield_expression(PyrexScanner s)
diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py
index 09c7139e..904d2fda 100644
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
@@ -1,4 +1,4 @@
-# cython: auto_cpdef=True, infer_types=True
+# cython: auto_cpdef=True, infer_types=True, language_level=3
 #
 #   Pyrex Parser
 #
@@ -269,6 +269,10 @@ def p_term(s):
 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
 
 def p_factor(s):
+    # little indirection for C-ification purposes
+    return _p_factor(s)
+
+def _p_factor(s):
     sy = s.sy
     if sy in ('+', '-', '~'):
         op = s.sy
@@ -1701,7 +1705,6 @@ def p_statement(s, ctx, first_statement = 0):
         return p_IF_statement(s, ctx)
     elif s.sy == 'DECORATOR':
         if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):
-            print ctx.level
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)
diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py
index dab1d6b7..ff719ca6 100644
--- a/Cython/Compiler/Scanning.py
+++ b/Cython/Compiler/Scanning.py
@@ -1,3 +1,4 @@
+# cython: infer_types=True, language_level=3
 #
 #   Cython Scanner
 #
@@ -7,7 +8,7 @@ import os
 import platform
 
 import cython
-cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object,
+cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=unicode,
                print_function=object)
 
 from Cython import Plex, Utils
diff --git a/setup.py b/setup.py
index 1db45e0f..4325a2d3 100644
--- a/setup.py
+++ b/setup.py
@@ -88,6 +88,7 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
     source_root = os.path.abspath(os.path.dirname(__file__))
     compiled_modules = ["Cython.Plex.Scanners",
                         "Cython.Plex.Actions",
+                        "Cython.Compiler.Lexicon",
                         "Cython.Compiler.Scanning",
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",
-- 
2.26.2