move Cython language keywords out of the way when parsing .py files

diff --git a/Cython/Compiler/Scanning.pxd b/Cython/Compiler/Scanning.pxd
index 4df5f3013a986ebdd1f85198c6027f78c5316c5b..d98b71328eca9255671d3e9be0187be22e0ba2ab 100644 (file)
--- a/Cython/Compiler/Scanning.pxd
+++ b/Cython/Compiler/Scanning.pxd
@@ -14,6 +14,7 @@ cdef class PyrexScanner(Scanner):
     cdef public bint compile_time_expr
     cdef public bint parse_comments
     cdef public source_encoding
+    cdef set keywords
     cdef public list indentation_stack
     cdef public indentation_char
     cdef public int bracket_nesting_level

diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py
index 8b1c36366456d6060d09f9743d61454c6f8c48ee..bdbf46c48053e07425955506de759044458f4ac8 100644 (file)
--- a/Cython/Compiler/Scanning.py
+++ b/Cython/Compiler/Scanning.py
@@ -37,13 +37,17 @@ def get_lexicon():
     
 #------------------------------------------------------------------
 
-reserved_words = [
-    "global", "include", "ctypedef", "cdef", "def", "class",
-    "print", "del", "pass", "break", "continue", "return",
-    "raise", "import", "exec", "try", "except", "finally",
-    "while", "if", "elif", "else", "for", "in", "assert",
-    "and", "or", "not", "is", "in", "lambda", "from", "yield",
-    "cimport", "by", "with", "cpdef", "DEF", "IF", "ELIF", "ELSE"
+py_reserved_words = [
+    "global", "def", "class", "print", "del", "pass", "break",
+    "continue", "return", "raise", "import", "exec", "try",
+    "except", "finally", "while", "if", "elif", "else", "for",
+    "in", "assert", "and", "or", "not", "is", "in", "lambda",
+    "from", "yield", "with", "nonlocal",
+]
+
+pyx_reserved_words = py_reserved_words + [
+    "include", "ctypedef", "cdef", "cpdef",
+    "cimport", "by", "DEF", "IF", "ELIF", "ELSE"
 ]
 
 class Method(object):
@@ -57,17 +61,6 @@ class Method(object):
 
 #------------------------------------------------------------------
 
-def build_resword_dict():
-    d = {}
-    for word in reserved_words:
-        d[word] = 1
-    return d
-
-cython.declare(resword_dict=dict)
-resword_dict = build_resword_dict()
-
-#------------------------------------------------------------------
-
 class CompileTimeScope(object):
 
     def __init__(self, outer = None):
@@ -123,11 +116,23 @@ class SourceDescriptor(object):
     """
     A SourceDescriptor should be considered immutable.
     """
+    _file_type = 'pyx'
+
     _escaped_description = None
     _cmp_name = ''
     def __str__(self):
         assert False # To catch all places where a descriptor is used directly as a filename
-    
+
+    def set_file_type_from_name(self, filename):
+        name, ext = os.path.splitext(filename)
+        self._file_type = ext in ('.pyx', '.pxd', '.py') and ext[1:] or 'pyx'
+
+    def is_cython_file(self):
+        return self._file_type in ('pyx', 'pxd')
+
+    def is_python_file(self):
+        return self._file_type == 'py'
+
     def get_escaped_description(self):
         if self._escaped_description is None:
             self._escaped_description = \
@@ -165,6 +170,7 @@ class FileSourceDescriptor(SourceDescriptor):
     """
     def __init__(self, filename):
         self.filename = filename
+        self.set_file_type_from_name(filename)
         self._cmp_name = filename
     
     def get_lines(self, encoding=None, error_handling=None):
@@ -196,6 +202,7 @@ class StringSourceDescriptor(SourceDescriptor):
     """
     def __init__(self, name, code):
         self.name = name
+        self.set_file_type_from_name(name)
         self.codelines = [x + "\n" for x in code.split("\n")]
         self._cmp_name = name
     
@@ -247,6 +254,10 @@ class PyrexScanner(Scanner):
             self.compile_time_expr = 0
         self.parse_comments = parse_comments
         self.source_encoding = source_encoding
+        if filename.is_python_file():
+            self.keywords = cython.set(py_reserved_words)
+        else:
+            self.keywords = cython.set(pyx_reserved_words)
         self.trace = trace_scanner
         self.indentation_stack = [0]
         self.indentation_char = None
@@ -346,7 +357,7 @@ class PyrexScanner(Scanner):
         except UnrecognizedInput:
             self.error("Unrecognized character")
         if sy == IDENT:
-            if systring in resword_dict:
+            if systring in self.keywords:
                 if systring == 'print' and \
                        print_function in self.context.future_directives:
                     systring = EncodedString(systring)