From b754b352d5e9133c7dc3944aeadc43ba081d352e Mon Sep 17 00:00:00 2001 From: Robert Bradshaw Date: Tue, 11 Nov 2008 02:17:30 -0800 Subject: [PATCH] Further bootstrapping of Scanner --- Cython/Compiler/Nodes.py | 8 ++++-- Cython/Compiler/Scanning.pxd | 15 +++++++++++ Cython/Compiler/Scanning.py | 14 ++++++---- Cython/Plex/Scanners.pxd | 23 +++++++++++++++++ Cython/Plex/Scanners.py | 50 +++++++++++++++++++++--------------- setup.py | 2 +- 6 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 Cython/Compiler/Scanning.pxd create mode 100644 Cython/Plex/Scanners.pxd diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index 10700cd2..0d1273db 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -2316,10 +2316,11 @@ class PyClassDefNode(ClassDefNode): elif len(bases) == 1: base = bases[0] path = [] - while isinstance(base, ExprNodes.AttributeNode): + from ExprNodes import AttributeNode, NameNode + while isinstance(base, AttributeNode): path.insert(0, base.attribute) base = base.obj - if isinstance(base, ExprNodes.NameNode): + if isinstance(base, NameNode): path.insert(0, base.name) base_class_name = path[-1] if len(path) > 1: @@ -4399,6 +4400,9 @@ class FromImportStatNode(StatNode): env.use_utility_code(ExprNodes.type_test_utility_code) break else: + entry = env.lookup(target.name) + if entry.is_type and entry.type.name == name and entry.type.module_name == self.module.module_name.value: + continue # already cimported self.interned_items.append( (env.intern_identifier(name), target)) target.analyse_target_expression(env, None) diff --git a/Cython/Compiler/Scanning.pxd b/Cython/Compiler/Scanning.pxd new file mode 100644 index 00000000..8af93ebf --- /dev/null +++ b/Cython/Compiler/Scanning.pxd @@ -0,0 +1,15 @@ +from Cython.Plex.Scanners cimport Scanner + +cdef class PyrexScanner(Scanner): + cdef public context + cdef public list included_files + cdef public compile_time_env + cdef public bint compile_time_eval + cdef public bint compile_time_expr + cdef public bint parse_comments + cdef public source_encoding + cdef public list indentation_stack + cdef public indentation_char + cdef public int bracket_nesting_level + cdef public sy + cdef public systring diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index fc7da80d..9a08a2a6 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -12,14 +12,17 @@ import sys from time import time from Cython import Plex, Utils -from Cython.Plex import Scanner +from Cython.Plex.Scanners import Scanner from Cython.Plex.Errors import UnrecognizedInput from Errors import CompileError, error from Lexicon import string_prefixes, raw_prefixes, make_lexicon from StringEncoding import EncodedString -plex_version = getattr(Plex, '_version', None) +try: + plex_version = Plex._version +except AttributeError: + plex_version = None #print "Plex version:", plex_version ### debug_scanner = 0 @@ -91,7 +94,7 @@ def try_to_unpickle_lexicon(): source_file = os.path.join(dir, "Lexicon.py") lexicon_hash = hash_source_file(source_file) lexicon_pickle = os.path.join(dir, "Lexicon.pickle") - f = open_pickled_lexicon(expected_hash = lexicon_hash) + f = open_pickled_lexicon(lexicon_hash) if f: if notify_lexicon_unpickling: t0 = time() @@ -165,6 +168,8 @@ def build_resword_dict(): d[word] = 1 return d +resword_dict = build_resword_dict() + #------------------------------------------------------------------ class CompileTimeScope(object): @@ -286,7 +291,6 @@ class PyrexScanner(Scanner): # compile_time_env dict Environment for conditional compilation # compile_time_eval boolean In a true conditional compilation context # compile_time_expr boolean In a compile-time expression context - resword_dict = build_resword_dict() def __init__(self, file, filename, parent_scanner = None, scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None): @@ -404,7 +408,7 @@ class PyrexScanner(Scanner): except UnrecognizedInput: self.error("Unrecognized character") if sy == 'IDENT': - if systring in self.resword_dict: + if systring in resword_dict: sy = systring else: systring = EncodedString(systring) diff --git a/Cython/Plex/Scanners.pxd b/Cython/Plex/Scanners.pxd new file mode 100644 index 00000000..081e73db --- /dev/null +++ b/Cython/Plex/Scanners.pxd @@ -0,0 +1,23 @@ +cdef class Scanner: + cdef public lexicon + cdef public stream + cdef public name + cdef public buffer + cdef public long buf_start_pos + cdef public long next_pos + cdef public long cur_pos + cdef public long cur_line + cdef public long cur_line_start + cdef public long start_pos + cdef public long start_line + cdef public long start_col + cdef public text + cdef public initial_state + cdef public state_name + cdef public list queue + cdef public bint trace + cdef public cur_char + cdef public input_state + + cdef public level + \ No newline at end of file diff --git a/Cython/Plex/Scanners.py b/Cython/Plex/Scanners.py index 3a7e8ddc..02d6b2d2 100644 --- a/Cython/Plex/Scanners.py +++ b/Cython/Plex/Scanners.py @@ -10,6 +10,8 @@ import Errors from Regexps import BOL, EOL, EOF +import cython + class Scanner: """ A Scanner is used to read tokens from a stream of characters @@ -42,23 +44,23 @@ class Scanner: """ - lexicon = None # Lexicon - stream = None # file-like object - name = '' - buffer = '' - buf_start_pos = 0 # position in input of start of buffer - next_pos = 0 # position in input of next char to read - cur_pos = 0 # position in input of current char - cur_line = 1 # line number of current char - cur_line_start = 0 # position in input of start of current line - start_pos = 0 # position in input of start of token - start_line = 0 # line number of start of token - start_col = 0 # position in line of start of token - text = None # text of last token read - initial_state = None # Node - state_name = '' # Name of initial state - queue = None # list of tokens to be returned - trace = 0 +# lexicon = None # Lexicon +# stream = None # file-like object +# name = '' +# buffer = '' +# buf_start_pos = 0 # position in input of start of buffer +# next_pos = 0 # position in input of next char to read +# cur_pos = 0 # position in input of current char +# cur_line = 1 # line number of current char +# cur_line_start = 0 # position in input of start of current line +# start_pos = 0 # position in input of start of token +# start_line = 0 # line number of start of token +# start_col = 0 # position in line of start of token +# text = None # text of last token read +# initial_state = None # Node +# state_name = '' # Name of initial state +# queue = None # list of tokens to be returned +# trace = 0 def __init__(self, lexicon, stream, name = '', initial_pos = None): """ @@ -73,6 +75,17 @@ class Scanner: |name| is optional, and may be the name of the file being scanned or any other identifying string. """ + self.buffer = '' + self.buf_start_pos = 0 + self.next_pos = 0 + self.cur_pos = 0 + self.cur_line = 1 + self.start_pos = 0 + self.start_line = 0 + self.start_col = 0 + self.text = None + self.state_name = None + self.lexicon = lexicon self.stream = stream self.name = name @@ -374,6 +387,3 @@ class Scanner: Override this method if you want something to be done at end of file. """ - -# For backward compatibility: -setattr(Scanner, "yield", Scanner.produce) diff --git a/setup.py b/setup.py index d011f8aa..16e1f0c3 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ except ValueError: try: from Cython.Compiler.Main import compile source_root = os.path.dirname(__file__) - compiled_modules = ["Cython.Plex.Scanners"] + compiled_modules = ["Cython.Plex.Scanners", "Cython.Compiler.Scanning"] extensions = [] for module in compiled_modules: source_file = os.path.join(source_root, *module.split('.')) -- 2.26.2