From 5cb5c6854df2bfff2977d5a101830211bf7f676b Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Wed, 14 May 2008 21:20:08 +0200 Subject: [PATCH] make it explicit in the generated source code what is an identifier string constant and what is a normal string constant that just happens to get interned --- Cython/Compiler/ExprNodes.py | 5 +++-- Cython/Compiler/ModuleNode.py | 3 ++- Cython/Compiler/Nodes.py | 37 +++++++++++++++++------------------ Cython/Compiler/Symtab.py | 25 +++++++++++++++-------- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index d0598559..d659c9c9 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -875,7 +875,8 @@ class NameNode(AtomicExprNode): self.type = type if entry.is_pyglobal or entry.is_builtin: assert type.is_pyobject, "Python global or builtin not a Python object" - self.interned_cname = self.entry.interned_cname = env.intern(self.entry.name) + self.interned_cname = self.entry.interned_cname = \ + env.intern_identifier(self.entry.name) def check_identifier_kind(self): #print "NameNode.check_identifier_kind:", self.entry.name ### @@ -1989,7 +1990,7 @@ class AttributeNode(ExprNode): if obj_type.is_pyobject: self.type = py_object_type self.is_py_attr = 1 - self.interned_attr_cname = env.intern(self.attribute) + self.interned_attr_cname = env.intern_identifier(self.attribute) else: if not obj_type.is_error: error(self.pos, diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 5528af40..6a0aaf07 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -1390,11 +1390,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): Naming.stringtab_cname) for entry in entries: code.putln( - "{&%s, %s, sizeof(%s), %d, %d}," % ( + "{&%s, %s, sizeof(%s), %d, %d, %d}," % ( entry.pystring_cname, entry.cname, entry.cname, entry.type.is_unicode, + entry.is_identifier, entry.is_interned )) code.putln( diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index e08cd219..e79062c2 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -1084,7 +1084,8 @@ class CFuncDefNode(FuncDefNode): self.entry.as_variable = self.py_func.entry # Reset scope entry the above cfunction env.entries[name] = self.entry - self.py_func.interned_attr_cname = env.intern(self.py_func.entry.name) + self.py_func.interned_attr_cname = env.intern_identifier( + self.py_func.entry.name) if not env.is_module_scope or Options.lookup_module_cpdef: self.override = OverrideCheckNode(self.pos, py_func = self.py_func) self.body = StatListNode(self.pos, stats=[self.override, self.body]) @@ -2119,10 +2120,11 @@ class PropertyNode(StatNode): entry = env.declare_property(self.name, self.doc, self.pos) if entry: if self.doc and Options.docstrings: - doc_entry = env.get_string_const(self.doc) + doc_entry = env.get_string_const( + self.doc, identifier = False) entry.doc_cname = doc_entry.cname self.body.analyse_declarations(entry.scope) - + def analyse_expressions(self, env): self.body.analyse_expressions(env) @@ -3679,7 +3681,8 @@ class FromImportStatNode(StatNode): self.item.allocate_temp(env) self.interned_items = [] for name, target in self.items: - self.interned_items.append((env.intern(name), target)) + self.interned_items.append( + (env.intern_identifier(name), target)) target.analyse_target_expression(env, None) #target.release_target_temp(env) # was release_temp ?!? self.module.release_temp(env) @@ -3713,7 +3716,7 @@ utility_function_predeclarations = \ #define INLINE #endif -typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern;} __Pyx_StringTabEntry; /*proto*/ +typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ """ + """ @@ -4295,23 +4298,19 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ """,""" static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { + #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { - #if PY_MAJOR_VERSION < 3 *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - #else - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - #endif + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + #else /* Python 3+ has unicode identifiers */ + if (t->is_identifier || (t->is_unicode && t->intern)) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->is_unicode) { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + #endif } else { - #if PY_MAJOR_VERSION < 3 - if (t->intern) - *t->p = PyString_InternFromString(t->s); - else - #endif - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } if (!*t->p) return -1; diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index a0beb711..eb9d3ba9 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -66,6 +66,7 @@ class Entry: # is_inherited boolean Is an inherited attribute of an extension type # pystring_cname string C name of Python version of string literal # is_interned boolean For string const entries, value is interned + # is_identifier boolean For string const entries, value is an identifier # used boolean # is_special boolean Is a special method or property accessor # of an extension type @@ -104,6 +105,7 @@ class Entry: as_module = None is_inherited = 0 pystring_cname = None + is_identifier = 0 is_interned = 0 used = 0 is_special = 0 @@ -186,6 +188,7 @@ class Scope: self.cname_to_entry = {} self.pow_function_used = 0 self.string_to_entry = {} + self.identifier_to_entry = {} self.num_to_entry = {} self.obj_to_entry = {} self.pystring_entries = [] @@ -203,8 +206,8 @@ class Scope: def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.qualified_name) - def intern(self, name): - return self.global_scope().intern(name) + def intern_identifier(self, name): + return self.global_scope().intern_identifier(name) def qualifying_scope(self): return self.parent_scope @@ -446,14 +449,19 @@ class Scope: self.const_entries.append(entry) return entry - def get_string_const(self, value): + def get_string_const(self, value, identifier = False): # Get entry for string constant. Returns an existing # one if possible, otherwise creates a new one. genv = self.global_scope() - entry = genv.string_to_entry.get(value) + if identifier: + string_map = genv.identifier_to_entry + else: + string_map = genv.string_to_entry + entry = string_map.get(value) if not entry: entry = self.add_string_const(value) - genv.string_to_entry[value] = entry + entry.is_identifier = identifier + string_map[value] = entry return entry def add_py_string(self, entry): @@ -669,6 +677,7 @@ class ModuleScope(Scope): # python_include_files [string] Standard Python headers to be included # include_files [string] Other C headers to be included # string_to_entry {string : Entry} Map string const to entry + # identifier_to_entry {string : Entry} Map identifier string const to entry # context Context # parent_module Scope Parent in the import namespace # module_entries {string : Entry} For cimport statements @@ -740,8 +749,8 @@ class ModuleScope(Scope): entry.is_builtin = 1 return entry - def intern(self, name): - string_entry = self.get_string_const(name) + def intern_identifier(self, name): + string_entry = self.get_string_const(name, identifier = True) self.add_py_string(string_entry) return string_entry.pystring_cname @@ -1238,7 +1247,7 @@ class CClassScope(ClassScope): # I keep it in for now. is_member should be enough # later on entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname - entry.interned_cname = self.intern(name) + entry.interned_cname = self.intern_identifier(name) return entry -- 2.26.2