From d30d8fb7b32e7bafe77f65a19aee886704ff0abc Mon Sep 17 00:00:00 2001 From: Dag Sverre Seljebotn Date: Fri, 30 May 2008 12:06:40 +0200 Subject: [PATCH] Merge --- Cython/Compiler/ExprNodes.py | 209 +++++++++++++++++++++------------- Cython/Compiler/Lexicon.py | 4 +- Cython/Compiler/ModuleNode.py | 159 +++++++++++++++++++++++++- Cython/Compiler/Naming.py | 2 + Cython/Compiler/Nodes.py | 77 ++++++++++--- Cython/Compiler/Parsing.py | 40 ++++--- Cython/Compiler/PyrexTypes.py | 12 +- Cython/Compiler/Symtab.py | 7 +- runtests.py | 25 +++- tests/compile/forfromelse.pyx | 5 + tests/compile/indices.pyx | 17 +++ tests/run/big_indices.pyx | 25 ++++ tests/run/inplace.pyx | 28 +++++ 13 files changed, 486 insertions(+), 124 deletions(-) create mode 100644 tests/compile/indices.pyx create mode 100644 tests/run/big_indices.pyx create mode 100644 tests/run/inplace.pyx diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 57526d7a..a410311f 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -183,7 +183,7 @@ class ExprNode(Node): print_call_chain(method_name, "not implemented") ### raise InternalError( "%s.%s not implemented" % - (self.__class__.__name__, method_name)) + (self.__class__.__name__, method_name)) def is_lvalue(self): return 0 @@ -966,7 +966,7 @@ class NameNode(AtomicExprNode): self.result_code, namespace, self.interned_cname, - code.error_goto_if_null(self.result_code, self.pos))) + code.error_goto_if_null(self.result_code, self.pos))) elif entry.is_local and False: # control flow not good enough yet assigned = entry.scope.control_flow.get_state((entry.name, 'initalized'), self.pos) @@ -995,7 +995,7 @@ class NameNode(AtomicExprNode): self.interned_cname, rhs.py_result())) # in Py2.6+, we need to invalidate the method cache - code.putln("__Pyx_TypeModified((PyTypeObject*)%s);" % + code.putln("__Pyx_TypeModified(%s);" % entry.scope.parent_type.typeptr_cname) else: code.put_error_if_neg(self.pos, @@ -1229,7 +1229,7 @@ class IndexNode(ExprNode): # base ExprNode # index ExprNode - subexprs = ['base', 'index', 'py_index'] + subexprs = ['base', 'index'] def compile_time_value(self, denv): base = self.base.compile_time_value(denv) @@ -1246,19 +1246,27 @@ class IndexNode(ExprNode): pass def analyse_types(self, env): + self.analyse_base_and_index_types(env, getting = 1) + + def analyse_target_types(self, env): + self.analyse_base_and_index_types(env, setting = 1) + + def analyse_base_and_index_types(self, env, getting = 0, setting = 0): self.base.analyse_types(env) self.index.analyse_types(env) if self.base.type.is_pyobject: if self.index.type.is_int: + self.original_index_type = self.index.type self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env) - self.py_index = CloneNode(self.index).coerce_to_pyobject(env) + if getting: + env.use_utility_code(getitem_int_utility_code) + if setting: + env.use_utility_code(setitem_int_utility_code) else: self.index = self.index.coerce_to_pyobject(env) - self.py_index = CloneNode(self.index) self.type = py_object_type self.is_temp = 1 else: - self.py_index = CloneNode(self.index) # so that it exists for subexpr processing if self.base.type.is_ptr or self.base.type.is_array: self.type = self.base.type.base_type else: @@ -1284,79 +1292,63 @@ class IndexNode(ExprNode): def calculate_result_code(self): return "(%s[%s])" % ( self.base.result_code, self.index.result_code) + + def index_unsigned_parameter(self): + if self.index.type.is_int: + if self.original_index_type.signed: + return ", 0" + else: + return ", sizeof(Py_ssize_t) <= sizeof(%s)" % self.original_index_type.declaration_code("") + else: + return "" def generate_subexpr_evaluation_code(self, code): - # do not evaluate self.py_index in case we don't need it self.base.generate_evaluation_code(code) self.index.generate_evaluation_code(code) def generate_subexpr_disposal_code(self, code): - # if we used self.py_index, it will be disposed of manually self.base.generate_disposal_code(code) self.index.generate_disposal_code(code) def generate_result_code(self, code): if self.type.is_pyobject: if self.index.type.is_int: - code.putln("if (PyList_CheckExact(%s) && 0 <= %s && %s < PyList_GET_SIZE(%s)) {" % ( - self.base.py_result(), - self.index.result_code, - self.index.result_code, - self.base.py_result())) - code.putln("%s = PyList_GET_ITEM(%s, %s); Py_INCREF(%s);" % ( - self.result_code, - self.base.py_result(), - self.index.result_code, - self.result_code)) - code.putln("} else if (PyTuple_CheckExact(%s) && 0 <= %s && %s < PyTuple_GET_SIZE(%s)) {" % ( - self.base.py_result(), - self.index.result_code, - self.index.result_code, - self.base.py_result())) - code.putln("%s = PyTuple_GET_ITEM(%s, %s); Py_INCREF(%s);" % ( - self.result_code, - self.base.py_result(), - self.index.result_code, - self.result_code)) - code.putln("} else {") - self.generate_generic_code_result(code) - code.putln("}") + function = "__Pyx_GetItemInt" + index_code = self.index.result_code else: - self.generate_generic_code_result(code) + function = "PyObject_GetItem" + index_code = self.index.py_result() + sign_code = "" + code.putln( + "%s = %s(%s, %s%s); if (!%s) %s" % ( + self.result_code, + function, + self.base.py_result(), + index_code, + self.index_unsigned_parameter(), + self.result_code, + code.error_goto(self.pos))) - def generate_generic_code_result(self, code): - self.py_index.generate_result_code(code) + def generate_setitem_code(self, value_code, code): + if self.index.type.is_int: + function = "__Pyx_SetItemInt" + index_code = self.index.result_code + else: + function = "PyObject_SetItem" + index_code = self.index.py_result() code.putln( - "%s = PyObject_GetItem(%s, %s); %s" % ( - self.result_code, + "if (%s(%s, %s, %s%s) < 0) %s" % ( + function, self.base.py_result(), - self.py_index.py_result(), - code.error_goto_if_null(self.result_code, self.pos))) - if self.is_temp: - self.py_index.generate_disposal_code(code) - + index_code, + value_code, + self.index_unsigned_parameter(), + code.error_goto(self.pos))) + def generate_assignment_code(self, rhs, code): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: - if self.index.type.is_int: - code.putln("if (PyList_CheckExact(%s) && 0 <= %s && %s < PyList_GET_SIZE(%s)) {" % ( - self.base.py_result(), - self.index.result_code, - self.index.result_code, - self.base.py_result())) - code.putln("Py_DECREF(PyList_GET_ITEM(%s, %s)); Py_INCREF(%s);" % ( - self.base.py_result(), - self.index.result_code, - rhs.py_result())) - code.putln("PyList_SET_ITEM(%s, %s, %s);" % ( - self.base.py_result(), - self.index.result_code, - rhs.py_result())) - code.putln("} else {") - self.generate_generic_assignment_code(rhs, code) - code.putln("}") - else: - self.generate_generic_assignment_code(rhs, code) + self.generate_setitem_code(rhs.py_result(), code) else: code.putln( "%s = %s;" % ( @@ -1364,25 +1356,22 @@ class IndexNode(ExprNode): self.generate_subexpr_disposal_code(code) rhs.generate_disposal_code(code) - def generate_generic_assignment_code(self, rhs, code): - self.py_index.generate_result_code(code) - code.put_error_if_neg(self.pos, - "PyObject_SetItem(%s, %s, %s)" % ( - self.base.py_result(), - self.py_index.py_result(), - rhs.py_result())) - if self.is_temp: - self.py_index.generate_disposal_code(code) - def generate_deletion_code(self, code): self.generate_subexpr_evaluation_code(code) - self.py_index.generate_evaluation_code(code) - code.put_error_if_neg(self.pos, - "PyObject_DelItem(%s, %s)" % ( + #if self.type.is_pyobject: + if self.index.type.is_int: + function = "PySequence_DelItem" + index_code = self.index.result_code + else: + function = "PyObject_DelItem" + index_code = self.index.py_result() + code.putln( + "if (%s(%s, %s) < 0) %s" % ( + function, self.base.py_result(), - self.py_index.py_result())) + index_code, + code.error_goto(self.pos))) self.generate_subexpr_disposal_code(code) - self.py_index.generate_disposal_code(code) class SliceIndexNode(ExprNode): @@ -2277,7 +2266,7 @@ class TupleNode(SequenceNode): # of generate_disposal_code, because values were stored # in the tuple using a reference-stealing operation. for arg in self.args: - arg.generate_post_assignment_code(code) + arg.generate_post_assignment_code(code) class ListNode(SequenceNode): @@ -2740,9 +2729,8 @@ class TypecastNode(ExprNode): if from_py and not to_py and self.operand.is_ephemeral() and not self.type.is_numeric: error(self.pos, "Casting temporary Python object to non-numeric non-Python type") if to_py and not from_py: - self.result_ctype = py_object_type - self.is_temp = 1 if self.operand.type.to_py_function: + self.result_ctype = py_object_type self.operand = self.operand.coerce_to_pyobject(env) else: warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.operand.type, self.type)) @@ -3853,9 +3841,6 @@ class CloneNode(CoercionNode): if hasattr(self.arg, 'entry'): self.entry = self.arg.entry - #def result_as_extension_type(self): - # return self.arg.result_as_extension_type() - def generate_evaluation_code(self, code): pass @@ -4110,6 +4095,7 @@ static INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) { type_cache_invalidation_code = [ """ #if PY_VERSION_HEX >= 0x02060000 +/* #define __Pyx_TypeModified(t) PyType_Modified(t) */ /* Py3.0beta1 */ static void __Pyx_TypeModified(PyTypeObject* type); /*proto*/ #else #define __Pyx_TypeModified(t) @@ -4140,3 +4126,62 @@ static void __Pyx_TypeModified(PyTypeObject* type) { #endif """ ] + +#------------------------------------------------------------------------------------ + +# If the is_unsigned flag is set, we need to do some extra work to make +# sure the index doesn't become negative. + +getitem_int_utility_code = [ +""" +static INLINE PyObject *__Pyx_GetItemInt(PyObject *o, Py_ssize_t i, int is_unsigned) { + PyObject *r; + if (PyList_CheckExact(o) && 0 <= i && i < PyList_GET_SIZE(o)) { + r = PyList_GET_ITEM(o, i); + Py_INCREF(r); + } + else if (PyTuple_CheckExact(o) && 0 <= i && i < PyTuple_GET_SIZE(o)) { + r = PyTuple_GET_ITEM(o, i); + Py_INCREF(r); + } + else if (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_item && (likely(i >= 0) || !is_unsigned)) + r = PySequence_GetItem(o, i); + else { + PyObject *j = (likely(i >= 0) || !is_unsigned) ? PyInt_FromLong(i) : PyLong_FromUnsignedLongLong((sizeof(unsigned long long) > sizeof(Py_ssize_t) ? (1ULL << (sizeof(Py_ssize_t)*8)) : 0) + i); + if (!j) + return 0; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + } + return r; +} +""", +""" +"""] + +#------------------------------------------------------------------------------------ + +setitem_int_utility_code = [ +""" +static INLINE int __Pyx_SetItemInt(PyObject *o, Py_ssize_t i, PyObject *v, int is_unsigned) { + int r; + if (PyList_CheckExact(o) && 0 <= i && i < PyList_GET_SIZE(o)) { + Py_DECREF(PyList_GET_ITEM(o, i)); + Py_INCREF(v); + PyList_SET_ITEM(o, i, v); + return 1; + } + else if (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_ass_item && (likely(i >= 0) || !is_unsigned)) + r = PySequence_SetItem(o, i, v); + else { + PyObject *j = (likely(i >= 0) || !is_unsigned) ? PyInt_FromLong(i) : PyLong_FromUnsignedLongLong((sizeof(unsigned long long) > sizeof(Py_ssize_t) ? (1ULL << (sizeof(Py_ssize_t)*8)) : 0) + i); + if (!j) + return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + } + return r; +} +""", +""" +"""] diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py index 0f8bc898..bfc7ea97 100644 --- a/Cython/Compiler/Lexicon.py +++ b/Cython/Compiler/Lexicon.py @@ -68,7 +68,9 @@ def make_lexicon(): bra = Any("([{") ket = Any(")]}") punct = Any(":,;+-*/|&<>=.%`~^?") - diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", "//") + diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//", + "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", + "<<=", ">>=", "**=", "//=") spaces = Rep1(Any(" \t\f")) comment = Str("#") + Rep(AnyBut("\n")) escaped_newline = Str("\\\n") diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 13580f2e..61df6912 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -230,6 +230,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_typeobj_definitions(env, code) self.generate_method_table(env, code) self.generate_filename_init_prototype(code) + if env.has_import_star: + self.generate_import_star(env, code) self.generate_module_init_func(modules[:-1], env, code) code.mark_pos(None) self.generate_module_cleanup_func(env, code) @@ -423,6 +425,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#if PY_MAJOR_VERSION >= 3") code.putln(" #define PyBaseString_Type PyUnicode_Type") + code.putln(" #define PyString_Type PyBytes_Type") code.putln(" #define PyInt_Type PyLong_Type") code.putln(" #define PyInt_Check(op) PyLong_Check(op)") code.putln(" #define PyInt_CheckExact(op) PyLong_CheckExact(op)") @@ -729,7 +732,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if scope.defines_any(["__setitem__", "__delitem__"]): self.generate_ass_subscript_function(scope, code) if scope.defines_any(["__setslice__", "__delslice__"]): - warning(self.pos, "__setslice__ and __delslice__ are not supported by Python 3") + warning(self.pos, "__setslice__ and __delslice__ are not supported by Python 3", 1) self.generate_ass_slice_function(scope, code) if scope.defines_any(["__getattr__","__getattribute__"]): self.generate_getattro_function(scope, code) @@ -1429,6 +1432,66 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_filename_init_prototype(self, code): code.putln(""); code.putln("static void %s(void); /*proto*/" % Naming.fileinit_cname) + + def generate_import_star(self, env, code): + code.putln() + code.putln("char* %s_type_names[] = {" % Naming.import_star) + for name, entry in env.entries.items(): + if entry.is_type: + code.putln('"%s",' % name) + code.putln("0") + code.putln("};") + code.putln() + code.putln("static int %s(PyObject *o, PyObject* py_name, char *name) {" % Naming.import_star_set) + code.putln("char** type_name = %s_type_names;" % Naming.import_star) + code.putln("while (*type_name) {") + code.putln("if (!strcmp(name, *type_name)) {") + code.putln('PyErr_Format(PyExc_TypeError, "Cannot overwrite C type %s", name);') + code.putln('goto bad;') + code.putln("}") + code.putln("type_name++;") + code.putln("}") + old_error_label = code.new_error_label() + code.putln("if (0);") # so the first one can be "else if" + for name, entry in env.entries.items(): + if entry.is_cglobal and entry.used: + code.putln('else if (!strcmp(name, "%s")) {' % name) + if entry.type.is_pyobject: + if entry.type.is_extension_type or entry.type.is_builtin_type: + code.putln("if (!(%s)) %s;" % ( + entry.type.type_test_code("o"), + code.error_goto(entry.pos))) + code.put_var_decref(entry) + code.putln("%s = %s;" % ( + entry.cname, + PyrexTypes.typecast(entry.type, py_object_type, "o"))) + elif entry.type.from_py_function: + rhs = "%s(o)" % entry.type.from_py_function + if entry.type.is_enum: + rhs = typecast(entry.type, c_long_type, rhs) + code.putln("%s = %s; if (%s) %s;" % ( + entry.cname, + rhs, + entry.type.error_condition(entry.cname), + code.error_goto(entry.pos))) + code.putln("Py_DECREF(o);") + else: + code.putln('PyErr_Format(PyExc_TypeError, "Cannot convert Python object %s to %s");' % (name, entry.type)) + code.putln(code.error_goto(entry.pos)) + code.putln("}") + code.putln("else {") + code.putln("if (PyObject_SetAttr(%s, py_name, o) < 0) goto bad;" % Naming.module_cname) + code.putln("}") + code.putln("return 0;") + code.put_label(code.error_label) + # This helps locate the offending name. + code.putln('__Pyx_AddTraceback("%s");' % self.full_module_name); + code.error_label = old_error_label + code.putln("bad:") + code.putln("Py_DECREF(o);") + code.putln("return -1;") + code.putln("}") + code.putln(import_star_utility_code) def generate_module_init_func(self, imported_modules, env, code): code.putln("") @@ -1543,6 +1606,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "if (!%s) %s;" % ( env.module_cname, code.error_goto(self.pos))); + code.putln( + "Py_INCREF(%s);" % + env.module_cname) code.putln( '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME);' % Naming.builtins_cname) @@ -2016,3 +2082,94 @@ bad: return ret; } """] + +import_star_utility_code = """ + +/* import_all_from is an unexposed function from ceval.c */ + +static int +__Pyx_import_all_from(PyObject *locals, PyObject *v) +{ + PyObject *all = PyObject_GetAttrString(v, "__all__"); + PyObject *dict, *name, *value; + int skip_leading_underscores = 0; + int pos, err; + + if (all == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + return -1; /* Unexpected error */ + PyErr_Clear(); + dict = PyObject_GetAttrString(v, "__dict__"); + if (dict == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + return -1; + PyErr_SetString(PyExc_ImportError, + "from-import-* object has no __dict__ and no __all__"); + return -1; + } + all = PyMapping_Keys(dict); + Py_DECREF(dict); + if (all == NULL) + return -1; + skip_leading_underscores = 1; + } + + for (pos = 0, err = 0; ; pos++) { + name = PySequence_GetItem(all, pos); + if (name == NULL) { + if (!PyErr_ExceptionMatches(PyExc_IndexError)) + err = -1; + else + PyErr_Clear(); + break; + } + if (skip_leading_underscores && + PyString_Check(name) && + PyString_AS_STRING(name)[0] == '_') + { + Py_DECREF(name); + continue; + } + value = PyObject_GetAttr(v, name); + if (value == NULL) + err = -1; + else if (PyDict_CheckExact(locals)) + err = PyDict_SetItem(locals, name, value); + else + err = PyObject_SetItem(locals, name, value); + Py_DECREF(name); + Py_XDECREF(value); + if (err != 0) + break; + } + Py_DECREF(all); + return err; +} + + +static int %s(PyObject* m) { + + int i; + int ret = -1; + PyObject *locals = 0; + PyObject *list = 0; + PyObject *name; + PyObject *item; + + locals = PyDict_New(); if (!locals) goto bad; + if (__Pyx_import_all_from(locals, m) < 0) goto bad; + list = PyDict_Items(locals); if (!list) goto bad; + + for(i=0; i>": "PyNumber_InPlaceRshift", + "**": "PyNumber_InPlacePower", + "//": "PyNumber_InPlaceFloorDivide", } def annotate(self, code): @@ -3663,10 +3680,14 @@ class FromCImportStatNode(StatNode): module_scope = env.find_module(self.module_name, self.pos) env.add_imported_module(module_scope) for pos, name, as_name in self.imported_names: - entry = module_scope.find(name, pos) - if entry: - local_name = as_name or name - env.add_imported_entry(local_name, entry, pos) + if name == "*": + for local_name, entry in module_scope.entries.items(): + env.add_imported_entry(local_name, entry, pos) + else: + entry = module_scope.find(name, pos) + if entry: + local_name = as_name or name + env.add_imported_entry(local_name, entry, pos) def analyse_expressions(self, env): pass @@ -3682,12 +3703,21 @@ class FromImportStatNode(StatNode): # items [(string, NameNode)] # interned_items [(string, NameNode)] # item PyTempNode used internally + # import_star boolean used internally child_attrs = ["module"] + import_star = 0 def analyse_declarations(self, env): - for _, target in self.items: - target.analyse_target_declaration(env) + for name, target in self.items: + if name == "*": + if not env.is_module_scope: + error(self.pos, "import * only allowed at module level") + return + env.has_import_star = 1 + self.import_star = 1 + else: + target.analyse_target_declaration(env) def analyse_expressions(self, env): import ExprNodes @@ -3696,15 +3726,27 @@ class FromImportStatNode(StatNode): self.item.allocate_temp(env) self.interned_items = [] for name, target in self.items: - self.interned_items.append( - (env.intern_identifier(name), target)) - target.analyse_target_expression(env, None) - #target.release_target_temp(env) # was release_temp ?!? + if name == '*': + for _, entry in env.entries.items(): + if not entry.is_type and entry.type.is_extension_type: + env.use_utility_code(ExprNodes.type_test_utility_code) + break + else: + self.interned_items.append( + (env.intern_identifier(name), target)) + target.analyse_target_expression(env, None) + #target.release_target_temp(env) # was release_temp ?!? self.module.release_temp(env) self.item.release_temp(env) def generate_execution_code(self, code): self.module.generate_evaluation_code(code) + if self.import_star: + code.putln( + 'if (%s(%s) < 0) %s;' % ( + Naming.import_star, + self.module.py_result(), + code.error_goto(self.pos))) for cname, target in self.interned_items: code.putln( '%s = PyObject_GetAttr(%s, %s); %s' % ( @@ -4253,7 +4295,11 @@ static void __Pyx_AddTraceback(char *funcname) { if (!py_funcname) goto bad; py_globals = PyModule_GetDict(%(GLOBALS)s); if (!py_globals) goto bad; + #if PY_MAJOR_VERSION < 3 empty_string = PyString_FromStringAndSize("", 0); + #else + empty_string = PyBytes_FromStringAndSize("", 0); + #endif if (!empty_string) goto bad; py_code = PyCode_New( 0, /*int argcount,*/ @@ -4366,17 +4412,18 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } #else /* Python 3+ has unicode identifiers */ if (t->is_identifier || (t->is_unicode && t->intern)) { *t->p = PyUnicode_InternFromString(t->s); } else if (t->is_unicode) { *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); } #endif - else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } if (!*t->p) return -1; ++t; diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 372593c2..7bdfa4d2 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -755,11 +755,11 @@ def p_expression_or_assignment(s): s.next() expr_list.append(p_expr(s)) if len(expr_list) == 1: - if re.match("[+*/\%^\&|-]=", s.sy): + if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy): lhs = expr_list[0] if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ): error(lhs.pos, "Illegal operand for inplace operation.") - operator = s.sy[0] + operator = s.sy[:-1] s.next() rhs = p_expr(s) return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs) @@ -944,8 +944,11 @@ def p_from_import_statement(s, first_statement = 0): else: s.error("Expected 'import' or 'cimport'") if s.sy == '*': - s.error("'import *' not supported") - imported_names = [p_imported_name(s)] +# s.error("'import *' not supported") + imported_names = [(s.position(), "*", None)] + s.next() + else: + imported_names = [p_imported_name(s)] while s.sy == ',': s.next() imported_names.append(p_imported_name(s)) @@ -1080,9 +1083,13 @@ def p_for_bounds(s): s.next() iterator = p_for_iterator(s) return { 'target': target, 'iterator': iterator } - elif s.sy == 'from': - s.next() - bound1 = p_bit_expr(s) + else: + if s.sy == 'from': + s.next() + bound1 = p_bit_expr(s) + else: + # Support shorter "for a <= x < b" syntax + bound1, target = target, None rel1 = p_for_from_relation(s) name2_pos = s.position() name2 = p_ident(s) @@ -1090,12 +1097,15 @@ def p_for_bounds(s): rel2 = p_for_from_relation(s) bound2 = p_bit_expr(s) step = p_for_from_step(s) - if not target.is_name: - error(target.pos, - "Target of for-from statement must be a variable name") - elif name2 != target.name: - error(name2_pos, - "Variable name in for-from range does not match target") + if target is None: + target = ExprNodes.NameNode(name2_pos, name = name2) + else: + if not target.is_name: + error(target.pos, + "Target of for-from statement must be a variable name") + elif name2 != target.name: + error(name2_pos, + "Variable name in for-from range does not match target") if rel1[0] != rel2[0]: error(rel2_pos, "Relation directions in for-from do not match") @@ -1105,8 +1115,6 @@ def p_for_bounds(s): 'relation2': rel2, 'bound2': bound2, 'step': step } - else: - s.error("Expected 'in' or 'from'") def p_for_from_relation(s): if s.sy in inequality_relations: @@ -1287,7 +1295,7 @@ def p_DEF_statement(s): return Nodes.PassStatNode(pos) def p_IF_statement(s, level, cdef_flag, visibility, api): - pos = s.position + pos = s.position() saved_eval = s.compile_time_eval current_eval = saved_eval denv = s.compile_time_env diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index b7950da7..cb71d828 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -929,8 +929,8 @@ class CStringType: is_string = 1 is_unicode = 0 - to_py_function = "PyString_FromString" - from_py_function = "PyString_AsString" + to_py_function = "__Pyx_PyBytes_FromString" + from_py_function = "__Pyx_PyBytes_AsString" exception_value = "NULL" def literal_code(self, value): @@ -1172,6 +1172,14 @@ def typecast(to_type, from_type, expr_code): type_conversion_predeclarations = """ /* Type Conversion Predeclarations */ +#if PY_MAJOR_VERSION < 3 +#define __Pyx_PyBytes_FromString PyString_FromString +#define __Pyx_PyBytes_AsString PyString_AsString +#else +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_AsString PyBytes_AsString +#endif + #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) static INLINE int __Pyx_PyObject_IsTrue(PyObject* x); static INLINE PY_LONG_LONG __pyx_PyInt_AsLongLong(PyObject* x); diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index 08edcff6..1fa1ede6 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -693,8 +693,10 @@ class ModuleScope(Scope): # interned_nums [int/long] Interned numeric constants # all_pystring_entries [Entry] Python string consts from all scopes # types_imported {PyrexType : 1} Set of types for which import code generated + # has_import_star boolean Module contains import * is_module_scope = 1 + has_import_star = 0 def __init__(self, name, parent_module, context): self.parent_module = parent_module @@ -734,7 +736,10 @@ class ModuleScope(Scope): def declare_builtin(self, name, pos): if not hasattr(__builtin__, name): - if self.outer_scope is not None: + if self.has_import_star: + entry = self.declare_var(name, py_object_type, pos) + return entry + elif self.outer_scope is not None: return self.outer_scope.declare_builtin(name, pos) else: error(pos, "undeclared name not builtin: %s"%name) diff --git a/runtests.py b/runtests.py index d66808d1..1d86bd0e 100644 --- a/runtests.py +++ b/runtests.py @@ -17,20 +17,31 @@ CFLAGS = os.getenv('CFLAGS', '').split() class ErrorWriter(object): - match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match + match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match def __init__(self): self.output = [] self.write = self.output.append - def geterrors(self): + def _collect(self, collect_errors, collect_warnings): s = ''.join(self.output) - errors = [] + result = [] for line in s.split('\n'): match = self.match_error(line) if match: - line, column, message = match.groups() - errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) ) - return errors + is_warning, line, column, message = match.groups() + if (is_warning and collect_warnings) or \ + (not is_warning and collect_errors): + result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) ) + return result + + def geterrors(self): + return self._collect(True, False) + + def getwarnings(self): + return self._collect(False, True) + + def getall(self): + return self._collect(True, True) class TestBuilder(object): def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir): @@ -259,6 +270,8 @@ if __name__ == '__main__': CompilationOptions, \ default_options as pyrex_default_options, \ compile as cython_compile + from Cython.Compiler import Errors + Errors.LEVEL = 0 # show all warnings # RUN ALL TESTS! ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests') diff --git a/tests/compile/forfromelse.pyx b/tests/compile/forfromelse.pyx index 014f42b5..11cad88b 100644 --- a/tests/compile/forfromelse.pyx +++ b/tests/compile/forfromelse.pyx @@ -5,3 +5,8 @@ cdef void spam(): else: k = j + # new syntax + for 0 <= i < 10: + j = i + else: + j = k diff --git a/tests/compile/indices.pyx b/tests/compile/indices.pyx new file mode 100644 index 00000000..527ad159 --- /dev/null +++ b/tests/compile/indices.pyx @@ -0,0 +1,17 @@ +cdef int* a +cdef object x + +cdef int f(int i): + print i + return i + +x[f(1)] = 3 +a[f(1)] = 3 + +x[f(2)] += 4 +a[f(2)] += 4 + +print x[1] +print a[1] + +x[f(1)] = 15 \ No newline at end of file diff --git a/tests/run/big_indices.pyx b/tests/run/big_indices.pyx new file mode 100644 index 00000000..d8f3a765 --- /dev/null +++ b/tests/run/big_indices.pyx @@ -0,0 +1,25 @@ +__doc__ = u""" + >>> test() + neg -1 + pos 4294967294 + neg + pos + neg + pos +""" + +def test(): + cdef long neg = -1 + cdef unsigned long pos = -2 # will be a large positive number + + print "neg", neg + print "pos", pos + + D = { neg: 'neg', pos: 'pos' } + + print D[neg] + print D[pos] + + print D[neg] + print D[pos] + diff --git a/tests/run/inplace.pyx b/tests/run/inplace.pyx new file mode 100644 index 00000000..22f8ac83 --- /dev/null +++ b/tests/run/inplace.pyx @@ -0,0 +1,28 @@ +__doc__ = u""" + >>> f(5, 7) + 29509034655744 + + >>> g(13, 4) + 32 + + >>> h(56, 7) + 105.0 +""" + +def f(a,b): + a += b + a *= b + a **= b + return a + +def g(int a, int b): + a -= b + a /= b + a <<= b + return a + +def h(double a, double b): + a /= b + a += b + a *= b + return a -- 2.26.2