From: Stefan Behnel Date: Mon, 15 Oct 2007 08:26:57 +0000 (+0200) Subject: preliminary merge of Pyrex 0.9.6.2 -> crashes X-Git-Tag: 0.9.6.14~29^2~119^2~11 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=465f158e6450611b7207bb8e92a283c8608ba926;p=cython.git preliminary merge of Pyrex 0.9.6.2 -> crashes --- diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 7c54d6ff..29457db1 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -21,10 +21,11 @@ class CCodeWriter: # in_try_finally boolean inside try of try...finally # filename_table {string : int} for finding filename table indexes # filename_list [string] filenames in filename table order + # exc_vars (string * 3) exception variables for reraise, or None # input_file_contents dict contents (=list of lines) of any file that was used as input # to create this output C code. This is # used to annotate the comments. - + in_try_finally = 0 def __init__(self, f): @@ -37,8 +38,9 @@ class CCodeWriter: self.error_label = None self.filename_table = {} self.filename_list = [] + self.exc_vars = None self.input_file_contents = {} - + def putln(self, code = ""): if self.marker and self.bol: self.emit_marker() @@ -186,8 +188,10 @@ class CCodeWriter: #print "Code.put_var_declaration:", entry.name, "definition =", definition ### visibility = entry.visibility if visibility == 'private' and not definition: + #print "...private and not definition, skipping" ### return if not entry.used and visibility == "private": + #print "not used and private, skipping" ### return storage_class = "" if visibility == 'extern': @@ -288,13 +292,6 @@ class CCodeWriter: # code = "((PyObject*)%s)" % code self.put_init_to_py_none(code, entry.type) - def put_py_gil_state_ensure(self, cname): - self.putln("PyGILState_STATE %s;" % cname) - self.putln("%s = PyGILState_Ensure();" % cname) - - def put_py_gil_state_release(self, cname): - self.putln("PyGILState_Release(%s);" % cname) - def put_pymethoddef(self, entry, term): if entry.doc: doc_code = entry.doc_cname @@ -316,6 +313,10 @@ class CCodeWriter: # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) # TODO this path is almost _never_ taken, yet this macro makes is slower! return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) + def put_h_guard(self, guard): + self.putln("#ifndef %s" % guard) + self.putln("#define %s" % guard) + def error_goto(self, pos): lbl = self.error_label self.use_label(lbl) diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 74e2efc7..41aeb637 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -2,13 +2,14 @@ # Pyrex - Parse tree nodes for expressions # +import operator from string import join from Errors import error, warning, InternalError import Naming from Nodes import Node import PyrexTypes -from PyrexTypes import py_object_type, c_long_type, typecast +from PyrexTypes import py_object_type, c_long_type, typecast, error_type import Symtab import Options @@ -208,6 +209,14 @@ class ExprNode(Node): # C type of the result_code expression). return self.result_ctype or self.type + def compile_time_value(self, denv): + # Return value of compile-time expression, or report error. + error(self.pos, "Invalid compile-time expression") + + def compile_time_value_error(self, e): + error(self.pos, "Error in compile-time expression: %s: %s" % ( + e.__class__.__name__, e)) + # ------------- Declaration Analysis ---------------- def analyse_target_declaration(self, env): @@ -489,7 +498,7 @@ class ExprNode(Node): if type.is_pyobject or type.is_ptr or type.is_float: return CoerceToBooleanNode(self, env) else: - if not type.is_int: + if not type.is_int and not type.is_error: error(self.pos, "Type '%s' not acceptable as a boolean" % type) return self @@ -551,12 +560,18 @@ class NoneNode(PyConstNode): value = "Py_None" + def compile_time_value(self, denv): + return None + class EllipsisNode(PyConstNode): # '...' in a subscript list. value = "Py_Ellipsis" + def compile_time_value(self, denv): + return Ellipsis + class ConstNode(AtomicExprNode): # Abstract base type for literal constant nodes. @@ -589,6 +604,9 @@ class NullNode(ConstNode): class CharNode(ConstNode): type = PyrexTypes.c_char_type + def compile_time_value(self, denv): + return ord(self.value) + def calculate_result_code(self): return "'%s'" % self.value @@ -613,14 +631,24 @@ class IntNode(ConstNode): else: return str(self.value) + def compile_time_value(self, denv): + return int(self.value) + + class FloatNode(ConstNode): type = PyrexTypes.c_double_type + def compile_time_value(self, denv): + return float(self.value) + class StringNode(ConstNode): # entry Symtab.Entry type = PyrexTypes.c_char_ptr_type + + def compile_time_value(self, denv): + return eval('"%s"' % self.value) def analyse_types(self, env): self.entry = env.add_string_const(self.value) @@ -656,6 +684,9 @@ class LongNode(AtomicExprNode): # # value string + def compile_time_value(self, denv): + return long(self.value) + def analyse_types(self, env): self.type = py_object_type self.is_temp = 1 @@ -673,6 +704,9 @@ class ImagNode(AtomicExprNode): # # value float imaginary part + def compile_time_value(self, denv): + return complex(0.0, self.value) + def analyse_types(self, env): self.type = py_object_type self.is_temp = 1 @@ -689,10 +723,34 @@ class NameNode(AtomicExprNode): # Reference to a local or global variable name. # # name string Python name of the variable + # # entry Entry Symbol table entry + # interned_cname string is_name = 1 + def compile_time_value(self, denv): + try: + return denv.lookup(self.name) + except KeyError: + error(self.pos, "Compile-time name '%s' not defined", self.name) + + def coerce_to(self, dst_type, env): + # If coercing to a generic pyobject and this is a builtin + # C function with a Python equivalent, manufacture a NameNode + # referring to the Python builtin. + #print "NameNode.coerce_to:", self.name, dst_type ### + if dst_type is py_object_type: + entry = self.entry + if entry.is_cfunction: + var_entry = entry.as_variable + if var_entry: + node = NameNode(self.pos, name = self.name) + node.entry = var_entry + node.analyse_rvalue_entry(env) + return node + return AtomicExprNode.coerce_to(self, dst_type, env) + def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. # Returns the module scope, or None. @@ -706,14 +764,13 @@ class NameNode(AtomicExprNode): # Returns the extension type, or None. entry = env.lookup(self.name) if entry and entry.is_type and entry.type.is_extension_type: - return entry.type - return None + return entry.type + else: + return None def analyse_target_declaration(self, env): self.entry = env.lookup_here(self.name) if not self.entry: - #print "NameNode.analyse_target_declaration:", self.name ### - #print "...declaring as py_object_type" ### self.entry = env.declare_var(self.name, py_object_type, self.pos) if self.entry.is_declared_generic: self.result_ctype = py_object_type @@ -722,19 +779,23 @@ class NameNode(AtomicExprNode): self.entry = env.lookup(self.name) if not self.entry: self.entry = env.declare_builtin(self.name, self.pos) + self.analyse_rvalue_entry(env) + + def analyse_target_types(self, env): self.analyse_entry(env) + if not self.is_lvalue(): + error(self.pos, "Assignment to non-lvalue '%s'" + % self.name) + self.type = PyrexTypes.error_type - def analyse_entry(self, env): - self.check_identifier_kind() + def analyse_rvalue_entry(self, env): + #print "NameNode.analyse_rvalue_entry:", self.name ### + #print "Entry:", self.entry.__dict__ ### + self.analyse_entry(env) entry = self.entry - self.type = entry.type if entry.is_declared_generic: self.result_ctype = py_object_type - ## Reference to C array turns into pointer to first element. - #while self.type.is_array: - # self.type = self.type.element_ptr_type() if entry.is_pyglobal or entry.is_builtin: - assert self.type.is_pyobject, "Python global or builtin not a Python object" if Options.cache_builtins and entry.is_builtin: self.is_temp = 0 else: @@ -744,17 +805,20 @@ class NameNode(AtomicExprNode): else: env.use_utility_code(get_name_utility_code) - def analyse_target_types(self, env): + def analyse_entry(self, env): + #print "NameNode.analyse_entry:", self.name ### self.check_identifier_kind() - if self.is_lvalue(): - self.type = self.entry.type - else: - error(self.pos, "Assignment to non-lvalue '%s'" - % self.name) - self.type = PyrexTypes.error_type - + entry = self.entry + type = entry.type + self.type = type + if entry.is_pyglobal or entry.is_builtin: + assert type.is_pyobject, "Python global or builtin not a Python object" + if Options.intern_names: + self.interned_cname = env.intern(self.name) + def check_identifier_kind(self): #print "NameNode.check_identifier_kind:", self.entry.name ### + #print self.entry.__dict__ ### entry = self.entry entry.used = 1 if not (entry.is_const or entry.is_variable @@ -798,8 +862,9 @@ class NameNode(AtomicExprNode): return self.entry.cname def generate_result_code(self, code): - if not hasattr(self, 'entry'): - error(self.pos, "INTERNAL ERROR: NameNode has no entry attribute during code generation") + assert hasattr(self, 'entry') + #if not hasattr(self, 'entry'): + # error(self.pos, "INTERNAL ERROR: NameNode has no entry attribute during code generation") entry = self.entry if entry is None: return # There was an error earlier @@ -816,7 +881,8 @@ class NameNode(AtomicExprNode): '%s = __Pyx_GetName(%s, %s); %s' % ( self.result_code, namespace, - entry.interned_cname, + #entry.interned_cname, + self.interned_cname, code.error_goto_if_null(self.result_code, self.pos))) else: code.putln( @@ -827,6 +893,7 @@ class NameNode(AtomicExprNode): code.error_goto_if_null(self.result_code, self.pos))) def generate_assignment_code(self, rhs, code): + #print "NameNode.generate_assignment_code:", self.name ### entry = self.entry if entry is None: return # There was an error earlier @@ -842,7 +909,7 @@ class NameNode(AtomicExprNode): code.put_error_if_neg(self.pos, 'PyDict_SetItem(%s->tp_dict, %s, %s)' % ( namespace, - entry.interned_cname, + self.interned_cname, rhs.py_result())) else: code.put_error_if_neg(self.pos, @@ -856,7 +923,7 @@ class NameNode(AtomicExprNode): code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % ( namespace, - entry.interned_cname, + self.interned_cname, rhs.py_result())) else: code.put_error_if_neg(self.pos, @@ -1019,24 +1086,23 @@ class NextNode(AtomicExprNode): code.putln("break;") code.putln("}") code.putln("}") - + class ExcValueNode(AtomicExprNode): # Node created during analyse_types phase # of an ExceptClauseNode to fetch the current # exception value. - def __init__(self, pos, env): + def __init__(self, pos, env, var): ExprNode.__init__(self, pos) self.type = py_object_type - self.is_temp = 1 - env.use_utility_code(get_exception_utility_code) + self.var = var + def calculate_result_code(self): + return self.var + def generate_result_code(self, code): - code.putln( - "%s = __Pyx_GetExcValue(); %s" % ( - self.result_code, - code.error_goto_if_null(self.result_code, self.pos))) + pass class TempNode(AtomicExprNode): @@ -1078,6 +1144,14 @@ class IndexNode(ExprNode): subexprs = ['base', 'index', 'py_index'] + def compile_time_value(self, denv): + base = self.base.compile_time_value(denv) + index = self.index.compile_time_value(denv) + try: + return base[index] + except Exception, e: + self.compile_time_value_error(e) + def is_ephemeral(self): return self.base.is_ephemeral() @@ -1133,7 +1207,7 @@ class IndexNode(ExprNode): # if we used self.py_index, it will be disposed of manually self.base.generate_disposal_code(code) self.index.generate_disposal_code(code) - + def generate_result_code(self, code): if self.type.is_pyobject: if self.index.type.is_int: @@ -1162,7 +1236,7 @@ class IndexNode(ExprNode): code.putln("}") else: self.generate_generic_code_result(code) - + def generate_generic_code_result(self, code): self.py_index.generate_result_code(code) code.putln( @@ -1173,7 +1247,7 @@ class IndexNode(ExprNode): code.error_goto_if_null(self.result_code, self.pos))) if self.is_temp: self.py_index.generate_disposal_code(code) - + def generate_assignment_code(self, rhs, code): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: @@ -1222,7 +1296,7 @@ class IndexNode(ExprNode): self.py_index.py_result())) self.generate_subexpr_disposal_code(code) self.py_index.generate_disposal_code(code) - + class SliceIndexNode(ExprNode): # 2-element slice indexing @@ -1233,6 +1307,15 @@ class SliceIndexNode(ExprNode): subexprs = ['base', 'start', 'stop'] + def compile_time_value(self, denv): + base = self.base.compile_time_value(denv) + start = self.start.compile_time_value(denv) + stop = self.stop.compile_time_value(denv) + try: + return base[start:stop] + except Exception, e: + self.compile_time_value_error(e) + def analyse_target_declaration(self, env): pass @@ -1290,7 +1373,7 @@ class SliceIndexNode(ExprNode): if self.stop: return self.stop.result_code else: - return "0x7fffffff" + return "PY_SSIZE_T_MAX" def calculate_result_code(self): # self.result_code is not used, but this method must exist @@ -1304,6 +1387,15 @@ class SliceNode(ExprNode): # stop ExprNode # step ExprNode + def compile_time_value(self, denv): + start = self.start.compile_time_value(denv) + stop = self.stop.compile_time_value(denv) + step = step.step.compile_time_value(denv) + try: + return slice(start, stop, step) + except Exception, e: + self.compile_time_value_error(e) + subexprs = ['start', 'stop', 'step'] def analyse_types(self, env): @@ -1340,6 +1432,14 @@ class SimpleCallNode(ExprNode): coerced_self = None arg_tuple = None + def compile_time_value(self, denv): + function = self.function.compile_time_value(denv) + args = [arg.compile_time_value(denv) for arg in self.args] + try: + return function(*args) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): function = self.function function.is_called = 1 @@ -1492,6 +1592,17 @@ class GeneralCallNode(ExprNode): subexprs = ['function', 'positional_args', 'keyword_args', 'starstar_arg'] + def compile_time_value(self, denv): + function = self.function.compile_time_value(denv) + positional_args = self.positional_args.compile_time_value(denv) + keyword_args = self.keyword_args.compile_time_value(denv) + starstar_arg = self.starstar_arg.compile_time_value(denv) + try: + keyword_args.update(starstar_arg) + return function(*positional_args, **keyword_args) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): self.function.analyse_types(env) self.positional_args.analyse_types(env) @@ -1545,6 +1656,13 @@ class AsTupleNode(ExprNode): subexprs = ['arg'] + def compile_time_value(self, denv): + arg = self.arg.compile_time_value(denv) + try: + return tuple(arg) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): self.arg.analyse_types(env) self.arg = self.arg.coerce_to_pyobject(env) @@ -1581,6 +1699,18 @@ class AttributeNode(ExprNode): entry = None is_called = 0 + def compile_time_value(self, denv): + attr = self.attribute + if attr.beginswith("__") and attr.endswith("__"): + self.error("Invalid attribute name '%s' in compile-time expression" + % attr) + return None + obj = self.arg.compile_time_value(denv) + try: + return getattr(obj, attr) + except Exception, e: + self.compile_time_value_error(e) + def analyse_target_declaration(self, env): pass @@ -1660,7 +1790,7 @@ class AttributeNode(ExprNode): if target: NameNode.analyse_target_types(self, env) else: - NameNode.analyse_entry(self, env) + NameNode.analyse_rvalue_entry(self, env) def analyse_as_ordinary_attribute(self, env, target): self.obj.analyse_types(env) @@ -1846,6 +1976,9 @@ class SequenceNode(ExprNode): is_sequence_constructor = 1 unpacked_items = None + def compile_time_value_list(self, denv): + return [arg.compile_time_value(denv) for arg in self.args] + def analyse_target_declaration(self, env): for arg in self.args: arg.analyse_target_declaration(env) @@ -1946,6 +2079,13 @@ class SequenceNode(ExprNode): class TupleNode(SequenceNode): # Tuple constructor. + def compile_time_value(self, denv): + values = self.compile_time_value_list(denv) + try: + return tuple(values) + except Exception, e: + self.compile_time_value_error(e) + def generate_operation_code(self, code): code.putln( "%s = PyTuple_New(%s); %s" % ( @@ -1973,6 +2113,9 @@ class TupleNode(SequenceNode): class ListNode(SequenceNode): # List constructor. + def compile_time_value(self, denv): + return self.compile_time_value_list(denv) + def generate_operation_code(self, code): code.putln("%s = PyList_New(%s); %s" % (self.result_code, @@ -2044,6 +2187,14 @@ class DictNode(ExprNode): # # key_value_pairs [(ExprNode, ExprNode)] + def compile_time_value(self, denv): + pairs = [(key.compile_time_value(denv), value.compile_time_value(denv)) + for (key, value) in self.key_value_pairs] + try: + return dict(pairs) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): new_pairs = [] for key, value in self.key_value_pairs: @@ -2174,6 +2325,13 @@ class PyCFunctionNode(AtomicExprNode): # #------------------------------------------------------------------- +compile_time_unary_operators = { + 'not': operator.not_, + '~': operator.inv, + '-': operator.neg, + '+': operator.pos, +} + class UnopNode(ExprNode): # operator string # operand ExprNode @@ -2188,6 +2346,18 @@ class UnopNode(ExprNode): subexprs = ['operand'] + def compile_time_value(self, denv): + func = compile_time_unary_operators.get(self.operator) + if not func: + error(self.pos, + "Unary '%s' not supported in compile-time expression" + % self.operator) + operand = self.operand.compile_time_value(denv) + try: + return func(operand) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): self.operand.analyse_types(env) if self.is_py_operation(): @@ -2234,6 +2404,13 @@ class NotNode(ExprNode): # # operand ExprNode + def compile_time_value(self, denv): + operand = self.operand.compile_time_value(denv) + try: + return not operand + except Exception, e: + self.compile_time_value_error(e) + subexprs = ['operand'] def analyse_types(self, env): @@ -2358,6 +2535,10 @@ class TypecastNode(ExprNode): def analyse_types(self, env): base_type = self.base_type.analyse(env) _, self.type = self.declarator.analyse(base_type, env) + if self.type.is_cfunction: + error(self.pos, + "Cannot cast to a function type") + self.type = PyrexTypes.error_type self.operand.analyse_types(env) to_py = self.type.is_pyobject from_py = self.operand.type.is_pyobject @@ -2455,6 +2636,40 @@ class SizeofVarNode(SizeofNode): # #------------------------------------------------------------------- +compile_time_binary_operators = { + '<': operator.lt, + '<=': operator.le, + '=': operator.eq, + '!=': operator.ne, + '>=': operator.ge, + '>': operator.gt, + 'is': operator.is_, + 'is_not': operator.is_not, + '+': operator.add, + '&': operator.and_, + '/': operator.div, + '//': operator.floordiv, + '<<': operator.lshift, + '%': operator.mod, + '*': operator.mul, + '|': operator.or_, + '**': operator.pow, + '>>': operator.rshift, + '-': operator.sub, + #'/': operator.truediv, + '^': operator.xor, + 'in': lambda x, y: x in y, + 'not_in': lambda x, y: x not in y, +} + +def get_compile_time_binop(node): + func = compile_time_binary_operators.get(node.operator) + if not func: + error(node.pos, + "Binary '%s' not supported in compile-time expression" + % self.operator) + return func + class BinopNode(ExprNode): # operator string # operand1 ExprNode @@ -2470,6 +2685,15 @@ class BinopNode(ExprNode): subexprs = ['operand1', 'operand2'] + def compile_time_value(self, denv): + func = get_compile_time_binop(self) + operand1 = self.operand1.compile_time_value(denv) + operand2 = self.operand2.compile_time_value(denv) + try: + return func(operand1, operand2) + except Exception, e: + self.compile_time_value_error(e) + def analyse_types(self, env): self.operand1.analyse_types(env) self.operand2.analyse_types(env) @@ -2529,10 +2753,10 @@ class NumBinopNode(BinopNode): def analyse_c_operation(self, env): type1 = self.operand1.type type2 = self.operand2.type - if type1.is_enum: - type1 = PyrexTypes.c_int_type - if type2.is_enum: - type2 = PyrexTypes.c_int_type + if self.operator == "**" and type1.is_int and type2.is_int: + error(self.pos, "** with two C int types is ambiguous") + self.type = error_type + return self.type = self.compute_c_result_type(type1, type2) if not self.type: self.type_error() @@ -2544,7 +2768,9 @@ class NumBinopNode(BinopNode): return None def c_types_okay(self, type1, type2): - return type1.is_numeric and type2.is_numeric + #print "NumBinopNode.c_types_okay:", type1, type2 ### + return (type1.is_numeric or type1.is_enum) \ + and (type2.is_numeric or type2.is_enum) def calculate_result_code(self): return "(%s %s %s)" % ( @@ -2575,7 +2801,9 @@ class IntBinopNode(NumBinopNode): # Binary operation taking integer arguments. def c_types_okay(self, type1, type2): - return type1.is_int and type2.is_int + #print "IntBinopNode.c_types_okay:", type1, type2 ### + return (type1.is_int or type1.is_enum) \ + and (type2.is_int or type2.is_enum) class AddNode(NumBinopNode): @@ -2589,9 +2817,10 @@ class AddNode(NumBinopNode): return NumBinopNode.is_py_operation(self) def compute_c_result_type(self, type1, type2): - if type1.is_ptr and type2.is_int: + #print "AddNode.compute_c_result_type:", type1, self.operator, type2 ### + if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum): return type1 - elif type1.is_int and type2.is_ptr: + elif (type2.is_ptr or type2.is_array) and (type1.is_int or type1.is_enum): return type2 else: return NumBinopNode.compute_c_result_type( @@ -2602,9 +2831,9 @@ class SubNode(NumBinopNode): # '-' operator. def compute_c_result_type(self, type1, type2): - if type1.is_ptr and type2.is_int: + if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum): return type1 - elif type1.is_ptr and type2.is_ptr: + elif (type1.is_ptr or type1.is_array) and (type2.is_ptr or type2.is_array): return PyrexTypes.c_int_type else: return NumBinopNode.compute_c_result_type( @@ -2645,20 +2874,20 @@ class ModNode(IntBinopNode): class PowNode(NumBinopNode): # '**' operator. - + def analyse_types(self, env): env.pow_function_used = 1 NumBinopNode.analyse_types(self, env) - + def compute_c_result_type(self, type1, type2): if self.c_types_okay(type1, type2): return PyrexTypes.c_double_type else: return None - + def c_types_okay(self, type1, type2): return type1.is_float or type2.is_float - + def type_error(self): if not (self.operand1.type.is_error or self.operand2.type.is_error): if self.operand1.type.is_int and self.operand2.type.is_int: @@ -2667,11 +2896,11 @@ class PowNode(NumBinopNode): else: NumBinopNode.type_error(self) self.type = PyrexTypes.error_type - + def calculate_result_code(self): return "pow(%s, %s)" % ( self.operand1.result_code, self.operand2.result_code) - + class BoolBinopNode(ExprNode): # Short-circuiting boolean operation. @@ -2685,6 +2914,14 @@ class BoolBinopNode(ExprNode): subexprs = ['operand1', 'operand2', 'temp_bool'] + def compile_time_value(self, denv): + if self.operator == 'and': + return self.operand1.compile_time_value(denv) \ + and self.operand2.compile_time_value(denv) + else: + return self.operand1.compile_time_value(denv) \ + or self.operand2.compile_time_value(denv) + def analyse_types(self, env): self.operand1.analyse_types(env) self.operand2.analyse_types(env) @@ -2870,6 +3107,19 @@ class CmpNode: # Mixin class containing code common to PrimaryCmpNodes # and CascadedCmpNodes. + def cascaded_compile_time_value(self, operand1, denv): + func = get_compile_time_binop(self) + operand2 = self.operand.compile_time_value(denv) + try: + result = func(operand1, operand2) + except Exception, e: + self.compile_time_value_error(e) + if result: + cascade = self.cascade + if cascade: + result = result and cascade.compile_time_value(operand2, denv) + return result + def is_python_comparison(self): return (self.has_python_operands() or (self.cascade and self.cascade.is_python_comparison()) @@ -2929,17 +3179,22 @@ class CmpNode: richcmp_constants[op], code.error_goto_if_null(result_code, self.pos))) else: - if operand1.type.is_pyobject: - res1, res2 = operand1.py_result(), operand2.py_result() + type1 = operand1.type + type2 = operand2.type + if (type1.is_extension_type or type2.is_extension_type) \ + and not type1.same_as(type2): + common_type = py_object_type else: - res1, res2 = operand1.result_code, operand2.result_code + common_type = type1 + code1 = operand1.result_as(common_type) + code2 = operand2.result_as(common_type) code.putln("%s = %s(%s %s %s);" % ( result_code, coerce_result, - res1, + code1, self.c_operator(op), - res2)) - + code2)) + def c_operator(self, op): if op == 'is': return "==" @@ -2965,6 +3220,10 @@ class PrimaryCmpNode(ExprNode, CmpNode): cascade = None + def compile_time_value(self, denv): + operand1 = self.operand.compile_time_value(denv) + return self.cascaded_compile_time_value(operand1, denv) + def analyse_types(self, env): self.operand1.analyse_types(env) self.operand2.analyse_types(env) @@ -3257,12 +3516,14 @@ class CoerceFromPyTypeNode(CoercionNode): rhs = typecast(self.type, c_long_type, rhs) code.putln('%s = %s; %s' % ( self.result_code, - rhs, + rhs, code.error_goto_if(self.error_cond(), self.pos))) - + def error_cond(self): conds = [] - if self.type.exception_value is not None: + if self.type.is_string: + conds.append("(!%s)" % self.result_code) + elif self.type.exception_value is not None: conds.append("(%s == %s)" % (self.result_code, self.type.exception_value)) if self.type.exception_check: conds.append("PyErr_Occurred()") @@ -3442,42 +3703,42 @@ bad: }] #------------------------------------------------------------------------------------ - -get_exception_utility_code = [ -""" -static PyObject *__Pyx_GetExcValue(void); /*proto*/ -""",""" -static PyObject *__Pyx_GetExcValue(void) { - PyObject *type = 0, *value = 0, *tb = 0; - PyObject *result = 0; - PyThreadState *tstate = PyThreadState_Get(); - PyErr_Fetch(&type, &value, &tb); - PyErr_NormalizeException(&type, &value, &tb); - if (PyErr_Occurred()) - goto bad; - if (!value) { - value = Py_None; - Py_INCREF(value); - } - Py_XDECREF(tstate->exc_type); - Py_XDECREF(tstate->exc_value); - Py_XDECREF(tstate->exc_traceback); - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - result = value; - Py_XINCREF(result); - type = 0; - value = 0; - tb = 0; -bad: - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tb); - return result; -} -"""] - +# +#get_exception_utility_code = [ +#""" +#static PyObject *__Pyx_GetExcValue(void); /*proto*/ +#""",""" +#static PyObject *__Pyx_GetExcValue(void) { +# PyObject *type = 0, *value = 0, *tb = 0; +# PyObject *result = 0; +# PyThreadState *tstate = PyThreadState_Get(); +# PyErr_Fetch(&type, &value, &tb); +# PyErr_NormalizeException(&type, &value, &tb); +# if (PyErr_Occurred()) +# goto bad; +# if (!value) { +# value = Py_None; +# Py_INCREF(value); +# } +# Py_XDECREF(tstate->exc_type); +# Py_XDECREF(tstate->exc_value); +# Py_XDECREF(tstate->exc_traceback); +# tstate->exc_type = type; +# tstate->exc_value = value; +# tstate->exc_traceback = tb; +# result = value; +# Py_XINCREF(result); +# type = 0; +# value = 0; +# tb = 0; +#bad: +# Py_XDECREF(type); +# Py_XDECREF(value); +# Py_XDECREF(tb); +# return result; +#} +#"""] +# #------------------------------------------------------------------------------------ unpacking_utility_code = [ diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index ec705046..54b17b0b 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -17,12 +17,14 @@ import Parsing from Symtab import BuiltinScope, ModuleScope import Code from Cython.Utils import replace_suffix +import Builtin +from Cython import Utils verbose = 0 class Context: # This class encapsulates the context needed for compiling - # one or more Pyrex implementation files along with their + # one or more Cython implementation files along with their # associated and imported declaration files. It includes # the root of the module import namespace and the list # of directories to search for include files. @@ -31,7 +33,8 @@ class Context: # include_directories [string] def __init__(self, include_directories): - self.modules = {"__builtin__" : BuiltinScope()} + #self.modules = {"__builtin__" : BuiltinScope()} + self.modules = {"__builtin__" : Builtin.builtin_scope} self.include_directories = include_directories def find_module(self, module_name, @@ -180,22 +183,29 @@ class Context: else: c_suffix = ".c" result.c_file = replace_suffix(source, c_suffix) + c_stat = None + if result.c_file: + try: + c_stat = os.stat(result.c_file) + except EnvironmentError: + pass module_name = self.extract_module_name(source, options) initial_pos = (source, 1, 0) scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0) errors_occurred = False try: tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name) - tree.process_implementation(scope, result) + tree.process_implementation(scope, options, result) except CompileError: errors_occurred = True Errors.close_listing_file() result.num_errors = Errors.num_errors if result.num_errors > 0: errors_occurred = True - if errors_occurred: + if errors_occurred and result.c_file: try: - os.unlink(result.c_file) + #os.unlink(result.c_file) + Utils.castrate_file(result.c_file, c_stat) except EnvironmentError: pass result.c_file = None @@ -225,6 +235,7 @@ class CompilationOptions: errors_to_stderr boolean Echo errors to stderr when using .lis include_path [string] Directories to search for include files output_file string Name of generated .c file + generate_pxi boolean Generate .pxi file for public declarations Following options are experimental and only used on MacOSX: @@ -238,7 +249,11 @@ class CompilationOptions: self.include_path = [] self.objects = [] if defaults: - self.__dict__.update(defaults.__dict__) + if isinstance(defaults, CompilationOptions): + defaults = defaults.__dict__ + else: + defaults = default_options + self.__dict__.update(defaults) self.__dict__.update(kw) @@ -249,6 +264,7 @@ class CompilationResult: c_file string or None The generated C source file h_file string or None The generated C header file i_file string or None The generated .pxi file + api_file string or None The generated C API .h file listing_file string or None File of error messages object_file string or None Result of compiling the C file extension_file string or None Result of linking the object file @@ -259,6 +275,7 @@ class CompilationResult: self.c_file = None self.h_file = None self.i_file = None + self.api_file = None self.listing_file = None self.object_file = None self.extension_file = None @@ -317,18 +334,19 @@ def main(command_line = 0): # #------------------------------------------------------------------------ -default_options = CompilationOptions( +default_options = dict( show_version = 0, use_listing_file = 0, errors_to_stderr = 1, c_only = 1, obj_only = 1, cplus = 0, - output_file = None) + output_file = None, + generate_pxi = 0) if sys.platform == "mac": from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError - default_options.use_listing_file = 1 + default_options['use_listing_file'] = 1 elif sys.platform == "darwin": from Cython.Mac.DarwinSystem import c_compile, c_link, CCompilerError else: diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 4aeff24f..9f8413de 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -4,6 +4,7 @@ import os, time from cStringIO import StringIO +from PyrexTypes import CPtrType import Code import Naming @@ -20,6 +21,9 @@ from Cython.Utils import open_new_file, replace_suffix class ModuleNode(Nodes.Node, Nodes.BlockNode): # doc string or None # body StatListNode + # + # referenced_modules [ModuleScope] + # module_temp_cname string def analyse_declarations(self, env): if Options.embed_pos_in_docstring: @@ -30,15 +34,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): env.doc = self.doc self.body.analyse_declarations(env) - def process_implementation(self, env, result): + def process_implementation(self, env, options, result): self.analyse_declarations(env) env.check_c_classes() self.body.analyse_expressions(env) env.return_type = PyrexTypes.c_void_type + self.referenced_modules = [] + self.find_referenced_modules(env, self.referenced_modules, {}) + if self.has_imported_c_functions(): + self.module_temp_cname = env.allocate_temp_pyobject() + env.release_temp(self.module_temp_cname) self.generate_c_code(env, result) - self.generate_h_code(env, result) + self.generate_h_code(env, options, result) + self.generate_api_code(env, result) - def generate_h_code(self, env, result): + def has_imported_c_functions(self): + for module in self.referenced_modules: + for entry in module.cfunc_entries: + if entry.defined_in_pxd: + return 1 + return 0 + + def generate_h_code(self, env, options, result): public_vars = [] public_funcs = [] public_extension_types = [] @@ -53,52 +70,125 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): public_extension_types.append(entry) if public_vars or public_funcs or public_extension_types: result.h_file = replace_suffix(result.c_file, ".h") - result.i_file = replace_suffix(result.c_file, ".pxi") h_code = Code.CCodeWriter(open_new_file(result.h_file)) - i_code = Code.PyrexCodeWriter(result.i_file) - header_barrier = "__HAS_PYX_" + env.module_name - h_code.putln("#ifndef %s" % header_barrier) - h_code.putln("#define %s" % header_barrier) + if options.generate_pxi: + result.i_file = replace_suffix(result.c_file, ".pxi") + i_code = Code.PyrexCodeWriter(result.i_file) + else: + i_code = None + guard = Naming.h_guard_prefix + env.qualified_name.replace(".", "__") + h_code.put_h_guard(guard) self.generate_extern_c_macro_definition(h_code) - for entry in public_vars: - h_code.putln("%s %s;" % ( - Naming.extern_c_macro, - entry.type.declaration_code( - entry.cname, dll_linkage = "DL_IMPORT"))) - i_code.putln("cdef extern %s" % - entry.type.declaration_code(entry.cname, pyrex = 1)) - for entry in public_extension_types: - self.generate_cclass_header_code(entry.type, h_code) - self.generate_cclass_include_code(entry.type, i_code) + self.generate_type_header_code(env, h_code) + h_code.putln("") + h_code.putln("#ifndef %s" % Naming.api_guard_prefix + self.api_name(env)) + if public_vars: + h_code.putln("") + for entry in public_vars: + self.generate_public_declaration(entry, h_code, i_code) if public_funcs: - sort_public_funcs = [ (func.cname, func) - for func in public_funcs ] - sort_public_funcs.sort() - public_funcs = [ func[1] for func in sort_public_funcs ] - for entry in public_funcs: - h_code.putln( - 'static %s;' % - entry.type.declaration_code("(*%s)" % entry.cname)) - i_code.putln("cdef extern %s" % - entry.type.declaration_code(entry.cname, pyrex = 1)) - h_code.putln( - "static struct {char *s; void **p;} _%s_API[] = {" % - env.module_name) + h_code.putln("") for entry in public_funcs: - h_code.putln('{"%s", (void*)(&%s)},' % ( - entry.cname, entry.cname)) - h_code.putln("{0, 0}") - h_code.putln("};") - self.generate_c_api_import_code(env, h_code) + self.generate_public_declaration(entry, h_code, i_code) + if public_extension_types: + h_code.putln("") + for entry in public_extension_types: + self.generate_cclass_header_code(entry.type, h_code) + if i_code: + self.generate_cclass_include_code(entry.type, i_code) + h_code.putln("") + h_code.putln("#endif") + h_code.putln("") h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name) - h_code.putln("#endif /* %s */" % header_barrier) + h_code.putln("") + h_code.putln("#endif") + + def generate_public_declaration(self, entry, h_code, i_code): + h_code.putln("%s %s;" % ( + Naming.extern_c_macro, + entry.type.declaration_code( + entry.cname, dll_linkage = "DL_IMPORT"))) + if i_code: + i_code.putln("cdef extern %s" % + entry.type.declaration_code(entry.cname, pyrex = 1)) + + def api_name(self, env): + return env.qualified_name.replace(".", "__") + + def generate_api_code(self, env, result): + api_funcs = [] + for entry in env.cfunc_entries: + if entry.api: + api_funcs.append(entry) + public_extension_types = [] + for entry in env.c_class_entries: + if entry.visibility == 'public': + public_extension_types.append(entry) + if api_funcs or public_extension_types: + result.api_file = replace_suffix(result.c_file, "_api.h") + h_code = Code.CCodeWriter(open_new_file(result.api_file)) + name = self.api_name(env) + guard = Naming.api_guard_prefix + name + h_code.put_h_guard(guard) + h_code.putln('#include "Python.h"') + if result.h_file: + h_code.putln('#include "%s"' % os.path.basename(result.h_file)) + for entry in public_extension_types: + type = entry.type + h_code.putln("") + h_code.putln("static PyTypeObject *%s;" % type.typeptr_cname) + h_code.putln("#define %s (*%s)" % ( + type.typeobj_cname, type.typeptr_cname)) + if api_funcs: + h_code.putln("") + for entry in api_funcs: + type = CPtrType(entry.type) + h_code.putln("static %s;" % type.declaration_code(entry.cname)) + h_code.putln("") + h_code.put_h_guard(Naming.api_func_guard + "import_module") + h_code.put(import_module_utility_code[1]) + h_code.putln("") + h_code.putln("#endif") + if api_funcs: + h_code.putln("") + h_code.put_h_guard(Naming.api_func_guard + "import_function") + h_code.put(function_import_utility_code[1]) + h_code.putln("") + h_code.putln("#endif") + if public_extension_types: + h_code.putln("") + h_code.put_h_guard(Naming.api_func_guard + "import_type") + h_code.put(type_import_utility_code[1]) + h_code.putln("") + h_code.putln("#endif") + h_code.putln("") + h_code.putln("static int import_%s(void) {" % name) + h_code.putln("PyObject *module = 0;") + h_code.putln('module = __Pyx_ImportModule("%s");' % env.qualified_name) + h_code.putln("if (!module) goto bad;") + for entry in api_funcs: + sig = entry.type.signature_string() + h_code.putln( + 'if (__Pyx_ImportFunction(module, "%s", (void**)&%s, "%s") < 0) goto bad;' % ( + entry.name, + entry.cname, + sig)) + h_code.putln("Py_DECREF(module);") + for entry in public_extension_types: + self.generate_type_import_call(entry.type, h_code, "goto bad") + h_code.putln("return 0;") + h_code.putln("bad:") + h_code.putln("Py_XDECREF(module);") + h_code.putln("return -1;") + h_code.putln("}") + h_code.putln("") + h_code.putln("#endif") def generate_cclass_header_code(self, type, h_code): - #h_code.putln("extern DL_IMPORT(PyTypeObject) %s;" % type.typeobj_cname) h_code.putln("%s DL_IMPORT(PyTypeObject) %s;" % ( Naming.extern_c_macro, type.typeobj_cname)) - self.generate_obj_struct_definition(type, h_code) + #self.generate_obj_struct_definition(type, h_code) def generate_cclass_include_code(self, type, i_code): i_code.putln("cdef extern class %s.%s:" % ( @@ -112,11 +202,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): else: i_code.putln("pass") i_code.dedent() - + def generate_c_code(self, env, result): - modules = [] - self.find_referenced_modules(env, modules, {}) - #code = Code.CCodeWriter(result.c_file) +# modules = [] +# self.find_referenced_modules(env, modules, {}) + modules = self.referenced_modules code = Code.CCodeWriter(StringIO()) code.h = Code.CCodeWriter(StringIO()) code.init_labels() @@ -132,7 +222,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.body.generate_function_definitions(env, code) self.generate_interned_name_table(env, code) self.generate_py_string_table(env, code) - self.generate_c_api_table(env, code) self.generate_typeobj_definitions(env, code) self.generate_method_table(env, code) self.generate_filename_init_prototype(code) @@ -177,18 +266,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(" #define PyNumber_Index(o) PyNumber_Int(o)") code.putln(" #define PyIndex_Check(o) PyNumber_Check(o)") code.putln("#endif") + code.putln("#ifndef WIN32") + code.putln(" #define __stdcall") + code.putln(" #define __cdecl") + code.putln("#endif") self.generate_extern_c_macro_definition(code) - code.putln("%s double pow(double, double);" % Naming.extern_c_macro) + code.putln("#include ") self.generate_includes(env, cimported_modules, code) - #for filename in env.include_files: - # code.putln('#include "%s"' % filename) code.putln('') code.put(Nodes.utility_function_predeclarations) code.put(Nodes.branch_prediction_macros) - #if Options.intern_names: - # code.putln(Nodes.get_name_interned_predeclaration) - #else: - # code.putln(get_name_predeclaration) code.putln('') code.putln('static PyObject *%s;' % env.module_cname) code.putln('static PyObject *%s;' % Naming.builtins_cname) @@ -238,28 +325,65 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_type_predeclarations(env, code) self.generate_type_definitions(env, code) self.generate_global_declarations(env, code, definition) - self.generate_cfunction_predeclarations(env, code) + self.generate_cfunction_predeclarations(env, code, definition) def generate_type_predeclarations(self, env, code): pass - def generate_type_definitions(self, env, code): - # Generate definitions of structs/unions/enums. - for entry in env.sue_entries: + def generate_type_header_code(self, env, code): + # Generate definitions of structs/unions/enums/typedefs/objstructs. + #self.generate_gcc33_hack(env, code) # Is this still needed? + for entry in env.type_entries: if not entry.in_cinclude: + #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_struct_or_union: + if type.is_typedef: # Must test this first! + self.generate_typedef(entry, code) + elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) - else: + elif type.is_enum: self.generate_enum_definition(entry, code) + elif type.is_extension_type: + self.generate_obj_struct_definition(type, code) + + def generate_type_definitions(self, env, code): + # Generate definitions of structs/unions/enums. +# self.generate_gcc33_hack(env, code) +# for entry in env.sue_entries: +# if not entry.in_cinclude: +# type = entry.type +# if type.is_struct_or_union: +# self.generate_struct_union_definition(entry, code) +# else: +# self.generate_enum_definition(entry, code) + self.generate_type_header_code(env, code) # Generate extension type object struct definitions. for entry in env.c_class_entries: if not entry.in_cinclude: self.generate_typeobject_predeclaration(entry, code) - self.generate_obj_struct_definition(entry.type, code) + #self.generate_obj_struct_definition(entry.type, code) self.generate_exttype_vtable_struct(entry, code) self.generate_exttype_vtabptr_declaration(entry, code) + def generate_gcc33_hack(self, env, code): + # Workaround for spurious warning generation in gcc 3.3 + code.putln("") + for entry in env.c_class_entries: + type = entry.type + if not type.typedef_flag: + name = type.objstruct_cname + if name.startswith("__pyx_"): + tail = name[6:] + else: + tail = name + code.putln("typedef struct %s __pyx_gcc33_%s;" % ( + name, tail)) + + def generate_typedef(self, entry, code): + base_type = entry.type.typedef_base_type + code.putln("") + code.putln("typedef %s;" % base_type.declaration_code(entry.cname)) + def sue_header_footer(self, type, kind, name): if type.typedef_flag: header = "typedef %s {" % kind @@ -399,14 +523,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_var_declarations(env.default_entries, static = 1, definition = definition) - def generate_cfunction_predeclarations(self, env, code): + def generate_cfunction_predeclarations(self, env, code, definition): for entry in env.cfunc_entries: - if not entry.in_cinclude: - if entry.visibility == 'public': + if not entry.in_cinclude and (definition + or entry.defined_in_pxd or entry.visibility == 'extern'): + if entry.visibility in ('public', 'extern'): dll_linkage = "DL_EXPORT" else: dll_linkage = None - header = entry.type.declaration_code(entry.cname, + type = entry.type + if not definition and entry.defined_in_pxd: + type = CPtrType(type) + header = type.declaration_code(entry.cname, dll_linkage = dll_linkage) if entry.visibility == 'private': storage_class = "static " @@ -468,11 +596,21 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.declaration_code(""))) def generate_new_function(self, scope, code): - base_type = scope.parent_type.base_type + type = scope.parent_type + base_type = type.base_type + py_attrs = [] + for entry in scope.var_entries: + if entry.type.is_pyobject: + py_attrs.append(entry) + need_self_cast = type.vtabslot_cname or py_attrs code.putln("") code.putln( "static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k) {" % scope.mangle_internal("tp_new")) + if need_self_cast: + code.putln( + "%s;" + % scope.parent_type.declaration_code("p")) if base_type: code.putln( "PyObject *o = %s->tp_new(t, a, k);" % @@ -480,13 +618,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): else: code.putln( "PyObject *o = (*t->tp_alloc)(t, 0);") - type = scope.parent_type - py_attrs = [] - for entry in scope.var_entries: - if entry.type.is_pyobject: - py_attrs.append(entry) - if type.vtabslot_cname or py_attrs: - self.generate_self_cast(scope, code) + code.putln( + "if (!o) return 0;") + if need_self_cast: + code.putln( + "p = %s;" + % type.cast_code("o")) + #if need_self_cast: + # self.generate_self_cast(scope, code) if type.vtabslot_cname: code.putln("*(struct %s **)&p->%s = %s;" % ( type.vtabstruct_cname, @@ -1085,74 +1224,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "{0, 0, 0, 0}") code.putln( "};") - - def generate_c_api_table(self, env, code): - public_funcs = [] - for entry in env.cfunc_entries: - if entry.visibility == 'public': - public_funcs.append(entry.cname) - if public_funcs: - env.use_utility_code(Nodes.c_api_import_code) - code.putln( - "static __Pyx_CApiTabEntry %s[] = {" % - Naming.c_api_tab_cname) - public_funcs.sort() - for entry_cname in public_funcs: - code.putln('{"%s", %s},' % (entry_cname, entry_cname)) - code.putln( - "{0, 0}") - code.putln( - "};") - def generate_c_api_import_code(self, env, h_code): - # this is written to the header file! - h_code.put(""" - /* Return -1 and set exception on error, 0 on success. */ - static int - import_%(name)s(PyObject *module) - { - if (module != NULL) - { - int (*init)(struct {const char *s; const void **p;}*); - PyObject* c_api_init; - - c_api_init = PyObject_GetAttrString(module, - "_import_c_api"); - if (!c_api_init) - return -1; - if (!PyCObject_Check(c_api_init)) - { - Py_DECREF(c_api_init); - PyErr_SetString(PyExc_RuntimeError, - "%(name)s module provided an invalid C-API reference"); - return -1; - } - - init = PyCObject_AsVoidPtr(c_api_init); - Py_DECREF(c_api_init); - if (!init) - { - PyErr_SetString(PyExc_RuntimeError, - "%(name)s module returned NULL pointer for C-API init function"); - return -1; - } - - if (init(_%(name)s_API)) - return -1; - } - return 0; - } - """.replace('\n ', '\n') % {'name' : env.module_name}) - - def generate_c_api_init_code(self, env, code): - public_funcs = [] - for entry in env.cfunc_entries: - if entry.visibility == 'public': - public_funcs.append(entry) - if public_funcs: - code.putln('if (__Pyx_InitCApi(%s) < 0) %s' % ( - Naming.module_cname, - code.error_goto(self.pos))) def generate_filename_init_prototype(self, code): code.putln(""); @@ -1164,21 +1236,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("%s; /*proto*/" % header) code.putln("%s {" % header) code.put_var_declarations(env.temp_entries) + #code.putln("/*--- Libary function declarations ---*/") env.generate_library_function_declarations(code) self.generate_filename_init_call(code) + #code.putln("/*--- Module creation code ---*/") self.generate_module_creation_code(env, code) + #code.putln("/*--- Intern code ---*/") self.generate_intern_code(env, code) + #code.putln("/*--- String init code ---*/") self.generate_string_init_code(env, code) - #code.putln("/*--- External C API setup code ---*/") - self.generate_c_api_init_code(env, code) + #code.putln("/*--- Builtin init code ---*/") - self.generate_builtin_init_code(env, code) + # FIXME !! + #self.generate_builtin_init_code(env, code) + #code.putln("/*--- Global init code ---*/") self.generate_global_init_code(env, code) + + #code.putln("/*--- Function export code ---*/") + self.generate_c_function_export_code(env, code) + + #code.putln("/*--- Function import code ---*/") + for module in imported_modules: + self.generate_c_function_import_code_for_module(module, env, code) #code.putln("/*--- Type init code ---*/") self.generate_type_init_code(env, code) @@ -1270,14 +1354,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): entry.cname, Naming.builtins_cname, entry.interned_cname, - entry.cname, + entry.cname, code.error_goto(entry.pos))) else: code.putln( '%s = __Pyx_GetName(%s, "%s"); if (!%s) %s' % ( entry.cname, Naming.builtins_cname, - self.entry.name, + entry.name, entry.cname, code.error_goto(entry.pos))) @@ -1285,17 +1369,56 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate code to initialise global PyObject * # variables to None. for entry in env.var_entries: - if entry.visibility <> 'extern': + if entry.visibility != 'extern': if entry.type.is_pyobject: code.put_init_var_to_py_none(entry) + + def generate_c_function_export_code(self, env, code): + # Generate code to create PyCFunction wrappers for exported C functions. + for entry in env.cfunc_entries: + if entry.api or entry.defined_in_pxd: + env.use_utility_code(function_export_utility_code) + signature = entry.type.signature_string() + code.putln('if (__Pyx_ExportFunction("%s", (void*)%s, "%s") < 0) %s' % ( + entry.name, + entry.cname, + signature, + code.error_goto(self.pos))) def generate_type_import_code_for_module(self, module, env, code): - # Generate type import code for all extension types in + # Generate type import code for all exported extension types in # an imported module. - if module.c_class_entries: - for entry in module.c_class_entries: + #if module.c_class_entries: + for entry in module.c_class_entries: + if entry.defined_in_pxd: self.generate_type_import_code(env, entry.type, entry.pos, code) + def generate_c_function_import_code_for_module(self, module, env, code): + # Generate import code for all exported C functions in a cimported module. + entries = [] + for entry in module.cfunc_entries: + if entry.defined_in_pxd: + entries.append(entry) + if entries: + env.use_utility_code(import_module_utility_code) + env.use_utility_code(function_import_utility_code) + temp = self.module_temp_cname + code.putln( + '%s = __Pyx_ImportModule("%s"); if (!%s) %s' % ( + temp, + module.qualified_name, + temp, + code.error_goto(self.pos))) + for entry in entries: + code.putln( + 'if (__Pyx_ImportFunction(%s, "%s", (void**)&%s, "%s") < 0) %s' % ( + temp, + entry.name, + entry.cname, + entry.type.signature_string(), + code.error_goto(self.pos))) + code.putln("Py_DECREF(%s); %s = 0;" % (temp, temp)) + def generate_type_init_code(self, env, code): # Generate type import code for extern extension types # and type ready code for non-extern ones. @@ -1315,8 +1438,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def use_type_import_utility_code(self, env): import ExprNodes - env.use_utility_code(Nodes.type_import_utility_code) - env.use_utility_code(ExprNodes.import_utility_code) + env.use_utility_code(type_import_utility_code) + env.use_utility_code(import_module_utility_code) def generate_type_import_code(self, env, type, pos, code): # If not already done, generate code to import the typeobject of an @@ -1328,12 +1451,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): objstruct = type.objstruct_cname else: objstruct = "struct %s" % type.objstruct_cname - code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); %s' % ( - type.typeptr_cname, - type.module_name, - type.name, - objstruct, - code.error_goto_if_null(type.typeptr_cname, pos))) +# code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); %s' % ( +# type.typeptr_cname, +# type.module_name, +# type.name, +# objstruct, +# code.error_goto_if_null(type.typeptr_cname, pos))) + self.generate_type_import_call(type, code, + code.error_goto_if_null(type.typeptr_cname, pos)) self.use_type_import_utility_code(env) if type.vtabptr_cname: code.putln( @@ -1343,7 +1468,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.error_goto(pos))) env.use_utility_code(Nodes.get_vtable_utility_code) env.types_imported[type] = 1 - + + def generate_type_import_call(self, type, code, error_code): + if type.typedef_flag: + objstruct = type.objstruct_cname + else: + objstruct = "struct %s" % type.objstruct_cname + code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); %s' % ( + type.typeptr_cname, + type.module_name, + type.name, + objstruct, + error_code)) + def generate_type_ready_code(self, env, entry, code): # Generate a call to PyType_Ready for an extension # type defined in this module. @@ -1401,7 +1538,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for meth_entry in type.scope.cfunc_entries: if meth_entry.func_cname: code.putln( - "*(void(**)())&%s.%s = (void(*)())%s;" % ( + "*(void(**)(void))&%s.%s = (void(*)(void))%s;" % ( type.vtable_cname, meth_entry.cname, meth_entry.func_cname)) @@ -1426,3 +1563,171 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for utility_code in env.utility_code_used: code.h.put(utility_code[0]) code.put(utility_code[1]) + +#------------------------------------------------------------------------------------ + +#type_import_utility_code = [ +#""" +#static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, long size); /*proto*/ +#""",""" +#static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, +# long size) +#{ +# PyObject *py_module_name = 0; +# PyObject *py_class_name = 0; +# PyObject *py_name_list = 0; +# PyObject *py_module = 0; +# PyObject *result = 0; +# +# py_module_name = PyString_FromString(module_name); +# if (!py_module_name) +# goto bad; +# py_class_name = PyString_FromString(class_name); +# if (!py_class_name) +# goto bad; +# py_name_list = PyList_New(1); +# if (!py_name_list) +# goto bad; +# Py_INCREF(py_class_name); +# if (PyList_SetItem(py_name_list, 0, py_class_name) < 0) +# goto bad; +# py_module = __Pyx_Import(py_module_name, py_name_list); +# if (!py_module) +# goto bad; +# result = PyObject_GetAttr(py_module, py_class_name); +# if (!result) +# goto bad; +# if (!PyType_Check(result)) { +# PyErr_Format(PyExc_TypeError, +# "%s.%s is not a type object", +# module_name, class_name); +# goto bad; +# } +# if (((PyTypeObject *)result)->tp_basicsize != size) { +# PyErr_Format(PyExc_ValueError, +# "%s.%s does not appear to be the correct type object", +# module_name, class_name); +# goto bad; +# } +# goto done; +#bad: +# Py_XDECREF(result); +# result = 0; +#done: +# Py_XDECREF(py_module_name); +# Py_XDECREF(py_class_name); +# Py_XDECREF(py_name_list); +# return (PyTypeObject *)result; +#} +#"""] + +#------------------------------------------------------------------------------------ + +import_module_utility_code = [ +""" +static PyObject *__Pyx_ImportModule(char *name); /*proto*/ +""",""" +static PyObject *__Pyx_ImportModule(char *name) { + PyObject *py_name = 0; + + py_name = PyString_FromString(name); + if (!py_name) + goto bad; + return PyImport_Import(py_name); +bad: + Py_XDECREF(py_name); + return 0; +} +"""] + +#------------------------------------------------------------------------------------ + +type_import_utility_code = [ +""" +static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, long size); /*proto*/ +""",""" +static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, + long size) +{ + PyObject *py_module = 0; + PyObject *result = 0; + + py_module = __Pyx_ImportModule(module_name); + if (!py_module) + goto bad; + result = PyObject_GetAttrString(py_module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%s.%s is not a type object", + module_name, class_name); + goto bad; + } + if (((PyTypeObject *)result)->tp_basicsize != size) { + PyErr_Format(PyExc_ValueError, + "%s.%s does not appear to be the correct type object", + module_name, class_name); + goto bad; + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return 0; +} +"""] + +#------------------------------------------------------------------------------------ + +function_export_utility_code = [ +""" +static int __Pyx_ExportFunction(char *n, void *f, char *s); /*proto*/ +""",r""" +static int __Pyx_ExportFunction(char *n, void *f, char *s) { + PyObject *p = 0; + p = PyCObject_FromVoidPtrAndDesc(f, s, 0); + if (!p) + goto bad; + if (PyModule_AddObject(%(MODULE)s, n, p) < 0) + goto bad; + return 0; +bad: + Py_XDECREF(p); + return -1; +} +""" % {'MODULE': Naming.module_cname}] + +#------------------------------------------------------------------------------------ + +function_import_utility_code = [ +""" +static int __Pyx_ImportFunction(PyObject *module, char *funcname, void **f, char *sig); /*proto*/ +""",""" +static int __Pyx_ImportFunction(PyObject *module, char *funcname, void **f, char *sig) { + PyObject *cobj = 0; + char *desc; + + cobj = PyObject_GetAttrString(module, funcname); + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%s does not export expected C function %s", + PyModule_GetName(module), funcname); + goto bad; + } + desc = (char *)PyCObject_GetDesc(cobj); + if (!desc) + goto bad; + if (strcmp(desc, sig) != 0) { + PyErr_Format(PyExc_TypeError, + "C function %s.%s has wrong signature (expected %s, got %s)", + PyModule_GetName(module), funcname, sig, desc); + goto bad; + } + *f = PyCObject_AsVoidPtr(cobj); + Py_DECREF(cobj); + return 0; +bad: + Py_XDECREF(cobj); + return -1; +} +"""] diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 152edb07..a0f5e366 100644 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -50,6 +50,7 @@ module_cname = pyrex_prefix + "m" moddoc_cname = pyrex_prefix + "mdoc" methtable_cname = pyrex_prefix + "methods" retval_cname = pyrex_prefix + "r" +reqd_kwds_cname = pyrex_prefix + "reqd_kwds" self_cname = pyrex_prefix + "self" stringtab_cname = pyrex_prefix + "string_tab" vtabslot_cname = pyrex_prefix + "vtab" @@ -58,6 +59,16 @@ gilstate_cname = pyrex_prefix + "state" extern_c_macro = pyrex_prefix.upper() + "EXTERN_C" +exc_type_name = pyrex_prefix + "exc_type" +exc_value_name = pyrex_prefix + "exc_value" +exc_tb_name = pyrex_prefix + "exc_tb" +exc_lineno_name = pyrex_prefix + "exc_lineno" + +exc_vars = (exc_type_name, exc_value_name, exc_tb_name) + +h_guard_prefix = "__PYX_HAVE__" +api_guard_prefix = "__PYX_HAVE_API__" +api_func_guard = "__PYX_HAVE_API_FUNC_" def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0): return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial) diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index 8c20c1c2..deea3d66 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -9,7 +9,7 @@ from Errors import error, warning, InternalError import Naming import PyrexTypes import TypeSlots -from PyrexTypes import py_object_type, error_type, CTypedefType +from PyrexTypes import py_object_type, error_type, CTypedefType, CFuncType from Symtab import ModuleScope, LocalScope, \ StructOrUnionScope, PyClassScope, CClassScope from Cython.Utils import open_new_file, replace_suffix @@ -212,7 +212,11 @@ class CDeclaratorNode(Node): # CNameDeclaratorNode of the name being declared # and type is the type it is being declared as. # - + # calling_convention string Calling convention of CFuncDeclaratorNode + # for which this is a base + + calling_convention = "" + def analyse_expressions(self, env): pass @@ -274,7 +278,6 @@ class CArrayDeclaratorNode(CDeclaratorNode): self.dimension.analyse_const_expression(env) if not self.dimension.type.is_int: error(self.dimension.pos, "Array dimension not integer") - #size = self.dimension.value size = self.dimension.result_code else: size = None @@ -284,6 +287,9 @@ class CArrayDeclaratorNode(CDeclaratorNode): if base_type.is_pyobject: error(self.pos, "Array element cannot be a Python object") + if base_type.is_cfunction: + error(self.pos, + "Array element cannot be a function") array_type = PyrexTypes.c_array_type(base_type, size) return self.base.analyse(array_type, env) @@ -294,7 +300,8 @@ class CFuncDeclaratorNode(CDeclaratorNode): # has_varargs boolean # exception_value ConstNode # exception_check boolean True if PyErr_Occurred check needed - # with_gil boolean True if GIL should be grabbed/released + # nogil boolean Can be called without gil + # with_gil boolean Acquire gil around function body def analyse(self, return_type, env): func_type_args = [] @@ -310,6 +317,9 @@ class CFuncDeclaratorNode(CDeclaratorNode): # Catch attempted C-style func(void) decl if type.is_void: error(arg_node.pos, "Function argument cannot be void") + if type.is_pyobject and self.nogil: + error(self.pos, + "Function with Python argument cannot be declared nogil") func_type_args.append( PyrexTypes.CFuncTypeArg(name, type, arg_node.pos)) if arg_node.default: @@ -328,10 +338,20 @@ class CFuncDeclaratorNode(CDeclaratorNode): error(self.exception_value.pos, "Exception value incompatible with function return type") exc_check = self.exception_check + if return_type.is_pyobject and self.nogil: + error(self.pos, + "Function with Python return type cannot be declared nogil") + if return_type.is_array: + error(self.pos, + "Function cannot return an array") + if return_type.is_cfunction: + error(self.pos, + "Function cannot return a function") func_type = PyrexTypes.CFuncType( return_type, func_type_args, self.has_varargs, exception_value = exc_val, exception_check = exc_check, - with_gil = self.with_gil) + calling_convention = self.base.calling_convention, + nogil = self.nogil, with_gil = self.with_gil) return self.base.analyse(func_type, env) @@ -344,11 +364,12 @@ class CArgDeclNode(Node): # default ExprNode or None # default_entry Symtab.Entry Entry for the variable holding the default value # is_self_arg boolean Is the "self" arg of an extension type method - # kw_only boolean Is a keyword-only argument - + # is_kw_only boolean Is a keyword-only argument + is_self_arg = 0 def analyse(self, env): + #print "CArgDeclNode.analyse: is_self_arg =", self.is_self_arg ### base_type = self.base_type.analyse(env) return self.declarator.analyse(base_type, env) @@ -374,6 +395,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): def analyse(self, env): # Return type descriptor. + #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ### type = None if self.is_basic_c_type: type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name) @@ -383,6 +405,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): type = py_object_type elif self.name is None: if self.is_self_arg and env.is_c_class_scope: + #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ### type = env.parent_type else: type = py_object_type @@ -425,6 +448,7 @@ class CVarDefNode(StatNode): # visibility 'private' or 'public' or 'extern' # base_type CBaseTypeNode # declarators [CDeclaratorNode] + # in_pxd boolean def analyse_declarations(self, env, dest_scope = None): if not dest_scope: @@ -445,9 +469,18 @@ class CVarDefNode(StatNode): error(declarator.pos, "Missing name in declaration.") return if type.is_cfunction: - dest_scope.declare_cfunction(name, type, declarator.pos, - cname = cname, visibility = self.visibility) + entry = dest_scope.declare_cfunction(name, type, declarator.pos, + cname = cname, visibility = self.visibility, in_pxd = self.in_pxd) + #if self.visibility <> 'extern': + # if self.in_pxd: + # entry.defined_in_pxd = 1 + # else: + # error(declarator.pos, + # "Non-extern C function declared but not defined") else: + if self.in_pxd and self.visibility != 'extern': + error(self.pos, + "Only 'extern' C variable declaration allowed in .pxd file") dest_scope.declare_var(name, type, declarator.pos, cname = cname, visibility = self.visibility, is_cdef = 1) @@ -465,6 +498,7 @@ class CStructOrUnionDefNode(StatNode): # cname string or None # kind "struct" or "union" # typedef_flag boolean + # visibility "public" or "private" # attributes [CVarDefNode] or None # entry Entry @@ -474,11 +508,11 @@ class CStructOrUnionDefNode(StatNode): scope = StructOrUnionScope() self.entry = env.declare_struct_or_union( self.name, self.kind, scope, self.typedef_flag, self.pos, - self.cname) + self.cname, visibility = self.visibility) if self.attributes is not None: for attr in self.attributes: attr.analyse_declarations(env, scope) - + def analyse_expressions(self, env): pass @@ -491,11 +525,13 @@ class CEnumDefNode(StatNode): # cname string or None # items [CEnumDefItemNode] # typedef_flag boolean + # visibility "public" or "private" # entry Entry def analyse_declarations(self, env): self.entry = env.declare_enum(self.name, self.pos, - cname = self.cname, typedef_flag = self.typedef_flag) + cname = self.cname, typedef_flag = self.typedef_flag, + visibility = self.visibility) for item in self.items: item.analyse_declarations(env, self.entry) @@ -525,15 +561,17 @@ class CEnumDefItemNode(StatNode): class CTypeDefNode(StatNode): # base_type CBaseTypeNode # declarator CDeclaratorNode + # visibility "public" or "private" def analyse_declarations(self, env): base = self.base_type.analyse(env) name_declarator, type = self.declarator.analyse(base, env) name = name_declarator.name cname = name_declarator.cname - if env.in_cinclude: - type = CTypedefType(cname or name, type) - env.declare_type(name, type, self.pos, cname = cname) + #if env.in_cinclude: + # type = CTypedefType(cname, type) + env.declare_typedef(name, type, self.pos, + cname = cname, visibility = self.visibility) def analyse_expressions(self, env): pass @@ -553,14 +591,15 @@ class FuncDefNode(StatNode, BlockNode): def analyse_expressions(self, env): pass + + def need_gil_acquisition(self, lenv): + return 0 def generate_function_definitions(self, env, code): # Generate C code for header and body of function genv = env.global_scope() lenv = LocalScope(name = self.entry.name, outer_scope = genv) - #lenv.function_name = self.function_name() lenv.return_type = self.return_type - #self.filename = lenv.get_filename_const(self.pos) code.init_labels() self.declare_arguments(lenv) self.body.analyse_declarations(lenv) @@ -597,8 +636,10 @@ class FuncDefNode(StatNode, BlockNode): self.generate_keyword_list(code) # ----- Extern library function declarations lenv.generate_library_function_declarations(code) - # ----- Grab GIL - self.generate_grab_gil(code) + # ----- GIL acquisition + acquire_gil = self.need_gil_acquisition(lenv) + if acquire_gil: + code.putln("PyGILState_STATE _save = PyGILState_Ensure();") # ----- Fetch arguments self.generate_argument_parsing_code(code) self.generate_argument_increfs(lenv, code) @@ -645,27 +686,29 @@ class FuncDefNode(StatNode, BlockNode): '__Pyx_WriteUnraisable("%s");' % self.entry.qualified_name) env.use_utility_code(unraisable_exception_utility_code) + #if not self.return_type.is_void: + default_retval = self.return_type.default_value + if default_retval: + code.putln( + "%s = %s;" % ( + Naming.retval_cname, + default_retval)) + #self.return_type.default_value)) # ----- Return cleanup code.put_label(code.return_label) code.put_var_decrefs(lenv.var_entries, used_only = 1) code.put_var_decrefs(lenv.arg_entries) self.put_stararg_decrefs(code) - # ----- Release GIL - self.generate_release_gil(code) + if acquire_gil: + code.putln("PyGILState_Release(_save);") # ----- Return if not self.return_type.is_void: - retval_code = Naming.retval_cname - #if self.return_type.is_extension_type: - # retval_code = "((%s)%s) " % ( - # self.return_type.declaration_code(""), - # retval_code) - code.putln("return %s;" % retval_code) + code.putln("return %s;" % Naming.retval_cname) code.putln("}") # ----- Python version if self.py_func: self.py_func.generate_function_definitions(env, code) - def put_stararg_decrefs(self, code): pass @@ -687,12 +730,6 @@ class FuncDefNode(StatNode, BlockNode): def generate_execution_code(self, code): pass - def generate_grab_gil(self, code): - pass - - def generate_release_gil(self, code): - pass - class CFuncDefNode(FuncDefNode): # C function definition. @@ -702,7 +739,9 @@ class CFuncDefNode(FuncDefNode): # base_type CBaseTypeNode # declarator CDeclaratorNode # body StatListNode + # api boolean # + # with_gil boolean Acquire GIL around body # type CFuncType def unqualified_name(self): @@ -711,20 +750,21 @@ class CFuncDefNode(FuncDefNode): def analyse_declarations(self, env): base_type = self.base_type.analyse(env) name_declarator, type = self.declarator.analyse(base_type, env) + if not type.is_cfunction: + error(self.pos, + "Suite attached to non-function declaration") # Remember the actual type according to the function header # written here, because the type in the symbol table entry # may be different if we're overriding a C method inherited # from the base type of an extension type. self.type = type - if not type.is_cfunction: - error(self.pos, - "Suite attached to non-function declaration") name = name_declarator.name cname = name_declarator.cname self.entry = env.declare_cfunction( name, type, self.pos, cname = cname, visibility = self.visibility, - defining = self.body is not None) + defining = self.body is not None, + api = self.api) self.return_type = type.return_type if self.overridable: @@ -756,6 +796,14 @@ class CFuncDefNode(FuncDefNode): error(arg.pos, "Missing argument name") self.declare_argument(env, arg) + def need_gil_acquisition(self, lenv): + with_gil = self.type.with_gil + if self.type.nogil and not with_gil: + for entry in lenv.var_entries + lenv.temp_entries: + if entry.type.is_pyobject: + error(self.pos, "Function declared nogil has Python locals or temporaries") + return with_gil + def generate_function_header(self, code, with_pymethdef): arg_decls = [] type = self.type @@ -820,18 +868,6 @@ class CFuncDefNode(FuncDefNode): error(arg.pos, "Cannot test type of extern C class " "without type object name specification") - def generate_grab_gil(self, code): - if self.entry.type.with_gil: - code.putln("") - code.put_py_gil_state_ensure(Naming.gilstate_cname) - code.putln("") - - def generate_release_gil(self, code): - if self.entry.type.with_gil: - code.putln("") - code.put_py_gil_state_release(Naming.gilstate_cname) - code.putln("") - def error_value(self): if self.return_type.is_pyobject: return "0" @@ -869,6 +905,16 @@ class DefNode(FuncDefNode): # assmt AssignmentNode Function construction/assignment assmt = None + num_kwonly_args = 0 + reqd_kw_flags_cname = "0" + + def __init__(self, pos, **kwds): + FuncDefNode.__init__(self, pos, **kwds) + n = 0 + for arg in self.args: + if arg.kw_only: + n += 1 + self.num_kwonly_args = n def analyse_declarations(self, env): for arg in self.args: @@ -966,20 +1012,25 @@ class DefNode(FuncDefNode): desc, self.name, len(self.args), expected_str)) def declare_pyfunction(self, env): - self.entry = env.declare_pyfunction(self.name, self.pos) - if Options.embed_pos_in_docstring: - self.entry.doc = 'File: %s (starting at line %s)'%relative_position(self.pos) - if not self.doc is None: - self.entry.doc = self.entry.doc + '\\n' + self.doc - else: - self.entry.doc = self.doc - self.entry.func_cname = \ - Naming.func_prefix + "py_" + env.scope_prefix + self.name - self.entry.doc_cname = \ - Naming.funcdoc_prefix + env.scope_prefix + self.name - self.entry.pymethdef_cname = \ - Naming.pymethdef_prefix + env.scope_prefix + self.name - + #print "DefNode.declare_pyfunction:", self.name, "in", env ### + name = self.name + entry = env.declare_pyfunction(self.name, self.pos) + self.entry = entry + prefix = env.scope_prefix + entry.func_cname = \ + Naming.func_prefix + prefix + name + entry.pymethdef_cname = \ + Naming.pymethdef_prefix + prefix + name + if not entry.is_special: + if Options.embed_pos_in_docstring: + entry.doc = 'File: %s (starting at line %s)'%relative_position(self.pos) + if not self.doc is None: + entry.doc = entry.doc + '\\n' + self.doc + else: + entry.doc = self.doc + entry.doc_cname = \ + Naming.funcdoc_prefix + prefix + name + def declare_arguments(self, env): for arg in self.args: if not arg.name: @@ -1088,6 +1139,8 @@ class DefNode(FuncDefNode): def generate_keyword_list(self, code): if self.entry.signature.has_generic_args: + reqd_kw_flags = [] + has_reqd_kwds = False code.put( "static char *%s[] = {" % Naming.kwdlist_cname) @@ -1096,22 +1149,38 @@ class DefNode(FuncDefNode): code.put( '"%s",' % arg.name) + if arg.kw_only and not arg.default: + has_reqd_kwds = 1 + flag = "1" + else: + flag = "0" + reqd_kw_flags.append(flag) code.putln( "0};") - + if has_reqd_kwds: + flags_name = Naming.reqd_kwds_cname + self.reqd_kw_flags_cname = flags_name + code.putln( + "static char %s[] = {%s};" % ( + flags_name, + ",".join(reqd_kw_flags))) + def generate_argument_parsing_code(self, code): # Generate PyArg_ParseTuple call for generic # arguments, if any. - if self.entry.signature.has_generic_args: + has_kwonly_args = self.num_kwonly_args > 0 + has_star_or_kw_args = self.star_arg is not None \ + or self.starstar_arg is not None or has_kwonly_args + if not self.entry.signature.has_generic_args: + if has_star_or_kw_args: + error(self.pos, "This method cannot have * or keyword arguments") + else: arg_addrs = [] arg_formats = [] default_seen = 0 - kw_only_args = [] for arg in self.args: arg_entry = arg.entry if arg.is_generic: - if arg.kw_only: - kw_only_args.append(arg_entry) if arg.default: code.putln( "%s = %s;" % ( @@ -1135,20 +1204,8 @@ class DefNode(FuncDefNode): "Cannot convert Python object argument to type '%s' (when parsing input arguments)" % arg.type) error_return_code = "return %s;" % self.error_value() - if kw_only_args: - max_normal_args = len(self.args) - len(kw_only_args) - code.putln("if (%s && PyTuple_GET_SIZE(%s) > %d) {" % ( - Naming.args_cname, - Naming.args_cname, - max_normal_args)) - code.putln('PyErr_Format(PyExc_TypeError, "function takes at most %d non-keyword arguments (%%d given)", PyTuple_GET_SIZE(%s));' % ( - max_normal_args, - Naming.args_cname)) - code.putln(error_return_code) - code.putln("}") argformat = '"%s"' % string.join(arg_formats, "") - has_starargs = self.star_arg is not None or self.starstar_arg is not None - if has_starargs: + if has_star_or_kw_args: self.generate_stararg_getting_code(code) pt_arglist = [Naming.args_cname, Naming.kwds_cname, argformat, Naming.kwdlist_cname] + arg_addrs @@ -1156,7 +1213,7 @@ class DefNode(FuncDefNode): code.put( 'if (unlikely(!PyArg_ParseTupleAndKeywords(%s))) ' % pt_argstring) - if has_starargs: + if has_star_or_kw_args: code.putln("{") code.put_xdecref(Naming.args_cname, py_object_type) code.put_xdecref(Naming.kwds_cname, py_object_type) @@ -1166,32 +1223,6 @@ class DefNode(FuncDefNode): code.putln("}") else: code.putln(error_return_code) - # check that all required keywords were passed - required_keyword_entries = [] - kw_checks = [] - for arg in self.args: - if arg.is_generic and arg.kw_only and not arg.default: - arg_entry = arg.entry - required_keyword_entries.append(arg_entry) - kw_checks.append("!" + arg_entry.cname) - if required_keyword_entries: - kw_check = ' || '.join(kw_checks) - code.putln("if (unlikely(%s)) {" % kw_check) - for entry in required_keyword_entries: - kw_checks.pop() - if kw_checks: - code.putln('if (!%s) {' % entry.cname) - code.putln('PyErr_SetString(PyExc_TypeError, "keyword argument \'%s\' is required");' % ( - entry.name)) - if kw_checks: - code.put("} else ") - if has_starargs: - code.put_xdecref(Naming.args_cname, py_object_type) - code.put_xdecref(Naming.kwds_cname, py_object_type) - self.generate_arg_xdecref(self.star_arg, code) - self.generate_arg_xdecref(self.starstar_arg, code) - code.putln(error_return_code) - code.putln("}") def put_stararg_decrefs(self, code): if self.star_arg or self.starstar_arg: @@ -1209,20 +1240,20 @@ class DefNode(FuncDefNode): return 0 def generate_stararg_getting_code(self, code): - if self.star_arg or self.starstar_arg: - if not self.entry.signature.has_generic_args: - error(self.pos, "This method cannot have * or ** arguments") - star_arg_addr = self.arg_address(self.star_arg) - starstar_arg_addr = self.arg_address(self.starstar_arg) - code.putln( - "if (__Pyx_GetStarArgs(&%s, &%s, %s, %s, %s, %s) < 0) return %s;" % ( - Naming.args_cname, - Naming.kwds_cname, - Naming.kwdlist_cname, - len(self.args) - self.entry.signature.num_fixed_args(), - star_arg_addr, - starstar_arg_addr, - self.error_value())) + num_kwonly = self.num_kwonly_args + nargs = len(self.args) - num_kwonly - self.entry.signature.num_fixed_args() + star_arg_addr = self.arg_address(self.star_arg) + starstar_arg_addr = self.arg_address(self.starstar_arg) + code.putln( + "if (__Pyx_GetStarArgs(&%s, &%s, %s, %s, %s, %s, %s) < 0) return %s;" % ( + Naming.args_cname, + Naming.kwds_cname, + Naming.kwdlist_cname, + nargs, + star_arg_addr, + starstar_arg_addr, + self.reqd_kw_flags_cname, + self.error_value())) def generate_argument_conversion_code(self, code): # Generate code to convert arguments from @@ -1495,14 +1526,14 @@ class CClassDefNode(StatNode): visibility = self.visibility, typedef_flag = self.typedef_flag) scope = self.entry.type.scope - + if self.doc: if Options.embed_pos_in_docstring: scope.doc = 'File: %s (starting at line %s)'%relative_position(self.pos) scope.doc = scope.doc + '\\n' + self.doc else: scope.doc = self.doc - + if has_body: self.body.analyse_declarations(scope) if self.in_pxd: @@ -2050,11 +2081,10 @@ class RaiseStatNode(StatNode): self.exc_value.release_temp(env) if self.exc_tb: self.exc_tb.release_temp(env) - #env.recycle_pending_temps() # TEMPORARY - if not (self.exc_type or self.exc_value or self.exc_tb): - env.use_utility_code(reraise_utility_code) - else: - env.use_utility_code(raise_utility_code) +# if not (self.exc_type or self.exc_value or self.exc_tb): +# env.use_utility_code(reraise_utility_code) +# else: + env.use_utility_code(raise_utility_code) def generate_execution_code(self, code): if self.exc_type: @@ -2091,6 +2121,20 @@ class RaiseStatNode(StatNode): code.error_goto(self.pos)) +class ReraiseStatNode(StatNode): + + def analyse_expressions(self, env): + env.use_utility_code(raise_utility_code) + + def generate_execution_code(self, code): + vars = code.exc_vars + if vars: + code.putln("__Pyx_Raise(%s, %s, %s);" % tuple(vars)) + code.putln(code.error_goto(self.pos)) + else: + error(self.pos, "Reraise not inside except clause") + + class AssertStatNode(StatNode): # assert statement # @@ -2132,7 +2176,6 @@ class AssertStatNode(StatNode): self.value.generate_disposal_code(code) code.putln("#endif") - class IfStatNode(StatNode): # if statement # @@ -2447,9 +2490,6 @@ class TryExceptStatNode(StatNode): self.else_clause.generate_execution_code(code) code.putln( "}") - #code.putln( - # "goto %s;" % - # end_label) code.put_goto(end_label) code.put_label(our_error_label) code.put_var_xdecrefs_clear(self.cleanup_list) @@ -2462,9 +2502,6 @@ class TryExceptStatNode(StatNode): error(except_clause.pos, "Default except clause not last") except_clause.generate_handling_code(code, end_label) if not default_clause_seen: - #code.putln( - # "goto %s;" % - # code.error_label) code.put_goto(code.error_label) code.put_label(end_label) @@ -2478,6 +2515,7 @@ class ExceptClauseNode(Node): # match_flag string result of exception match # exc_value ExcValueNode used internally # function_name string qualified name of enclosing function + # exc_vars (string * 3) local exception variables def analyse_declarations(self, env): if self.target: @@ -2494,15 +2532,15 @@ class ExceptClauseNode(Node): self.match_flag = env.allocate_temp(PyrexTypes.c_int_type) self.pattern.release_temp(env) env.release_temp(self.match_flag) - self.exc_value = ExprNodes.ExcValueNode(self.pos, env) - self.exc_value.allocate_temps(env) + self.exc_vars = [env.allocate_temp(py_object_type) for i in xrange(3)] if self.target: + self.exc_value = ExprNodes.ExcValueNode(self.pos, env, self.exc_vars[1]) + self.exc_value.allocate_temps(env) self.target.analyse_target_expression(env, self.exc_value) - else: - self.exc_value.release_temp(env) - #if self.target: - # self.target.release_target_temp(env) self.body.analyse_expressions(env) + for var in self.exc_vars: + env.release_temp(var) + env.use_utility_code(get_exception_utility_code) def generate_handling_code(self, code, end_label): code.mark_pos(self.pos) @@ -2524,15 +2562,18 @@ class ExceptClauseNode(Node): # We always have to fetch the exception value even if # there is no target, because this also normalises the # exception and stores it in the thread state. - self.exc_value.generate_evaluation_code(code) + exc_args = "&%s, &%s, &%s" % tuple(self.exc_vars) + code.putln("if (__Pyx_GetException(%s) < 0) %s" % (exc_args, + code.error_goto(self.pos))) if self.target: + self.exc_value.generate_evaluation_code(code) self.target.generate_assignment_code(self.exc_value, code) - else: - self.exc_value.generate_disposal_code(code) + old_exc_vars = code.exc_vars + code.exc_vars = self.exc_vars self.body.generate_execution_code(code) - #code.putln( - # "goto %s;" - # % end_label) + code.exc_vars = old_exc_vars + for var in self.exc_vars: + code.putln("Py_DECREF(%s);" % var) code.put_goto(end_label) code.putln( "}") @@ -2543,8 +2584,8 @@ class TryFinallyStatNode(StatNode): # # body StatNode # finally_clause StatNode + # # cleanup_list [Entry] temps to clean up on error - # exc_vars 3*(string,) temps to hold saved exception # # The plan is that we funnel all continue, break # return and error gotos into the beginning of the @@ -2555,6 +2596,8 @@ class TryFinallyStatNode(StatNode): # exception on entry to the finally block and restore # it on exit. + preserve_exception = 1 + disallow_continue_in_try_finally = 0 # There doesn't seem to be any point in disallowing # continue in the try block, since we have no problem @@ -2567,15 +2610,15 @@ class TryFinallyStatNode(StatNode): def analyse_expressions(self, env): self.body.analyse_expressions(env) self.cleanup_list = env.free_temp_entries[:] - self.exc_vars = ( - env.allocate_temp(PyrexTypes.py_object_type), - env.allocate_temp(PyrexTypes.py_object_type), - env.allocate_temp(PyrexTypes.py_object_type)) - self.lineno_var = \ - env.allocate_temp(PyrexTypes.c_int_type) + #self.exc_vars = ( + # env.allocate_temp(PyrexTypes.py_object_type), + # env.allocate_temp(PyrexTypes.py_object_type), + # env.allocate_temp(PyrexTypes.py_object_type)) + #self.lineno_var = \ + # env.allocate_temp(PyrexTypes.c_int_type) self.finally_clause.analyse_expressions(env) - for var in self.exc_vars: - env.release_temp(var) + #for var in self.exc_vars: + # env.release_temp(var) def generate_execution_code(self, code): old_error_label = code.error_label @@ -2595,47 +2638,69 @@ class TryFinallyStatNode(StatNode): "}") code.putln( "/*finally:*/ {") - code.putln( - "int __pyx_why;") - #code.putln( - # "PyObject *%s, *%s, *%s;" % - # self.exc_vars) - #code.putln( - # "int %s;" % - # self.lineno_var) - code.use_label(catch_label) - code.putln( - "__pyx_why = 0; goto %s;" % - catch_label) - for i in range(len(new_labels)): - new_label = new_labels[i] - if new_label and new_label <> "": - if new_label in code.labels_used: - if new_label == new_error_label: - self.put_error_catcher(code, - new_error_label, i+1, catch_label) - else: - code.putln( - "%s: __pyx_why = %s; goto %s;" % ( - new_label, - i+1, - catch_label)) - code.put_label(catch_label) + cases_used = [] + error_label_used = 0 + for i, new_label in enumerate(new_labels): + if new_label in code.labels_used: + cases_used.append(i) + if new_label == new_error_label: + error_label_used = 1 + error_label_case = i + if cases_used: + code.putln( + "int __pyx_why;") + if error_label_used and self.preserve_exception: + code.putln( + "PyObject *%s, *%s, *%s;" % Naming.exc_vars) + code.putln( + "int %s;" % Naming.exc_lineno_name) + code.use_label(catch_label) + code.putln( + "__pyx_why = 0; goto %s;" % catch_label) + for i in cases_used: + new_label = new_labels[i] + #if new_label and new_label <> "": + if new_label == new_error_label and self.preserve_exception: + self.put_error_catcher(code, + new_error_label, i+1, catch_label) + else: + code.putln( + "%s: __pyx_why = %s; goto %s;" % ( + new_label, + i+1, + catch_label)) + code.put_label(catch_label) code.set_all_labels(old_labels) + if error_label_used: + code.new_error_label() + finally_error_label = code.error_label self.finally_clause.generate_execution_code(code) - code.putln( + if error_label_used: + if finally_error_label in code.labels_used and self.preserve_exception: + over_label = code.new_label() + code.put_goto(over_label); + code.put_label(finally_error_label) + code.putln("if (__pyx_why == %d) {" % (error_label_case + 1)) + for var in Naming.exc_vars: + code.putln("Py_XDECREF(%s);" % var) + code.putln("}") + code.put_goto(old_error_label) + code.put_label(over_label) + code.error_label = old_error_label + if cases_used: + code.putln( "switch (__pyx_why) {") - for i in range(len(old_labels)): - if old_labels[i]: - if old_labels[i] == old_error_label: + for i in cases_used: + old_label = old_labels[i] + if old_label == old_error_label and self.preserve_exception: self.put_error_uncatcher(code, i+1, old_error_label) else: - code.use_label(old_labels[i]) + code.use_label(old_label) code.putln( "case %s: goto %s;" % ( i+1, - old_labels[i])) - code.putln( + old_label)) + code.putln( "}") code.putln( "}") @@ -2650,10 +2715,10 @@ class TryFinallyStatNode(StatNode): code.put_var_xdecrefs_clear(self.cleanup_list) code.putln( "PyErr_Fetch(&%s, &%s, &%s);" % - self.exc_vars) + Naming.exc_vars) code.putln( "%s = %s;" % ( - self.lineno_var, Naming.lineno_cname)) + Naming.exc_lineno_name, Naming.lineno_cname)) #code.putln( # "goto %s;" % # catch_label) @@ -2667,22 +2732,71 @@ class TryFinallyStatNode(StatNode): i) code.putln( "PyErr_Restore(%s, %s, %s);" % - self.exc_vars) + Naming.exc_vars) code.putln( "%s = %s;" % ( - Naming.lineno_cname, self.lineno_var)) - for var in self.exc_vars: + Naming.lineno_cname, Naming.exc_lineno_name)) + for var in Naming.exc_vars: code.putln( "%s = 0;" % var) - #code.putln( - # "goto %s;" % - # error_label) code.put_goto(error_label) code.putln( "}") +class GILStatNode(TryFinallyStatNode): + # 'with gil' or 'with nogil' statement + # + # state string 'gil' or 'nogil' + + preserve_exception = 0 + + def __init__(self, pos, state, body): + self.state = state + TryFinallyStatNode.__init__(self, pos, + body = body, + finally_clause = GILExitNode(pos, state = state)) + + def generate_execution_code(self, code): + code.putln("/*with %s:*/ {" % self.state) + if self.state == 'gil': + code.putln("PyGILState_STATE _save = PyGILState_Ensure();") + else: + code.putln("PyThreadState *_save;") + code.putln("Py_UNBLOCK_THREADS") + TryFinallyStatNode.generate_execution_code(self, code) + code.putln("}") + +#class GILEntryNode(StatNode): +# # state string 'gil' or 'nogil' +# +# def analyse_expressions(self, env): +# pass +# +# def generate_execution_code(self, code): +# if self.state == 'gil': +# code.putln("PyGILState_STATE _save = PyGILState_Ensure();") +# else: +# code.putln("PyThreadState *_save;") +# code.putln("Py_UNBLOCK_THREADS") + + +class GILExitNode(StatNode): + # Used as the 'finally' block in a GILStatNode + # + # state string 'gil' or 'nogil' + + def analyse_expressions(self, env): + pass + + def generate_execution_code(self, code): + if self.state == 'gil': + code.putln("PyGILState_Release();") + else: + code.putln("Py_BLOCK_THREADS") + + class CImportStatNode(StatNode): # cimport statement # @@ -2801,7 +2915,6 @@ utility_function_predeclarations = \ #define INLINE #endif -typedef struct {const char *s; const void **p;} __Pyx_CApiTabEntry; /*proto*/ typedef struct {PyObject **p; char *s;} __Pyx_InternTabEntry; /*proto*/ typedef struct {PyObject **p; char *s; long n; int is_unicode;} __Pyx_StringTabEntry; /*proto*/ @@ -2875,7 +2988,7 @@ static int __Pyx_PrintItem(PyObject *v) { return -1; if (PyString_Check(v)) { char *s = PyString_AsString(v); - int len = PyString_Size(v); + Py_ssize_t len = PyString_Size(v); if (len > 0 && isspace(Py_CHARMASK(s[len-1])) && s[len-1] != ' ') @@ -2923,21 +3036,12 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { value = Py_None; Py_INCREF(value); } - /* Next, repeatedly, replace a tuple exception with its first item */ - while (PyTuple_Check(type) && PyTuple_Size(type) > 0) { - PyObject *tmp = type; - type = PyTuple_GET_ITEM(type, 0); - Py_INCREF(type); - Py_DECREF(tmp); - } - if (PyString_Check(type)) { - if (PyErr_Warn(PyExc_DeprecationWarning, - "raising a string exception is deprecated")) - goto raise_error; - } - else if (PyType_Check(type) || PyClass_Check(type)) - ; /*PyErr_NormalizeException(&type, &value, &tb);*/ - else { + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { /* Raising an instance. The value should be a dummy. */ if (value != Py_None) { PyErr_SetString(PyExc_TypeError, @@ -2947,11 +3051,25 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { /* Normalize to raise , */ Py_DECREF(value); value = type; - if (PyInstance_Check(type)) - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - else + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else type = (PyObject*) type->ob_type; - Py_INCREF(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif } PyErr_Restore(type, value, tb); return; @@ -3014,21 +3132,28 @@ static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed # *kwds == 0, it is not changed. If kwds2 == 0 and *kwds != 0, a new # reference to the same dictionary is passed back in *kwds. # +# If rqd_kwds is not 0, it is an array of booleans corresponding to the +# names in kwd_list, indicating required keyword arguments. If any of +# these are not present in kwds, an exception is raised. +# get_starargs_utility_code = [ """ -static int __Pyx_GetStarArgs(PyObject **args, PyObject **kwds,\ - char *kwd_list[], int nargs, PyObject **args2, PyObject **kwds2); /*proto*/ +static int __Pyx_GetStarArgs(PyObject **args, PyObject **kwds, char *kwd_list[], \ + Py_ssize_t nargs, PyObject **args2, PyObject **kwds2, char rqd_kwds[]); /*proto*/ """,""" static int __Pyx_GetStarArgs( PyObject **args, PyObject **kwds, char *kwd_list[], - int nargs, + Py_ssize_t nargs, PyObject **args2, - PyObject **kwds2) + PyObject **kwds2, + char rqd_kwds[]) { PyObject *x = 0, *args1 = 0, *kwds1 = 0; + int i; + char **p; if (args2) *args2 = 0; @@ -3039,25 +3164,37 @@ static int __Pyx_GetStarArgs( args1 = PyTuple_GetSlice(*args, 0, nargs); if (!args1) goto bad; - *args2 = PyTuple_GetSlice(*args, nargs, PyTuple_Size(*args)); + *args2 = PyTuple_GetSlice(*args, nargs, PyTuple_GET_SIZE(*args)); if (!*args2) goto bad; } + else if (PyTuple_GET_SIZE(*args) > nargs) { + int m = nargs; + int n = PyTuple_GET_SIZE(*args); + PyErr_Format(PyExc_TypeError, + "function takes at most %d positional arguments (%d given)", + m, n); + goto bad; + } else { args1 = *args; Py_INCREF(args1); } + if (rqd_kwds && !*kwds) + for (i = 0, p = kwd_list; *p; i++, p++) + if (rqd_kwds[i]) + goto missing_kwarg; + if (kwds2) { if (*kwds) { - char **p; kwds1 = PyDict_New(); if (!kwds1) goto bad; *kwds2 = PyDict_Copy(*kwds); if (!*kwds2) goto bad; - for (p = kwd_list; *p; p++) { + for (i = 0, p = kwd_list; *p; i++, p++) { x = PyDict_GetItemString(*kwds, *p); if (x) { if (PyDict_SetItemString(kwds1, *p, x) < 0) @@ -3065,6 +3202,8 @@ static int __Pyx_GetStarArgs( if (PyDict_DelItemString(*kwds2, *p) < 0) goto bad; } + else if (rqd_kwds && rqd_kwds[i]) + goto missing_kwarg; } } else { @@ -3076,18 +3215,25 @@ static int __Pyx_GetStarArgs( else { kwds1 = *kwds; Py_XINCREF(kwds1); + if (rqd_kwds && *kwds) + for (i = 0, p = kwd_list; *p; i++, p++) + if (rqd_kwds[i] && !PyDict_GetItemString(*kwds, *p)) + goto missing_kwarg; } *args = args1; *kwds = kwds1; return 0; +missing_kwarg: + PyErr_Format(PyExc_TypeError, + "required keyword argument '%s' is missing", *p); bad: Py_XDECREF(args1); Py_XDECREF(kwds1); - if (*args2) { + if (args2) { Py_XDECREF(*args2); } - if (*kwds2) { + if (kwds2) { Py_XDECREF(*kwds2); } return -1; @@ -3183,63 +3329,6 @@ bad: #------------------------------------------------------------------------------------ -type_import_utility_code = [ -""" -static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, long size); /*proto*/ -""",""" -static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name, - long size) -{ - PyObject *py_module_name = 0; - PyObject *py_class_name = 0; - PyObject *py_name_list = 0; - PyObject *py_module = 0; - PyObject *result = 0; - - py_module_name = PyString_FromString(module_name); - if (!py_module_name) - goto bad; - py_class_name = PyString_FromString(class_name); - if (!py_class_name) - goto bad; - py_name_list = PyList_New(1); - if (!py_name_list) - goto bad; - Py_INCREF(py_class_name); - if (PyList_SetItem(py_name_list, 0, py_class_name) < 0) - goto bad; - py_module = __Pyx_Import(py_module_name, py_name_list); - if (!py_module) - goto bad; - result = PyObject_GetAttr(py_module, py_class_name); - if (!result) - goto bad; - if (!PyType_Check(result)) { - PyErr_Format(PyExc_TypeError, - "%s.%s is not a type object", - module_name, class_name); - goto bad; - } - if (((PyTypeObject *)result)->tp_basicsize != size) { - PyErr_Format(PyExc_ValueError, - "%s.%s does not appear to be the correct type object", - module_name, class_name); - goto bad; - } - goto done; -bad: - Py_XDECREF(result); - result = 0; -done: - Py_XDECREF(py_module_name); - Py_XDECREF(py_class_name); - Py_XDECREF(py_name_list); - return (PyTypeObject *)result; -} -"""] - -#------------------------------------------------------------------------------------ - set_vtable_utility_code = [ """ static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ @@ -3331,40 +3420,32 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { #------------------------------------------------------------------------------------ -c_api_import_code = [ +get_exception_utility_code = [ """ -static int __Pyx_InitCApi(PyObject *module); /*proto*/ -static int __Pyx_ImportModuleCApi(__Pyx_CApiTabEntry *t); /*proto*/ +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ """,""" -static int __Pyx_ImportModuleCApi(__Pyx_CApiTabEntry *t) { - __Pyx_CApiTabEntry *api_t; - while (t->s) { - if (*t->s == '\\0') - continue; /* shortcut for erased string entries */ - api_t = %(API_TAB)s; - while ((api_t->s) && (strcmp(api_t->s, t->s) < 0)) - ++api_t; - if ((!api_t->p) || (strcmp(api_t->s, t->s) != 0)) { - PyErr_Format(PyExc_ValueError, - "Unknown function name in C API: %%s", t->s); - return -1; - } - *t->p = api_t->p; - ++t; - } +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { + PyThreadState *tstate = PyThreadState_Get(); + PyErr_Fetch(type, value, tb); + PyErr_NormalizeException(type, value, tb); + if (PyErr_Occurred()) + goto bad; + Py_INCREF(*type); + Py_INCREF(*value); + Py_INCREF(*tb); + Py_XDECREF(tstate->exc_type); + Py_XDECREF(tstate->exc_value); + Py_XDECREF(tstate->exc_traceback); + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; return 0; +bad: + Py_XDECREF(*type); + Py_XDECREF(*value); + Py_XDECREF(*tb); + return -1; } +"""] -static int __Pyx_InitCApi(PyObject *module) { - int result; - PyObject* cobj = PyCObject_FromVoidPtr(&__Pyx_ImportModuleCApi, NULL); - if (!cobj) - return -1; - - result = PyObject_SetAttrString(module, "_import_c_api", cobj); - Py_DECREF(cobj); - return result; -} -""" % {'API_TAB' : Naming.c_api_tab_cname} -] #------------------------------------------------------------------------------------ diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py index 3b426f24..ccb5f216 100644 --- a/Cython/Compiler/Options.py +++ b/Cython/Compiler/Options.py @@ -3,7 +3,7 @@ # intern_names = 1 # Intern global variable and attribute names -cache_builtins = 1 # Perform lookups on builtin names only once +cache_builtins = 0 # Perform lookups on builtin names only once embed_pos_in_docstring = 0 gcc_branch_hints = 1 diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 4b594367..3bcc01b1 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -5,7 +5,7 @@ import os, re from string import join, replace from types import ListType, TupleType -from Scanning import PyrexScanner, function_contexts +from Scanning import PyrexScanner import Nodes import ExprNodes from ModuleNode import ModuleNode @@ -463,13 +463,35 @@ def p_atom(s): if name == "None": return ExprNodes.NoneNode(pos) else: - return ExprNodes.NameNode(pos, name=name) + return p_name(s, name) elif sy == 'NULL': s.next() return ExprNodes.NullNode(pos) else: s.error("Expected an identifier or literal") +def p_name(s, name): + pos = s.position() + if not s.compile_time_expr: + try: + value = s.compile_time_env.lookup_here(name) + except KeyError: + pass + else: + rep = repr(value) + if isinstance(value, int): + return ExprNodes.IntNode(pos, value = rep) + elif isinstance(value, long): + return ExprNodes.LongNode(pos, value = rep) + elif isinstance(value, float): + return ExprNodes.FloatNode(pos, value = rep) + elif isinstance(value, str): + return ExprNodes.StringNode(pos, value = rep[1:-1]) + else: + error(pos, "Invalid type for compile-time constant: %s" + % value.__class__.__name__) + return ExprNodes.NameNode(pos, name = name) + def p_cat_string_literal(s): # A sequence of one or more adjacent string literals. # Returns (kind, value) where kind in ('', 'c', 'r') @@ -746,8 +768,10 @@ def p_expression_or_assignment(s): s.next() rhs = p_expr(s) return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs) + expr = expr_list[0] + if isinstance(expr, ExprNodes.StringNode): + return Nodes.PassStatNode(expr.pos) else: - expr = expr_list[0] return Nodes.ExprStatNode(expr.pos, expr = expr) else: expr_list_list = [] @@ -872,10 +896,13 @@ def p_raise_statement(s): if s.sy == ',': s.next() exc_tb = p_simple_expr(s) - return Nodes.RaiseStatNode(pos, - exc_type = exc_type, - exc_value = exc_value, - exc_tb = exc_tb) + if exc_type or exc_value or exc_tb: + return Nodes.RaiseStatNode(pos, + exc_type = exc_type, + exc_value = exc_value, + exc_tb = exc_tb) + else: + return Nodes.ReraiseStatNode(pos) def p_import_statement(s): # s.sy in ('import', 'cimport') @@ -893,13 +920,18 @@ def p_import_statement(s): module_name = dotted_name, as_name = as_name) else: + if as_name and "." in dotted_name: + name_list = ExprNodes.ListNode(pos, args = [ + ExprNodes.StringNode(pos, value = "*")]) + else: + name_list = None stat = Nodes.SingleAssignmentNode(pos, lhs = ExprNodes.NameNode(pos, name = as_name or target_name), rhs = ExprNodes.ImportNode(pos, module_name = ExprNodes.StringNode(pos, value = dotted_name), - name_list = None)) + name_list = name_list)) stats.append(stat) return Nodes.StatListNode(pos, stats = stats) @@ -962,8 +994,6 @@ def p_dotted_name(s, as_allowed): names.append(p_ident(s)) if as_allowed: as_name = p_as_name(s) - else: - as_name = None return (pos, target_name, join(names, "."), as_name) def p_as_name(s): @@ -1072,7 +1102,7 @@ def p_for_from_relation(s): return op else: s.error("Expected one of '<', '<=', '>' '>='") - + def p_for_from_step(s): if s.sy == 'by': s.next() @@ -1147,17 +1177,32 @@ def p_include_statement(s, level): s.next() # 'include' _, include_file_name = p_string_literal(s) s.expect_newline("Syntax error in include statement") - include_file_path = s.context.find_include_file(include_file_name, pos) - if include_file_path: - f = open(include_file_path, "rU") - s2 = PyrexScanner(f, include_file_path, s) - try: - tree = p_statement_list(s2, level) - finally: - f.close() - return tree + if s.compile_time_eval: + include_file_path = s.context.find_include_file(include_file_name, pos) + if include_file_path: + f = open(include_file_path, "rU") + s2 = PyrexScanner(f, include_file_path, s) + try: + tree = p_statement_list(s2, level) + finally: + f.close() + return tree + else: + return None else: - return None + return Nodes.PassStatNode(pos) + +def p_with_statement(s): + pos = s.position() + s.next() # 'with' +# if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'): + if s.sy == 'IDENT' and s.systring == 'nogil': + state = s.systring + s.next() + body = p_suite(s) + return Nodes.GILStatNode(pos, state = state, body = body) + else: + s.error(pos, "Only 'with gil' and 'with nogil' implemented") def p_simple_statement(s): #print "p_simple_statement:", s.sy, s.systring ### @@ -1203,69 +1248,131 @@ def p_simple_statement_list(s): s.expect_newline("Syntax error in simple statement list") return stat -def p_statement(s, level, cdef_flag = 0, visibility = 'private'): - #print "p_statement:", s.sy, s.systring ### +def p_compile_time_expr(s): + old = s.compile_time_expr + s.compile_time_expr = 1 + expr = p_expr(s) + s.compile_time_expr = old + return expr + +def p_DEF_statement(s): + pos = s.position() + denv = s.compile_time_env + s.next() # 'DEF' + name = p_ident(s) + s.expect('=') + expr = p_compile_time_expr(s) + value = expr.compile_time_value(denv) + #print "p_DEF_statement: %s = %r" % (name, value) ### + denv.declare(name, value) + s.expect_newline() + return Nodes.PassStatNode(pos) + +def p_IF_statement(s, level, cdef_flag, visibility, api): + pos = s.position + saved_eval = s.compile_time_eval + current_eval = saved_eval + denv = s.compile_time_env + result = None + while 1: + s.next() # 'IF' or 'ELIF' + expr = p_compile_time_expr(s) + s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) + body = p_suite(s, level, cdef_flag, visibility, api = api) + if s.compile_time_eval: + result = body + current_eval = 0 + if s.sy <> 'ELIF': + break + if s.sy == 'ELSE': + s.next() + s.compile_time_eval = current_eval + body = p_suite(s, level, cdef_flag, visibility, api = api) + if current_eval: + result = body + if not result: + result = PassStatNode(pos) + s.compile_time_eval = saved_eval + return result + +def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0): if s.sy == 'ctypedef': if level not in ('module', 'module_pxd'): s.error("ctypedef statement not allowed here") + if api: + error(s.pos, "'api' not allowed with 'ctypedef'") return p_ctypedef_statement(s, level, visibility) - overridable = 0 - if s.sy == 'cdef': - cdef_flag = 1 - s.next() - if s.sy == 'rdef': - cdef_flag = 1 - overridable = 1 - s.next() - if cdef_flag: - if level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): - s.error('cdef statement not allowed here') - return p_cdef_statement(s, level, visibility, overridable = overridable) -# elif s.sy == 'rdef': -# s.next() -# return p_c_func_or_var_declaration(s, level, s.position(), visibility = visibility, overridable = True) - elif s.sy == 'def': - if level not in ('module', 'class', 'c_class', 'property'): - s.error('def statement not allowed here') - return p_def_statement(s) - elif s.sy == 'class': - if level <> 'module': - s.error("class definition not allowed here") - return p_class_statement(s) - elif s.sy == 'include': - if level not in ('module', 'module_pxd'): - s.error("include statement not allowed here") - return p_include_statement(s, level) - elif level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': - return p_property_decl(s) + elif s.sy == 'DEF': + return p_DEF_statement(s) + elif s.sy == 'IF': + return p_IF_statement(s, level, cdef_flag, visibility, api) else: - if level in ('c_class', 'c_class_pxd'): - if s.sy == 'pass': - return p_pass_statement(s, with_newline = 1) - if s.sy == 'if': - return p_if_statement(s) - elif s.sy == 'while': - return p_while_statement(s) - elif s.sy == 'for': - return p_for_statement(s) - elif s.sy == 'try': - return p_try_statement(s) + overridable = 0 + if s.sy == 'cdef': + cdef_flag = 1 + s.next() + if s.sy == 'rdef': + cdef_flag = 1 + overridable = 1 + s.next() + if cdef_flag: + if level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): + s.error('cdef statement not allowed here') + return p_cdef_statement(s, level, visibility = visibility, + api = api, overridable = overridable) + # elif s.sy == 'rdef': + # s.next() + # return p_c_func_or_var_declaration(s, level, s.position(), visibility = visibility, api = api, overridable = True) else: - return p_simple_statement_list(s) + if api: + error(s.pos, "'api' not allowed with this statement") + elif s.sy == 'def': + if level not in ('module', 'class', 'c_class', 'property'): + s.error('def statement not allowed here') + return p_def_statement(s) + elif s.sy == 'class': + if level <> 'module': + s.error("class definition not allowed here") + return p_class_statement(s) + elif s.sy == 'include': + if level not in ('module', 'module_pxd'): + s.error("include statement not allowed here") + return p_include_statement(s, level) + elif level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': + return p_property_decl(s) + elif s.sy == 'pass' and level <> 'property': + return p_pass_statement(s, with_newline = 1) + else: + if level in ('c_class', 'c_class_pxd', 'property'): + s.error("Executable statement not allowed here") + if s.sy == 'if': + return p_if_statement(s) + elif s.sy == 'while': + return p_while_statement(s) + elif s.sy == 'for': + return p_for_statement(s) + elif s.sy == 'try': + return p_try_statement(s) + elif s.sy == 'with': + return p_with_statement(s) + else: + return p_simple_statement_list(s) def p_statement_list(s, level, - cdef_flag = 0, visibility = 'private'): + cdef_flag = 0, visibility = 'private', api = 0): # Parse a series of statements separated by newlines. - #print "p_statement_list:", s.sy, s.systring ### pos = s.position() stats = [] while s.sy not in ('DEDENT', 'EOF'): stats.append(p_statement(s, level, - cdef_flag = cdef_flag, visibility = visibility)) - return Nodes.StatListNode(pos, stats = stats) + cdef_flag = cdef_flag, visibility = visibility, api = api)) + if len(stats) == 1: + return stats[0] + else: + return Nodes.StatListNode(pos, stats = stats) def p_suite(s, level = 'other', cdef_flag = 0, - visibility = 'private', with_doc = 0): + visibility = 'private', with_doc = 0, with_pseudo_doc = 0, api = 0): pos = s.position() s.expect(':') doc = None @@ -1273,14 +1380,17 @@ def p_suite(s, level = 'other', cdef_flag = 0, if s.sy == 'NEWLINE': s.next() s.expect_indent() - if with_doc: + if with_doc or with_pseudo_doc: doc = p_doc_string(s) body = p_statement_list(s, level = level, cdef_flag = cdef_flag, - visibility = visibility) + visibility = visibility, + api = api) s.expect_dedent() else: + if api: + error(s.pos, "'api' not allowed with this statement") if level in ('module', 'class', 'function', 'other'): body = p_simple_statement_list(s) else: @@ -1299,6 +1409,16 @@ def p_c_base_type(s, self_flag = 0): else: return p_c_simple_base_type(s, self_flag) +def p_calling_convention(s): + if s.sy == 'IDENT' and s.systring in calling_convention_words: + result = s.systring + s.next() + return result + else: + return "" + +calling_convention_words = ("__stdcall", "__cdecl") + def p_c_complex_base_type(s): # s.sy == '(' pos = s.position() @@ -1314,13 +1434,11 @@ def p_c_simple_base_type(s, self_flag): is_basic = 0 signed = 1 longness = 0 - pos = s.position() module_path = [] + pos = s.position() if looking_at_base_type(s): #print "p_c_simple_base_type: looking_at_base_type at", s.position() is_basic = 1 - #signed = p_signed_or_unsigned(s) - #longness = p_short_or_long(s) signed, longness = p_sign_and_longness(s) if s.sy == 'IDENT' and s.systring in basic_c_type_names: name = s.systring @@ -1360,16 +1478,12 @@ def looking_at_dotted_name(s): else: return 0 -#base_type_start_words = ( -# "char", "short", "int", "long", "float", "double", -# "void", "signed", "unsigned" -#) - basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint") sign_and_longness_words = ("short", "long", "signed", "unsigned") -base_type_start_words = basic_c_type_names + sign_and_longness_words +base_type_start_words = \ + basic_c_type_names + sign_and_longness_words def p_sign_and_longness(s): signed = 1 @@ -1377,6 +1491,8 @@ def p_sign_and_longness(s): while s.sy == 'IDENT' and s.systring in sign_and_longness_words: if s.systring == 'unsigned': signed = 0 + elif s.systring == 'signed': + signed = 2 elif s.systring == 'short': longness = -1 elif s.systring == 'long': @@ -1384,27 +1500,6 @@ def p_sign_and_longness(s): s.next() return signed, longness -#def p_signed_or_unsigned(s): -# signed = 1 -# if s.sy == 'IDENT': -# if s.systring == 'signed': -# s.next() -# elif s.systring == 'unsigned': -# signed = 0 -# s.next() -# return signed -# -#def p_short_or_long(s): -# longness = 0 -# if s.sy == 'IDENT' and s.systring == 'short': -# longness = -1 -# s.next() -# else: -# while s.sy == 'IDENT' and s.systring == 'long': -# longness += 1 -# s.next() -# return longness - def p_opt_cname(s): literal = p_opt_string_literal(s) if literal: @@ -1413,68 +1508,113 @@ def p_opt_cname(s): cname = None return cname -def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0 , assignable = 0): - # If empty is true, the declarator must be - # empty, otherwise we don't care. +def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0, + nonempty = 0, calling_convention_allowed = 0): + # If empty is true, the declarator must be empty. If nonempty is true, + # the declarator must be nonempty. Otherwise we don't care. # If cmethod_flag is true, then if this declarator declares # a function, it's a C method of an extension type. pos = s.position() + if s.sy == '(': + s.next() + if s.sy == ')' or looking_at_type(s): + base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None) + result = p_c_func_declarator(s, pos, base, cmethod_flag) + else: + result = p_c_declarator(s, empty, is_type, cmethod_flag, nonempty, + calling_convention_allowed = 1) + s.expect(')') + else: + result = p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) + if not calling_convention_allowed and result.calling_convention and s.sy <> '(': + error(s.position(), "%s on something that is not a function" + % result.calling_convention) + while s.sy in ('[', '('): + pos = s.position() + if s.sy == '[': + result = p_c_array_declarator(s, result) + else: # sy == '(' + s.next() + result = p_c_func_declarator(s, pos, result, cmethod_flag) + cmethod_flag = 0 + return result + +def p_c_array_declarator(s, base): + pos = s.position() + s.next() # '[' + if s.sy <> ']': + dim = p_expr(s) + else: + dim = None + s.expect(']') + return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) + +def p_c_func_declarator(s, pos, base, cmethod_flag): + # Opening paren has already been skipped + args = p_c_arg_list(s, in_pyfunc = 0, cmethod_flag = cmethod_flag, + nonempty_declarators = 0) + ellipsis = p_optional_ellipsis(s) + s.expect(')') + nogil = p_nogil(s) + exc_val, exc_check = p_exception_value_clause(s) + with_gil = p_with_gil(s) + return Nodes.CFuncDeclaratorNode(pos, + base = base, args = args, has_varargs = ellipsis, + exception_value = exc_val, exception_check = exc_check, + nogil = nogil or with_gil, with_gil = with_gil) + +def p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty): + pos = s.position() + calling_convention = p_calling_convention(s) if s.sy == '*': s.next() - base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable) + base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) result = Nodes.CPtrDeclaratorNode(pos, base = base) elif s.sy == '**': # scanner returns this as a single token s.next() - base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable) + base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) result = Nodes.CPtrDeclaratorNode(pos, base = Nodes.CPtrDeclaratorNode(pos, base = base)) else: - if s.sy == '(': + rhs = None + if s.sy == 'IDENT': + name = s.systring + if is_type: + s.add_type_name(name) + if empty: + error(s.position(), "Declarator should be empty") s.next() - result = p_c_declarator(s, empty, is_type, cmethod_flag) - s.expect(')') + cname = p_opt_cname(s) else: - if s.sy == 'IDENT': - name = s.systring - if is_type: - s.add_type_name(name) - if empty: - error(s.position(), "Declarator should be empty") - s.next() - cname = p_opt_cname(s) - else: - name = "" - cname = None + if nonempty: + error(s.position(), "Empty declarator") + name = "" + cname = None if s.sy == '=' and assignable: s.next() rhs = p_simple_expr(s) - else: rhs = None - result = Nodes.CNameDeclaratorNode(pos, - name = name, cname = cname, rhs = rhs) - while s.sy in ('[', '('): - if s.sy == '[': - s.next() - if s.sy <> ']': - dim = p_expr(s) - else: - dim = None - s.expect(']') - result = Nodes.CArrayDeclaratorNode(pos, - base = result, dimension = dim) - else: # sy == '(' - s.next() - args = p_c_arg_list(s, in_pyfunc = 0, cmethod_flag = cmethod_flag) - ellipsis = p_optional_ellipsis(s) - s.expect(')') - options = p_c_func_options(s) - result = Nodes.CFuncDeclaratorNode(pos, - base = result, args = args, has_varargs = ellipsis, - **options) - cmethod_flag = 0 + result = Nodes.CNameDeclaratorNode(pos, + name = name, cname = cname, rhs = rhs) + result.calling_convention = calling_convention return result +def p_nogil(s): + if s.sy == 'IDENT' and s.systring == 'nogil': + s.next() + return 1 + else: + return 0 + +def p_with_gil(s): + if s.sy == 'with': + s.next() + s.expect_keyword('gil') + return 1 + else: + return 0 + def p_exception_value_clause(s): exc_val = None exc_check = 0 @@ -1487,63 +1627,38 @@ def p_exception_value_clause(s): if s.sy == '?': exc_check = 1 s.next() - exc_val = p_simple_expr(s) #p_exception_value(s) + exc_val = p_simple_expr(s) return exc_val, exc_check -def p_c_with(s): - if s.sy == 'with': - s.next() - return p_ident_list(s) - return () - -def p_c_func_options(s): - exc_val = None - exc_check = 0 - contexts = [] - - if s.sy == 'except': - exc_val, exc_check = p_exception_value_clause(s) - contexts = p_c_with(s) - elif s.sy == 'with': - contexts = p_c_with(s) - exc_val, exc_check = p_exception_value_clause(s) - - for context in contexts: - if context not in function_contexts: - s.error("Unknown context: " + context) - return None - - ret = { - 'exception_value': exc_val, - 'exception_check': exc_check, - 'with_gil': 'GIL' in contexts, - } - - return ret - -#def p_exception_value(s): -# sign = "" -# if s.sy == "-": -# sign = "-" -# s.next() -# if s.sy in ('INT', 'LONG', 'FLOAT', 'NULL'): -# s.systring = sign + s.systring -# return p_atom(s) -# else: -# s.error("Exception value must be an int or float literal or NULL") - c_arg_list_terminators = ('*', '**', '.', ')') -c_arg_list_trailers = ('.', '*', '**') -def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, kw_only = 0): +#def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0, +# kw_only = 0): +# args = [] +# if s.sy not in c_arg_list_terminators: +# args.append(p_c_arg_decl(s, in_pyfunc, cmethod_flag, +# nonempty = nonempty_declarators, kw_only = kw_only)) +# while s.sy == ',': +# s.next() +# if s.sy in c_arg_list_terminators: +# break +# args.append(p_c_arg_decl(s, in_pyfunc), nonempty = nonempty_declarators, +# kw_only = kw_only) +# return args + +def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0, + kw_only = 0): + # Comma-separated list of C argument declarations, possibly empty. + # May have a trailing comma. args = [] - if s.sy not in c_arg_list_terminators: - args.append(p_c_arg_decl(s, in_pyfunc, cmethod_flag, kw_only)) - while s.sy == ',': - s.next() - if s.sy in c_arg_list_terminators: - break - args.append(p_c_arg_decl(s, in_pyfunc, kw_only = kw_only)) + is_self_arg = cmethod_flag + while s.sy not in c_arg_list_terminators: + args.append(p_c_arg_decl(s, in_pyfunc, is_self_arg, + nonempty = nonempty_declarators, kw_only = kw_only)) + if s.sy != ',': + break + s.next() + is_self_arg = 0 return args def p_optional_ellipsis(s): @@ -1553,12 +1668,12 @@ def p_optional_ellipsis(s): else: return 0 -def p_c_arg_decl(s, in_pyfunc, cmethod_flag = 0, kw_only = 0): +def p_c_arg_decl(s, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0): pos = s.position() not_none = 0 default = None base_type = p_c_base_type(s, cmethod_flag) - declarator = p_c_declarator(s) + declarator = p_c_declarator(s, nonempty = nonempty) if s.sy == 'not': s.next() if s.sy == 'IDENT' and s.systring == 'None': @@ -1578,32 +1693,55 @@ def p_c_arg_decl(s, in_pyfunc, cmethod_flag = 0, kw_only = 0): default = default, kw_only = kw_only) -def p_cdef_statement(s, level, visibility = 'private', overridable = False): +def p_api(s): + if s.sy == 'IDENT' and s.systring == 'api': + s.next() + return 1 + else: + return 0 + +def p_cdef_statement(s, level, visibility = 'private', api = 0, + overridable = False): pos = s.position() if overridable and level not in ('c_class', 'c_class_pxd'): error(pos, "Overridable cdef function not allowed here") visibility = p_visibility(s, visibility) - if visibility == 'extern' and s.sy in ('from' ,':'): + api = api or p_api(s) + if api: + if visibility not in ('private', 'public'): + error(pos, "Cannot combine 'api' with '%s'" % visibility) + if visibility == 'extern' and s.sy == 'from': return p_cdef_extern_block(s, level, pos) + elif s.sy == ':': + p_cdef_block(s, level, visibility, api) elif s.sy == 'class': if level not in ('module', 'module_pxd'): error(pos, "Extension type definition not allowed here") + if api: + error(pos, "'api' not allowed with extension class") return p_c_class_definition(s, level, pos, visibility = visibility) elif s.sy == 'IDENT' and s.systring in struct_union_or_enum: if level not in ('module', 'module_pxd'): error(pos, "C struct/union/enum definition not allowed here") - if visibility == 'public': - error(pos, "Public struct/union/enum definition not implemented") + #if visibility == 'public': + # error(pos, "Public struct/union/enum definition not implemented") + if api: + error(pos, "'api' not allowed with '%s'" % s.systring) if s.systring == "enum": - return p_c_enum_definition(s, pos) + return p_c_enum_definition(s, pos, visibility) else: - return p_c_struct_or_union_definition(s, pos) + return p_c_struct_or_union_definition(s, pos, visibility) elif s.sy == 'pass': node = p_pass_statement(s) s.expect_newline('Expected a newline') return node else: - return p_c_func_or_var_declaration(s, level, pos, visibility, overridable) + return p_c_func_or_var_declaration(s, level, pos, visibility, api, + overridable) + +def p_cdef_block(s, level, visibility, api): + body = p_suite(s, level, cdef_flag = 1, visibility = 'extern', api = api) + return Nodes.StatListNode(pos, stats = body) def p_cdef_extern_block(s, level, pos): include_file = None @@ -1621,7 +1759,7 @@ struct_union_or_enum = ( "struct", "union", "enum" ) -def p_c_enum_definition(s, pos, typedef_flag = 0): +def p_c_enum_definition(s, pos, visibility, typedef_flag = 0): # s.sy == ident 'enum' s.next() if s.sy == 'IDENT': @@ -1644,7 +1782,7 @@ def p_c_enum_definition(s, pos, typedef_flag = 0): p_c_enum_line(s, items) s.expect_dedent() return Nodes.CEnumDefNode(pos, name = name, cname = cname, - items = items, typedef_flag = typedef_flag) + items = items, typedef_flag = typedef_flag, visibility = visibility) def p_c_enum_line(s, items): if s.sy <> 'pass': @@ -1669,7 +1807,7 @@ def p_c_enum_item(s, items): items.append(Nodes.CEnumDefItemNode(pos, name = name, cname = cname, value = value)) -def p_c_struct_or_union_definition(s, pos, typedef_flag = 0): +def p_c_struct_or_union_definition(s, pos, visibility, typedef_flag = 0): # s.sy == ident 'struct' or 'union' kind = s.systring s.next() @@ -1694,7 +1832,7 @@ def p_c_struct_or_union_definition(s, pos, typedef_flag = 0): s.expect_newline("Syntax error in struct or union definition") return Nodes.CStructOrUnionDefNode(pos, name = name, cname = cname, kind = kind, attributes = attributes, - typedef_flag = typedef_flag) + typedef_flag = typedef_flag, visibility = visibility) def p_visibility(s, prev_visibility): pos = s.position() @@ -1714,38 +1852,40 @@ def p_c_modifiers(s): return [modifier] + p_c_modifiers(s) return [] -def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', overridable = False): +def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', api = 0, + overridable = False): cmethod_flag = level in ('c_class', 'c_class_pxd') modifiers = p_c_modifiers(s) base_type = p_c_base_type(s) - declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1) + declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1) if s.sy == ':': if level not in ('module', 'c_class'): s.error("C function definition not allowed here") - suite = p_suite(s, 'function') + suite = p_suite(s, 'function', with_pseudo_doc = 1) result = Nodes.CFuncDefNode(pos, visibility = visibility, base_type = base_type, declarator = declarator, body = suite, modifiers = modifiers, + api = api, overridable = overridable) else: - if level == 'module_pxd' and visibility <> 'extern': - error(pos, - "Only 'extern' C function or variable declaration allowed in .pxd file") + if api: + error(s.pos, "'api' not allowed with variable declaration") declarators = [declarator] while s.sy == ',': s.next() if s.sy == 'NEWLINE': break - declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1) + declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1) declarators.append(declarator) s.expect_newline("Syntax error in C variable declaration") result = Nodes.CVarDefNode(pos, visibility = visibility, base_type = base_type, - declarators = declarators) + declarators = declarators, + in_pxd = level == 'module_pxd') return result def p_ctypedef_statement(s, level, visibility = 'private'): @@ -1759,24 +1899,24 @@ def p_ctypedef_statement(s, level, visibility = 'private'): typedef_flag = 1) elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'): if s.systring == 'enum': - return p_c_enum_definition(s, pos, typedef_flag = 1) + return p_c_enum_definition(s, pos, visibility, typedef_flag = 1) else: - return p_c_struct_or_union_definition(s, pos, typedef_flag = 1) + return p_c_struct_or_union_definition(s, pos, visibility, typedef_flag = 1) else: base_type = p_c_base_type(s) - declarator = p_c_declarator(s, is_type = 1) + declarator = p_c_declarator(s, is_type = 1, nonempty = 1) s.expect_newline("Syntax error in ctypedef statement") return Nodes.CTypeDefNode(pos, - base_type = base_type, declarator = declarator) + base_type = base_type, declarator = declarator, visibility = visibility) def p_def_statement(s): # s.sy == 'def' pos = s.position() s.next() name = p_ident(s) - args = [] + #args = [] s.expect('('); - args = p_c_arg_list(s, in_pyfunc = 1) + args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1) star_arg = None starstar_arg = None if s.sy == '*': @@ -1785,15 +1925,16 @@ def p_def_statement(s): star_arg = p_py_arg_decl(s) if s.sy == ',': s.next() - if s.sy == 'IDENT': - args.extend(p_c_arg_list(s, in_pyfunc = 1, kw_only = 1)) - if s.sy == '**': - s.next() - starstar_arg = p_py_arg_decl(s) - elif s.sy == '**': + args.extend(p_c_arg_list(s, in_pyfunc = 1, + nonempty_declarators = 1, kw_only = 1)) + elif s.sy != ')': + s.error("Syntax error in Python function argument list") + if s.sy == '**': s.next() starstar_arg = p_py_arg_decl(s) s.expect(')') + if p_nogil(s): + error(s.pos, "Python function cannot be declared nogil") doc, body = p_suite(s, 'function', with_doc = 1) return Nodes.DefNode(pos, name = name, args = args, star_arg = star_arg, starstar_arg = starstar_arg, diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index c98fd801..c95ada92 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -5,7 +5,21 @@ import string import Naming -class PyrexType: +class BaseType: + # + # Base class for all Pyrex types including pseudo-types. + + def cast_code(self, expr_code): + return "((%s)%s)" % (self.declaration_code(""), expr_code) + + def base_declaration_code(self, base_code, entity_code): + if entity_code: + return "%s %s" % (base_code, entity_code) + else: + return base_code + + +class PyrexType(BaseType): # # Base class for all Pyrex types. # @@ -21,6 +35,7 @@ class PyrexType: # is_cfunction boolean Is a C function type # is_struct_or_union boolean Is a C struct or union type # is_enum boolean Is a C enum type + # is_typedef boolean Is a typedef type # is_string boolean Is a C char * type # is_returncode boolean Is used only to signal exceptions # is_error boolean Is the dummy error type @@ -66,6 +81,7 @@ class PyrexType: is_cfunction = 0 is_struct_or_union = 0 is_enum = 0 + is_typedef = 0 is_string = 0 is_returncode = 0 is_error = 0 @@ -111,17 +127,20 @@ class PyrexType: # A type is incomplete if it is an unsized array, # a struct whose attributes are not defined, etc. return 1 - - def cast_code(self, expr_code): - return "((%s)%s)" % (self.declaration_code(""), expr_code) -class CTypedefType: +class CTypedefType(BaseType): # - # Type defined with a ctypedef statement in a + # Pseudo-type defined with a ctypedef statement in a # 'cdef extern from' block. Delegates most attribute - # lookups to the base type. + # lookups to the base type. ANYTHING NOT DEFINED + # HERE IS DELEGATED! # + # qualified_name string + # typedef_cname string + # typedef_base_type PyrexType + + is_typedef = 1 def __init__(self, cname, base_type): self.typedef_cname = cname @@ -132,10 +151,23 @@ class CTypedefType: def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): - return "%s %s" % (self.typedef_cname, entity_code) + name = self.declaration_name(for_display, pyrex) + return self.base_declaration_code(name, entity_code) + + def declaration_name(self, for_display = 0, pyrex = 0): + if pyrex or for_display: + return self.qualified_name + else: + return self.typedef_cname + + def as_argument_type(self): + return self + + def __repr__(self): + return "" % self.typedef_cname def __str__(self): - return self.typedef_cname + return self.declaration_name(for_display = 1) def __getattr__(self, name): return getattr(self.typedef_base_type, name) @@ -155,15 +187,15 @@ class PyObjectType(PyrexType): return "Python object" def __repr__(self): - return "PyObjectType" + return "" def assignable_from(self, src_type): return 1 # Conversion will be attempted def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): - if pyrex: - return "object %s" % entity_code + if pyrex or for_display: + return self.base_declaration_code("object", entity_code) else: return "%s *%s" % (public_decl("PyObject", dll_linkage), entity_code) @@ -226,8 +258,8 @@ class PyExtensionType(PyObjectType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0, deref = 0): - if pyrex: - return "%s %s" % (self.name, entity_code) + if pyrex or for_display: + return self.base_declaration_code(self.name, entity_code) else: if self.typedef_flag: base_format = "%s" @@ -246,8 +278,8 @@ class PyExtensionType(PyObjectType): return self.name def __repr__(self): - return "PyExtensionType(%s%s)" % (self.scope.class_name, - ("", ".typedef_flag=1")[self.typedef_flag]) + return "" % (self.scope.class_name, + ("", " typedef")[self.typedef_flag]) class CType(PyrexType): @@ -264,13 +296,6 @@ class CType(PyrexType): exception_check = 1 -#class CSimpleType(CType): -# # -# # Base class for all unstructured C types. -# # -# pass - - class CVoidType(CType): is_void = 1 @@ -280,7 +305,7 @@ class CVoidType(CType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): base = public_decl("void", dll_linkage) - return "%s %s" % (base, entity_code) + return self.base_declaration_code(base, entity_code) def is_complete(self): return 0 @@ -291,17 +316,20 @@ class CNumericType(CType): # Base class for all C numeric types. # # rank integer Relative size - # signed boolean + # signed integer 0 = unsigned, 1 = unspecified, 2 = explicitly signed # is_numeric = 1 default_value = "0" parsetuple_formats = ( # rank -> format - "?HIkKn???", # unsigned - "chilLnfd?", # signed + "BHIkK????", # unsigned + "bhilL?fd?", # assumed signed + "bhilL?fd?", # explicitly signed ) + sign_words = ("unsigned ", "", "signed ") + def __init__(self, rank, signed = 1, pymemberdef_typecode = None): self.rank = rank self.signed = signed @@ -311,21 +339,18 @@ class CNumericType(CType): self.parsetuple_format = ptf self.pymemberdef_typecode = pymemberdef_typecode + def sign_and_name(self): + s = self.sign_words[self.signed] + n = rank_to_type_name[self.rank] + return s + n + def __repr__(self): - if self.signed: - u = "" - else: - u = "unsigned " - return "" % (u, rank_to_type_name[self.rank]) + return "" % self.sign_and_name() def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): - if self.signed: - u = "" - else: - u = "unsigned " - base = public_decl(u + rank_to_type_name[self.rank], dll_linkage) - return "%s %s" % (base, entity_code) + base = public_decl(self.sign_and_name(), dll_linkage) + return self.base_declaration_code(base, entity_code) class CIntType(CNumericType): @@ -342,20 +367,19 @@ class CIntType(CNumericType): def assignable_from_resolved_type(self, src_type): return src_type.is_int or src_type.is_enum or src_type is error_type - + class CBIntType(CIntType): to_py_function = "__Pyx_PyBool_FromLong" from_py_function = "__Pyx_PyObject_IsTrue" exception_check = 0 - -class CPySSizeTType(CIntType): - to_py_function = "PyInt_FromSsize_t" - from_py_function = "__pyx_PyIndex_AsSsize_t" - exception_value = None +class CAnonEnumType(CIntType): + + is_enum = 1 + class CUIntType(CIntType): @@ -382,6 +406,12 @@ class CULongLongType(CUIntType): from_py_function = "PyInt_AsUnsignedLongLongMask" +class CPySSizeTType(CIntType): + + to_py_function = "PyInt_FromSsize_t" + from_py_function = "PyInt_AsSsize_t" + + class CFloatType(CNumericType): is_float = 1 @@ -408,7 +438,7 @@ class CArrayType(CType): self.is_string = 1 def __repr__(self): - return "CArrayType(%s,%s)" % (self.size, repr(self.base_type)) + return "" % (self.size, repr(self.base_type)) def same_as_resolved_type(self, other_type): return ((other_type.is_array and @@ -428,8 +458,10 @@ class CArrayType(CType): dimension_code = self.size else: dimension_code = "" + if entity_code.startswith("*"): + entity_code = "(%s)" % entity_code return self.base_type.declaration_code( - "(%s[%s])" % (entity_code, dimension_code), + "%s[%s]" % (entity_code, dimension_code), for_display, dll_linkage, pyrex) def as_argument_type(self): @@ -443,13 +475,13 @@ class CPtrType(CType): # base_type CType Referenced type is_ptr = 1 - default_value = 0 + default_value = "0" def __init__(self, base_type): self.base_type = base_type def __repr__(self): - return "CPtrType(%s)" % repr(self.base_type) + return "" % repr(self.base_type) def same_as_resolved_type(self, other_type): return ((other_type.is_ptr and @@ -460,24 +492,20 @@ class CPtrType(CType): for_display = 0, dll_linkage = None, pyrex = 0): #print "CPtrType.declaration_code: pointer to", self.base_type ### return self.base_type.declaration_code( - "(*%s)" % entity_code, + "*%s" % entity_code, for_display, dll_linkage, pyrex) def assignable_from_resolved_type(self, other_type): if other_type is error_type: return 1 - elif self.base_type.is_cfunction and other_type.is_cfunction: - return self.base_type.same_as(other_type) - elif other_type.is_array: - return self.base_type.same_as(other_type.base_type) - elif not other_type.is_ptr: - return 0 - elif self.base_type.is_void: - return 1 elif other_type.is_null_ptr: return 1 + elif self.base_type.is_cfunction and other_type.is_cfunction: + return self.base_type.same_as(other_type) + elif other_type.is_array or other_type.is_ptr: + return self.base_type.is_void or self.base_type.same_as(other_type.base_type) else: - return self.base_type.same_as(other_type.base_type) + return 0 class CNullPtrType(CPtrType): @@ -490,33 +518,48 @@ class CFuncType(CType): # args [CFuncTypeArg] # has_varargs boolean # exception_value string - # exception_check boolean True if PyErr_Occurred check needed - # with_gil boolean True if GIL should be grabbed/released + # exception_check boolean True if PyErr_Occurred check needed + # calling_convention string Function calling convention + # nogil boolean Can be called without gil + # with_gil boolean Acquire gil around function body is_cfunction = 1 - def __init__(self, return_type, args, has_varargs, - exception_value = None, exception_check = 0, with_gil = False): + def __init__(self, return_type, args, has_varargs = 0, + exception_value = None, exception_check = 0, calling_convention = "", + nogil = 0, with_gil = 0): self.return_type = return_type self.args = args self.has_varargs = has_varargs self.exception_value = exception_value self.exception_check = exception_check + self.calling_convention = calling_convention + self.nogil = nogil self.with_gil = with_gil def __repr__(self): arg_reprs = map(repr, self.args) if self.has_varargs: arg_reprs.append("...") - return "CFuncType(%s,[%s])" % ( + return "" % ( repr(self.return_type), + self.calling_convention_prefix(), string.join(arg_reprs, ",")) + def calling_convention_prefix(self): + cc = self.calling_convention + if cc: + return cc + " " + else: + return "" + def same_c_signature_as(self, other_type, as_cmethod = 0): return self.same_c_signature_as_resolved_type( other_type.resolve(), as_cmethod) def same_c_signature_as_resolved_type(self, other_type, as_cmethod): + #print "CFuncType.same_c_signature_as_resolved_type:", \ + # self, other_type, "as_cmethod =", as_cmethod ### if other_type is error_type: return 1 if not other_type.is_cfunction: @@ -535,6 +578,8 @@ class CFuncType(CType): return 0 if not self.return_type.same_as(other_type.return_type): return 0 + if not self.same_calling_convention_as(other_type): + return 0 return 1 def narrower_c_signature_than(self, other_type, as_cmethod = 0): @@ -560,6 +605,10 @@ class CFuncType(CType): return 0 return 1 + def same_calling_convention_as(self, other): + sc1 = self.calling_convention == '__stdcall' + sc2 = other.calling_convention == '__stdcall' + return sc1 == sc2 def same_exception_signature_as(self, other_type): return self.same_exception_signature_as_resolved_type( @@ -585,21 +634,25 @@ class CFuncType(CType): if not arg_decl_code and not pyrex: arg_decl_code = "void" exc_clause = "" - with_gil_clause = "" - if pyrex or for_display: + if (pyrex or for_display) and not self.return_type.is_pyobject: if self.exception_value and self.exception_check: exc_clause = " except? %s" % self.exception_value elif self.exception_value: exc_clause = " except %s" % self.exception_value elif self.exception_check: exc_clause = " except *" - if self.with_gil: - with_gil_clause = " with GIL" + cc = self.calling_convention_prefix() + if (not entity_code and cc) or entity_code.startswith("*"): + entity_code = "(%s%s)" % (cc, entity_code) + cc = "" return self.return_type.declaration_code( - "(%s(%s)%s%s)" % (entity_code, arg_decl_code, - exc_clause, with_gil_clause), + "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, exc_clause), for_display, dll_linkage, pyrex) + def signature_string(self): + s = self.declaration_code("") + return s + class CFuncTypeArg: # name string @@ -640,13 +693,13 @@ class CStructOrUnionType(CType): self.typedef_flag = typedef_flag def __repr__(self): - return "CStructOrUnionType(%s,%s%s)" % (self.name, self.cname, - ("", ",typedef_flag=1")[self.typedef_flag]) + return "" % (self.name, self.cname, + ("", " typedef")[self.typedef_flag]) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): if pyrex: - return "%s %s" % (self.name, entity_code) + return self.base_declaration_code(self.name, entity_code) else: if for_display: base = self.name @@ -654,7 +707,7 @@ class CStructOrUnionType(CType): base = self.cname else: base = "%s %s" % (self.kind, self.cname) - return "%s %s" % (public_decl(base, dll_linkage), entity_code) + return self.base_declaration_code(public_decl(base, dll_linkage), entity_code) def __cmp__(self, other): try: @@ -678,8 +731,8 @@ class CEnumType(CType): # typedef_flag boolean is_enum = 1 - #signed = 1 - #rank = 2 + signed = 1 + rank = -1 # Ranks below any integer type to_py_function = "PyInt_FromLong" from_py_function = "PyInt_AsLong" @@ -689,20 +742,23 @@ class CEnumType(CType): self.values = [] self.typedef_flag = typedef_flag + def __str__(self): + return self.name + def __repr__(self): - return "CEnumType(%s,%s%s)" % (self.name, self.cname, - ("", ",typedef_flag=1")[self.typedef_flag]) + return "" % (self.name, self.cname, + ("", " typedef")[self.typedef_flag]) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): if pyrex: - return "%s %s" % (self.cname, entity_code) + return self.base_declaration_code(self.cname, entity_code) else: if self.typedef_flag: base = self.cname else: base = "enum %s" % self.cname - return "%s %s" % (public_decl(base, dll_linkage), entity_code) + return self.base_declaration_code(public_decl(base, dll_linkage), entity_code) class CStringType: @@ -763,23 +819,29 @@ c_void_type = CVoidType() c_void_ptr_type = CPtrType(c_void_type) c_void_ptr_ptr_type = CPtrType(c_void_ptr_type) -c_char_type = CIntType(0, 1, "T_CHAR") -c_short_type = CIntType(1, 1, "T_SHORT") -c_int_type = CIntType(2, 1, "T_INT") -c_long_type = CIntType(3, 1, "T_LONG") -c_longlong_type = CLongLongType(4, 1, "T_LONGLONG") -c_py_ssize_t_type = CPySSizeTType(5, 1) -c_bint_type = CBIntType(2, 1, "T_INT") - -c_uchar_type = CIntType(0, 0, "T_UBYTE") -c_ushort_type = CIntType(1, 0, "T_USHORT") -c_uint_type = CUIntType(2, 0, "T_UINT") -c_ulong_type = CULongType(3, 0, "T_ULONG") -c_ulonglong_type = CULongLongType(4, 0, "T_ULONGLONG") - -c_float_type = CFloatType(6, "T_FLOAT") -c_double_type = CFloatType(7, "T_DOUBLE") -c_longdouble_type = CFloatType(8) +c_uchar_type = CIntType(0, 0, "T_UBYTE") +c_ushort_type = CIntType(1, 0, "T_USHORT") +c_uint_type = CUIntType(2, 0, "T_UINT") +c_ulong_type = CULongType(3, 0, "T_ULONG") +c_ulonglong_type = CULongLongType(4, 0, "T_ULONGLONG") + +c_char_type = CIntType(0, 1, "T_CHAR") +c_short_type = CIntType(1, 1, "T_SHORT") +c_int_type = CIntType(2, 1, "T_INT") +c_long_type = CIntType(3, 1, "T_LONG") +c_longlong_type = CLongLongType(4, 1, "T_LONGLONG") +c_py_ssize_t_type = CPySSizeTType(5, 1) +c_bint_type = CBIntType(2, 1, "T_INT") + +c_schar_type = CIntType(0, 2, "T_CHAR") +c_sshort_type = CIntType(1, 2, "T_SHORT") +c_sint_type = CIntType(2, 2, "T_INT") +c_slong_type = CIntType(3, 2, "T_LONG") +c_slonglong_type = CLongLongType(4, 2, "T_LONGLONG") + +c_float_type = CFloatType(6, "T_FLOAT") +c_double_type = CFloatType(7, "T_DOUBLE") +c_longdouble_type = CFloatType(8) c_null_ptr_type = CNullPtrType(c_void_type) c_char_array_type = CCharArrayType(None) @@ -789,6 +851,8 @@ c_int_ptr_type = CPtrType(c_int_type) c_returncode_type = CIntType(2, 1, "T_INT", is_returncode = 1) +c_anon_enum_type = CAnonEnumType(-1, 1) + error_type = ErrorType() lowest_float_rank = 6 @@ -819,6 +883,12 @@ sign_and_rank_to_type = { (1, 3): c_long_type, (1, 4): c_longlong_type, (1, 5): c_py_ssize_t_type, + (2, 0): c_schar_type, + (2, 1): c_sshort_type, + (2, 2): c_sint_type, + (2, 3): c_slong_type, + (2, 4): c_slonglong_type, + (2, 5): c_py_ssize_t_type, (1, 6): c_float_type, (1, 7): c_double_type, (1, 8): c_longdouble_type, @@ -842,18 +912,25 @@ modifiers_and_name_to_type = { (1, 0, "double"): c_double_type, (1, 1, "double"): c_longdouble_type, (1, 0, "object"): py_object_type, - (1, 0, "bint"): c_bint_type, + (2, 0, "char"): c_schar_type, + (2, -1, "int"): c_sshort_type, + (2, 0, "int"): c_sint_type, + (2, 1, "int"): c_slong_type, + (2, 2, "int"): c_slonglong_type, + (2, 0, "Py_ssize_t"): c_py_ssize_t_type, } def widest_numeric_type(type1, type2): # Given two numeric types, return the narrowest type # encompassing both of them. - signed = type1.signed - rank = max(type1.rank, type2.rank) - if rank >= lowest_float_rank: - signed = 1 - return sign_and_rank_to_type[signed, rank] + if type1.is_enum and type2.is_enum: + widest_type = c_int_type + elif type2.rank > type1.rank: + widest_type = type2 + else: + widest_type = type1 + return widest_type def simple_c_type(signed, longness, name): # Find type descriptor for simple type given name and modifiers. diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index 4bb1b867..821f661d 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -6,6 +6,7 @@ import cPickle as pickle import os +import platform import stat import sys from time import time @@ -138,11 +139,7 @@ reserved_words = [ "raise", "import", "exec", "try", "except", "finally", "while", "if", "elif", "else", "for", "in", "assert", "and", "or", "not", "is", "in", "lambda", "from", - "NULL", "cimport", "by", "with", "rdef" -] - -function_contexts = [ # allowed arguments to the "with" option - "GIL" + "NULL", "cimport", "by", "with", "rdef", "DEF", "IF", "ELIF", "ELSE" ] class Method: @@ -164,7 +161,53 @@ def build_resword_dict(): #------------------------------------------------------------------ +class CompileTimeScope(object): + + def __init__(self, outer = None): + self.entries = {} + self.outer = outer + + def declare(self, name, value): + self.entries[name] = value + + def lookup_here(self, name): + return self.entries[name] + + def lookup(self, name): + try: + return self.lookup_here(name) + except KeyError: + outer = self.outer + if outer: + return outer.lookup(name) + else: + raise + +def initial_compile_time_env(): + benv = CompileTimeScope() + names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE', + 'UNAME_VERSION', 'UNAME_MACHINE') + for name, value in zip(names, platform.uname()): + benv.declare(name, value) + import __builtin__ + names = ('False', 'True', + 'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate', + 'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min', + 'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str', + 'sum', 'tuple', 'xrange', 'zip') + for name in names: + benv.declare(name, getattr(__builtin__, name)) + denv = CompileTimeScope(benv) + return denv + +#------------------------------------------------------------------ + class PyrexScanner(Scanner): + # context Context Compilation context + # type_names set Identifiers to be treated as type names + # compile_time_env dict Environment for conditional compilation + # compile_time_eval boolean In a true conditional compilation context + # compile_time_expr boolean In a compile-time expression context resword_dict = build_resword_dict() @@ -174,9 +217,15 @@ class PyrexScanner(Scanner): if parent_scanner: self.context = parent_scanner.context self.type_names = parent_scanner.type_names + self.compile_time_env = parent_scanner.compile_time_env + self.compile_time_eval = parent_scanner.compile_time_eval + self.compile_time_expr = parent_scanner.compile_time_expr else: self.context = context self.type_names = type_names + self.compile_time_env = initial_compile_time_env() + self.compile_time_eval = 1 + self.compile_time_expr = 0 self.trace = trace_scanner self.indentation_stack = [0] self.indentation_char = None @@ -307,10 +356,19 @@ class PyrexScanner(Scanner): if self.sy == what: self.next() else: - if message: - self.error(message) - else: - self.error("Expected '%s'" % what) + self.expected(what, message) + + def expect_keyword(self, what, message = None): + if self.sy == 'IDENT' and self.systring == what: + self.next() + else: + self.expected(what, message) + + def expected(self, what, message): + if message: + self.error(message) + else: + self.error("Expected '%s'" % what) def expect_indent(self): self.expect('INDENT', @@ -320,7 +378,7 @@ class PyrexScanner(Scanner): self.expect('DEDENT', "Expected a decrease in indentation level") - def expect_newline(self, message): + def expect_newline(self, message = "Expected a newline"): # Expect either a newline or end of file if self.sy <> 'EOF': self.expect('NEWLINE', message) diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index f18ce2b5..9a84ca8b 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -3,9 +3,10 @@ # import re -from Errors import error, InternalError, warning +from Errors import warning, error, InternalError import Options import Naming +import PyrexTypes from PyrexTypes import * import TypeSlots from TypeSlots import \ @@ -24,12 +25,11 @@ class Entry: # doc string Doc string # init string Initial value # visibility 'private' or 'public' or 'extern' - # is_builtin boolean Is a Python builtin name + # is_builtin boolean Is an entry in the Python builtins dict # is_cglobal boolean Is a C global variable # is_pyglobal boolean Is a Python module-level variable # or class attribute during # class construction - # is_special boolean Is a special class method # is_member boolean Is an assigned class member # is_variable boolean Is a variable # is_cfunction boolean Is a C function @@ -51,7 +51,7 @@ class Entry: # signature Signature Arg & return types for Python func # init_to_none boolean True if initial value should be None # as_variable Entry Alternative interpretation of extension - # type name as a variable + # type name or builtin C function as a variable # xdecref_cleanup boolean Use Py_XDECREF for error cleanup # in_cinclude boolean Suppress C declaration code # enum_values [Entry] For enum types, list of values @@ -61,10 +61,14 @@ class Entry: # type is an extension type # as_module None Module scope, if a cimported module # is_inherited boolean Is an inherited attribute of an extension type - # interned_cname string C name of interned name string + # #interned_cname string C name of interned name string # pystring_cname string C name of Python version of string literal # is_interned boolean For string const entries, value is interned # used boolean + # is_special boolean Is a special method or property accessor + # of an extension type + # defined_in_pxd boolean Is defined in a .pxd file (not just declared) + # api boolean Generate C API for C function borrowed = 0 init = "" @@ -72,7 +76,6 @@ class Entry: is_builtin = 0 is_cglobal = 0 is_pyglobal = 0 - is_special = 0 is_member = 0 is_variable = 0 is_cfunction = 0 @@ -95,11 +98,14 @@ class Entry: in_cinclude = 0 as_module = None is_inherited = 0 - interned_cname = None + #interned_cname = None pystring_cname = None is_interned = 0 used = 0 - + is_special = 0 + defined_in_pxd = 0 + api = 0 + def __init__(self, name, cname, type, pos = None, init = None): self.name = name self.cname = cname @@ -113,6 +119,7 @@ class Scope: # outer_scope Scope or None Enclosing scope # entries {string : Entry} Python name to entry, non-types # const_entries [Entry] Constant entries + # type_entries [Entry] Struct/union/enum/typedef/exttype entries # sue_entries [Entry] Struct/union/enum entries # arg_entries [Entry] Function argument entries # var_entries [Entry] User-defined variable entries @@ -155,6 +162,7 @@ class Scope: self.scope_prefix = mangled_name self.entries = {} self.const_entries = [] + self.type_entries = [] self.sue_entries = [] self.arg_entries = [] self.var_entries = [] @@ -236,27 +244,41 @@ class Scope: return entry def declare_type(self, name, type, pos, - cname = None, visibility = 'private'): + cname = None, visibility = 'private', defining = 1): # Add an entry for a type definition. if not cname: cname = name entry = self.declare(name, cname, type, pos) entry.visibility = visibility entry.is_type = 1 + if defining: + self.type_entries.append(entry) return entry + + def declare_typedef(self, name, base_type, pos, cname = None, + visibility = 'private'): + if not cname: + if self.in_cinclude or visibility == 'public': + cname = name + else: + cname = self.mangle(Naming.type_prefix, name) + type = PyrexTypes.CTypedefType(cname, base_type) + entry = self.declare_type(name, type, pos, cname, visibility) + type.qualified_name = entry.qualified_name def declare_struct_or_union(self, name, kind, scope, - typedef_flag, pos, cname = None): + typedef_flag, pos, cname = None, visibility = 'private'): # Add an entry for a struct or union definition. if not cname: - if self.in_cinclude: + if self.in_cinclude or visibility == 'public': cname = name else: cname = self.mangle(Naming.type_prefix, name) entry = self.lookup_here(name) if not entry: type = CStructOrUnionType(name, kind, scope, typedef_flag, cname) - entry = self.declare_type(name, type, pos, cname) + entry = self.declare_type(name, type, pos, cname, + visibility = visibility, defining = scope is not None) self.sue_entries.append(entry) else: if not (entry.is_type and entry.type.is_struct_or_union): @@ -265,8 +287,10 @@ class Scope: warning(pos, "'%s' already defined (ignoring second definition)" % name, 0) else: self.check_previous_typedef_flag(entry, typedef_flag, pos) + self.check_previous_visibility(entry, visibility, pos) if scope: entry.type.scope = scope + self.type_entries.append(entry) if not scope and not entry.type.scope: self.check_for_illegal_incomplete_ctypedef(typedef_flag, pos) return entry @@ -276,17 +300,24 @@ class Scope: error(pos, "'%s' previously declared using '%s'" % ( entry.name, ("cdef", "ctypedef")[entry.type.typedef_flag])) - def declare_enum(self, name, pos, cname, typedef_flag): + def check_previous_visibility(self, entry, visibility, pos): + if entry.visibility <> visibility: + error(pos, "'%s' previously declared as '%s'" % ( + entry.name, entry.visibility)) + + def declare_enum(self, name, pos, cname, typedef_flag, + visibility = 'private'): if name: if not cname: - if self.in_cinclude: + if self.in_cinclude or visibility == 'public': cname = name else: cname = self.mangle(Naming.type_prefix, name) type = CEnumType(name, cname, typedef_flag) else: - type = c_int_type - entry = self.declare_type(name, type, pos, cname = cname) + type = PyrexTypes.c_anon_enum_type + entry = self.declare_type(name, type, pos, cname = cname, + visibility = visibility) entry.enum_values = [] self.sue_entries.append(entry) return entry @@ -318,15 +349,26 @@ class Scope: self.pyfunc_entries.append(entry) def declare_cfunction(self, name, type, pos, - cname = None, visibility = 'private', defining = 0): + cname = None, visibility = 'private', defining = 0, api = 0, in_pxd = 0): # Add an entry for a C function. - if not cname: - if visibility <> 'private': - cname = name - else: - cname = self.mangle(Naming.func_prefix, name) - entry = self.add_cfunction(name, type, pos, cname, visibility) - entry.func_cname = cname + entry = self.lookup_here(name) + if entry: + if not entry.type.same_as(type): + error(pos, "Function signature does not match previous declaration") + else: + if not cname: + if api or visibility <> 'private': + cname = name + else: + cname = self.mangle(Naming.func_prefix, name) + entry = self.add_cfunction(name, type, pos, cname, visibility) + entry.func_cname = cname + if in_pxd and visibility <> 'extern': + entry.defined_in_pxd = 1 + if api: + entry.api = 1 + if not defining and not in_pxd and visibility <> 'extern': + error(pos, "Non-extern C function declared but not defined") return entry def add_cfunction(self, name, type, pos, cname, visibility): @@ -531,7 +573,16 @@ class BuiltinScope(Scope): else: entry.is_builtin = 1 return entry - + + def declare_builtin_cfunction(self, name, type, cname, with_python_equiv = 0): + entry = self.declare_cfunction(name, type, None, cname) + if with_python_equiv: + var_entry = Entry(name, name, py_object_type) + var_entry.is_variable = 1 + var_entry.is_builtin = 1 + entry.as_variable = var_entry + return entry + def builtin_scope(self): return self @@ -580,7 +631,7 @@ class BuiltinScope(Scope): "type": ["((PyObject*)&PyType_Type)", py_object_type], "slice": ["((PyObject*)&PySlice_Type)", py_object_type], "file": ["((PyObject*)&PyFile_Type)", py_object_type], - + "None": ["Py_None", py_object_type], "False": ["Py_False", py_object_type], "True": ["Py_True", py_object_type], @@ -645,7 +696,7 @@ class ModuleScope(Scope): def declare_builtin(self, name, pos): entry = Scope.declare_builtin(self, name, pos) - entry.interned_cname = self.intern(name) + #entry.interned_cname = self.intern(name) return entry def intern(self, name): @@ -736,8 +787,8 @@ class ModuleScope(Scope): "Non-cdef global variable is not a generic Python object") entry.is_pyglobal = 1 entry.namespace_cname = self.module_cname - if Options.intern_names: - entry.interned_cname = self.intern(name) + #if Options.intern_names: + # entry.interned_cname = self.intern(name) else: entry.is_cglobal = 1 self.var_entries.append(entry) @@ -774,9 +825,6 @@ class ModuleScope(Scope): module_name, base_type, objstruct_cname, typeobj_cname, visibility, typedef_flag): # - #print "declare_c_class:", name - #print "...visibility =", visibility - # # Look for previous declaration as a type # entry = self.lookup_here(name) @@ -798,7 +846,8 @@ class ModuleScope(Scope): else: type.module_name = self.qualified_name type.typeptr_cname = self.mangle(Naming.typeptr_prefix, name) - entry = self.declare_type(name, type, pos, visibility = visibility) + entry = self.declare_type(name, type, pos, visibility = visibility, + defining = 0) if objstruct_cname: type.objstruct_cname = objstruct_cname elif not entry.in_cinclude: @@ -818,6 +867,7 @@ class ModuleScope(Scope): if base_type: scope.declare_inherited_c_attributes(base_type.scope) type.set_scope(scope) + self.type_entries.append(entry) else: self.check_for_illegal_incomplete_ctypedef(typedef_flag, pos) else: @@ -828,11 +878,13 @@ class ModuleScope(Scope): # # Fill in options, checking for compatibility with any previous declaration # + if defining: + entry.defined_in_pxd = 1 if implementing: # So that filenames in runtime exceptions refer to entry.pos = pos # the .pyx file and not the .pxd file if entry.visibility <> visibility: error(pos, "Declaration of '%s' as '%s' conflicts with previous " - "declaration as '%s'" % (class_name, visibility, entry.visibility)) + "declaration as '%s'" % (name, visibility, entry.visibility)) if objstruct_cname: if type.objstruct_cname and type.objstruct_cname <> objstruct_cname: error(pos, "Object struct name differs from previous declaration") @@ -1023,8 +1075,8 @@ class PyClassScope(ClassScope): cname, visibility, is_cdef) entry.is_pyglobal = 1 entry.namespace_cname = self.class_obj_cname - if Options.intern_names: - entry.interned_cname = self.intern(name) + #if Options.intern_names: + # entry.interned_cname = self.intern(name) return entry def allocate_temp(self, type): @@ -1131,6 +1183,10 @@ class CClassScope(ClassScope): # Add an entry for a method. if name in ('__eq__', '__ne__', '__lt__', '__gt__', '__le__', '__ge__'): error(pos, "Special method %s must be implemented via __richcmp__" % name) + if name == "__new__": + warning(pos, "__new__ method of extension type will change semantics " + "in a future version of Pyrex and Cython. Use __cinit__ instead.") + name = "__cinit__" entry = self.declare_var(name, py_object_type, pos) special_sig = get_special_method_signature(name) if special_sig: @@ -1144,9 +1200,14 @@ class CClassScope(ClassScope): self.pyfunc_entries.append(entry) return entry - + + def lookup_here(self, name): + if name == "__new__": + name = "__cinit__" + return ClassScope.lookup_here(self, name) + def declare_cfunction(self, name, type, pos, - cname = None, visibility = 'private', defining = 0): + cname = None, visibility = 'private', defining = 0, api = 0, in_pxd = 0): if get_special_method_signature(name): error(pos, "Special methods must be declared with 'def', not 'cdef'") args = type.args @@ -1161,6 +1222,7 @@ class CClassScope(ClassScope): else: if defining and entry.func_cname: error(pos, "'%s' already defined" % name) + #print "CClassScope.declare_cfunction: checking signature" ### if type.same_c_signature_as(entry.type, as_cmethod = 1): pass # if type.narrower_c_signature_than(entry.type, as_cmethod = 1): @@ -1243,8 +1305,8 @@ class PropertyScope(Scope): signature = get_property_accessor_signature(name) if signature: entry = self.declare(name, name, py_object_type, pos) - entry.signature = signature entry.is_special = 1 + entry.signature = signature return entry else: error(pos, "Only __get__, __set__ and __del__ methods allowed " diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index b272341f..e7edf053 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -83,7 +83,20 @@ class Signature: def return_type(self): return self.format_map[self.ret_format] - + + def exception_value(self): + return self.error_value_map.get(self.ret_format) + + def function_type(self): + # Construct a C function type descriptor for this signature + args = [] + for i in xrange(self.num_fixed_args()): + arg_type = self.fixed_arg_type(i) + args.append(PyrexTypes.CFuncTypeArg("", arg_type, None)) + ret_type = self.return_type() + exc_value = self.exception_value() + return PyrexTypes.CFuncType(ret_type, args, exception_value = exc_value) + def method_flags(self): if self.ret_format == "O": full_args = self.fixed_arg_format @@ -104,22 +117,23 @@ class SlotDescriptor: # # slot_name string Member name of the slot in the type object # is_initialised_dynamically Is initialised by code in the module init function - - def __init__(self, slot_name, dynamic = 0, min_python_version = None): + # flag Py_TPFLAGS_XXX value indicating presence of slot + + def __init__(self, slot_name, dynamic = 0, flag = None): self.slot_name = slot_name self.is_initialised_dynamically = dynamic - self.min_python_version = min_python_version + self.flag = flag def generate(self, scope, code): if self.is_initialised_dynamically: value = 0 else: value = self.slot_code(scope) - if self.min_python_version is not None: - code.putln("#if PY_VERSION_HEX >= " + - code.get_py_version_hex(self.min_python_version)) + flag = self.flag + if flag: + code.putln("#if Py_TPFLAGS_DEFAULT & %s" % flag) code.putln("%s, /*%s*/" % (value, self.slot_name)) - if self.min_python_version is not None: + if flag: code.putln("#endif") # Some C implementations have trouble statically @@ -182,10 +196,8 @@ class MethodSlot(SlotDescriptor): # method_name string The __xxx__ name of the method # default string or None Default value of the slot - def __init__(self, signature, slot_name, method_name, - default = None, min_python_version = None): - SlotDescriptor.__init__(self, slot_name, - min_python_version = min_python_version) + def __init__(self, signature, slot_name, method_name, default = None, flag = None): + SlotDescriptor.__init__(self, slot_name, flag = flag) self.signature = signature self.slot_name = slot_name self.method_name = method_name @@ -411,7 +423,7 @@ getcharbufferproc = Signature("TiS", 'i') # typedef int (*getcharbufferproc)(Py readbufferproc = Signature("TZP", "Z") # typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **); writebufferproc = Signature("TZP", "Z") # typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **); segcountproc = Signature("TZ", "Z") # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); -writebufferproc = Signature("TZS", "Z") # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); +charbufferproc = Signature("TZS", "Z") # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); objargproc = Signature("TO", 'r') # typedef int (*objobjproc)(PyObject *, PyObject *); # typedef int (*visitproc)(PyObject *, void *); # typedef int (*traverseproc)(PyObject *, visitproc, void *); @@ -502,11 +514,11 @@ PyNumberMethods = ( MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__"), # Added in release 2.5 - MethodSlot(unaryfunc, "nb_index", "__index__", min_python_version=(2,5)), + MethodSlot(unaryfunc, "nb_index", "__index__", flag = "Py_TPFLAGS_HAVE_INDEX") ) PySequenceMethods = ( - MethodSlot(lenfunc, "sq_length", "__len__"), # EmptySlot("sq_length"), # mp_length used instead + MethodSlot(lenfunc, "sq_length", "__len__"), EmptySlot("sq_concat"), # nb_add used instead EmptySlot("sq_repeat"), # nb_multiply used instead SyntheticSlot("sq_item", ["__getitem__"], "0"), #EmptySlot("sq_item"), # mp_subscript used instead @@ -610,7 +622,7 @@ slot_table = ( # #------------------------------------------------------------------------------------------ -MethodSlot(initproc, "", "__new__") +MethodSlot(initproc, "", "__cinit__") MethodSlot(destructor, "", "__dealloc__") MethodSlot(objobjargproc, "", "__setitem__") MethodSlot(objargproc, "", "__delitem__") diff --git a/Cython/Debugging.py b/Cython/Debugging.py index 3c538b64..1fbccd88 100644 --- a/Cython/Debugging.py +++ b/Cython/Debugging.py @@ -7,7 +7,7 @@ def print_call_chain(*args): import sys print " ".join(map(str, args)) - f = sys._getframe(2) + f = sys._getframe(1) while f: name = f.f_code.co_name s = f.f_locals.get('self', None) diff --git a/Cython/Distutils/__init__.py b/Cython/Distutils/__init__.py index 5033ac04..49493bf2 100644 --- a/Cython/Distutils/__init__.py +++ b/Cython/Distutils/__init__.py @@ -1,22 +1,11 @@ # July 2002, Graham Fawcett - # - # this hack was inspired by the way Thomas Heller got py2exe - # to appear as a distutil command - # - # we replace distutils.command.build_ext with our own version - # and keep the old one under the module name _build_ext, - # so that *our* build_ext can make use of it. - - from build_ext import build_ext - - - +from extension import Extension diff --git a/Cython/Distutils/build_ext.py b/Cython/Distutils/build_ext.py index c6acffef..f30f47c3 100644 --- a/Cython/Distutils/build_ext.py +++ b/Cython/Distutils/build_ext.py @@ -1,74 +1,197 @@ -# Subclasses disutils.command.build_ext, -# replacing it with a Pyrex version that compiles pyx->c -# before calling the original build_ext command. -# July 2002, Graham Fawcett -# Modified by Darrell Gallion -# to allow inclusion of .c files along with .pyx files. -# Pyrex is (c) Greg Ewing. +"""Cython.Distutils.build_ext -import distutils.command.build_ext -#import Cython.Compiler.Main -from Cython.Compiler.Main import CompilationOptions, default_options, compile -from Cython.Compiler.Errors import PyrexError -from distutils.dep_util import newer -import os -import sys +Implements a version of the Distutils 'build_ext' command, for +building Cython extension modules.""" -def replace_suffix(path, new_suffix): - return os.path.splitext(path)[0] + new_suffix +# This module should be kept compatible with Python 2.1. -class build_ext (distutils.command.build_ext.build_ext): +__revision__ = "$Id:$" - description = "compile Cython scripts, then build C/C++ extensions (compile/link to build directory)" +import sys, os, string, re +from types import * +from distutils.core import Command +from distutils.errors import * +from distutils.sysconfig import customize_compiler, get_python_version +from distutils.dep_util import newer_group +from distutils import log +from distutils.dir_util import mkpath +try: + from Cython.Compiler.Main \ + import CompilationOptions, \ + default_options as pyrex_default_options, \ + compile as cython_compile + from Cython.Compiler.Errors import PyrexError +except ImportError, e: + print e + PyrexError = None + +from distutils.command import build_ext as _build_ext + +extension_name_re = _build_ext.extension_name_re + +show_compilers = _build_ext.show_compilers + +class build_ext(_build_ext.build_ext): + + description = "build C/C++ and Cython extensions (compile/link to build directory)" + + sep_by = _build_ext.build_ext.sep_by + user_options = _build_ext.build_ext.user_options + boolean_options = _build_ext.build_ext.boolean_options + help_options = _build_ext.build_ext.help_options + + # Add the pyrex specific data. + user_options.extend([ + ('pyrex-cplus', None, + "generate C++ source files"), + ('pyrex-create-listing', None, + "write errors to a listing file"), + ('pyrex-include-dirs=', None, + "path to the Cython include files" + sep_by), + ('pyrex-c-in-temp', None, + "put generated C files in temp directory"), + ('pyrex-gen-pxi', None, + "generate .pxi file for public declarations"), + ]) + + boolean_options.extend([ + 'pyrex-cplus', 'pyrex-create-listing', 'pyrex-c-in-temp' + ]) + + def initialize_options(self): + _build_ext.build_ext.initialize_options(self) + self.pyrex_cplus = 0 + self.pyrex_create_listing = 0 + self.pyrex_include_dirs = None + self.pyrex_c_in_temp = 0 + self.pyrex_gen_pxi = 0 def finalize_options (self): - distutils.command.build_ext.build_ext.finalize_options(self) - - # The following hack should no longer be needed. - if 0: - # compiling with mingw32 gets an "initializer not a constant" error - # doesn't appear to happen with MSVC! - # so if we are compiling with mingw32, - # switch to C++ mode, to avoid the problem - if self.compiler == 'mingw32': - self.swig_cpp = 1 - - def swig_sources (self, sources, extension = None): - if not self.extensions: - return - - #suffix = self.swig_cpp and '.cpp' or '.c' - suffix = '.c' - cplus = 0 - include_dirs = [] - if extension is not None: - module_name = extension.name - include_dirs = extension.include_dirs or [] - if extension.language == "c++": - cplus = 1 - suffix = ".cpp" + _build_ext.build_ext.finalize_options(self) + if self.pyrex_include_dirs is None: + self.pyrex_include_dirs = [] + elif type(self.pyrex_include_dirs) is StringType: + self.pyrex_include_dirs = \ + string.split(self.pyrex_include_dirs, os.pathsep) + # finalize_options () + + def build_extensions(self): + # First, sanity-check the 'extensions' list + self.check_extensions_list(self.extensions) + for ext in self.extensions: + ext.sources = self.cython_sources(ext.sources, ext) + self.build_extension(ext) + + def cython_sources(self, sources, extension): + + """ + Walk the list of source files in 'sources', looking for Cython + source (.pyx) files. Run Cython on all that are found, and return + a modified 'sources' list with Cython source files replaced by the + generated C (or C++) files. + """ + + if PyrexError == None: + raise DistutilsPlatformError, \ + ("Cython does not appear to be installed " + "on platform '%s'") % os.name + + new_sources = [] + pyrex_sources = [] + pyrex_targets = {} + + # Setup create_list and cplus from the extension options if + # Cython.Distutils.extension.Extension is used, otherwise just + # use what was parsed from the command-line or the configuration file. + # cplus will also be set to true is extension.language is equal to + # 'C++' or 'c++'. + #try: + # create_listing = self.pyrex_create_listing or \ + # extension.pyrex_create_listing + # cplus = self.pyrex_cplus or \ + # extension.pyrex_cplus or \ + # (extension.language != None and \ + # extension.language.lower() == 'c++') + #except AttributeError: + # create_listing = self.pyrex_create_listing + # cplus = self.pyrex_cplus or \ + # (extension.language != None and \ + # extension.language.lower() == 'c++') + + create_listing = self.pyrex_create_listing or \ + getattr(extension, 'pyrex_create_listing', 0) + cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \ + (extension.language and extension.language.lower() == 'c++') + pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0) + + # Set up the include_path for the Cython compiler: + # 1. Start with the command line option. + # 2. Add in any (unique) paths from the extension + # pyrex_include_dirs (if Cython.Distutils.extension is used). + # 3. Add in any (unique) paths from the extension include_dirs + includes = self.pyrex_include_dirs + try: + for i in extension.pyrex_include_dirs: + if not i in includes: + includes.append(i) + except AttributeError: + pass + for i in extension.include_dirs: + if not i in includes: + includes.append(i) + + # Set the target_ext to '.c'. Cython will change this to '.cpp' if + # needed. + if cplus: + target_ext = '.cpp' else: - module_name = None - - # collect the names of the source (.pyx) files - pyx_sources = [source for source in sources if source.endswith('.pyx')] - other_sources = [source for source in sources if not source.endswith('.pyx')] - - for pyx in pyx_sources: - # should I raise an exception if it doesn't exist? - if os.path.exists(pyx): - source = pyx - target = replace_suffix(source, suffix) - if newer(source, target) or self.force: - self.cython_compile(source, module_name, include_dirs, cplus) - - return [replace_suffix(src, suffix) for src in pyx_sources] + other_sources - - def cython_compile(self, source, module_name, include_dirs, cplus): - options = CompilationOptions( - default_options, - include_path = include_dirs + self.include_dirs, - cplus=cplus) - result = compile(source, options, full_module_name=module_name) - if result.num_errors <> 0: - sys.exit(1) + target_ext = '.c' + + # Decide whether to drop the generated C files into the temp dir + # or the source tree. + + if not self.inplace and (self.pyrex_c_in_temp + or getattr(extension, 'pyrex_c_in_temp', 0)): + target_dir = os.path.join(self.build_temp, "pyrex") + else: + target_dir = "" + + for source in sources: + (base, ext) = os.path.splitext(source) + if ext == ".pyx": # Cython source file + new_sources.append(os.path.join(target_dir, base + target_ext)) + pyrex_sources.append(source) + pyrex_targets[source] = new_sources[-1] + else: + new_sources.append(source) + + if not pyrex_sources: + return new_sources + + module_name = extension.name + + for source in pyrex_sources: + target = pyrex_targets[source] + source_time = os.stat(source).st_mtime + try: + target_time = os.stat(target).st_mtime + newer = source_time > target_time + except EnvironmentError: + newer = 1 + if newer: + log.info("cythoning %s to %s", source, target) + self.mkpath(os.path.dirname(target)) + options = CompilationOptions(pyrex_default_options, + use_listing_file = create_listing, + include_path = includes, + output_file = target, + cplus = cplus, + generate_pxi = pyrex_gen_pxi) + result = cython_compile(source, options=options, + full_module_name=module_name) + + return new_sources + + # cython_sources () + +# class build_ext diff --git a/Cython/Mac/DarwinSystem.py b/Cython/Mac/DarwinSystem.py index a3efd9f6..0b1a655c 100644 --- a/Cython/Mac/DarwinSystem.py +++ b/Cython/Mac/DarwinSystem.py @@ -6,15 +6,20 @@ verbose = 0 gcc_pendantic = True gcc_warnings_are_errors = True gcc_all_warnings = True +gcc_optimize = False -import os +import os, sys from Cython.Utils import replace_suffix from Cython.Compiler.Errors import PyrexError +version_string = "%s.%s" % sys.version_info[:2] + py_include_dirs = [ - "/Library/Frameworks/Python.framework/Headers" + "/Library/Frameworks/Python.framework/Versions/%s/Headers" % version_string ] +os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.3" + compilers = ["gcc", "g++"] compiler_options = \ "-g -c -fno-strict-aliasing -Wno-long-double -no-cpp-precomp " \ @@ -27,11 +32,16 @@ if gcc_warnings_are_errors: if gcc_all_warnings: compiler_options.append("-Wall") compiler_options.append("-Wno-unused-function") +if gcc_optimize: + compiler_options.append("-O") linkers = ["gcc", "g++"] linker_options = \ - "-Wl,-F.,-w -bundle -framework Python" \ + "-Wl,-F.,-w -bundle -undefined dynamic_lookup" \ .split() +#linker_options = \ +# "-Wl,-F.,-w -bundle -framework Python" \ +# .split() class CCompilerError(PyrexError): pass diff --git a/Cython/Unix/LinuxSystem.py b/Cython/Unix/LinuxSystem.py index 466b61ec..1f71e3fd 100644 --- a/Cython/Unix/LinuxSystem.py +++ b/Cython/Unix/LinuxSystem.py @@ -7,7 +7,7 @@ gcc_pendantic = True gcc_warnings_are_errors = True gcc_all_warnings = True -import os +import os, sys from Cython.Utils import replace_suffix from Cython.Compiler.Errors import PyrexError diff --git a/Cython/Utils.py b/Cython/Utils.py index 6cccd39d..1b4b07d3 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -14,3 +14,21 @@ def open_new_file(path): # preserve metadata on the Mac. return open(path, "w+") +def castrate_file(path, st): + # Remove junk contents from an output file after a + # failed compilation, but preserve metadata on Mac. + # Also sets access and modification times back to + # those specified by st (a stat struct). + try: + f = open(path, "r+") + except EnvironmentError: + pass + else: + #st = os.stat(path) + f.seek(0, 0) + f.truncate() + f.write( + "#error Do not use this file, it is the result of a failed Pyrex compilation.\n") + f.close() + if st: + os.utime(path, (st.st_atime, st.st_mtime))