From 026931cfc282ecb567d884e84230c431592b1dc1 Mon Sep 17 00:00:00 2001 From: Dag Sverre Seljebotn Date: Mon, 5 Jan 2009 12:39:48 +0100 Subject: [PATCH] Refcount nanny framework in place; smaller fixes and better build needed --- Cython/Compiler/Code.py | 46 +++++++++++--------- Cython/Compiler/DebugFlags.py | 2 +- Cython/Compiler/ExprNodes.py | 4 +- Cython/Compiler/ModuleNode.py | 41 +++++++++++++++++- Cython/Compiler/Nodes.py | 26 ++++++++++-- Cython/Runtime/build.sh | 21 ++++++++++ Cython/Runtime/refnanny.pyx | 79 +++++++++++++++++++++++++++++++++++ runtests.py | 8 +++- 8 files changed, 199 insertions(+), 28 deletions(-) create mode 100755 Cython/Runtime/build.sh create mode 100644 Cython/Runtime/refnanny.pyx diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index df1ac58f..4a49a407 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -673,50 +673,52 @@ class CCodeWriter(object): return typecast(py_object_type, type, cname) def put_gotref(self, cname): - if DebugFlags.debug_ref_check_code: - self.putln("__Pyx_GOTREF(%s);" % cname) + self.putln("__Pyx_GOTREF(%s);" % cname) + + def put_giveref(self, cname): + self.putln("__Pyx_GIVEREF(%s);" % cname) def put_incref(self, cname, type): - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) def put_decref(self, cname, type): - self.putln("Py_DECREF(%s);" % self.as_pyobject(cname, type)) + self.putln("__Pyx_DECREF(%s);" % self.as_pyobject(cname, type)) def put_var_incref(self, entry): if entry.type.is_pyobject: - self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) + self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) def put_decref_clear(self, cname, type): - self.putln("Py_DECREF(%s); %s = 0;" % ( + self.putln("__Pyx_DECREF(%s); %s = 0;" % ( typecast(py_object_type, type, cname), cname)) #self.as_pyobject(cname, type), cname)) def put_xdecref(self, cname, type): - self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type)) + self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type)) def put_xdecref_clear(self, cname, type): - self.putln("Py_XDECREF(%s); %s = 0;" % ( + self.putln("__Pyx_XDECREF(%s); %s = 0;" % ( self.as_pyobject(cname, type), cname)) def put_var_decref(self, entry): if entry.type.is_pyobject: if entry.init_to_none is False: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) + self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) else: - self.putln("Py_DECREF(%s);" % self.entry_as_pyobject(entry)) + self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry)) def put_var_decref_clear(self, entry): if entry.type.is_pyobject: - self.putln("Py_DECREF(%s); %s = 0;" % ( + self.putln("__Pyx_DECREF(%s); %s = 0;" % ( self.entry_as_pyobject(entry), entry.cname)) def put_var_xdecref(self, entry): if entry.type.is_pyobject: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) + self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) def put_var_xdecref_clear(self, entry): if entry.type.is_pyobject: - self.putln("Py_XDECREF(%s); %s = 0;" % ( + self.putln("__Pyx_XDECREF(%s); %s = 0;" % ( self.entry_as_pyobject(entry), entry.cname)) def put_var_decrefs(self, entries, used_only = 0): @@ -737,7 +739,7 @@ class CCodeWriter(object): def put_init_to_py_none(self, cname, type): py_none = typecast(type, py_object_type, "Py_None") - self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none)) + self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none)) def put_init_var_to_py_none(self, entry, template = "%s"): code = template % entry.cname @@ -775,21 +777,25 @@ class CCodeWriter(object): return 'unlikely(%s)' % cond else: return cond - - def error_goto(self, pos): - lbl = self.funcstate.error_label - self.funcstate.use_label(lbl) + + def set_error_info(self, pos): if Options.c_line_in_traceback: cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro) else: cinfo = "" - return "{%s = %s[%s]; %s = %s;%s goto %s;}" % ( + return "%s = %s[%s]; %s = %s;%s" % ( Naming.filename_cname, Naming.filetable_cname, self.lookup_filename(pos[0]), Naming.lineno_cname, pos[1], - cinfo, + cinfo) + + def error_goto(self, pos): + lbl = self.funcstate.error_label + self.funcstate.use_label(lbl) + return "{%s goto %s;}" % ( + self.set_error_info(pos), lbl) def error_goto_if(self, cond, pos): diff --git a/Cython/Compiler/DebugFlags.py b/Cython/Compiler/DebugFlags.py index 2d096a4c..958ce269 100644 --- a/Cython/Compiler/DebugFlags.py +++ b/Cython/Compiler/DebugFlags.py @@ -2,4 +2,4 @@ debug_disposal_code = 0 debug_temp_alloc = 0 debug_coercion = 0 -debug_ref_check_code = 1 +debug_refnanny = 1 diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 58341265..5f832cdc 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -1678,7 +1678,6 @@ class IndexNode(ExprNode): value_code, self.index_unsigned_parameter(), code.error_goto(self.pos))) - code.put_gotref(self.base.py_result()) def generate_buffer_setitem_code(self, rhs, code, op=""): # Used from generate_assignment_code and InPlaceAssignmentNode @@ -2785,6 +2784,7 @@ class TupleNode(SequenceNode): self.result(), i, arg.py_result())) + code.put_giveref(arg.py_result()) def generate_subexpr_disposal_code(self, code): # We call generate_post_assignment_code here instead @@ -2864,6 +2864,7 @@ class ListNode(SequenceNode): (self.result(), len(self.args), code.error_goto_if_null(self.result(), self.pos))) + code.put_gotref(self.result()) for i in range(len(self.args)): arg = self.args[i] #if not arg.is_temp: @@ -2873,6 +2874,7 @@ class ListNode(SequenceNode): (self.result(), i, arg.py_result())) + code.put_giveref(arg.py_result()) elif self.type.is_array: for i, arg in enumerate(self.args): code.putln("%s[%s] = %s;" % ( diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 36c8b020..452d71e2 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -19,6 +19,7 @@ import Options import PyrexTypes import TypeSlots import Version +import DebugFlags from Errors import error, warning from PyrexTypes import py_object_type @@ -250,6 +251,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.globalstate.module_pos = self.pos code.globalstate.directives = self.directives + code.globalstate.use_utility_code(refcount_utility_code) + code.putln("") code.putln("/* Implementation of %s */" % env.qualified_name) self.generate_const_definitions(env, code) @@ -535,7 +538,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('') code.putln('static char %s[] = "%s";' % ( env.doc_cname, escape_byte_string(docstr))) - + def generate_extern_c_macro_definition(self, code): name = Naming.extern_c_macro code.putln("#ifdef __cplusplus") @@ -1559,6 +1562,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln("{") tempdecl_code = code.insertion_point() + + code.putln('__Pyx_SetupRefcountContext("%s");' % header3) + code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos))); code.putln("/*--- Library function declarations ---*/") @@ -2274,3 +2280,36 @@ bad: } """ % {'IMPORT_STAR' : Naming.import_star, 'IMPORT_STAR_SET' : Naming.import_star_set } + +refcount_utility_code = UtilityCode(proto=""" +#ifdef CYTHON_REFNANNY + +void __Pyx_Refnanny_INCREF(void*, PyObject*, int); +void __Pyx_Refnanny_GOTREF(void*, PyObject*, int); +void __Pyx_Refnanny_GIVEREF(void*, PyObject*, int); +void __Pyx_Refnanny_INCREF(void*, PyObject*, int); +void __Pyx_Refnanny_DECREF(void*, PyObject*, int); +void* __Pyx_Refnanny_NewContext(char*, int); +int __Pyx_Refnanny_FinishContext(void*); + +#define __Pyx_INCREF(r) __Pyx_Refnanny_INCREF(__pyx_refchk, r, __LINE__) +#define __Pyx_GOTREF(r) __Pyx_Refnanny_GOTREF(__pyx_refchk, r, __LINE__) +#define __Pyx_GIVEREF(r) __Pyx_Refnanny_GIVEREF(__pyx_refchk, r, __LINE__) +#define __Pyx_DECREF(r) __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__) +#define __Pyx_XDECREF(r) (r ? __Pyx_Refnanny_DECREF(__pyx_refchk, r, __LINE__) : 0) +#define __Pyx_SetupRefcountContext(name) \ + void* __pyx_refchk = __Pyx_Refnanny_NewContext(name, __LINE__) +#define __Pyx_FinishRefcountContext() __Pyx_Refnanny_FinishContext(__pyx_refchk) + +#else + +#define __Pyx_INCREF(r) Py_INCREF(r) +#define __Pyx_GOTREF(r) +#define __Pyx_GIVEREF(r) +#define __Pyx_DECREF(r) Py_DECREF(r) +#define __Pyx_XDECREF(r) Py_XDECREF(r) +#define __Pyx_SetupRefcountContext(name) +#define __Pyx_FinishRefcountContext() 0 + +#endif /* CYTHON_REFNANNY */ +""") diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index 7f64bdcf..a2dd7989 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -16,6 +16,7 @@ from Cython.Utils import open_new_file, replace_suffix, UtilityCode from StringEncoding import EncodedString, escape_byte_string, split_docstring import Options import ControlFlow +import DebugFlags from DebugFlags import debug_disposal_code @@ -1010,6 +1011,7 @@ class FuncDefNode(StatNode, BlockNode): # ----- Automatic lead-ins for certain special functions if is_getbuffer_slot: self.getbuffer_init(code) + code.putln('__Pyx_SetupRefcountContext("%s");' % self.entry.name) # ----- Fetch arguments self.generate_argument_parsing_code(env, code) # If an argument is assigned to in the body, we must @@ -1119,8 +1121,24 @@ class FuncDefNode(StatNode, BlockNode): code.putln("PyGILState_Release(_save);") # code.putln("/* TODO: decref scope object */") # ----- Return + default_retval = self.return_type.default_value + err_val = self.error_value() + if err_val is None and default_retval: + err_val = default_retval + if self.return_type.is_pyobject: + code.put_giveref(Naming.retval_cname) + if err_val is None: + code.putln('__Pyx_FinishRefcountContext();') + else: + code.putln('if (__Pyx_FinishRefcountContext() == -1) {') + code.putln(code.set_error_info(self.pos)) + code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name) + code.putln('%s = %s;' % (Naming.retval_cname, err_val)) + code.putln('}') + if not self.return_type.is_void: code.putln("return %s;" % Naming.retval_cname) + code.putln("}") # ----- Go back and insert temp variable declarations tempvardecl_code.put_var_declarations(lenv.temp_entries) @@ -1171,16 +1189,16 @@ class FuncDefNode(StatNode, BlockNode): # getbuffer with a NULL parameter. For now we work around this; # the following line should be removed when this bug is fixed. code.putln("if (%s == NULL) return 0;" % info) - code.putln("%s->obj = Py_None; Py_INCREF(Py_None);" % info) + code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info) def getbuffer_error_cleanup(self, code): info = self.local_scope.arg_entries[1].cname - code.putln("Py_DECREF(%s->obj); %s->obj = NULL;" % + code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;" % (info, info)) def getbuffer_normal_cleanup(self, code): info = self.local_scope.arg_entries[1].cname - code.putln("if (%s->obj == Py_None) { Py_DECREF(Py_None); %s->obj = NULL; }" % + code.putln("if (%s->obj == Py_None) { __Pyx_DECREF(Py_None); %s->obj = NULL; }" % (info, info)) class CFuncDefNode(FuncDefNode): @@ -4040,7 +4058,7 @@ class ExceptClauseNode(Node): self.body.generate_execution_code(code) code.funcstate.exc_vars = old_exc_vars for var in self.exc_vars: - code.putln("Py_DECREF(%s); %s = 0;" % (var, var)) + code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var)) code.put_goto(end_label) code.putln( "}") diff --git a/Cython/Runtime/build.sh b/Cython/Runtime/build.sh new file mode 100755 index 00000000..7f369b91 --- /dev/null +++ b/Cython/Runtime/build.sh @@ -0,0 +1,21 @@ +cat < ../../Cython/Compiler/DebugFlags.py +debug_disposal_code = 0 +debug_temp_alloc = 0 +debug_coercion = 0 + +debug_refnanny = 0 +EOF + + +python ../../cython.py refnanny.pyx +gcc -shared -pthread -fPIC -fwrapv -O2 -Wall \ + -fno-strict-aliasing -I/local/include/python2.5 \ + -o refnanny.so -I. refnanny.c + +cat < ../../Cython/Compiler/DebugFlags.py +debug_disposal_code = 0 +debug_temp_alloc = 0 +debug_coercion = 0 + +debug_refnanny = 1 +EOF diff --git a/Cython/Runtime/refnanny.pyx b/Cython/Runtime/refnanny.pyx new file mode 100644 index 00000000..a56823cd --- /dev/null +++ b/Cython/Runtime/refnanny.pyx @@ -0,0 +1,79 @@ +from python_ref cimport Py_INCREF, Py_DECREF + +loglevel = 0 +reflog = [] + +cdef log(level, action, obj, lineno): + if loglevel >= level: + reflog.append((lineno, action, id(obj))) + +LOG_NONE, LOG_ALL = range(2) + +class RefnannyException(Exception): + pass + +class RefnannyContext(object): + def __init__(self): + self.refs = {} # id -> (count, [lineno]) + self.errors = [] + + def regref(self, obj, lineno): + log(LOG_ALL, 'regref', obj, lineno) + id_ = id(obj) + count, linenumbers = self.refs.get(id_, (0, [])) + self.refs[id_] = (count + 1, linenumbers) + linenumbers.append(lineno) + + def delref(self, obj, lineno): + log(LOG_ALL, 'delref', obj, lineno) + id_ = id(obj) + count, linenumbers = self.refs.get(id_, (0, [])) + if count == 0: + self.errors.append("Too many decrefs on line %d, reference acquired on lines %r" % + (lineno, linenumbers)) + elif count == 1: + del self.refs[id_] + else: + self.refs[id_] = (count - 1, linenumbers) + + def end(self): + if len(self.refs) > 0: + msg = "" + for count, linenos in self.refs.itervalues(): + msg += "\n Acquired on lines: " + ", ".join(["%d" % x for x in linenos]) + self.errors.append("References leaked: %s" % msg) + if self.errors: + raise RefnannyException("\n".join(self.errors)) + +cdef public void* __Pyx_Refnanny_NewContext(char* funcname, int lineno) except NULL: + ctx = RefnannyContext() + Py_INCREF(ctx) + return ctx + +cdef public void __Pyx_Refnanny_GOTREF(void* ctx, object obj, int lineno): + if ctx == NULL: return + (ctx).regref(obj, lineno) + +cdef public void __Pyx_Refnanny_GIVEREF(void* ctx, object obj, int lineno): + if ctx == NULL: return + (ctx).delref(obj, lineno) + +cdef public void __Pyx_Refnanny_INCREF(void* ctx, object obj, int lineno): + Py_INCREF(obj) + __Pyx_Refnanny_GOTREF(ctx, obj, lineno) + +cdef public void __Pyx_Refnanny_DECREF(void* ctx, object obj, int lineno): + # GIVEREF raises exception if we hit 0 + # + __Pyx_Refnanny_GIVEREF(ctx, obj, lineno) + Py_DECREF(obj) + +cdef public int __Pyx_Refnanny_FinishContext(void* ctx) except -1: + obj = ctx + try: + obj.end() + finally: + Py_DECREF(obj) + return 0 + + diff --git a/runtests.py b/runtests.py index 7538cf70..1e7ef958 100644 --- a/runtests.py +++ b/runtests.py @@ -1,6 +1,6 @@ #!/usr/bin/python -import os, sys, re, shutil, unittest, doctest +import os, sys, re, shutil, unittest, doctest, ctypes WITH_CYTHON = True @@ -26,6 +26,10 @@ VER_DEP_MODULES = { INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ] CFLAGS = os.getenv('CFLAGS', '').split() +ctypes.PyDLL("Cython/Runtime/refnanny.so", mode=ctypes.RTLD_GLOBAL) +sys.path.append("Cython/Runtime") +import refnanny +#CFLAGS.append("-DCYTHON_REFNANNY") class ErrorWriter(object): match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match @@ -594,3 +598,5 @@ if __name__ == '__main__': sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n") for test in missing_dep_excluder.tests_missing_deps: sys.stderr.write(" %s\n" % test) + +print "\n".join([repr(x) for x in refnanny.reflog]) -- 2.26.2