From: Mark Florisson Date: Sat, 25 Dec 2010 21:32:04 +0000 (+0100) Subject: Debugger: Fix closure support + tests X-Git-Tag: 0.14.1rc0~12^2~6 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=7d98e4a1088a685bb74a0fb780daf1403dd9cfbb;p=cython.git Debugger: Fix closure support + tests Debugger: Recognition of module-level Cython code (initmodulename and PyInit_modulename) Fix debug flag (import Parsing when needed) --- diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 13b0b49e..1f1939ce 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -18,7 +18,11 @@ from time import time import Code import Errors -import Parsing +# Do not import Parsing here, import it when needed, because Parsing imports +# Nodes, which globally needs debug command line options initialized to set a +# conditional metaclass. These options are processed by CmdLine called from +# main() in this file. +# import Parsing import Version from Scanning import PyrexScanner, FileSourceDescriptor from Errors import PyrexError, CompileError, InternalError, AbortError, error, warning @@ -493,6 +497,7 @@ class Context(object): try: f = Utils.open_source_file(source_filename, "rU") try: + import Parsing s = PyrexScanner(f, source_desc, source_encoding = f.encoding, scope = scope, context = self) tree = Parsing.p_module(s, pxd, full_module_name) diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index ad3b390b..ce9d53fd 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1591,6 +1591,10 @@ class DebugTransform(CythonTransform): #self.c_output_file = options.output_file self.c_output_file = result.c_file + # Closure support, basically treat nested functions as if the AST were + # never nested + self.nested_funcdefs = [] + # tells visit_NameNode whether it should register step-into functions self.register_stepinto = False @@ -1605,7 +1609,17 @@ class DebugTransform(CythonTransform): # serialize functions self.tb.start('Functions') + # First, serialize functions normally... self.visitchildren(node) + + # ... then, serialize nested functions + for nested_funcdef in self.nested_funcdefs: + self.visit_FuncDefNode(nested_funcdef) + + self.register_stepinto = True + self.serialize_modulenode_as_function(node) + self.register_stepinto = False + self.tb.end('Functions') # 2.3 compatibility. Serialize global variables @@ -1625,8 +1639,16 @@ class DebugTransform(CythonTransform): # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map return node - def visit_FuncDefNode(self, node): + def visit_FuncDefNode(self, node): self.visited.add(node.local_scope.qualified_name) + + if getattr(node, 'is_wrapper', False): + return node + + if self.register_stepinto: + self.nested_funcdefs.append(node) + return node + # node.entry.visibility = 'extern' if node.py_func is None: pf_cname = '' @@ -1678,6 +1700,51 @@ class DebugTransform(CythonTransform): self.visitchildren(node) return node + def serialize_modulenode_as_function(self, node): + """ + Serialize the module-level code as a function so the debugger will know + it's a "relevant frame" and it will know where to set the breakpoint + for 'break modulename'. + """ + name = node.full_module_name.rpartition('.')[-1] + + cname_py2 = 'init' + name + cname_py3 = 'PyInit_' + name + + py2_attrs = dict( + name=name, + cname=cname_py2, + pf_cname='', + # Ignore the qualified_name, breakpoints should be set using + # `cy break modulename:lineno` for module-level breakpoints. + qualified_name='', + lineno='1', + is_initmodule_function="True", + ) + + py3_attrs = dict(py2_attrs, cname=cname_py3) + + self._serialize_modulenode_as_function(node, py2_attrs) + self._serialize_modulenode_as_function(node, py3_attrs) + + def _serialize_modulenode_as_function(self, node, attrs): + self.tb.start('Function', attrs=attrs) + + self.tb.start('Locals') + self.serialize_local_variables(node.scope.entries) + self.tb.end('Locals') + + self.tb.start('Arguments') + self.tb.end('Arguments') + + self.tb.start('StepIntoFunctions') + self.register_stepinto = True + self.visitchildren(node) + self.register_stepinto = False + self.tb.end('StepIntoFunctions') + + self.tb.end('Function') + def serialize_local_variables(self, entries): for entry in entries.values(): if entry.type.is_pyobject: @@ -1685,10 +1752,19 @@ class DebugTransform(CythonTransform): else: vartype = 'CObject' - cname = entry.cname - # if entry.type.is_extension_type: - # cname = entry.type.typeptr_cname - + if entry.from_closure: + # We're dealing with a closure where a variable from an outer + # scope is accessed, get it from the scope object. + cname = '%s->%s' % (Naming.cur_scope_cname, + entry.outer_entry.cname) + + qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name, + entry.scope.name, + entry.name) + else: + cname = entry.cname + qname = entry.qualified_name + if not entry.pos: # this happens for variables that are not in the user's code, # e.g. for the global __builtins__, __doc__, etc. We can just @@ -1696,11 +1772,11 @@ class DebugTransform(CythonTransform): lineno = '0' else: lineno = str(entry.pos[1]) - + attrs = dict( name=entry.name, cname=cname, - qualified_name=entry.qualified_name, + qualified_name=qname, type=vartype, lineno=lineno) diff --git a/Cython/Compiler/Tests/TestParseTreeTransforms.py b/Cython/Compiler/Tests/TestParseTreeTransforms.py index adf8105f..9361de58 100644 --- a/Cython/Compiler/Tests/TestParseTreeTransforms.py +++ b/Cython/Compiler/Tests/TestParseTreeTransforms.py @@ -182,7 +182,8 @@ class TestDebugTransform(DebuggerTestCase): self.assertEqual('PythonObject', xml_globals.get('python_var')) # test functions - funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs' + funcnames = ('codefile.spam', 'codefile.ham', 'codefile.eggs', + 'codefile.closure', 'codefile.inner') required_xml_attrs = 'name', 'cname', 'qualified_name' assert all([f in xml_funcs for f in funcnames]) spam, ham, eggs = [xml_funcs[funcname] for funcname in funcnames] diff --git a/Cython/Debugger/Tests/codefile b/Cython/Debugger/Tests/codefile index 53d6518d..6bbae82b 100644 --- a/Cython/Debugger/Tests/codefile +++ b/Cython/Debugger/Tests/codefile @@ -21,16 +21,34 @@ def spam(a=0): puts("spam") os.path.join("foo", "bar") some_c_function() + +cpdef eggs(): + pass cdef ham(): pass -cpdef eggs(): - pass - cdef class SomeClass(object): def spam(self): pass +def closure(): + a = 1 + def inner(): + b = 2 + # access closed over variables + print a, b + return inner + +def closure_without_closing_variables(): + a = 1 + def inner2(): + b = 2 + print b + return inner2 + +closure()() +closure_without_closing_variables()() + spam() print "bye!" diff --git a/Cython/Debugger/Tests/test_libcython_in_gdb.py b/Cython/Debugger/Tests/test_libcython_in_gdb.py index 2c9e7ef5..6af4183d 100644 --- a/Cython/Debugger/Tests/test_libcython_in_gdb.py +++ b/Cython/Debugger/Tests/test_libcython_in_gdb.py @@ -98,7 +98,7 @@ class TestDebugInformationClasses(DebugTestCase): 'codefile.SomeClass.spam') self.assertEqual(self.spam_func.module, self.module) - assert self.eggs_func.pf_cname + assert self.eggs_func.pf_cname, (self.eggs_func, self.eggs_func.pf_cname) assert not self.ham_func.pf_cname assert not self.spam_func.pf_cname assert not self.spam_meth.pf_cname @@ -135,11 +135,21 @@ class TestBreak(DebugTestCase): self.assertEqual(bp.type, gdb.BP_BREAKPOINT) assert self.spam_func.cname in bp.location assert bp.enabled - + def test_python_break(self): gdb.execute('cy break -p join') assert 'def join(' in gdb.execute('cy run', to_string=True) + def test_break_lineno(self): + beginline = 'import os' + nextline = 'cdef int c_var = 12' + + self.break_and_run(beginline) + self.lineno_equals(beginline) + step_result = gdb.execute('cy step', to_string=True) + self.lineno_equals(nextline) + assert step_result.rstrip().endswith(nextline) + class TestKilled(DebugTestCase): @@ -341,6 +351,19 @@ class TestExec(DebugTestCase): gdb.execute('cy exec some_random_var = 14') self.assertEqual('14', self.eval_command('some_random_var')) +class TestClosure(DebugTestCase): + + def test_cython_closure(self): + self.break_and_run('def inner():') + + self.assertEqual(str(self.read_var('a')), '1') + print_result = gdb.execute('cy print a', to_string=True).strip() + self.assertEqual(print_result, 'a = 1') + + def test_cython_closure_no_closing_variables(self): + self.break_and_run('def inner2():') + self.assertEqual(gdb.execute('cy locals', to_string=True), '') + _do_debug = os.environ.get('GDB_DEBUG') if _do_debug: @@ -381,7 +404,7 @@ def runtests(): returned to the parent test process. """ from Cython.Debugger.Tests import test_libpython_in_gdb - + success_libcython = run_unittest_in_module(__name__) success_libpython = run_unittest_in_module(test_libpython_in_gdb.__name__) diff --git a/Cython/Debugger/libcython.py b/Cython/Debugger/libcython.py index 09961442..7ce9c5bc 100644 --- a/Cython/Debugger/libcython.py +++ b/Cython/Debugger/libcython.py @@ -154,9 +154,6 @@ class CythonModule(object): self.lineno_c2cy = {} self.functions = {} - def qualified_name(self, varname): - return '.'.join(self.name, varname) - class CythonVariable(object): def __init__(self, name, cname, qualified_name, type, lineno): @@ -174,7 +171,8 @@ class CythonFunction(CythonVariable): pf_cname, qualified_name, lineno, - type=CObject): + type=CObject, + is_initmodule_function="False"): super(CythonFunction, self).__init__(name, cname, qualified_name, @@ -182,6 +180,7 @@ class CythonFunction(CythonVariable): lineno) self.module = module self.pf_cname = pf_cname + self.is_initmodule_function = is_initmodule_function == "True" self.locals = {} self.arguments = [] self.step_into_functions = set() @@ -243,7 +242,8 @@ class CythonBase(object): pyframeobject = libpython.Frame(frame).get_pyop() if not pyframeobject: - raise gdb.GdbError('Unable to read information on python frame') + raise gdb.GdbError( + 'Unable to read information on python frame') filename = pyframeobject.filename() lineno = pyframeobject.current_line_num() @@ -752,14 +752,20 @@ class CyBreak(CythonCommand): def _break_funcname(self, funcname): func = self.cy.functions_by_qualified_name.get(funcname) + + if func and func.is_initmodule_function: + func = None + break_funcs = [func] if not func: - funcs = self.cy.functions_by_name.get(funcname) + funcs = self.cy.functions_by_name.get(funcname) or [] + funcs = [f for f in funcs if not f.is_initmodule_function] + if not funcs: gdb.execute('break ' + funcname) return - + if len(funcs) > 1: # multiple functions, let the user pick one print 'There are multiple such functions:' @@ -811,18 +817,28 @@ class CyBreak(CythonCommand): @dont_suppress_errors def complete(self, text, word): - names = self.cy.functions_by_qualified_name + # Filter init-module functions (breakpoints can be set using + # modulename:linenumber). + names = [n for n, L in self.cy.functions_by_name.iteritems() + if any(not f.is_initmodule_function for f in L)] + qnames = [n for n, f in self.cy.functions_by_qualified_name.iteritems() + if not f.is_initmodule_function] + if parameters.complete_unqualified: - names = itertools.chain(names, self.cy.functions_by_name) + all_names = itertools.chain(qnames, names) + else: + all_names = qnames words = text.strip().split() - if words and '.' in words[-1]: - lastword = words[-1] - compl = [n for n in self.cy.functions_by_qualified_name - if n.startswith(lastword)] - else: + if not words or '.' not in words[-1]: + # complete unqualified seen = set(text[:-len(word)].split()) - return [n for n in names if n.startswith(word) and n not in seen] + return [n for n in all_names + if n.startswith(word) and n not in seen] + + # complete qualified name + lastword = words[-1] + compl = [n for n in qnames if n.startswith(lastword)] if len(lastword) > len(word): # readline sees something (e.g. a '.') as a word boundary, so don't @@ -862,6 +878,7 @@ class CythonInfo(CythonBase, libpython.PythonInfo): def runtime_break_functions(self): if self.is_cython_function(): return self.get_cython_function().step_into_functions + return () def static_break_functions(self): result = ['PyEval_EvalFrameEx'] @@ -1091,7 +1108,13 @@ class CyLocals(CythonCommand): @dispatch_on_frame(c_command='info locals', python_command='py-locals') def invoke(self, args, from_tty): - local_cython_vars = self.get_cython_function().locals + cython_function = self.get_cython_function() + + if cython_function.is_initmodule_function: + self.cy.globals.invoke(args, from_tty) + return + + local_cython_vars = cython_function.locals max_name_length = len(max(local_cython_vars, key=len)) for name, cyvar in sorted(local_cython_vars.iteritems(), key=sortkey): if self.is_initialized(self.get_cython_function(), cyvar.name): diff --git a/Cython/Debugger/libpython.py b/Cython/Debugger/libpython.py index ee696afe..6df87f85 100644 --- a/Cython/Debugger/libpython.py +++ b/Cython/Debugger/libpython.py @@ -1906,8 +1906,6 @@ class ExecutionControlCommandBase(gdb.Command): for location in all_locations: result = gdb.execute('break %s' % location, to_string=True) yield re.search(r'Breakpoint (\d+)', result).group(1) - - def delete_breakpoints(self, breakpoint_list): for bp in breakpoint_list: