From f1e63d52189d9bfe148e420185cc864e6bdfc240 Mon Sep 17 00:00:00 2001 From: Mark Florisson Date: Thu, 28 Oct 2010 00:23:28 +0200 Subject: [PATCH] Tests! (run: python runtests.py Cython.Tests.TestStringIOTree \ Cython.Debugger.Tests.TestLibCython \ Cython.Compiler.Tests.TestParseTreeTransforms) --HG-- rename : Cython/Debugger/cygdb.py => Cython/Debugger/Cygdb.py --- Cython/Compiler/Main.py | 5 +- Cython/Compiler/ParseTreeTransforms.py | 2 +- .../Compiler/Tests/TestParseTreeTransforms.py | 66 +++ Cython/Debugger/{cygdb.py => Cygdb.py} | 29 +- Cython/Debugger/Tests/TestLibCython.py | 128 +++++ Cython/Debugger/Tests/__init__.py | 0 .../Debugger/Tests/test_libcython_in_gdb.py | 151 ++++++ Cython/Debugger/libcython.py | 472 +++++++++++------- Cython/Debugger/libpython.py | 72 +-- Cython/StringIOTree.py | 3 +- bin/cygdb | 2 +- cygdb.py | 2 +- runtests.py | 9 +- 13 files changed, 713 insertions(+), 228 deletions(-) rename Cython/Debugger/{cygdb.py => Cygdb.py} (76%) create mode 100644 Cython/Debugger/Tests/TestLibCython.py create mode 100644 Cython/Debugger/Tests/__init__.py create mode 100644 Cython/Debugger/Tests/test_libcython_in_gdb.py diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index b84e13ff..6275fb12 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -181,9 +181,10 @@ class Context(object): test_support.append(TreeAssertVisitor()) if options.debug: - from Cython.Debugger import debug_output + from Cython.Debugger import DebugWriter from ParseTreeTransforms import DebugTransform - self.debug_outputwriter = debug_output.CythonDebugWriter(options) + self.debug_outputwriter = DebugWriter.CythonDebugWriter( + options.output_dir) debug_transform = [DebugTransform(self)] else: debug_transform = [] diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index bb64a8db..cfc9f90c 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1523,7 +1523,7 @@ class DebugTransform(CythonTransform): def serialize_local_variables(self, entries): for entry in entries.values(): if entry.type.is_pyobject: - vartype = 'PyObject' + vartype = 'PythonObject' else: vartype = 'CObject' diff --git a/Cython/Compiler/Tests/TestParseTreeTransforms.py b/Cython/Compiler/Tests/TestParseTreeTransforms.py index bda4896f..7857986c 100644 --- a/Cython/Compiler/Tests/TestParseTreeTransforms.py +++ b/Cython/Compiler/Tests/TestParseTreeTransforms.py @@ -1,6 +1,12 @@ +import os + +from Cython.Debugger import DebugWriter +from Cython.Compiler import Main +from Cython.Compiler import CmdLine from Cython.TestUtils import TransformTest from Cython.Compiler.ParseTreeTransforms import * from Cython.Compiler.Nodes import * +from Cython.Debugger.Tests import TestLibCython class TestNormalizeTree(TransformTest): def test_parserbehaviour_is_what_we_coded_for(self): @@ -139,6 +145,66 @@ class TestWithTransform(object): # (TransformTest): # Disabled! """, t) + +class TestDebugTransform(TestLibCython.DebuggerTestCase): + + def elem_hasattrs(self, elem, attrs): + return all(attr in elem.attrib for attr in attrs) + + def test_debug_info(self): + try: + assert os.path.exists(self.debug_dest) + + t = DebugWriter.etree.parse(self.debug_dest) + # the xpath of the standard ElementTree is primitive, don't use + # anything fancy + L = list(t.find('/Module/Globals')) + # assertTrue is retarded, use the normal assert statement + assert L + xml_globals = dict((e.attrib['name'], e.attrib['type']) for e in L) + self.assertEqual(len(L), len(xml_globals)) + + L = list(t.find('/Module/Functions')) + assert L + xml_funcs = dict((e.attrib['qualified_name'], e) for e in L) + self.assertEqual(len(L), len(xml_funcs)) + + # test globals + self.assertEqual('CObject', xml_globals.get('c_var')) + self.assertEqual('PythonObject', xml_globals.get('python_var')) + + # test functions + funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs' + required_xml_attrs = 'name', 'cname', 'qualified_name' + assert all(f in xml_funcs for f in funcnames) + spam, ham, eggs = (xml_funcs[funcname] for funcname in funcnames) + + self.assertEqual(spam.attrib['name'], 'spam') + self.assertNotEqual('spam', spam.attrib['cname']) + assert self.elem_hasattrs(spam, required_xml_attrs) + + # test locals of functions + spam_locals = list(spam.find('Locals')) + assert spam_locals + spam_locals.sort(key=lambda e: e.attrib['name']) + names = [e.attrib['name'] for e in spam_locals] + self.assertEqual(list('abcd'), names) + assert self.elem_hasattrs(spam_locals[0], required_xml_attrs) + + # test arguments of functions + spam_arguments = list(spam.find('Arguments')) + assert spam_arguments + self.assertEqual(1, len(list(spam_arguments))) + + # test step-into functions + spam_stepinto = list(spam.find('StepIntoFunctions')) + assert spam_stepinto + self.assertEqual(1, len(list(spam_stepinto))) + self.assertEqual('puts', list(spam_stepinto)[0].attrib['name']) + except: + print open(self.debug_dest).read() + raise + if __name__ == "__main__": import unittest diff --git a/Cython/Debugger/cygdb.py b/Cython/Debugger/Cygdb.py similarity index 76% rename from Cython/Debugger/cygdb.py rename to Cython/Debugger/Cygdb.py index f9d6f429..67f5be91 100644 --- a/Cython/Debugger/cygdb.py +++ b/Cython/Debugger/Cygdb.py @@ -20,36 +20,33 @@ import subprocess def usage(): print("Usage: cygdb [PATH GDB_ARGUMENTS]") -def main(gdb_argv=[], import_libpython=False, path_to_debug_info=os.curdir): - """ - Start the Cython debugger. This tells gdb to import the Cython and Python - extensions (libpython.py and libcython.py) and it enables gdb's pending - breakpoints - - import_libpython indicates whether we should just 'import libpython', - or import it from Cython.Debugger - - path_to_debug_info is the path to the cython_debug directory - """ +def make_command_file(path_to_debug_info): debug_files = glob.glob( os.path.join(path_to_debug_info, 'cython_debug/cython_debug_info_*')) if not debug_files: usage() sys.exit('No debug files were found in %s. Aborting.' % ( - os.path.abspath(path_to_debug_info))) + os.path.abspath(path_to_debug_info))) fd, tempfilename = tempfile.mkstemp() f = os.fdopen(fd, 'w') f.write('set breakpoint pending on\n') f.write('python from Cython.Debugger import libcython\n') - if import_libpython: - f.write('python import libpython') - else: - f.write('python from Cython.Debugger import libpython\n') f.write('\n'.join('cy import %s\n' % fn for fn in debug_files)) f.close() + + return tempfilename +def main(gdb_argv=[], path_to_debug_info=os.curdir): + """ + Start the Cython debugger. This tells gdb to import the Cython and Python + extensions (libpython.py and libcython.py) and it enables gdb's pending + breakpoints + + path_to_debug_info is the path to the cython_debug directory + """ + tempfilename = make_command_file(path_to_debug_info) p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv) while True: try: diff --git a/Cython/Debugger/Tests/TestLibCython.py b/Cython/Debugger/Tests/TestLibCython.py new file mode 100644 index 00000000..f83b1f84 --- /dev/null +++ b/Cython/Debugger/Tests/TestLibCython.py @@ -0,0 +1,128 @@ +import os +import re +import sys +import uuid +import shutil +import textwrap +import unittest +import tempfile +import subprocess +import distutils.core +from distutils import sysconfig + +import Cython.Distutils.extension +from Cython.Debugger import Cygdb as cygdb + + +class DebuggerTestCase(unittest.TestCase): + + def setUp(self): + """ + Run gdb and have cygdb import the debug information from the code + defined in TestParseTreeTransforms's setUp method + """ + self.tempdir = tempfile.mkdtemp() + self.destfile = os.path.join(self.tempdir, 'codefile.pyx') + self.debug_dest = os.path.join(self.tempdir, + 'cython_debug', + 'cython_debug_info_codefile') + + code = textwrap.dedent(""" + cdef extern from "stdio.h": + int puts(char *s) + + cdef int c_var = 0 + python_var = 0 + + def spam(a=0): + cdef: + int b, c, d + + b = c = d = 0 + + b = 1 + c = 2 + d = 3 + int(10) + puts("spam") + + cdef ham(): + pass + + cpdef eggs(): + pass + + cdef class SomeClass(object): + def spam(self): + pass + + spam() + """) + + self.cwd = os.getcwd() + os.chdir(self.tempdir) + + open(self.destfile, 'w').write(code) + + ext = Cython.Distutils.extension.Extension( + 'codefile', + ['codefile.pyx'], + pyrex_debug=True) + + distutils.core.setup( + script_args=['build_ext', '--inplace'], + ext_modules=[ext], + cmdclass=dict(build_ext=Cython.Distutils.build_ext) + ) + + def tearDown(self): + os.chdir(self.cwd) + shutil.rmtree(self.tempdir) + + +class GdbDebuggerTestCase(DebuggerTestCase): + def setUp(self): + super(GdbDebuggerTestCase, self).setUp() + + self.gdb_command_file = cygdb.make_command_file(self.tempdir) + with open(self.gdb_command_file, 'a') as f: + f.write('python ' + 'from Cython.Debugger.Tests import test_libcython_in_gdb;' + 'test_libcython_in_gdb.main()\n') + + args = ['gdb', '-batch', '-x', self.gdb_command_file, '-n', '--args', + sys.executable, '-c', 'import codefile'] + + paths = [] + path = os.environ.get('PYTHONPATH') + if path: + paths.append(path) + paths.append(os.path.dirname(os.path.dirname( + os.path.abspath(Cython.__file__)))) + env = dict(os.environ, PYTHONPATH=os.pathsep.join(paths)) + self.p = subprocess.Popen( + args, + stdout=open(os.devnull, 'w'), + stderr=subprocess.PIPE, + env=env) + + def tearDown(self): + super(GdbDebuggerTestCase, self).tearDown() + self.p.stderr.close() + self.p.wait() + os.remove(self.gdb_command_file) + + +class TestAll(GdbDebuggerTestCase): + + def test_all(self): + out, err = self.p.communicate() + border = '*' * 30 + start = '%s v INSIDE GDB v %s' % (border, border) + end = '%s ^ INSIDE GDB ^ %s' % (border, border) + errmsg = '\n%s\n%s%s' % (start, err, end) + self.assertEquals(0, self.p.wait(), errmsg) + sys.stderr.write(err) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/Cython/Debugger/Tests/__init__.py b/Cython/Debugger/Tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Cython/Debugger/Tests/test_libcython_in_gdb.py b/Cython/Debugger/Tests/test_libcython_in_gdb.py new file mode 100644 index 00000000..5a065b68 --- /dev/null +++ b/Cython/Debugger/Tests/test_libcython_in_gdb.py @@ -0,0 +1,151 @@ +""" +Tests that run inside GDB. + +Note: debug information is already imported by the file generated by +Cython.Debugger.Cygdb.make_command_file() +""" + +import sys + +# First, fix gdb's python. Make sure to do this before importing modules +# that bind output streams as default parameters + +# for some reason sys.argv is missing in gdb +sys.argv = ['gdb'] + +# Allow gdb to capture output, but have errors end up on stderr +# sys.stdout = sys.__stdout__ +sys.stderr = sys.__stderr__ + +import os +import warnings +import unittest +import traceback +from test import test_support + +import gdb + +from Cython.Debugger import libcython + +class DebugTestCase(unittest.TestCase): + + def __init__(self, name): + super(DebugTestCase, self).__init__(name) + self.cy = libcython.cy + self.module = libcython.cy.cython_namespace['codefile'] + self.spam_func, self.spam_meth = libcython.cy.functions_by_name['spam'] + self.ham_func = libcython.cy.functions_by_qualified_name[ + 'codefile.ham'] + self.eggs_func = libcython.cy.functions_by_qualified_name[ + 'codefile.eggs'] + + def read_var(self, varname): + return gdb.parse_and_eval('$cy_cname("%s")' % varname) + + def local_info(self): + return gdb.execute('info locals', to_string=True) + +class TestDebugInformationClasses(DebugTestCase): + + + def test_CythonModule(self): + "test that debug information was parsed properly into data structures" + self.assertEqual(self.module.name, 'codefile') + global_vars = ('c_var', 'python_var', 'SomeClass', '__name__', + '__builtins__', '__doc__', '__file__') + assert set(global_vars).issubset(self.module.globals) + + def test_CythonVariable(self): + module_globals = self.module.globals + c_var = module_globals['c_var'] + python_var = module_globals['python_var'] + self.assertEqual(c_var.type, libcython.CObject) + self.assertEqual(python_var.type, libcython.PythonObject) + self.assertEqual(c_var.qualified_name, 'codefile.c_var') + + def test_CythonFunction(self): + self.assertEqual(self.spam_func.qualified_name, 'codefile.spam') + self.assertEqual(self.spam_meth.qualified_name, + 'codefile.SomeClass.spam') + self.assertEqual(self.spam_func.module, self.module) + + assert self.eggs_func.pf_cname + assert not self.ham_func.pf_cname + assert not self.spam_func.pf_cname + assert not self.spam_meth.pf_cname + + self.assertEqual(self.spam_func.type, libcython.CObject) + self.assertEqual(self.ham_func.type, libcython.CObject) + + self.assertEqual(self.spam_func.arguments, ['a']) + self.assertEqual(self.spam_func.step_into_functions, set(['puts'])) + + self.assertEqual(self.spam_func.lineno, 8) + self.assertEqual(sorted(self.spam_func.locals), list('abcd')) + + +class TestParameters(unittest.TestCase): + + def test_parameters(self): + assert libcython.parameters.colorize_code + gdb.execute('set cy_colorize_code off') + assert not libcython.parameters.colorize_code + + +class TestBreak(DebugTestCase): + + def test_break(self): + result = gdb.execute('cy break codefile.spam', to_string=True) + assert self.spam_func.cname in result + + self.assertEqual(len(gdb.breakpoints()), 1) + bp, = gdb.breakpoints() + self.assertEqual(bp.type, gdb.BP_BREAKPOINT) + self.assertEqual(bp.location, self.spam_func.cname) + assert bp.enabled + + +class TestStep(DebugTestCase): + + def test_step(self): + # Note: breakpoint for spam is still set + gdb.execute('run') + + gdb.execute('cy step', to_string=True) # b = c = d = 0 + gdb.execute('cy step', to_string=True) # b = 1 + self.assertEqual(self.read_var('b'), 1, self.local_info()) + self.assertRaises(RuntimeError, self.read_var('b')) + gdb.execute('cont') + + +class TestNext(DebugTestCase): + + def test_next(self): + pass + +def main(): + try: + # unittest.main(module=__import__(__name__, fromlist=[''])) + try: + gdb.lookup_type('PyModuleObject') + except RuntimeError: + msg = ("Unable to run tests, Python was not compiled with " + "debugging information. Either compile python with " + "-g or get a debug build (configure with --with-pydebug).") + warnings.warn(msg) + else: + tests = ( + TestDebugInformationClasses, + TestParameters, + TestBreak, + TestStep + ) + # test_support.run_unittest(tests) + test_loader = unittest.TestLoader() + suite = unittest.TestSuite( + [test_loader.loadTestsFromTestCase(cls) for cls in tests]) + + unittest.TextTestRunner(verbosity=1).run(suite) + except Exception: + traceback.print_exc() + os._exit(1) \ No newline at end of file diff --git a/Cython/Debugger/libcython.py b/Cython/Debugger/libcython.py index c2369fa1..b0aa69f6 100644 --- a/Cython/Debugger/libcython.py +++ b/Cython/Debugger/libcython.py @@ -45,30 +45,17 @@ else: from Cython.Debugger import libpython - -# Cython module namespace -cython_namespace = {} - # C or Python type -CObject = object() -PythonObject = object() - -# maps (unique) qualified function names (e.g. -# cythonmodule.ClassName.method_name) to the CythonFunction object -functions_by_qualified_name = {} - -# unique cnames of Cython functions -functions_by_cname = {} - -# map function names like method_name to a list of all such CythonFunction -# objects -functions_by_name = collections.defaultdict(list) +CObject = 'CObject' +PythonObject = 'PythonObject' +_data_types = dict(CObject=CObject, PythonObject=PythonObject) _filesystemencoding = sys.getfilesystemencoding() or 'UTF-8' # decorators def dont_suppress_errors(function): + "*sigh*, readline" @functools.wraps(function) def wrapper(*args, **kwargs): try: @@ -79,17 +66,31 @@ def dont_suppress_errors(function): return wrapper -def default_selected_gdb_frame(function): - @functools.wraps(function) - def wrapper(self, frame=None, **kwargs): - frame = frame or gdb.selected_frame() - if frame.name() is None: - raise NoFunctionNameInFrameError() - - return function(self, frame) - return wrapper - +def default_selected_gdb_frame(err=True): + def decorator(function): + @functools.wraps(function) + def wrapper(self, frame=None, **kwargs): + try: + frame = frame or gdb.selected_frame() + except RuntimeError: + raise gdb.GdbError("No frame is currently selected.") + + if err and frame.name() is None: + raise NoFunctionNameInFrameError() + + return function(self, frame, **kwargs) + return wrapper + return decorator +def require_cython_frame(function): + @functools.wraps(function) + def wrapper(self, *args, **kwargs): + if not self.is_cython_function(): + raise gdb.GdbError('Selected frame does not correspond with a ' + 'Cython function we know about.') + return function(self, *args, **kwargs) + return wrapper + # Classes that represent the debug information # Don't rename the parameters of these classes, they come directly from the XML @@ -97,12 +98,14 @@ class CythonModule(object): def __init__(self, module_name, filename): self.name = module_name self.filename = filename - self.functions = {} self.globals = {} # {cython_lineno: min(c_linenos)} self.lineno_cy2c = {} # {c_lineno: cython_lineno} self.lineno_c2cy = {} + + def qualified_name(self, varname): + return '.'.join(self.name, varname) class CythonVariable(object): @@ -127,15 +130,78 @@ class CythonFunction(CythonVariable): type) self.module = module self.pf_cname = pf_cname - self.lineno = lineno + self.lineno = int(lineno) self.locals = {} self.arguments = [] self.step_into_functions = set() + +# General purpose classes + +class CythonBase(object): + + @default_selected_gdb_frame(err=False) + def is_cython_function(self, frame): + return frame.name() in self.cy.functions_by_cname + + @default_selected_gdb_frame(err=False) + def is_python_function(self, frame): + return frame.name() == 'PyEval_EvalFrameEx' + + @default_selected_gdb_frame() + def is_python_function(self, frame): + return libpython.Frame(frame).is_evalframeex() + + @default_selected_gdb_frame() + def get_c_function_name(self, frame): + return frame.name() + + @default_selected_gdb_frame() + def get_c_lineno(self, frame): + return frame.find_sal().line + + @default_selected_gdb_frame() + def get_cython_function(self, frame): + result = self.cy.functions_by_cname.get(frame.name()) + if result is None: + raise NoCythonFunctionInFrameError() + + return result + + @default_selected_gdb_frame() + def get_cython_lineno(self, frame): + cyfunc = self.get_cython_function(frame) + return cyfunc.module.lineno_c2cy.get(self.get_c_lineno(frame)) + + @default_selected_gdb_frame() + def get_source_desc(self, frame): + filename = lineno = lexer = None + if self.is_cython_function(): + filename = self.get_cython_function(frame).module.filename + lineno = self.get_cython_lineno(frame) + if pygments: + lexer = pygments.lexers.CythonLexer() + elif self.is_python_function(): + pyframeobject = libpython.Frame(frame).get_pyop() + + if not pyframeobject: + raise GdbError('Unable to read information on python frame') + + filename = pyframeobject.filename() + lineno = pyframeobject.current_line_num() + if pygments: + lexer = pygments.lexers.PythonLexer() + + return SourceFileDescriptor(filename, lexer), lineno + + @default_selected_gdb_frame() + def get_source_line(self, frame): + source_desc, lineno = self.get_source_desc() + return source_desc.get_source(lineno) + class SourceFileDescriptor(object): - def __init__(self, filename, lineno, lexer, formatter=None): + def __init__(self, filename, lexer, formatter=None): self.filename = filename - self.lineno = lineno self.lexer = lexer self.formatter = formatter @@ -143,8 +209,8 @@ class SourceFileDescriptor(object): return self.filename is not None def lex(self, code): - if pygments and parameter.colorize_code: - bg = parameter.terminal_background.value + if pygments and self.lexer and parameters.colorize_code: + bg = parameters.terminal_background.value if self.formatter is None: formatter = pygments.formatters.TerminalFormatter(bg=bg) else: @@ -154,25 +220,31 @@ class SourceFileDescriptor(object): return code - def get_source(self, start=0, stop=None, lex_source=True): - # todo: have it detect the source file's encoding - if not self.filename: - return 'Unable to retrieve source code' - - start = max(self.lineno + start, 0) - if stop is None: - stop = self.lineno + 1 - else: - stop = self.lineno + stop - + def _get_source(self, start, stop, lex_source, mark_line): with open(self.filename) as f: - source = itertools.islice(f, start, stop) - if lex_source: - return [self.lex(line) for line in source] + # to provide proper colouring, the entire code needs to be + # lexed + lines = self.lex(f.read()).splitlines() else: - return list(source) + lines = f + + for idx, line in enumerate(itertools.islice(lines, start - 1, stop - 1)): + if start + idx == mark_line: + prefix = '>' + else: + prefix = ' ' + + yield '%s %4d %s' % (prefix, start + idx, line) + def get_source(self, start, stop=None, lex_source=True, mark_line=0): + if not self.filename: + raise GdbError('Unable to retrieve source code') + + if stop is None: + stop = start + 1 + return '\n'.join(self._get_source(start, stop, lex_source, mark_line)) + # Errors @@ -235,32 +307,41 @@ class TerminalBackground(CythonParameter): Tell cygdb about the user's terminal background (light or dark) """ -class Parameter(object): +class CythonParameters(object): """ Simple container class that might get more functionality in the distant future (mostly to remind us that we're dealing with parameters) """ - complete_unqualified = CompleteUnqualifiedFunctionNames( - 'cy_complete_unqualified', - gdb.COMMAND_BREAKPOINTS, - gdb.PARAM_BOOLEAN, - True) - colorize_code = ColorizeSourceCode( - 'cy_colorize_code', - gdb.COMMAND_FILES, - gdb.PARAM_BOOLEAN, - True) - terminal_background = TerminalBackground( - 'cy_terminal_background_color', - gdb.COMMAND_FILES, - gdb.PARAM_STRING, - "dark") - -parameter = Parameter() + + def __init__(self): + self.complete_unqualified = CompleteUnqualifiedFunctionNames( + 'cy_complete_unqualified', + gdb.COMMAND_BREAKPOINTS, + gdb.PARAM_BOOLEAN, + True) + self.colorize_code = ColorizeSourceCode( + 'cy_colorize_code', + gdb.COMMAND_FILES, + gdb.PARAM_BOOLEAN, + True) + self.terminal_background = TerminalBackground( + 'cy_terminal_background_color', + gdb.COMMAND_FILES, + gdb.PARAM_STRING, + "dark") + +parameters = CythonParameters() + # Commands -class CythonCommand(gdb.Command): +class CythonCommand(gdb.Command, CythonBase): + """ + Base class for Cython commands + """ + + +class CyCy(CythonCommand): """ Invoke a Cython command. Available commands are: @@ -272,53 +353,49 @@ class CythonCommand(gdb.Command): cy locals cy globals cy backtrace - cy info line + cy up + cy down """ - def is_cython_function(self, frame=None): - func_name = (frame or gdb.selected_frame()).name() - return func_name is not None and func_name in functions_by_cname - - @default_selected_gdb_frame - def is_python_function(self, frame): - return libpython.Frame(frame).is_evalframeex() - - @default_selected_gdb_frame - def get_c_function_name(self, frame): - return frame.name() - - @default_selected_gdb_frame - def get_c_lineno(self, frame): - return frame.find_sal().line - - @default_selected_gdb_frame - def get_cython_function(self, frame): - result = functions_by_cname.get(frame.name()) - if result is None: - raise NoCythonFunctionInFrameError() + def __init__(self): + super(CythonCommand, self).__init__( + 'cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True) + + self.import_ = CyImport( + 'cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME) - return result - - @default_selected_gdb_frame - def get_cython_lineno(self, frame): - cyfunc = self.get_cython_function(frame) - return cyfunc.module.lineno_c2cy.get(self.get_c_lineno(frame)) - - @default_selected_gdb_frame - def get_source_desc(self, frame): - if self.is_cython_function(): - filename = self.get_cython_function(frame).module.filename - lineno = self.get_cython_lineno(frame) - lexer = pygments.lexers.CythonLexer() - else: - filename = None - lineno = -1 - lexer = None - - return SourceFileDescriptor(filename, lineno, lexer) + self.break_ = CyBreak('cy break', gdb.COMMAND_BREAKPOINTS) + self.step = CyStep('cy step', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE) + self.next = CyNext('cy next', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE) + self.list = CyList('cy list', gdb.COMMAND_FILES, gdb.COMPLETE_NONE) + self.print_ = CyPrint('cy print', gdb.COMMAND_DATA) + + self.locals = CyLocals( + 'cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE) + self.globals = CyGlobals( + 'cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE) + + self.cy_cname = CyCName('cy_cname') + + objs = (self.import_, self.break_, self.step, self.list, self.print_, + self.locals, self.globals, self.cy_cname) - -cy = CythonCommand('cy', gdb.COMMAND_NONE, gdb.COMPLETE_COMMAND, prefix=True) + for obj in objs: + obj.cy = self + + # Cython module namespace + self.cython_namespace = {} + + # maps (unique) qualified function names (e.g. + # cythonmodule.ClassName.method_name) to the CythonFunction object + self.functions_by_qualified_name = {} + + # unique cnames of Cython functions + self.functions_by_cname = {} + + # map function names like method_name to a list of all such + # CythonFunction objects + self.functions_by_name = collections.defaultdict(list) class CyImport(CythonCommand): @@ -340,7 +417,7 @@ class CyImport(CythonCommand): for module in t.getroot(): cython_module = CythonModule(**module.attrib) - cython_namespace[cython_module.name] = cython_module + self.cy.cython_namespace[cython_module.name] = cython_module for variable in module.find('Globals'): d = variable.attrib @@ -349,15 +426,14 @@ class CyImport(CythonCommand): for function in module.find('Functions'): cython_function = CythonFunction(module=cython_module, **function.attrib) - cython_module.functions[cython_function.name] = \ - cython_function # update the global function mappings - functions_by_name[cython_function.name].append( + self.cy.functions_by_name[cython_function.name].append( cython_function) - functions_by_qualified_name[ + self.cy.functions_by_qualified_name[ cython_function.qualified_name] = cython_function - functions_by_cname[cython_function.cname] = cython_function + self.cy.functions_by_cname[ + cython_function.cname] = cython_function for local in function.find('Locals'): d = local.attrib @@ -377,9 +453,6 @@ class CyImport(CythonCommand): for c_lineno in c_linenos: cython_module.lineno_c2cy[c_lineno] = cython_lineno - -cy.import_ = CyImport('cy import', gdb.COMMAND_STATUS, gdb.COMPLETE_FILENAME) - class CyBreak(CythonCommand): """ @@ -399,20 +472,21 @@ class CyBreak(CythonCommand): def _break_pyx(self, name): modulename, _, lineno = name.partition(':') lineno = int(lineno) - cython_module = cython_namespace[modulename] + cython_module = self.cy.cython_namespace[modulename] if lineno in cython_module.lineno_cy2c: c_lineno = cython_module.lineno_cy2c[lineno] breakpoint = '%s:%s' % (cython_module.name, c_lineno) gdb.execute('break ' + breakpoint) else: - sys.stderr.write("Not a valid line number (does it contain actual code?)\n") + raise GdbError("Not a valid line number. " + "Does it contain actual code?") def _break_funcname(self, funcname): - func = functions_by_qualified_name.get(funcname) + func = self.cy.functions_by_qualified_name.get(funcname) break_funcs = [func] if not func: - funcs = functions_by_name.get(funcname) + funcs = self.cy.functions_by_name.get(funcname) if not funcs: gdb.execute('break ' + funcname) return @@ -459,13 +533,13 @@ class CyBreak(CythonCommand): @dont_suppress_errors def complete(self, text, word): - names = functions_by_qualified_name - if parameter.complete_unqualified: - names = itertools.chain(names, functions_by_name) + names = self.cy.functions_by_qualified_name + if parameters.complete_unqualified: + names = itertools.chain(names, self.cy.functions_by_name) words = text.strip().split() if words and '.' in words[-1]: - compl = [n for n in functions_by_qualified_name + compl = [n for n in self.cy.functions_by_qualified_name if n.startswith(lastword)] else: seen = set(text[:-len(word)].split()) @@ -479,27 +553,37 @@ class CyBreak(CythonCommand): return compl -cy.break_ = CyBreak('cy break', gdb.COMMAND_BREAKPOINTS) + +class CodeStepperMixin(object): + + def init_stepping(self): + self.cython_func = self.get_cython_function() + self.beginline = self.get_cython_lineno() + self.curframe = gdb.selected_frame() + + def next_step(self, command): + "returns whether to continue stepping" + result = gdb.execute(command, to_string=True) + newframe = gdb.selected_frame() + + c1 = result.startswith('Breakpoint') + c2 = (newframe == self.curframe and + self.get_cython_lineno() > self.beginline) + return not c1 and not c2 + + def end_stepping(self): + sys.stdout.write(self.get_source_line()) -class CyStep(CythonCommand): +class CyStep(CythonCommand, CodeStepperMixin): - def step(self, from_tty=True, nsteps=1): + def step(self, nsteps=1): for nthstep in xrange(nsteps): - cython_func = self.get_cython_function() - beginline = self.get_cython_lineno() - curframe = gdb.selected_frame() - - while True: - result = gdb.execute('step', False, True) - if result.startswith('Breakpoint'): - break + self.init_stepping() + + while self.next_step('step'): newframe = gdb.selected_frame() - if newframe == curframe: - # still in the same function - if self.get_cython_lineno() > beginline: - break - else: + if newframe != self.curframe: # we entered a function funcname = self.get_c_function_name(newframe) if (self.is_cython_function() or @@ -507,64 +591,73 @@ class CyStep(CythonCommand): funcname in cython_function.step_into_functions): break - line, = self.get_source_desc().get_source() - sys.stdout.write(line) + self.end_stepping() def invoke(self, steps, from_tty): if self.is_cython_function(): if steps: - self.step(from_tty, int(steps)) + self.step(int(steps)) else: - self.step(from_tty) + self.step() else: - gdb.execute('step ' + steps) + gdb.execute('step ' + steps, from_tty) -cy.step = CyStep('cy step', gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE) + +class CyNext(CythonCommand, CodeStepperMixin): + + def next(self, nsteps=1): + for nthstep in xrange(nsteps): + self.init_stepping() + + while self.next_step('next'): + pass + + self.end_stepping() + + def invoke(self, steps, from_tty): + if self.is_cython_function(): + if steps: + self.next(int(steps)) + else: + self.next() + else: + gdb.execute('next ' + steps, from_tty) class CyList(CythonCommand): def invoke(self, _, from_tty): - sd = self.get_source_desc() - it = enumerate(sd.get_source(-5, +5)) - sys.stdout.write( - ''.join('%4d %s' % (sd.lineno + i, line) for i, line in it)) - -cy.list = CyList('cy list', gdb.COMMAND_FILES, gdb.COMPLETE_NONE) + sd, lineno = self.get_source_desc() + source = sd.get_source(lineno - 5, lineno + 5, mark_line=lineno) + print source class CyPrint(CythonCommand): """ Print a Cython variable using 'cy-print x' or 'cy-print module.function.x' """ - def invoke(self, name, from_tty): - cname = None - if self.is_cython_function(): - cython_function = self.get_cython_function() - if name in cython_function.locals: - cname = cython_function.locals[name].cname - elif name in cython_function.module.globals: - cname = cython_function.module.globals[name].cname - - # let the pretty printers do the work - cname = cname or name + try: + cname = cy.cy_cname.invoke(name) + except gdb.GdbError: + cname = name + gdb.execute('print ' + cname) + def complete(self): if self.is_cython_function(): - cf = self.get_cython_function() - return list(itertools.chain(cf.locals, cf.globals)) + f = self.get_cython_function() + return list(itertools.chain(f.locals, f.globals)) return [] - -cy.print_ = CyPrint('cy print', gdb.COMMAND_DATA) class CyLocals(CythonCommand): def ns(self): return self.get_cython_function().locals - + + @require_cython_frame def invoke(self, name, from_tty): try: ns = self.ns() @@ -573,9 +666,10 @@ class CyLocals(CythonCommand): return if ns is None: - print ('Information of Cython locals could not be obtained. ' - 'Is this an actual Cython function and did you ' - "'cy import' the debug information?") + raise gdb.GdbError( + 'Information of Cython locals could not be obtained. ' + 'Is this an actual Cython function and did you ' + "'cy import' the debug information?") for var in ns.itervalues(): val = gdb.parse_and_eval(var.cname) @@ -591,6 +685,7 @@ class CyGlobals(CythonCommand): def ns(self): return self.get_cython_function().globals + @require_cython_frame def invoke(self, name, from_tty): # include globals from the debug info XML file! m = gdb.parse_and_eval('__pyx_m') @@ -610,5 +705,30 @@ class CyGlobals(CythonCommand): print d.get_truncated_repr(1000) -cy.locals = CyLocals('cy locals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE) -cy.globals = CyGlobals('cy globals', gdb.COMMAND_STACK, gdb.COMPLETE_NONE) +# Functions + +class CyCName(gdb.Function, CythonBase): + """ + Get the C name of a Cython variable. + """ + + @require_cython_frame + def invoke(self, cyname, frame=None): + frame = frame or gdb.selected_frame() + cname = None + + cyname = cyname.string() + if self.is_cython_function(frame): + cython_function = self.get_cython_function(frame) + if cyname in cython_function.locals: + cname = cython_function.locals[cyname].cname + elif cyname in cython_function.module.globals: + cname = cython_function.module.globals[cyname].cname + + if not cname: + raise gdb.GdbError('No such Cython variable: %s' % cyname) + + return cname + + +cy = CyCy() \ No newline at end of file diff --git a/Cython/Debugger/libpython.py b/Cython/Debugger/libpython.py index 06d26ed8..323ae0cf 100644 --- a/Cython/Debugger/libpython.py +++ b/Cython/Debugger/libpython.py @@ -1,4 +1,8 @@ #!/usr/bin/python + +# NOTE: this file is taken from the Python source distribution +# It can be found under Tools/gdb/libpython.py + ''' From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb to be extended with Python code e.g. for library-specific data visualizations, @@ -41,6 +45,7 @@ The module also extends gdb with some python-specific commands. ''' from __future__ import with_statement import gdb +import sys # Look up the gdb.Type for some standard types: _type_char_ptr = gdb.lookup_type('char').pointer() # char* @@ -1009,6 +1014,18 @@ class PyTypeObjectPtr(PyObjectPtr): _typename = 'PyTypeObject' +if sys.maxunicode >= 0x10000: + _unichr = unichr +else: + # Needed for proper surrogate support if sizeof(Py_UNICODE) is 2 in gdb + def _unichr(x): + if x < 0x10000: + return unichr(x) + x -= 0x10000 + ch1 = 0xD800 | (x >> 10) + ch2 = 0xDC00 | (x & 0x3FF) + return unichr(ch1) + unichr(ch2) + class PyUnicodeObjectPtr(PyObjectPtr): _typename = 'PyUnicodeObject' @@ -1025,37 +1042,36 @@ class PyUnicodeObjectPtr(PyObjectPtr): # Gather a list of ints from the Py_UNICODE array; these are either # UCS-2 or UCS-4 code points: - Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)] + if self.char_width() > 2: + Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)] + else: + # A more elaborate routine if sizeof(Py_UNICODE) is 2 in the + # inferior process: we must join surrogate pairs. + Py_UNICODEs = [] + i = 0 + limit = safety_limit(field_length) + while i < limit: + ucs = int(field_str[i]) + i += 1 + if ucs < 0xD800 or ucs >= 0xDC00 or i == field_length: + Py_UNICODEs.append(ucs) + continue + # This could be a surrogate pair. + ucs2 = int(field_str[i]) + if ucs2 < 0xDC00 or ucs2 > 0xDFFF: + continue + code = (ucs & 0x03FF) << 10 + code |= ucs2 & 0x03FF + code += 0x00010000 + Py_UNICODEs.append(code) + i += 1 # Convert the int code points to unicode characters, and generate a - # local unicode instance: - result = u''.join([unichr(ucs) for ucs in Py_UNICODEs]) + # local unicode instance. + # This splits surrogate pairs if sizeof(Py_UNICODE) is 2 here (in gdb). + result = u''.join([_unichr(ucs) for ucs in Py_UNICODEs]) return result - def write_repr(self, out, visited): - proxy = self.proxyval(visited) - if self.char_width() == 2: - # sizeof(Py_UNICODE)==2: join surrogates - proxy2 = [] - i = 0 - while i < len(proxy): - ch = proxy[i] - i += 1 - if (i < len(proxy) - and 0xD800 <= ord(ch) < 0xDC00 \ - and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): - # Get code point from surrogate pair - ch2 = proxy[i] - code = (ord(ch) & 0x03FF) << 10 - code |= ord(ch2) & 0x03FF - code += 0x00010000 - i += 1 - proxy2.append(unichr(code)) - else: - proxy2.append(ch) - proxy = u''.join(proxy2) - out.write(repr(proxy)) - def int_from_int(gdbval): return int(str(gdbval)) @@ -1431,4 +1447,4 @@ class PyLocals(gdb.Command): % (pyop_name.proxyval(set()), pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) -PyLocals() \ No newline at end of file +PyLocals() diff --git a/Cython/StringIOTree.py b/Cython/StringIOTree.py index 80bf8351..0d4012bf 100644 --- a/Cython/StringIOTree.py +++ b/Cython/StringIOTree.py @@ -16,7 +16,6 @@ class StringIOTree(object): def getvalue(self): content = [x.getvalue() for x in self.prepended_children] content.append(self.stream.getvalue()) - print self.linenumber_map() return "".join(content) def copyto(self, target): @@ -33,6 +32,8 @@ class StringIOTree(object): # itself is empty -- this makes it ready for insertion if self.stream.tell(): self.prepended_children.append(StringIOTree(self.stream)) + self.prepended_children[-1].markers = self.markers + self.markers = [] self.stream = StringIO() self.write = self.stream.write diff --git a/bin/cygdb b/bin/cygdb index cad7d089..7a7f5158 100755 --- a/bin/cygdb +++ b/bin/cygdb @@ -2,7 +2,7 @@ import sys -from Cython.Debugger import cygdb +from Cython.Debugger import Cygdb as cygdb if __name__ == '__main__': if len(sys.argv) > 1: diff --git a/cygdb.py b/cygdb.py index cad7d089..7a7f5158 100644 --- a/cygdb.py +++ b/cygdb.py @@ -2,7 +2,7 @@ import sys -from Cython.Debugger import cygdb +from Cython.Debugger import Cygdb as cygdb if __name__ == '__main__': if len(sys.argv) > 1: diff --git a/runtests.py b/runtests.py index 89d22793..61168dd8 100644 --- a/runtests.py +++ b/runtests.py @@ -606,8 +606,13 @@ def collect_doctests(path, module_prefix, suite, selectors): def package_matches(dirname): return dirname not in ("Mac", "Distutils", "Plex") def file_matches(filename): - return (filename.endswith(".py") and not ('~' in filename - or '#' in filename or filename.startswith('.'))) + filename, ext = os.path.splitext(filename) + blacklist = ('libcython', 'libpython', 'test_libcython_in_gdb') + return (ext == '.py' and not + '~' in filename and not + '#' in filename and not + filename.startswith('.') and not + filename in blacklist) import doctest, types for dirpath, dirnames, filenames in os.walk(path): parentname = os.path.split(dirpath)[-1] -- 2.26.2