Merge branch 'master' of https://github.com/markflorisson88/cython into markflorisson88-master

Cython/Compiler/Main.py

@@ -18,7 +18,11 @@ from time import time
 
 import Code
 import Errors
-import Parsing
+# Do not import Parsing here, import it when needed, because Parsing imports
+# Nodes, which globally needs debug command line options initialized to set a
+# conditional metaclass. These options are processed by CmdLine called from 
+# main() in this file.
+# import Parsing
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
 from Errors import PyrexError, CompileError, InternalError, AbortError, error, warning
@@ -496,6 +500,7 @@ class Context(object):
         try:
             f = Utils.open_source_file(source_filename, "rU")
             try:
+                import Parsing
                 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
                                  scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)

Cython/Compiler/ParseTreeTransforms.py

@@ -1591,6 +1591,10 @@ class DebugTransform(CythonTransform):
         #self.c_output_file = options.output_file 
         self.c_output_file = result.c_file
         
+        # Closure support, basically treat nested functions as if the AST were
+        # never nested
+        self.nested_funcdefs = []
+        
         # tells visit_NameNode whether it should register step-into functions
         self.register_stepinto = False
         
@@ -1605,7 +1609,16 @@ class DebugTransform(CythonTransform):
         
         # serialize functions
         self.tb.start('Functions')
+        # First, serialize functions normally...
         self.visitchildren(node)
+        
+        # ... then, serialize nested functions
+        for nested_funcdef in self.nested_funcdefs:
+            self.visit_FuncDefNode(nested_funcdef)
+        
+        self.register_stepinto = True
+        self.serialize_modulenode_as_function(node)
+        self.register_stepinto = False
         self.tb.end('Functions')
         
         # 2.3 compatibility. Serialize global variables
@@ -1627,6 +1640,14 @@ class DebugTransform(CythonTransform):
     
     def visit_FuncDefNode(self, node):        
         self.visited.add(node.local_scope.qualified_name)
+
+        if getattr(node, 'is_wrapper', False):
+            return node
+
+        if self.register_stepinto:
+            self.nested_funcdefs.append(node)
+            return node
+
         # node.entry.visibility = 'extern'
         if node.py_func is None:
             pf_cname = ''
@@ -1678,6 +1699,51 @@ class DebugTransform(CythonTransform):
         self.visitchildren(node)
         return node
     
+    def serialize_modulenode_as_function(self, node):
+        """
+        Serialize the module-level code as a function so the debugger will know
+        it's a "relevant frame" and it will know where to set the breakpoint
+        for 'break modulename'.
+        """
+        name = node.full_module_name.rpartition('.')[-1]
+        
+        cname_py2 = 'init' + name
+        cname_py3 = 'PyInit_' + name
+        
+        py2_attrs = dict(
+            name=name,
+            cname=cname_py2,
+            pf_cname='',
+            # Ignore the qualified_name, breakpoints should be set using 
+            # `cy break modulename:lineno` for module-level breakpoints.
+            qualified_name='',
+            lineno='1',
+            is_initmodule_function="True",
+        )
+        
+        py3_attrs = dict(py2_attrs, cname=cname_py3)
+        
+        self._serialize_modulenode_as_function(node, py2_attrs)
+        self._serialize_modulenode_as_function(node, py3_attrs)
+    
+    def _serialize_modulenode_as_function(self, node, attrs):
+        self.tb.start('Function', attrs=attrs)
+        
+        self.tb.start('Locals')
+        self.serialize_local_variables(node.scope.entries)
+        self.tb.end('Locals')
+
+        self.tb.start('Arguments')
+        self.tb.end('Arguments')
+
+        self.tb.start('StepIntoFunctions')
+        self.register_stepinto = True
+        self.visitchildren(node)
+        self.register_stepinto = False
+        self.tb.end('StepIntoFunctions')
+        
+        self.tb.end('Function')
+    
     def serialize_local_variables(self, entries):
         for entry in entries.values():
             if entry.type.is_pyobject:
@@ -1685,9 +1751,22 @@ class DebugTransform(CythonTransform):
             else:
                 vartype = 'CObject'
             
+            if entry.from_closure:
+                # We're dealing with a closure where a variable from an outer
+                # scope is accessed, get it from the scope object.
+                cname = '%s->%s' % (Naming.cur_scope_cname, 
+                                    entry.outer_entry.cname)
+                
+                qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
+                                      entry.scope.name, 
+                                      entry.name)
+            elif entry.in_closure:
+                cname = '%s->%s' % (Naming.cur_scope_cname, 
+                                    entry.cname)
+                qname = entry.qualified_name
+            else:
                 cname = entry.cname
-            # if entry.type.is_extension_type:
-                # cname = entry.type.typeptr_cname
+                qname = entry.qualified_name
 
             if not entry.pos:
                 # this happens for variables that are not in the user's code,
@@ -1700,7 +1779,7 @@ class DebugTransform(CythonTransform):
             attrs = dict(
                 name=entry.name,
                 cname=cname,
-                qualified_name=entry.qualified_name,
+                qualified_name=qname,
                 type=vartype,
                 lineno=lineno)
             

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -182,7 +182,8 @@ class TestDebugTransform(DebuggerTestCase):
             self.assertEqual('PythonObject', xml_globals.get('python_var'))
             
             # test functions
-            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            funcnames = ('codefile.spam', 'codefile.ham', 'codefile.eggs', 
+                         'codefile.closure', 'codefile.inner')
             required_xml_attrs = 'name', 'cname', 'qualified_name'
             assert all([f in xml_funcs for f in funcnames])
             spam, ham, eggs = [xml_funcs[funcname] for funcname in funcnames]

Cython/Debugger/Tests/TestLibCython.py

@@ -16,6 +16,7 @@ from distutils import ccompiler
 
 import runtests
 import Cython.Distutils.extension
+import Cython.Distutils.build_ext
 from Cython.Debugger import Cygdb as cygdb
 
 root = os.path.dirname(os.path.abspath(__file__))
@@ -24,6 +25,10 @@ cfuncs_file = os.path.join(root, 'cfuncs.c')
 with open(codefile) as f:
     source_to_lineno = dict((line.strip(), i + 1) for i, line in enumerate(f))
 
+# Cython.Distutils.__init__ imports build_ext from build_ext which means we
+# can't access the module anymore. Get it from sys.modules instead.
+build_ext = sys.modules['Cython.Distutils.build_ext']
+
 class DebuggerTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -52,6 +57,9 @@ class DebuggerTestCase(unittest.TestCase):
             module='codefile',
         )
 
+        optimization_disabler = build_ext.Optimization()
+        optimization_disabler.disable_optimization()
+
         cython_compile_testcase = runtests.CythonCompileTestCase(
             workdir=self.tempdir,
             # we clean up everything (not only compiled files)
@@ -77,6 +85,8 @@ class DebuggerTestCase(unittest.TestCase):
             **opts
         )
 
+        optimization_disabler.restore_state()
+
         # ext = Cython.Distutils.extension.Extension(
             # 'codefile',
             # ['codefile.pyx'],
@@ -95,6 +105,7 @@ class DebuggerTestCase(unittest.TestCase):
 
 
 class GdbDebuggerTestCase(DebuggerTestCase):
+
     def setUp(self):
         super(GdbDebuggerTestCase, self).setUp()
 

Cython/Debugger/Tests/codefile

@@ -22,15 +22,26 @@ def spam(a=0):
     os.path.join("foo", "bar")
     some_c_function()
 
-cdef ham():
+cpdef eggs():
     pass    
     
-cpdef eggs():
+cdef ham():
     pass
     
 cdef class SomeClass(object):
     def spam(self):
         pass
 
+def outer():
+    cdef object a = "an object"
+    def inner():
+        b = 2
+        # access closed over variables
+        print a, b
+    return inner
+
+
+outer()()
+
 spam()
 print "bye!"
\ No newline at end of file

Cython/Debugger/Tests/test_libcython_in_gdb.py

@@ -14,6 +14,7 @@ import warnings
 import unittest
 import textwrap
 import tempfile
+import functools
 import traceback
 import itertools
 from test import test_support
@@ -28,12 +29,35 @@ from Cython.Debugger.Tests import TestLibCython as test_libcython
 sys.argv = ['gdb']
 
 
+def print_on_call_decorator(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        _debug(type(self).__name__, func.__name__)
+
+        try:
+            return func(self, *args, **kwargs)
+        except Exception, e:
+            _debug("An exception occurred:", traceback.format_exc(e))
+            raise
+
+    return wrapper
+
+class TraceMethodCallMeta(type):
+
+    def __init__(self, name, bases, dict):
+        for func_name, func in dict.iteritems():
+            if inspect.isfunction(func):
+                setattr(self, func_name, print_on_call_decorator(func))
+
+
 class DebugTestCase(unittest.TestCase):
     """
     Base class for test cases. On teardown it kills the inferior and unsets
     all breakpoints.
     """
 
+    __metaclass__ = TraceMethodCallMeta
+
     def __init__(self, name):
         super(DebugTestCase, self).__init__(name)
         self.cy = libcython.cy
@@ -58,7 +82,7 @@ class DebugTestCase(unittest.TestCase):
         if source_line is not None:
             lineno = test_libcython.source_to_lineno[source_line]
         frame = gdb.selected_frame()
-        self.assertEqual(libcython.cy.step.lineno(frame), lineno)
+        self.assertEqual(libcython.cython_info.lineno(frame), lineno)
 
     def break_and_run(self, source_line):
         break_lineno = test_libcython.source_to_lineno[source_line]
@@ -74,9 +98,6 @@ class DebugTestCase(unittest.TestCase):
 
         gdb.execute('set args -c "import codefile"')
 
-        libcython.cy.step.static_breakpoints.clear()
-        libcython.cy.step.runtime_breakpoints.clear()
-        libcython.cy.step.init_breakpoints()
 
 class TestDebugInformationClasses(DebugTestCase):
 
@@ -101,7 +122,7 @@ class TestDebugInformationClasses(DebugTestCase):
                          'codefile.SomeClass.spam')
         self.assertEqual(self.spam_func.module, self.module)
 
-        assert self.eggs_func.pf_cname
+        assert self.eggs_func.pf_cname, (self.eggs_func, self.eggs_func.pf_cname)
         assert not self.ham_func.pf_cname
         assert not self.spam_func.pf_cname
         assert not self.spam_meth.pf_cname
@@ -130,7 +151,7 @@ class TestParameters(unittest.TestCase):
 class TestBreak(DebugTestCase):
 
     def test_break(self):
-        breakpoint_amount = len(gdb.breakpoints())
+        breakpoint_amount = len(gdb.breakpoints() or ())
         gdb.execute('cy break codefile.spam')
 
         self.assertEqual(len(gdb.breakpoints()), breakpoint_amount + 1)
@@ -143,6 +164,16 @@ class TestBreak(DebugTestCase):
         gdb.execute('cy break -p join')
         assert 'def join(' in gdb.execute('cy run', to_string=True)
 
+    def test_break_lineno(self):
+        beginline = 'import os'
+        nextline = 'cdef int c_var = 12'
+
+        self.break_and_run(beginline)
+        self.lineno_equals(beginline)
+        step_result = gdb.execute('cy step', to_string=True)
+        self.lineno_equals(nextline)
+        assert step_result.rstrip().endswith(nextline)
+
 
 class TestKilled(DebugTestCase):
 
@@ -151,6 +182,7 @@ class TestKilled(DebugTestCase):
         output = gdb.execute('cy run', to_string=True)
         assert 'abort' in output.lower()
 
+
 class DebugStepperTestCase(DebugTestCase):
 
     def step(self, varnames_and_values, source_line=None, lineno=None):
@@ -262,7 +294,7 @@ class TestBacktrace(DebugTestCase):
 
         gdb.execute('cy bt')
         result = gdb.execute('cy bt -a', to_string=True)
-        assert re.search(r'\#0 *0x.* in main\(\) at', result), result
+        assert re.search(r'\#0 *0x.* in main\(\)', result), result
 
 
 class TestFunctions(DebugTestCase):
@@ -344,6 +376,36 @@ class TestExec(DebugTestCase):
         gdb.execute('cy exec some_random_var = 14')
         self.assertEqual('14', self.eval_command('some_random_var'))
 
+class TestClosure(DebugTestCase):
+
+    def break_and_run_func(self, funcname):
+        gdb.execute('cy break ' + funcname)
+        gdb.execute('cy run')
+
+    def test_inner(self):
+        self.break_and_run_func('inner')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+        
+        # Allow the Cython-generated code to initialize the scope variable
+        gdb.execute('cy step')
+
+        self.assertEqual(str(self.read_var('a')), "'an object'")
+        print_result = gdb.execute('cy print a', to_string=True).strip()
+        self.assertEqual(print_result, "a = 'an object'")
+
+    def test_outer(self):
+        self.break_and_run_func('outer')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+
+        # Initialize scope with 'a' uninitialized
+        gdb.execute('cy step')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+        
+        # Initialize 'a' to 1
+        gdb.execute('cy step')
+        print_result = gdb.execute('cy print a', to_string=True).strip()
+        self.assertEqual(print_result, "a = 'an object'")
+
 
 _do_debug = os.environ.get('GDB_DEBUG')
 if _do_debug:

Cython/Debugger/libcython.py

@@ -154,9 +154,6 @@ class CythonModule(object):
         self.lineno_c2cy = {}
         self.functions = {}
         
-    def qualified_name(self, varname):
-        return '.'.join(self.name, varname)
-
 class CythonVariable(object):
 
     def __init__(self, name, cname, qualified_name, type, lineno):
@@ -174,7 +171,8 @@ class CythonFunction(CythonVariable):
                  pf_cname,
                  qualified_name, 
                  lineno, 
-                 type=CObject):
+                 type=CObject,
+                 is_initmodule_function="False"):
         super(CythonFunction, self).__init__(name, 
                                              cname, 
                                              qualified_name, 
@@ -182,6 +180,7 @@ class CythonFunction(CythonVariable):
                                              lineno)
         self.module = module
         self.pf_cname = pf_cname
+        self.is_initmodule_function = is_initmodule_function == "True"
         self.locals = {}
         self.arguments = []
         self.step_into_functions = set()
@@ -243,7 +242,8 @@ class CythonBase(object):
             pyframeobject = libpython.Frame(frame).get_pyop()
 
             if not pyframeobject:
-                raise gdb.GdbError('Unable to read information on python frame')
+                raise gdb.GdbError(
+                            'Unable to read information on python frame')
 
             filename = pyframeobject.filename()
             lineno = pyframeobject.current_line_num()
@@ -256,7 +256,7 @@ class CythonBase(object):
                 filename = None
                 lineno = 0
             else:
-                filename = symbol_and_line_obj.symtab.filename
+                filename = symbol_and_line_obj.symtab.fullname()
                 lineno = symbol_and_line_obj.line
                 if pygments:
                     lexer = pygments.lexers.CLexer(stripall=False)
@@ -389,10 +389,20 @@ class CythonBase(object):
                                      value)
 
     def is_initialized(self, cython_func, local_name):
+        islocal = local_name in cython_func.locals
+        if islocal:
+            cyvar = cython_func.locals[local_name]
+            if '->' in cyvar.cname:
+                # Closed over free variable
+                if self.get_cython_lineno() >= cython_func.lineno + 1:
+                    if cyvar.type == PythonObject:
+                        return long(gdb.parse_and_eval(cyvar.cname))
+                    return True
+                return False
+        
         cur_lineno = self.get_cython_lineno()
         return (local_name in cython_func.arguments or
-                (local_name in cython_func.locals and
-                 cur_lineno > cython_func.locals[local_name].lineno))
+                (islocal and cur_lineno > cyvar.lineno))
 
 class SourceFileDescriptor(object):
     def __init__(self, filename, lexer, formatter=None):
@@ -706,8 +716,6 @@ class CyImport(CythonCommand):
                     for c_lineno in c_linenos:
                         cython_module.lineno_c2cy[c_lineno] = cython_lineno
 
-        self.cy.step.init_breakpoints()
-
 
 class CyBreak(CythonCommand):
     """
@@ -754,10 +762,16 @@ class CyBreak(CythonCommand):
     
     def _break_funcname(self, funcname):
         func = self.cy.functions_by_qualified_name.get(funcname)
+        
+        if func and func.is_initmodule_function:
+            func = None
+        
         break_funcs = [func]
         
         if not func:
-            funcs = self.cy.functions_by_name.get(funcname)
+            funcs = self.cy.functions_by_name.get(funcname) or []
+            funcs = [f for f in funcs if not f.is_initmodule_function]
+            
             if not funcs:
                 gdb.execute('break ' + funcname)
                 return
@@ -813,18 +827,28 @@ class CyBreak(CythonCommand):
     
     @dont_suppress_errors
     def complete(self, text, word):
-        names = self.cy.functions_by_qualified_name
+        # Filter init-module functions (breakpoints can be set using 
+        # modulename:linenumber).
+        names =  [n for n, L in self.cy.functions_by_name.iteritems() 
+                        if any(not f.is_initmodule_function for f in L)]
+        qnames = [n for n, f in self.cy.functions_by_qualified_name.iteritems()
+                        if not f.is_initmodule_function]
+        
         if parameters.complete_unqualified:
-            names = itertools.chain(names, self.cy.functions_by_name)
+            all_names = itertools.chain(qnames, names)
+        else:
+            all_names = qnames
 
         words = text.strip().split()
-        if words and '.' in words[-1]:
-            lastword = words[-1]
-            compl = [n for n in self.cy.functions_by_qualified_name
-                           if n.startswith(lastword)]
-        else:
+        if not words or '.' not in words[-1]:
+            # complete unqualified
             seen = set(text[:-len(word)].split())
-            return [n for n in names if n.startswith(word) and n not in seen]
+            return [n for n in all_names 
+                          if n.startswith(word) and n not in seen]
+        
+        # complete qualified name
+        lastword = words[-1]
+        compl = [n for n in qnames if n.startswith(lastword)]
         
         if len(lastword) > len(word):
             # readline sees something (e.g. a '.') as a word boundary, so don't
@@ -835,10 +859,9 @@ class CyBreak(CythonCommand):
         return compl
 
 
-class CythonCodeStepper(CythonCommand, libpython.GenericCodeStepper):
+class CythonInfo(CythonBase, libpython.PythonInfo):
     """
-    Base class for CyStep and CyNext. It implements the interface dictated by
-    libpython.GenericCodeStepper.
+    Implementation of the interface dictated by libpython.LanguageInfo.
     """
     
     def lineno(self, frame):
@@ -848,32 +871,49 @@ class CythonCodeStepper(CythonCommand, libpython.GenericCodeStepper):
         # related code. The C level should be dispatched to the 'step' command.
         if self.is_cython_function(frame):
             return self.get_cython_lineno(frame)
-        else:
-            return libpython.py_step.lineno(frame)
+        return super(CythonInfo, self).lineno(frame)
     
     def get_source_line(self, frame):
         try:
-            line = super(CythonCodeStepper, self).get_source_line(frame)
+            line = super(CythonInfo, self).get_source_line(frame)
         except gdb.GdbError:
             return None
         else:
             return line.strip() or None
 
-    @classmethod
-    def register(cls):
-        return cls(cls.name, stepinto=getattr(cls, 'stepinto', False))
+    def exc_info(self, frame):
+        if self.is_python_function:
+            return super(CythonInfo, self).exc_info(frame)
 
     def runtime_break_functions(self):
         if self.is_cython_function():
             return self.get_cython_function().step_into_functions
+        return ()
     
     def static_break_functions(self):
         result = ['PyEval_EvalFrameEx']
         result.extend(self.cy.functions_by_cname)
         return result
 
+
+class CythonExecutionControlCommand(CythonCommand, 
+                                    libpython.ExecutionControlCommandBase):
+    
+    @classmethod
+    def register(cls):
+        return cls(cls.name, cython_info)
+
+
+class CyStep(CythonExecutionControlCommand, libpython.PythonStepperMixin):
+    "Step through Cython, Python or C code."
+    
+    name = 'cy -step'
+    stepinto = True
+    
     def invoke(self, args, from_tty):
-        if not self.is_cython_function() and not self.is_python_function():
+        if self.is_python_function():
+            self.python_step(self.stepinto)
+        elif not self.is_cython_function():
             if self.stepinto:
                 command = 'step'
             else:
@@ -881,24 +921,17 @@ class CythonCodeStepper(CythonCommand, libpython.GenericCodeStepper):
                 
             self.finish_executing(gdb.execute(command, to_string=True))
         else:
-            self.step()
-
-
-class CyStep(CythonCodeStepper):
-    "Step through Cython, Python or C code."
-
-    name = 'cy step'
-    stepinto = True
+            self.step(stepinto=self.stepinto)
 
 
-class CyNext(CythonCodeStepper):
-    "Step-over Python code."
+class CyNext(CyStep):
+    "Step-over Cython, Python or C code."
 
-    name = 'cy next'
+    name = 'cy -next'
     stepinto = False
 
 
-class CyRun(CythonCodeStepper):
+class CyRun(CythonExecutionControlCommand):
     """
     Run a Cython program. This is like the 'run' command, except that it 
     displays Cython or Python source lines as well
@@ -906,26 +939,26 @@ class CyRun(CythonCodeStepper):
     
     name = 'cy run'
     
-    invoke = CythonCodeStepper.run
+    invoke = CythonExecutionControlCommand.run
 
 
-class CyCont(CyRun):
+class CyCont(CythonExecutionControlCommand):
     """
     Continue a Cython program. This is like the 'run' command, except that it 
     displays Cython or Python source lines as well.
     """
     
     name = 'cy cont'
-    invoke = CythonCodeStepper.cont
+    invoke = CythonExecutionControlCommand.cont
 
 
-class CyFinish(CyRun):
+class CyFinish(CythonExecutionControlCommand):
     """
     Execute until the function returns.
     """
     name = 'cy finish'
 
-    invoke = CythonCodeStepper.finish
+    invoke = CythonExecutionControlCommand.finish
 
 
 class CyUp(CythonCommand):
@@ -961,7 +994,7 @@ class CyDown(CyUp):
     _command = 'down'
 
 
-class CySelect(CythonCodeStepper):
+class CySelect(CythonCommand):
     """
     Select a frame. Use frame numbers as listed in `cy backtrace`.
     This command is useful because `cy backtrace` prints a reversed backtrace.
@@ -979,7 +1012,7 @@ class CySelect(CythonCodeStepper):
         while frame.newer():
             frame = frame.newer()
         
-        stackdepth = self._stackdepth(frame)
+        stackdepth = libpython.stackdepth(frame)
         
         try:
             gdb.execute('select %d' % (stackdepth - stackno - 1,))
@@ -1033,7 +1066,7 @@ class CyList(CythonCommand):
     command_class = gdb.COMMAND_FILES
     completer_class = gdb.COMPLETE_NONE
     
-    @dispatch_on_frame(c_command='list')
+    # @dispatch_on_frame(c_command='list')
     def invoke(self, _, from_tty):
         sd, lineno = self.get_source_desc()
         source = sd.get_source(lineno - 5, lineno + 5, mark_line=lineno, 
@@ -1085,7 +1118,13 @@ class CyLocals(CythonCommand):
     
     @dispatch_on_frame(c_command='info locals', python_command='py-locals')
     def invoke(self, args, from_tty):
-        local_cython_vars = self.get_cython_function().locals
+        cython_function = self.get_cython_function()
+        
+        if cython_function.is_initmodule_function:
+            self.cy.globals.invoke(args, from_tty)
+            return
+
+        local_cython_vars = cython_function.locals
         max_name_length = len(max(local_cython_vars, key=len))
         for name, cyvar in sorted(local_cython_vars.iteritems(), key=sortkey):
             if self.is_initialized(self.get_cython_function(), cyvar.name):
@@ -1263,17 +1302,16 @@ class CyCValue(CyCName):
     @require_cython_frame
     @gdb_function_value_to_unicode
     def invoke(self, cyname, frame=None):
-        try:
+        globals_dict = self.get_cython_globals_dict()
+        cython_function = self.get_cython_function(frame)
+
+        if self.is_initialized(cython_function, cyname):
             cname = super(CyCValue, self).invoke(cyname, frame=frame)
             return gdb.parse_and_eval(cname)
-        except (gdb.GdbError, RuntimeError), e:
-            # variable exists but may not have been initialized yet, or may be
-            # in the globals dict of the Cython module
-            d = self.get_cython_globals_dict()
-            if cyname in d:
-                return d[cyname]._gdbval
-
-            raise gdb.GdbError(str(e))
+        elif cyname in globals_dict:
+            return globals_dict[cyname]._gdbval
+        else:
+            raise gdb.GdbError("Variable %s is not initialized." % cyname)
 
 
 class CyLine(gdb.Function, CythonBase):
@@ -1285,5 +1323,27 @@ class CyLine(gdb.Function, CythonBase):
     def invoke(self):
         return self.get_cython_lineno()
 
-
+cython_info = CythonInfo()
 cy = CyCy.register()
+cython_info.cy = cy
+
+def register_defines():
+    libpython.source_gdb_script(textwrap.dedent("""\
+        define cy step
+        cy -step
+        end
+        
+        define cy next
+        cy -next
+        end
+        
+        document cy step
+        %s
+        end
+        
+        document cy next
+        %s
+        end
+    """) % (CyStep.__doc__, CyNext.__doc__))
+
+register_defines()
\ No newline at end of file

Cython/Debugger/libpython.py

@@ -369,8 +369,8 @@ class PyObjectPtr(object):
         if tp_name in name_map:
             return name_map[tp_name]
         
-        if tp_flags & Py_TPFLAGS_HEAPTYPE:
-            return HeapTypeObjectPtr
+        if tp_flags & (Py_TPFLAGS_HEAPTYPE|Py_TPFLAGS_TYPE_SUBCLASS):
+            return PyTypeObjectPtr
 
         if tp_flags & Py_TPFLAGS_INT_SUBCLASS:
             return PyIntObjectPtr
@@ -392,8 +392,6 @@ class PyObjectPtr(object):
             return PyDictObjectPtr
         if tp_flags & Py_TPFLAGS_BASE_EXC_SUBCLASS:
             return PyBaseExceptionObjectPtr
-        #if tp_flags & Py_TPFLAGS_TYPE_SUBCLASS:
-        #    return PyTypeObjectPtr
 
         # Use the base class:
         return cls
@@ -484,8 +482,8 @@ def _PyObject_VAR_SIZE(typeobj, nitems):
              ) & ~(SIZEOF_VOID_P - 1)
            ).cast(gdb.lookup_type('size_t'))
 
-class HeapTypeObjectPtr(PyObjectPtr):
-    _typename = 'PyObject'
+class PyTypeObjectPtr(PyObjectPtr):
+    _typename = 'PyTypeObject'
     
     def get_attr_dict(self):
         '''
@@ -546,9 +544,16 @@ class HeapTypeObjectPtr(PyObjectPtr):
             return
         visited.add(self.as_address())
         
-        pyop_attrdict = self.get_attr_dict()
-        _write_instance_repr(out, visited,
-                             self.safe_tp_name(), pyop_attrdict, self.as_address())
+        try:
+            tp_name = self.field('tp_name').string()
+        except RuntimeError:
+            tp_name = 'unknown'
+        
+        out.write('<type %s at remote 0x%x>' % (tp_name, 
+                                                self.as_address()))
+        # pyop_attrdict = self.get_attr_dict()
+        # _write_instance_repr(out, visited,
+                             # self.safe_tp_name(), pyop_attrdict, self.as_address())
 
 class ProxyException(Exception):
     def __init__(self, tp_name, args):
@@ -1136,9 +1141,6 @@ class PyTupleObjectPtr(PyObjectPtr):
         else:
             out.write(')')
 
-class PyTypeObjectPtr(PyObjectPtr):
-    _typename = 'PyTypeObject'
-
 
 def _unichr_is_printable(char):
     # Logic adapted from Python 3's Tools/unicode/makeunicodedata.py
@@ -1715,6 +1717,8 @@ class PyNameEquals(gdb.Function):
         if frame.is_evalframeex():
             pyframe = frame.get_pyop()
             if pyframe is None:
+                warnings.warn("Use a Python debug build, Python breakpoints "
+                              "won't work otherwise.")
                 return None
             
             return getattr(pyframe, attr).proxyval(set())
@@ -1839,132 +1843,98 @@ def get_selected_inferior():
             if thread == selected_thread:
                 return inferior
 
-
-class GenericCodeStepper(gdb.Command):
+def source_gdb_script(script_contents, to_string=False):
     """
-    Superclass for code stepping. Subclasses must implement the following
-    methods:
-
-        lineno(frame)
-            tells the current line number (only called for a relevant frame)
-
-        is_relevant_function(frame)
-            tells whether we care about frame 'frame'
-
-        get_source_line(frame)
-            get the line of source code for the current line (only called for a
-            relevant frame). If the source code cannot be retrieved this
-            function should return None
-
-        static_break_functions()
-            returns an iterable of function names that are considered relevant
-            and should halt step-into execution. This is needed to provide a
-            performing step-into
-
-        runtime_break_functions
-            list of functions that we should break into depending on the
-            context
-
-    This class provides an 'invoke' method that invokes a 'step' or 'step-over'
-    depending on the 'stepinto' argument.
+    Source a gdb script with script_contents passed as a string. This is useful
+    to provide defines for py-step and py-next to make them repeatable (this is
+    not possible with gdb.execute()). See 
+    http://sourceware.org/bugzilla/show_bug.cgi?id=12216
     """
+    fd, filename = tempfile.mkstemp()
+    f = os.fdopen(fd, 'w')
+    f.write(script_contents)
+    f.close()
+    gdb.execute("source %s" % filename, to_string=to_string)
+    os.remove(filename)
+
+def register_defines():
+    source_gdb_script(textwrap.dedent("""\
+        define py-step
+        -py-step
+        end
+        
+        define py-next
+        -py-next
+        end
+        
+        document py-step
+        %s
+        end
+        
+        document py-next
+        %s
+        end
+    """) % (PyStep.__doc__, PyNext.__doc__))
+    
+
+def stackdepth(frame):
+    "Tells the stackdepth of a gdb frame."
+    depth = 0
+    while frame:
+        frame = frame.older()
+        depth += 1
         
-    stepper = False
-    static_breakpoints = {}
-    runtime_breakpoints = {}
-
-    def __init__(self, name, stepinto=False):
-        super(GenericCodeStepper, self).__init__(name,
-                                                 gdb.COMMAND_RUNNING,
-                                                 gdb.COMPLETE_NONE)
-        self.stepinto = stepinto
-
-    def _break_func(self, funcname):
-        result = gdb.execute('break %s' % funcname, to_string=True)
-        return re.search(r'Breakpoint (\d+)', result).group(1)
-
-    def init_breakpoints(self):
-        """
-        Keep all breakpoints around and simply disable/enable them each time
-        we are stepping. We need this because if you set and delete a
-        breakpoint, gdb will not repeat your command (this is due to 'delete').
-        We also can't use the breakpoint API because there's no option to make
-        breakpoint setting silent.
-
-        This method must be called whenever the list of functions we should
-        step into changes. It can be called on any GenericCodeStepper instance.
-        """
-        break_funcs = set(self.static_break_functions())
-
-        for funcname in break_funcs:
-            if funcname not in self.static_breakpoints:
-                try:
-                    gdb.Breakpoint('', gdb.BP_BREAKPOINT, internal=True)
-                except (AttributeError, TypeError):
-                    # gdb.Breakpoint does not take an 'internal' argument, or
-                    # gdb.Breakpoint does not exist.
-                    breakpoint = self._break_func(funcname)
-                except RuntimeError:
-                    # gdb.Breakpoint does take an 'internal' argument, use it
-                    # and hide output
-                    result = gdb.execute(textwrap.dedent("""\
-                        python bp = gdb.Breakpoint(%r, gdb.BP_BREAKPOINT, \
-                                                   internal=True); \
-                               print bp.number""",
-                        to_string=True))
-
-                    breakpoint = int(result)
-
-                self.static_breakpoints[funcname] = breakpoint
-
-        for bp in set(self.static_breakpoints) - break_funcs:
-            gdb.execute("delete " + self.static_breakpoints[bp])
-
-        self.disable_breakpoints()
-
-    def enable_breakpoints(self):
-        for bp in self.static_breakpoints.itervalues():
-            gdb.execute('enable ' + bp)
-
-        runtime_break_functions = self.runtime_break_functions()
-        if runtime_break_functions is None:
-            return
-
-        for funcname in runtime_break_functions:
-            if (funcname not in self.static_breakpoints and
-                funcname not in self.runtime_breakpoints):
-                self.runtime_breakpoints[funcname] = self._break_func(funcname)
-            elif funcname in self.runtime_breakpoints:
-                gdb.execute('enable ' + self.runtime_breakpoints[funcname])
-
-    def disable_breakpoints(self):
-        chain = itertools.chain(self.static_breakpoints.itervalues(),
-                                self.runtime_breakpoints.itervalues())
-        for bp in chain:
-            gdb.execute('disable ' + bp)
+    return depth
 
-    def runtime_break_functions(self):
+class ExecutionControlCommandBase(gdb.Command):
     """
-        Implement this if the list of step-into functions depends on the
-        context.
+    Superclass for language specific execution control. Language specific
+    features should be implemented by lang_info using the LanguageInfo 
+    interface. 'name' is the name of the command.
     """
     
-    def stopped(self, result):
-        match = re.search('^Program received signal .*', result, re.MULTILINE)
+    def __init__(self, name, lang_info):
+        super(ExecutionControlCommandBase, self).__init__(
+                                name, gdb.COMMAND_RUNNING, gdb.COMPLETE_NONE)
+        self.lang_info = lang_info
+
+    def install_breakpoints(self):
+        all_locations = itertools.chain(
+            self.lang_info.static_break_functions(),
+            self.lang_info.runtime_break_functions())
+            
+        for location in all_locations:
+            result = gdb.execute('break %s' % location, to_string=True)
+            yield re.search(r'Breakpoint (\d+)', result).group(1)
+    
+    def delete_breakpoints(self, breakpoint_list):
+        for bp in breakpoint_list:
+            gdb.execute("delete %s" % bp)
+    
+    def filter_output(self, result):
+        output = []
+        
+        match_finish = re.search(r'^Value returned is \$\d+ = (.*)', result,
+                                 re.MULTILINE)
+        if match_finish:
+            output.append('Value returned: %s' % match_finish.group(1))
+        
+        reflags = re.MULTILINE
+        regexes = [
+            (r'^Program received signal .*', reflags|re.DOTALL),
+            (r'.*[Ww]arning.*', 0),
+            (r'^Program exited .*', reflags),
+        ]
+        
+        for regex, flags in regexes:
+            match = re.search(regex, result, flags)
             if match:
-            return match.group(0)
-        elif get_selected_inferior().pid == 0:
-            return result
-        else:
-            return None
+                output.append(match.group(0))
         
-    def _stackdepth(self, frame):
-        depth = 0
-        while frame:
-            frame = frame.older()
-            depth += 1
+        return '\n'.join(output)
         
-        return depth
+    def stopped(self):
+        return get_selected_inferior().pid == 0
     
     def finish_executing(self, result):
         """
@@ -1972,32 +1942,19 @@ class GenericCodeStepper(gdb.Command):
         of source code or the result of the last executed gdb command (passed
         in as the `result` argument).
         """
-        result = self.stopped(result)
-        if result:
+        result = self.filter_output(result)
+        
+        if self.stopped():
             print result.strip()
-            # check whether the program was killed by a signal, it should still
-            # have a stack.
-            try:
-                frame = gdb.selected_frame()
-            except RuntimeError:
-                pass
-            else:
-                print self.get_source_line(frame)
         else:
             frame = gdb.selected_frame()
-            output = None
-
-            if self.is_relevant_function(frame):
-                output = self.get_source_line(frame)
-
-            if output is None:
-                pframe = getattr(self, 'print_stackframe', None)
-                if pframe:
-                    pframe(frame, index=0)
-                else:
-                    print result.strip()
+            if self.lang_info.is_relevant_function(frame):
+                raised_exception = self.lang_info.exc_info(frame)
+                if raised_exception:
+                    print raised_exception
+                print self.lang_info.get_source_line(frame) or result
             else:
-                print output
+                print result
     
     def _finish(self):
         """
@@ -2010,11 +1967,10 @@ class GenericCodeStepper(gdb.Command):
             # outermost frame, continue
             return gdb.execute('cont', to_string=True)
     
-    def finish(self, *args):
+    def _finish_frame(self):
         """
         Execute until the function returns to a relevant caller.
         """
-
         while True:
             result = self._finish()
             
@@ -2024,39 +1980,64 @@ class GenericCodeStepper(gdb.Command):
                 break
             
             hitbp = re.search(r'Breakpoint (\d+)', result)
-            is_relavant = self.is_relevant_function(frame)
-            if hitbp or is_relavant or self.stopped(result):
+            is_relevant = self.lang_info.is_relevant_function(frame)
+            if hitbp or is_relevant or self.stopped():
                 break
         
+        return result
+    
+    def finish(self, *args):
+        "Implements the finish command."
+        result = self._finish_frame()
         self.finish_executing(result)
     
-    def _step(self):
+    def step(self, stepinto, stepover_command='next'):
         """
         Do a single step or step-over. Returns the result of the last gdb
         command that made execution stop.
+        
+        This implementation, for stepping, sets (conditional) breakpoints for 
+        all functions that are deemed relevant. It then does a step over until
+        either something halts execution, or until the next line is reached.
+        
+        If, however, stepover_command is given, it should be a string gdb 
+        command that continues execution in some way. The idea is that the 
+        caller has set a (conditional) breakpoint or watchpoint that can work 
+        more efficiently than the step-over loop. For Python this means setting
+        a watchpoint for f->f_lasti, which means we can then subsequently 
+        "finish" frames.
+        We want f->f_lasti instead of f->f_lineno, because the latter only 
+        works properly with local trace functions, see 
+        PyFrameObjectPtr.current_line_num and PyFrameObjectPtr.addr2line.
         """
-        if self.stepinto:
-            self.enable_breakpoints()
+        if stepinto:
+            breakpoint_list = list(self.install_breakpoints())
         
         beginframe = gdb.selected_frame()
-        beginline = self.lineno(beginframe)
-        if not self.stepinto:
-            depth = self._stackdepth(beginframe)
+        
+        if self.lang_info.is_relevant_function(beginframe):
+            # If we start in a relevant frame, initialize stuff properly. If
+            # we don't start in a relevant frame, the loop will halt 
+            # immediately. So don't call self.lang_info.lineno() as it may 
+            # raise for irrelevant frames.
+            beginline = self.lang_info.lineno(beginframe)
+            
+            if not stepinto:
+                depth = stackdepth(beginframe)
         
         newframe = beginframe
-        result = ''
         
         while True:
-            if self.is_relevant_function(newframe):
-                result = gdb.execute('next', to_string=True)
+            if self.lang_info.is_relevant_function(newframe):
+                result = gdb.execute(stepover_command, to_string=True)
             else:
-                result = self._finish()
+                result = self._finish_frame()
             
-            if self.stopped(result):
+            if self.stopped():
                 break
             
             newframe = gdb.selected_frame()
-            is_relevant_function = self.is_relevant_function(newframe)
+            is_relevant_function = self.lang_info.is_relevant_function(newframe)
             try:
                 framename = newframe.name()
             except RuntimeError:
@@ -2064,8 +2045,7 @@ class GenericCodeStepper(gdb.Command):
             
             m = re.search(r'Breakpoint (\d+)', result)
             if m:
-                bp = self.runtime_breakpoints.get(framename)
-                if bp is None or (m.group(1) == bp and is_relevant_function):
+                if is_relevant_function and m.group(1) in breakpoint_list:
                     # although we hit a breakpoint, we still need to check
                     # that the function, in case hit by a runtime breakpoint,
                     # is in the right context
@@ -2074,25 +2054,25 @@ class GenericCodeStepper(gdb.Command):
             if newframe != beginframe:
                 # new function
                 
-                if not self.stepinto:
+                if not stepinto:
                     # see if we returned to the caller
-                    newdepth = self._stackdepth(newframe)
+                    newdepth = stackdepth(newframe)
                     is_relevant_function = (newdepth < depth and 
                                             is_relevant_function)
                 
                 if is_relevant_function:
                     break
             else:
-                if self.lineno(newframe) > beginline:
+                # newframe equals beginframe, check for a difference in the
+                # line number
+                lineno = self.lang_info.lineno(newframe)
+                if lineno and lineno != beginline:
                     break
         
-        if self.stepinto:
-            self.disable_breakpoints()
-
-        return result
+        if stepinto:
+            self.delete_breakpoints(breakpoint_list)
 
-    def step(self, *args):
-        return self.finish_executing(self._step())
+        self.finish_executing(result)
     
     def run(self, *args):
         self.finish_executing(gdb.execute('run', to_string=True))
@@ -2101,14 +2081,57 @@ class GenericCodeStepper(gdb.Command):
         self.finish_executing(gdb.execute('cont', to_string=True))
 
 
-class PythonCodeStepper(GenericCodeStepper):
+class LanguageInfo(object):
+    """
+    This class defines the interface that ExecutionControlCommandBase needs to
+    provide language-specific execution control.
+    
+    Classes that implement this interface should implement:
+    
+        lineno(frame)
+            Tells the current line number (only called for a relevant frame).
+            If lineno is a false value it is not checked for a difference.
+        
+        is_relevant_function(frame)
+            tells whether we care about frame 'frame'
+        
+        get_source_line(frame)
+            get the line of source code for the current line (only called for a
+            relevant frame). If the source code cannot be retrieved this 
+            function should return None
+        
+        exc_info(frame) -- optional
+            tells whether an exception was raised, if so, it should return a
+            string representation of the exception value, None otherwise.
+        
+        static_break_functions() 
+            returns an iterable of function names that are considered relevant 
+            and should halt step-into execution. This is needed to provide a 
+            performing step-into
+      
+        runtime_break_functions() -- optional
+            list of functions that we should break into depending on the 
+            context
+    """
+
+    def exc_info(self, frame):
+        "See this class' docstring."
+
+    def runtime_break_functions(self):
+        """
+        Implement this if the list of step-into functions depends on the 
+        context.
+        """
+        return ()
+
+class PythonInfo(LanguageInfo):
     
     def pyframe(self, frame):
         pyframe = Frame(frame).get_pyop()
         if pyframe:
             return pyframe
         else:
-            raise gdb.GdbError(
+            raise gdb.RuntimeError(
                 "Unable to find the Python frame, run your code with a debug "
                 "build (configure with --with-pydebug or compile with -g).")
         
@@ -2126,47 +2149,67 @@ class PythonCodeStepper(GenericCodeStepper):
         except IOError, e:
             return None
     
+    def exc_info(self, frame):
+        try:
+            tstate = frame.read_var('tstate').dereference()
+            if gdb.parse_and_eval('tstate->frame == f'):
+                # tstate local variable initialized
+                inf_type = tstate['curexc_type']
+                inf_value = tstate['curexc_value']
+                if inf_type:
+                    return 'An exception was raised: %s(%s)' % (inf_type, 
+                                                                inf_value)
+        except (ValueError, RuntimeError), e:
+            # Could not read the variable tstate or it's memory, it's ok
+            pass
+        
     def static_break_functions(self):
         yield 'PyEval_EvalFrameEx'
 
 
-class PyStep(PythonCodeStepper):
+class PythonStepperMixin(object):
+    """
+    Make this a mixin so CyStep can also inherit from this and use a 
+    CythonCodeStepper at the same time.
+    """
+    
+    def python_step(self, stepinto):
+        frame = gdb.selected_frame()
+        framewrapper = Frame(frame)
+        
+        output = gdb.execute('watch f->f_lasti', to_string=True)
+        watchpoint = int(re.search(r'[Ww]atchpoint (\d+):', output).group(1))
+        self.step(stepinto=stepinto, stepover_command='finish')
+        gdb.execute('delete %s' % watchpoint)
+
+
+class PyStep(ExecutionControlCommandBase, PythonStepperMixin):
     "Step through Python code."
     
-    invoke = PythonCodeStepper.step
+    stepinto = True
+    
+    def invoke(self, args, from_tty):
+        self.python_step(stepinto=self.stepinto)
     
-class PyNext(PythonCodeStepper):
+class PyNext(PyStep):
     "Step-over Python code."
     
-    invoke = PythonCodeStepper.step
+    stepinto = False
     
-class PyFinish(PythonCodeStepper):
+class PyFinish(ExecutionControlCommandBase):
     "Execute until function returns to a caller."
     
-    invoke = PythonCodeStepper.finish
+    invoke = ExecutionControlCommandBase.finish
 
-class PyRun(PythonCodeStepper):
+class PyRun(ExecutionControlCommandBase):
     "Run the program."
     
-    invoke = PythonCodeStepper.run
-
-class PyCont(PythonCodeStepper):
+    invoke = ExecutionControlCommandBase.run
 
-    invoke = PythonCodeStepper.cont
+class PyCont(ExecutionControlCommandBase):
     
+    invoke = ExecutionControlCommandBase.cont
 
-py_step = PyStep('py-step', stepinto=True)
-py_next = PyNext('py-next', stepinto=False)
-py_finish = PyFinish('py-finish')
-py_run = PyRun('py-run')
-py_cont = PyCont('py-cont')
-
-gdb.execute('set breakpoint pending on')
-py_step.init_breakpoints()
-
-Py_single_input = 256
-Py_file_input = 257
-Py_eval_input = 258
 
 def _pointervalue(gdbval):
     """
@@ -2210,6 +2253,10 @@ def get_inferior_unicode_postfix():
         
 class PythonCodeExecutor(object):
     
+    Py_single_input = 256
+    Py_file_input = 257
+    Py_eval_input = 258
+    
     def malloc(self, size):
         chunk = (gdb.parse_and_eval("(void *) malloc((size_t) %d)" % size))
         
@@ -2382,7 +2429,7 @@ class PyExec(gdb.Command):
     
     def readcode(self, expr):
         if expr:
-            return expr, Py_single_input
+            return expr, PythonCodeExecutor.Py_single_input
         else:
             lines = []
             while True:
@@ -2412,5 +2459,19 @@ class PyExec(gdb.Command):
         
         executor.evalcode(expr, input_type, global_dict, local_dict)
 
-py_exec = FixGdbCommand('py-exec', '-py-exec')
-_py_exec = PyExec("-py-exec", gdb.COMMAND_DATA, gdb.COMPLETE_NONE)
+
+gdb.execute('set breakpoint pending on')
+
+if hasattr(gdb, 'GdbError'):
+     # Wrap py-step and py-next in gdb defines to make them repeatable.
+    py_step = PyStep('-py-step', PythonInfo())
+    py_next = PyNext('-py-next', PythonInfo())
+    register_defines()
+    py_finish = PyFinish('py-finish', PythonInfo())
+    py_run = PyRun('py-run', PythonInfo())
+    py_cont = PyCont('py-cont', PythonInfo())
+
+    py_exec = FixGdbCommand('py-exec', '-py-exec')
+    _py_exec = PyExec("-py-exec", gdb.COMMAND_DATA, gdb.COMPLETE_NONE)
+else:
+    warnings.warn("Use gdb 7.2 or higher to use the py-exec command.")
\ No newline at end of file

runtests.py

@@ -675,12 +675,12 @@ class CythonPyregrTestCase(CythonRunTestCase):
         except (unittest.SkipTest, support.ResourceDenied):
             result.addSkip(self, 'ok')
 
-
-try:
-    import gdb
-    include_debugger = sys.version_info[:2] > (2, 5)
-except:
-    include_debugger = False
+# Someone wrapped this in a:
+# 'try: import gdb; ... except: include_debugger = False' thing, but don't do 
+# this, it doesn't work as gdb is a builtin module in GDB. The tests themselves
+# are doing the skipping. If there's a problem with the tests, please file an 
+# issue.
+include_debugger = sys.version_info[:2] > (2, 5)
 
 def collect_unittests(path, module_prefix, suite, selectors):
     def file_matches(filename):

setup.py

@@ -100,7 +100,8 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",
                         "Cython.Compiler.Code",
-                        "Cython.Runtime.refnanny"]
+                        "Cython.Runtime.refnanny",
+                        "Cython.Debugger.do_repeat",]
     if compile_more:
         compiled_modules.extend([
             "Cython.Compiler.ParseTreeTransforms",