Merge branch 'master' of https://github.com/markflorisson88/cython into markflorisson88-master

Cython/Compiler/Main.py

@@ -18,7 +18,11 @@ from time import time
 
 import Code
 import Errors
-import Parsing
+# Do not import Parsing here, import it when needed, because Parsing imports
+# Nodes, which globally needs debug command line options initialized to set a
+# conditional metaclass. These options are processed by CmdLine called from 
+# main() in this file.
+# import Parsing
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
 from Errors import PyrexError, CompileError, InternalError, AbortError, error, warning
@@ -496,6 +500,7 @@ class Context(object):
         try:
             f = Utils.open_source_file(source_filename, "rU")
             try:
+                import Parsing
                 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
                                  scope = scope, context = self)
                 tree = Parsing.p_module(s, pxd, full_module_name)

Cython/Compiler/ParseTreeTransforms.py

@@ -1591,6 +1591,10 @@ class DebugTransform(CythonTransform):
         #self.c_output_file = options.output_file 
         self.c_output_file = result.c_file
         
+        # Closure support, basically treat nested functions as if the AST were
+        # never nested
+        self.nested_funcdefs = []
+        
         # tells visit_NameNode whether it should register step-into functions
         self.register_stepinto = False
         
@@ -1605,7 +1609,16 @@ class DebugTransform(CythonTransform):
         
         # serialize functions
         self.tb.start('Functions')
+        # First, serialize functions normally...
         self.visitchildren(node)
+        
+        # ... then, serialize nested functions
+        for nested_funcdef in self.nested_funcdefs:
+            self.visit_FuncDefNode(nested_funcdef)
+        
+        self.register_stepinto = True
+        self.serialize_modulenode_as_function(node)
+        self.register_stepinto = False
         self.tb.end('Functions')
         
         # 2.3 compatibility. Serialize global variables
@@ -1627,6 +1640,14 @@ class DebugTransform(CythonTransform):
     
     def visit_FuncDefNode(self, node):        
         self.visited.add(node.local_scope.qualified_name)
+
+        if getattr(node, 'is_wrapper', False):
+            return node
+
+        if self.register_stepinto:
+            self.nested_funcdefs.append(node)
+            return node
+
         # node.entry.visibility = 'extern'
         if node.py_func is None:
             pf_cname = ''
@@ -1678,6 +1699,51 @@ class DebugTransform(CythonTransform):
         self.visitchildren(node)
         return node
     
+    def serialize_modulenode_as_function(self, node):
+        """
+        Serialize the module-level code as a function so the debugger will know
+        it's a "relevant frame" and it will know where to set the breakpoint
+        for 'break modulename'.
+        """
+        name = node.full_module_name.rpartition('.')[-1]
+        
+        cname_py2 = 'init' + name
+        cname_py3 = 'PyInit_' + name
+        
+        py2_attrs = dict(
+            name=name,
+            cname=cname_py2,
+            pf_cname='',
+            # Ignore the qualified_name, breakpoints should be set using 
+            # `cy break modulename:lineno` for module-level breakpoints.
+            qualified_name='',
+            lineno='1',
+            is_initmodule_function="True",
+        )
+        
+        py3_attrs = dict(py2_attrs, cname=cname_py3)
+        
+        self._serialize_modulenode_as_function(node, py2_attrs)
+        self._serialize_modulenode_as_function(node, py3_attrs)
+    
+    def _serialize_modulenode_as_function(self, node, attrs):
+        self.tb.start('Function', attrs=attrs)
+        
+        self.tb.start('Locals')
+        self.serialize_local_variables(node.scope.entries)
+        self.tb.end('Locals')
+
+        self.tb.start('Arguments')
+        self.tb.end('Arguments')
+
+        self.tb.start('StepIntoFunctions')
+        self.register_stepinto = True
+        self.visitchildren(node)
+        self.register_stepinto = False
+        self.tb.end('StepIntoFunctions')
+        
+        self.tb.end('Function')
+    
     def serialize_local_variables(self, entries):
         for entry in entries.values():
             if entry.type.is_pyobject:
@@ -1685,9 +1751,22 @@ class DebugTransform(CythonTransform):
             else:
                 vartype = 'CObject'
             
+            if entry.from_closure:
+                # We're dealing with a closure where a variable from an outer
+                # scope is accessed, get it from the scope object.
+                cname = '%s->%s' % (Naming.cur_scope_cname, 
+                                    entry.outer_entry.cname)
+                
+                qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
+                                      entry.scope.name, 
+                                      entry.name)
+            elif entry.in_closure:
+                cname = '%s->%s' % (Naming.cur_scope_cname, 
+                                    entry.cname)
+                qname = entry.qualified_name
+            else:
                 cname = entry.cname
-            # if entry.type.is_extension_type:
-                # cname = entry.type.typeptr_cname
+                qname = entry.qualified_name
 
             if not entry.pos:
                 # this happens for variables that are not in the user's code,
@@ -1700,7 +1779,7 @@ class DebugTransform(CythonTransform):
             attrs = dict(
                 name=entry.name,
                 cname=cname,
-                qualified_name=entry.qualified_name,
+                qualified_name=qname,
                 type=vartype,
                 lineno=lineno)
             

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -182,7 +182,8 @@ class TestDebugTransform(DebuggerTestCase):
             self.assertEqual('PythonObject', xml_globals.get('python_var'))
             
             # test functions
-            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            funcnames = ('codefile.spam', 'codefile.ham', 'codefile.eggs', 
+                         'codefile.closure', 'codefile.inner')
             required_xml_attrs = 'name', 'cname', 'qualified_name'
             assert all([f in xml_funcs for f in funcnames])
             spam, ham, eggs = [xml_funcs[funcname] for funcname in funcnames]

Cython/Debugger/Tests/TestLibCython.py

@@ -16,6 +16,7 @@ from distutils import ccompiler
 
 import runtests
 import Cython.Distutils.extension
+import Cython.Distutils.build_ext
 from Cython.Debugger import Cygdb as cygdb
 
 root = os.path.dirname(os.path.abspath(__file__))
@@ -24,6 +25,10 @@ cfuncs_file = os.path.join(root, 'cfuncs.c')
 with open(codefile) as f:
     source_to_lineno = dict((line.strip(), i + 1) for i, line in enumerate(f))
 
+# Cython.Distutils.__init__ imports build_ext from build_ext which means we
+# can't access the module anymore. Get it from sys.modules instead.
+build_ext = sys.modules['Cython.Distutils.build_ext']
+
 class DebuggerTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -52,6 +57,9 @@ class DebuggerTestCase(unittest.TestCase):
             module='codefile',
         )
 
+        optimization_disabler = build_ext.Optimization()
+        optimization_disabler.disable_optimization()
+
         cython_compile_testcase = runtests.CythonCompileTestCase(
             workdir=self.tempdir,
             # we clean up everything (not only compiled files)
@@ -77,6 +85,8 @@ class DebuggerTestCase(unittest.TestCase):
             **opts
         )
 
+        optimization_disabler.restore_state()
+
         # ext = Cython.Distutils.extension.Extension(
             # 'codefile',
             # ['codefile.pyx'],
@@ -95,6 +105,7 @@ class DebuggerTestCase(unittest.TestCase):
 
 
 class GdbDebuggerTestCase(DebuggerTestCase):
+
     def setUp(self):
         super(GdbDebuggerTestCase, self).setUp()
 

Cython/Debugger/Tests/codefile

@@ -22,15 +22,26 @@ def spam(a=0):
     os.path.join("foo", "bar")
     some_c_function()
 
-cdef ham():
+cpdef eggs():
     pass    
     
-cpdef eggs():
+cdef ham():
     pass
     
 cdef class SomeClass(object):
     def spam(self):
         pass
 
+def outer():
+    cdef object a = "an object"
+    def inner():
+        b = 2
+        # access closed over variables
+        print a, b
+    return inner
+
+
+outer()()
+
 spam()
 print "bye!"
\ No newline at end of file

Cython/Debugger/Tests/test_libcython_in_gdb.py

@@ -14,6 +14,7 @@ import warnings
 import unittest
 import textwrap
 import tempfile
+import functools
 import traceback
 import itertools
 from test import test_support
@@ -28,12 +29,35 @@ from Cython.Debugger.Tests import TestLibCython as test_libcython
 sys.argv = ['gdb']
 
 
+def print_on_call_decorator(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        _debug(type(self).__name__, func.__name__)
+
+        try:
+            return func(self, *args, **kwargs)
+        except Exception, e:
+            _debug("An exception occurred:", traceback.format_exc(e))
+            raise
+
+    return wrapper
+
+class TraceMethodCallMeta(type):
+
+    def __init__(self, name, bases, dict):
+        for func_name, func in dict.iteritems():
+            if inspect.isfunction(func):
+                setattr(self, func_name, print_on_call_decorator(func))
+
+
 class DebugTestCase(unittest.TestCase):
     """
     Base class for test cases. On teardown it kills the inferior and unsets
     all breakpoints.
     """
 
+    __metaclass__ = TraceMethodCallMeta
+
     def __init__(self, name):
         super(DebugTestCase, self).__init__(name)
         self.cy = libcython.cy
@@ -58,7 +82,7 @@ class DebugTestCase(unittest.TestCase):
         if source_line is not None:
             lineno = test_libcython.source_to_lineno[source_line]
         frame = gdb.selected_frame()
-        self.assertEqual(libcython.cy.step.lineno(frame), lineno)
+        self.assertEqual(libcython.cython_info.lineno(frame), lineno)
 
     def break_and_run(self, source_line):
         break_lineno = test_libcython.source_to_lineno[source_line]
@@ -74,9 +98,6 @@ class DebugTestCase(unittest.TestCase):
 
         gdb.execute('set args -c "import codefile"')
 
-        libcython.cy.step.static_breakpoints.clear()
-        libcython.cy.step.runtime_breakpoints.clear()
-        libcython.cy.step.init_breakpoints()
 
 class TestDebugInformationClasses(DebugTestCase):
 
@@ -101,7 +122,7 @@ class TestDebugInformationClasses(DebugTestCase):
                          'codefile.SomeClass.spam')
         self.assertEqual(self.spam_func.module, self.module)
 
-        assert self.eggs_func.pf_cname
+        assert self.eggs_func.pf_cname, (self.eggs_func, self.eggs_func.pf_cname)
         assert not self.ham_func.pf_cname
         assert not self.spam_func.pf_cname
         assert not self.spam_meth.pf_cname
@@ -130,7 +151,7 @@ class TestParameters(unittest.TestCase):
 class TestBreak(DebugTestCase):
 
     def test_break(self):
-        breakpoint_amount = len(gdb.breakpoints())
+        breakpoint_amount = len(gdb.breakpoints() or ())
         gdb.execute('cy break codefile.spam')
 
         self.assertEqual(len(gdb.breakpoints()), breakpoint_amount + 1)
@@ -143,6 +164,16 @@ class TestBreak(DebugTestCase):
         gdb.execute('cy break -p join')
         assert 'def join(' in gdb.execute('cy run', to_string=True)
 
+    def test_break_lineno(self):
+        beginline = 'import os'
+        nextline = 'cdef int c_var = 12'
+
+        self.break_and_run(beginline)
+        self.lineno_equals(beginline)
+        step_result = gdb.execute('cy step', to_string=True)
+        self.lineno_equals(nextline)
+        assert step_result.rstrip().endswith(nextline)
+
 
 class TestKilled(DebugTestCase):
 
@@ -151,6 +182,7 @@ class TestKilled(DebugTestCase):
         output = gdb.execute('cy run', to_string=True)
         assert 'abort' in output.lower()
 
+
 class DebugStepperTestCase(DebugTestCase):
 
     def step(self, varnames_and_values, source_line=None, lineno=None):
@@ -262,7 +294,7 @@ class TestBacktrace(DebugTestCase):
 
         gdb.execute('cy bt')
         result = gdb.execute('cy bt -a', to_string=True)
-        assert re.search(r'\#0 *0x.* in main\(\) at', result), result
+        assert re.search(r'\#0 *0x.* in main\(\)', result), result
 
 
 class TestFunctions(DebugTestCase):
@@ -344,6 +376,36 @@ class TestExec(DebugTestCase):
         gdb.execute('cy exec some_random_var = 14')
         self.assertEqual('14', self.eval_command('some_random_var'))
 
+class TestClosure(DebugTestCase):
+
+    def break_and_run_func(self, funcname):
+        gdb.execute('cy break ' + funcname)
+        gdb.execute('cy run')
+
+    def test_inner(self):
+        self.break_and_run_func('inner')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+        
+        # Allow the Cython-generated code to initialize the scope variable
+        gdb.execute('cy step')
+
+        self.assertEqual(str(self.read_var('a')), "'an object'")
+        print_result = gdb.execute('cy print a', to_string=True).strip()
+        self.assertEqual(print_result, "a = 'an object'")
+
+    def test_outer(self):
+        self.break_and_run_func('outer')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+
+        # Initialize scope with 'a' uninitialized
+        gdb.execute('cy step')
+        self.assertEqual('', gdb.execute('cy locals', to_string=True))
+        
+        # Initialize 'a' to 1
+        gdb.execute('cy step')
+        print_result = gdb.execute('cy print a', to_string=True).strip()
+        self.assertEqual(print_result, "a = 'an object'")
+
 
 _do_debug = os.environ.get('GDB_DEBUG')
 if _do_debug:

runtests.py

@@ -675,12 +675,12 @@ class CythonPyregrTestCase(CythonRunTestCase):
         except (unittest.SkipTest, support.ResourceDenied):
             result.addSkip(self, 'ok')
 
-
-try:
-    import gdb
-    include_debugger = sys.version_info[:2] > (2, 5)
-except:
-    include_debugger = False
+# Someone wrapped this in a:
+# 'try: import gdb; ... except: include_debugger = False' thing, but don't do 
+# this, it doesn't work as gdb is a builtin module in GDB. The tests themselves
+# are doing the skipping. If there's a problem with the tests, please file an 
+# issue.
+include_debugger = sys.version_info[:2] > (2, 5)
 
 def collect_unittests(path, module_prefix, suite, selectors):
     def file_matches(filename):

setup.py

@@ -100,7 +100,8 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",
                         "Cython.Compiler.Code",
-                        "Cython.Runtime.refnanny"]
+                        "Cython.Runtime.refnanny",
+                        "Cython.Debugger.do_repeat",]
     if compile_more:
         compiled_modules.extend([
             "Cython.Compiler.ParseTreeTransforms",