Tests!

(run: python runtests.py Cython.Tests.TestStringIOTree \
                         Cython.Debugger.Tests.TestLibCython \
                         Cython.Compiler.Tests.TestParseTreeTransforms)

--HG--
rename : Cython/Debugger/cygdb.py => Cython/Debugger/Cygdb.py

Cython/Compiler/Main.py

@@ -181,9 +181,10 @@ class Context(object):
             test_support.append(TreeAssertVisitor())
 
         if options.debug:
-            from Cython.Debugger import debug_output
+            from Cython.Debugger import DebugWriter
             from ParseTreeTransforms import DebugTransform
-            self.debug_outputwriter = debug_output.CythonDebugWriter(options)
+            self.debug_outputwriter = DebugWriter.CythonDebugWriter(
+                options.output_dir)
             debug_transform = [DebugTransform(self)]
         else:
             debug_transform = []

Cython/Compiler/ParseTreeTransforms.py

@@ -1523,7 +1523,7 @@ class DebugTransform(CythonTransform):
     def serialize_local_variables(self, entries):
         for entry in entries.values():
             if entry.type.is_pyobject:
-                vartype = 'PyObject'
+                vartype = 'PythonObject'
             else:
                 vartype = 'CObject'
             

Cython/Compiler/Tests/TestParseTreeTransforms.py

+import os
+
+from Cython.Debugger import DebugWriter
+from Cython.Compiler import Main
+from Cython.Compiler import CmdLine
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.Nodes import *
+from Cython.Debugger.Tests import TestLibCython
 
 class TestNormalizeTree(TransformTest):
     def test_parserbehaviour_is_what_we_coded_for(self):
@@ -140,6 +146,66 @@ class TestWithTransform(object): # (TransformTest): # Disabled!
         """, t)
                           
         
+class TestDebugTransform(TestLibCython.DebuggerTestCase):
+    
+    def elem_hasattrs(self, elem, attrs):
+        return all(attr in elem.attrib for attr in attrs)
+    
+    def test_debug_info(self):
+        try:
+            assert os.path.exists(self.debug_dest)
+            
+            t = DebugWriter.etree.parse(self.debug_dest)
+            # the xpath of the standard ElementTree is primitive, don't use
+            # anything fancy
+            L = list(t.find('/Module/Globals'))
+            # assertTrue is retarded, use the normal assert statement
+            assert L
+            xml_globals = dict((e.attrib['name'], e.attrib['type']) for e in L)
+            self.assertEqual(len(L), len(xml_globals))
+            
+            L = list(t.find('/Module/Functions'))
+            assert L
+            xml_funcs = dict((e.attrib['qualified_name'], e) for e in L)
+            self.assertEqual(len(L), len(xml_funcs))
+            
+            # test globals
+            self.assertEqual('CObject', xml_globals.get('c_var'))
+            self.assertEqual('PythonObject', xml_globals.get('python_var'))
+            
+            # test functions
+            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            required_xml_attrs = 'name', 'cname', 'qualified_name'
+            assert all(f in xml_funcs for f in funcnames)
+            spam, ham, eggs = (xml_funcs[funcname] for funcname in funcnames) 
+            
+            self.assertEqual(spam.attrib['name'], 'spam')
+            self.assertNotEqual('spam', spam.attrib['cname'])
+            assert self.elem_hasattrs(spam, required_xml_attrs)
+
+            # test locals of functions
+            spam_locals = list(spam.find('Locals'))
+            assert spam_locals
+            spam_locals.sort(key=lambda e: e.attrib['name'])
+            names = [e.attrib['name'] for e in spam_locals]
+            self.assertEqual(list('abcd'), names)
+            assert self.elem_hasattrs(spam_locals[0], required_xml_attrs)
+            
+            # test arguments of functions
+            spam_arguments = list(spam.find('Arguments'))
+            assert spam_arguments
+            self.assertEqual(1, len(list(spam_arguments)))
+            
+            # test step-into functions
+            spam_stepinto = list(spam.find('StepIntoFunctions'))
+            assert spam_stepinto
+            self.assertEqual(1, len(list(spam_stepinto)))
+            self.assertEqual('puts', list(spam_stepinto)[0].attrib['name'])
+        except:
+            print open(self.debug_dest).read()
+            raise
+            
+
 if __name__ == "__main__":
     import unittest
     unittest.main()

Cython/Debugger/cygdb.py → Cython/Debugger/Cygdb.py

@@ -20,17 +20,7 @@ import subprocess
 def usage():
     print("Usage: cygdb [PATH GDB_ARGUMENTS]")
 
-def main(gdb_argv=[], import_libpython=False, path_to_debug_info=os.curdir):
-    """
-    Start the Cython debugger. This tells gdb to import the Cython and Python
-    extensions (libpython.py and libcython.py) and it enables gdb's pending 
-    breakpoints
-    
-    import_libpython indicates whether we should just 'import libpython',
-    or import it from Cython.Debugger
-    
-    path_to_debug_info is the path to the cython_debug directory
-    """
+def make_command_file(path_to_debug_info):
     debug_files = glob.glob(
         os.path.join(path_to_debug_info, 'cython_debug/cython_debug_info_*'))
 
@@ -43,13 +33,20 @@ def main(gdb_argv=[], import_libpython=False, path_to_debug_info=os.curdir):
     f = os.fdopen(fd, 'w')
     f.write('set breakpoint pending on\n')
     f.write('python from Cython.Debugger import libcython\n')
-    if import_libpython:
-        f.write('python import libpython')
-    else:
-        f.write('python from Cython.Debugger import libpython\n')
     f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
     f.close()
     
+    return tempfilename
+
+def main(gdb_argv=[], path_to_debug_info=os.curdir):
+    """
+    Start the Cython debugger. This tells gdb to import the Cython and Python
+    extensions (libpython.py and libcython.py) and it enables gdb's pending 
+    breakpoints
+    
+    path_to_debug_info is the path to the cython_debug directory
+    """
+    tempfilename = make_command_file(path_to_debug_info)
     p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv)
     while True:
         try:

Cython/Debugger/Tests/TestLibCython.py

+import os
+import re
+import sys
+import uuid
+import shutil
+import textwrap
+import unittest
+import tempfile
+import subprocess
+import distutils.core
+from distutils import sysconfig
+
+import Cython.Distutils.extension
+from Cython.Debugger import Cygdb as cygdb
+
+
+class DebuggerTestCase(unittest.TestCase):
+    
+    def setUp(self):
+        """
+        Run gdb and have cygdb import the debug information from the code
+        defined in TestParseTreeTransforms's setUp method
+        """
+        self.tempdir = tempfile.mkdtemp()
+        self.destfile = os.path.join(self.tempdir, 'codefile.pyx')
+        self.debug_dest = os.path.join(self.tempdir, 
+                                      'cython_debug', 
+                                      'cython_debug_info_codefile')
+        
+        code = textwrap.dedent("""
+            cdef extern from "stdio.h":
+                int puts(char *s)
+                
+            cdef int c_var = 0
+            python_var = 0
+            
+            def spam(a=0):
+                cdef:
+                    int b, c, d
+                
+                b = c = d = 0
+                
+                b = 1
+                c = 2
+                d = 3
+                int(10)
+                puts("spam")
+                
+            cdef ham():
+                pass
+                
+            cpdef eggs():
+                pass
+            
+            cdef class SomeClass(object):
+                def spam(self):
+                    pass
+            
+            spam()
+        """)
+        
+        self.cwd = os.getcwd()
+        os.chdir(self.tempdir)
+        
+        open(self.destfile, 'w').write(code)
+        
+        ext = Cython.Distutils.extension.Extension(
+            'codefile',
+            ['codefile.pyx'], 
+            pyrex_debug=True)
+            
+        distutils.core.setup(
+            script_args=['build_ext', '--inplace'],
+            ext_modules=[ext], 
+            cmdclass=dict(build_ext=Cython.Distutils.build_ext)
+        )
+    
+    def tearDown(self):
+        os.chdir(self.cwd)
+        shutil.rmtree(self.tempdir)
+
+
+class GdbDebuggerTestCase(DebuggerTestCase):
+    def setUp(self):
+        super(GdbDebuggerTestCase, self).setUp()
+        
+        self.gdb_command_file = cygdb.make_command_file(self.tempdir)
+        with open(self.gdb_command_file, 'a') as f:
+            f.write('python '
+                'from Cython.Debugger.Tests import test_libcython_in_gdb;'
+                'test_libcython_in_gdb.main()\n')
+                
+        args = ['gdb', '-batch', '-x', self.gdb_command_file, '-n', '--args',
+                sys.executable, '-c', 'import codefile']
+        
+        paths = []
+        path = os.environ.get('PYTHONPATH')
+        if path:
+            paths.append(path)
+        paths.append(os.path.dirname(os.path.dirname(
+            os.path.abspath(Cython.__file__))))
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(paths))
+        self.p = subprocess.Popen(
+            args,
+            stdout=open(os.devnull, 'w'),
+            stderr=subprocess.PIPE,
+            env=env)
+        
+    def tearDown(self):
+        super(GdbDebuggerTestCase, self).tearDown()
+        self.p.stderr.close()
+        self.p.wait()
+        os.remove(self.gdb_command_file)
+        
+   
+class TestAll(GdbDebuggerTestCase):
+    
+    def test_all(self):
+        out, err = self.p.communicate()
+        border = '*' * 30
+        start = '%s   v INSIDE GDB v   %s' % (border, border)
+        end   = '%s   ^ INSIDE GDB ^   %s' % (border, border)
+        errmsg = '\n%s\n%s%s' % (start, err, end)
+        self.assertEquals(0, self.p.wait(), errmsg)
+        sys.stderr.write(err)
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

Cython/Debugger/Tests/test_libcython_in_gdb.py

+"""
+Tests that run inside GDB.
+
+Note: debug information is already imported by the file generated by
+Cython.Debugger.Cygdb.make_command_file()
+"""
+
+import sys
+
+# First, fix gdb's python. Make sure to do this before importing modules
+# that bind output streams as default parameters
+
+# for some reason sys.argv is missing in gdb
+sys.argv = ['gdb']
+
+# Allow gdb to capture output, but have errors end up on stderr
+# sys.stdout = sys.__stdout__
+sys.stderr = sys.__stderr__
+
+import os
+import warnings
+import unittest
+import traceback
+from test import test_support
+
+import gdb
+
+from Cython.Debugger import libcython
+
+class DebugTestCase(unittest.TestCase):
+    
+    def __init__(self, name):
+        super(DebugTestCase, self).__init__(name)
+        self.cy = libcython.cy
+        self.module = libcython.cy.cython_namespace['codefile']
+        self.spam_func, self.spam_meth = libcython.cy.functions_by_name['spam']
+        self.ham_func = libcython.cy.functions_by_qualified_name[
+            'codefile.ham']
+        self.eggs_func = libcython.cy.functions_by_qualified_name[
+            'codefile.eggs']
+    
+    def read_var(self, varname):
+        return gdb.parse_and_eval('$cy_cname("%s")' % varname)
+    
+    def local_info(self):
+        return gdb.execute('info locals', to_string=True)
+    
+class TestDebugInformationClasses(DebugTestCase):
+    
+    
+    def test_CythonModule(self):
+        "test that debug information was parsed properly into data structures"
+        self.assertEqual(self.module.name, 'codefile')
+        global_vars = ('c_var', 'python_var', 'SomeClass', '__name__', 
+                       '__builtins__', '__doc__', '__file__')
+        assert set(global_vars).issubset(self.module.globals)
+        
+    def test_CythonVariable(self):
+        module_globals = self.module.globals
+        c_var = module_globals['c_var']
+        python_var = module_globals['python_var']
+        self.assertEqual(c_var.type, libcython.CObject)
+        self.assertEqual(python_var.type, libcython.PythonObject)
+        self.assertEqual(c_var.qualified_name, 'codefile.c_var')
+    
+    def test_CythonFunction(self):
+        self.assertEqual(self.spam_func.qualified_name, 'codefile.spam')
+        self.assertEqual(self.spam_meth.qualified_name, 
+                         'codefile.SomeClass.spam')
+        self.assertEqual(self.spam_func.module, self.module)
+        
+        assert self.eggs_func.pf_cname
+        assert not self.ham_func.pf_cname
+        assert not self.spam_func.pf_cname
+        assert not self.spam_meth.pf_cname
+        
+        self.assertEqual(self.spam_func.type, libcython.CObject)
+        self.assertEqual(self.ham_func.type, libcython.CObject)
+        
+        self.assertEqual(self.spam_func.arguments, ['a'])
+        self.assertEqual(self.spam_func.step_into_functions, set(['puts']))
+        
+        self.assertEqual(self.spam_func.lineno, 8)
+        self.assertEqual(sorted(self.spam_func.locals), list('abcd'))
+
+
+class TestParameters(unittest.TestCase):
+    
+    def test_parameters(self):
+        assert libcython.parameters.colorize_code
+        gdb.execute('set cy_colorize_code off')
+        assert not libcython.parameters.colorize_code
+
+
+class TestBreak(DebugTestCase):
+
+    def test_break(self):
+        result = gdb.execute('cy break codefile.spam', to_string=True)
+        assert self.spam_func.cname in result
+        
+        self.assertEqual(len(gdb.breakpoints()), 1)
+        bp, = gdb.breakpoints()
+        self.assertEqual(bp.type, gdb.BP_BREAKPOINT)
+        self.assertEqual(bp.location, self.spam_func.cname)
+        assert bp.enabled
+
+
+class TestStep(DebugTestCase):
+    
+    def test_step(self):
+        # Note: breakpoint for spam is still set
+        gdb.execute('run')
+        
+        gdb.execute('cy step', to_string=True) # b = c = d = 0
+        gdb.execute('cy step', to_string=True) # b = 1
+        self.assertEqual(self.read_var('b'), 1, self.local_info())
+        self.assertRaises(RuntimeError, self.read_var('b'))
+        gdb.execute('cont')
+
+
+class TestNext(DebugTestCase):
+    
+    def test_next(self):
+        pass
+    
+def main():
+    try:
+        # unittest.main(module=__import__(__name__, fromlist=['']))
+        try:
+            gdb.lookup_type('PyModuleObject')
+        except RuntimeError:
+            msg = ("Unable to run tests, Python was not compiled with "
+                   "debugging information. Either compile python with "
+                   "-g or get a debug build (configure with --with-pydebug).")
+            warnings.warn(msg)
+        else:
+            tests = (
+                TestDebugInformationClasses,
+                TestParameters,
+                TestBreak,
+                TestStep
+            )
+            # test_support.run_unittest(tests)
+            test_loader = unittest.TestLoader()
+            suite = unittest.TestSuite(
+                [test_loader.loadTestsFromTestCase(cls) for cls in tests])
+                
+            unittest.TextTestRunner(verbosity=1).run(suite)
+    except Exception:
+        traceback.print_exc()
+        os._exit(1)
\ No newline at end of file

Cython/Debugger/libpython.py

 #!/usr/bin/python
+
+# NOTE: this file is taken from the Python source distribution
+# It can be found under Tools/gdb/libpython.py
+
 '''
 From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb
 to be extended with Python code e.g. for library-specific data visualizations,
@@ -41,6 +45,7 @@ The module also extends gdb with some python-specific commands.
 '''
 from __future__ import with_statement
 import gdb
+import sys
 
 # Look up the gdb.Type for some standard types:
 _type_char_ptr = gdb.lookup_type('char').pointer() # char*
@@ -1009,6 +1014,18 @@ class PyTypeObjectPtr(PyObjectPtr):
     _typename = 'PyTypeObject'
 
 
+if sys.maxunicode >= 0x10000:
+    _unichr = unichr
+else:
+    # Needed for proper surrogate support if sizeof(Py_UNICODE) is 2 in gdb
+    def _unichr(x):
+        if x < 0x10000:
+            return unichr(x)
+        x -= 0x10000
+        ch1 = 0xD800 | (x >> 10)
+        ch2 = 0xDC00 | (x & 0x3FF)
+        return unichr(ch1) + unichr(ch2)
+
 class PyUnicodeObjectPtr(PyObjectPtr):
     _typename = 'PyUnicodeObject'
 
@@ -1025,36 +1042,35 @@ class PyUnicodeObjectPtr(PyObjectPtr):
 
         # Gather a list of ints from the Py_UNICODE array; these are either
         # UCS-2 or UCS-4 code points:
+        if self.char_width() > 2:
             Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)]
-
-        # Convert the int code points to unicode characters, and generate a
-        # local unicode instance:
-        result = u''.join([unichr(ucs) for ucs in Py_UNICODEs])
-        return result
-
-    def write_repr(self, out, visited):
-        proxy = self.proxyval(visited)
-        if self.char_width() == 2:
-            # sizeof(Py_UNICODE)==2: join surrogates
-            proxy2 = []
+        else:
+            # A more elaborate routine if sizeof(Py_UNICODE) is 2 in the
+            # inferior process: we must join surrogate pairs.
+            Py_UNICODEs = []
             i = 0
-            while i < len(proxy):
-                ch = proxy[i]
+            limit = safety_limit(field_length)
+            while i < limit:
+                ucs = int(field_str[i])
                 i += 1
-                if (i < len(proxy)
-                and 0xD800 <= ord(ch) < 0xDC00 \
-                and 0xDC00 <= ord(proxy[i]) <= 0xDFFF):
-                    # Get code point from surrogate pair
-                    ch2 = proxy[i]
-                    code = (ord(ch) & 0x03FF) << 10
-                    code |= ord(ch2) & 0x03FF
+                if ucs < 0xD800 or ucs >= 0xDC00 or i == field_length:
+                    Py_UNICODEs.append(ucs)
+                    continue
+                # This could be a surrogate pair.
+                ucs2 = int(field_str[i])
+                if ucs2 < 0xDC00 or ucs2 > 0xDFFF:
+                    continue
+                code = (ucs & 0x03FF) << 10
+                code |= ucs2 & 0x03FF
                 code += 0x00010000
+                Py_UNICODEs.append(code)
                 i += 1
-                    proxy2.append(unichr(code))
-                else:
-                    proxy2.append(ch)
-            proxy = u''.join(proxy2)
-        out.write(repr(proxy))
+
+        # Convert the int code points to unicode characters, and generate a
+        # local unicode instance.
+        # This splits surrogate pairs if sizeof(Py_UNICODE) is 2 here (in gdb).
+        result = u''.join([_unichr(ucs) for ucs in Py_UNICODEs])
+        return result
 
 
 def int_from_int(gdbval):

Cython/StringIOTree.py

@@ -16,7 +16,6 @@ class StringIOTree(object):
     def getvalue(self):
         content = [x.getvalue() for x in self.prepended_children]
         content.append(self.stream.getvalue())
-        print self.linenumber_map()
         return "".join(content)
 
     def copyto(self, target):
@@ -33,6 +32,8 @@ class StringIOTree(object):
         # itself is empty -- this makes it ready for insertion
         if self.stream.tell():
             self.prepended_children.append(StringIOTree(self.stream))
+            self.prepended_children[-1].markers = self.markers
+            self.markers = []
             self.stream = StringIO()
             self.write = self.stream.write
 

bin/cygdb

@@ -2,7 +2,7 @@
 
 import sys
 
-from Cython.Debugger import cygdb
+from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:

cygdb.py

@@ -2,7 +2,7 @@
 
 import sys
 
-from Cython.Debugger import cygdb
+from Cython.Debugger import Cygdb as cygdb
 
 if __name__ == '__main__':
     if len(sys.argv) > 1:

runtests.py

@@ -606,8 +606,13 @@ def collect_doctests(path, module_prefix, suite, selectors):
     def package_matches(dirname):
         return dirname not in ("Mac", "Distutils", "Plex")
     def file_matches(filename):
-        return (filename.endswith(".py") and not ('~' in filename
-                or '#' in filename or filename.startswith('.')))
+        filename, ext = os.path.splitext(filename)
+        blacklist = ('libcython', 'libpython', 'test_libcython_in_gdb')
+        return (ext == '.py' and not
+                '~' in filename and not
+                '#' in filename and not
+                filename.startswith('.') and not
+                filename in blacklist)
     import doctest, types
     for dirpath, dirnames, filenames in os.walk(path):
         parentname = os.path.split(dirpath)[-1]