merge

Cython/Debugger/Tests/TestLibCython.py

@@ -45,7 +45,7 @@ class DebuggerTestCase(unittest.TestCase):
         shutil.copy(cfuncs_file, self.cfuncs_destfile + '.c')
         
         compiler = ccompiler.new_compiler()
-        compiler.compile(['cfuncs.c'], debug=True)
+        compiler.compile(['cfuncs.c'], debug=True, extra_postargs=['-fPIC'])
         
         opts = dict(
             test_directory=self.tempdir,
@@ -122,14 +122,16 @@ class GdbDebuggerTestCase(DebuggerTestCase):
             python
             
             from Cython.Debugger.Tests import test_libcython_in_gdb
-            test_libcython_in_gdb.main()
+            test_libcython_in_gdb.main(version=%r)
             
             end
-            ''')
+            ''' % (sys.version_info[:2],))
         
         self.gdb_command_file = cygdb.make_command_file(self.tempdir, 
                                                         prefix_code)
-        open(self.gdb_command_file, 'a').write(code)
+        
+        with open(self.gdb_command_file, 'a') as f:
+            f.write(code)
         
         args = ['gdb', '-batch', '-x', self.gdb_command_file, '-n', '--args',
                 sys.executable, '-c', 'import codefile']
@@ -149,7 +151,7 @@ class GdbDebuggerTestCase(DebuggerTestCase):
             # gdb was not installed
             have_gdb = False
         else:
-            gdb_version = p.stdout.read()
+            gdb_version = p.stdout.read().decode('ascii')
             p.wait()
             p.stdout.close()
         
@@ -158,7 +160,8 @@ class GdbDebuggerTestCase(DebuggerTestCase):
             regex = "^GNU gdb [^\d]*(\d+)\.(\d+)"
             gdb_version_number = re.search(regex, gdb_version).groups()
 
-        if not have_gdb or map(int, gdb_version_number) < [7, 2]:
+        # Be Python 3 compatible
+        if not have_gdb or list(map(int, gdb_version_number)) < [7, 2]:
             self.p = None
             warnings.warn('Skipping gdb tests, need gdb >= 7.2')
         else:
@@ -186,7 +189,7 @@ class TestAll(GdbDebuggerTestCase):
         border = '*' * 30
         start = '%s   v INSIDE GDB v   %s' % (border, border)
         end   = '%s   ^ INSIDE GDB ^   %s' % (border, border)
-        errmsg = '\n%s\n%s%s' % (start, err, end)
+        errmsg = '\n%s\n%s%s' % (start, err.decode('UTF-8'), end)
         self.assertEquals(0, self.p.wait(), errmsg)
         sys.stderr.write(err)
 

Cython/Debugger/Tests/test_libcython_in_gdb.py

@@ -24,7 +24,6 @@ from Cython.Debugger import libcython
 from Cython.Debugger import libpython
 from Cython.Debugger.Tests import TestLibCython as test_libcython
 
-
 # for some reason sys.argv is missing in gdb
 sys.argv = ['gdb']
 
@@ -204,6 +203,7 @@ class TestStep(DebugStepperTestCase):
         self.assertEqual(str(pyframe.co_name), 'join')
         assert re.match(r'\d+    def join\(', result), result
 
+
 class TestNext(DebugStepperTestCase):
     
     def test_cython_next(self):
@@ -345,17 +345,18 @@ class TestExec(DebugTestCase):
         self.assertEqual('14', self.eval_command('some_random_var'))
 
 
-_do_debug = os.environ.get('CYTHON_GDB_DEBUG')
+_do_debug = os.environ.get('GDB_DEBUG')
 if _do_debug:
     _debug_file = open('/dev/tty', 'w')
 
 def _debug(*messages):
     if _do_debug:
-        messages = itertools.chain([sys._getframe(1).f_code.co_name], 
+        messages = itertools.chain([sys._getframe(1).f_code.co_name, ':'], 
                                    messages)
         _debug_file.write(' '.join(str(msg) for msg in messages) + '\n')
 
-def _main():
+
+def run_unittest_in_module(modulename):
     try:
         gdb.lookup_type('PyModuleObject')
     except RuntimeError:
@@ -365,7 +366,7 @@ def _main():
         warnings.warn(msg)
         os._exit(1)
     else:
-        m = __import__(__name__, fromlist=[''])
+        m = __import__(modulename, fromlist=[''])
         tests = inspect.getmembers(m, inspect.isclass)
         
         # test_support.run_unittest(tests)
@@ -375,15 +376,29 @@ def _main():
             [test_loader.loadTestsFromTestCase(cls) for name, cls in tests])
         
         result = unittest.TextTestRunner(verbosity=1).run(suite)
-        if not result.wasSuccessful():
-            os._exit(1)
+        return result.wasSuccessful()
 
-def main(trace_code=False):
+def runtests():
+    """
+    Run the libcython and libpython tests. Ensure that an appropriate status is
+    returned to the parent test process.
+    """
+    from Cython.Debugger.Tests import test_libpython_in_gdb
+    
+    success_libcython = run_unittest_in_module(__name__)
+    success_libpython = run_unittest_in_module(test_libpython_in_gdb.__name__)
+    
+    if not success_libcython or not success_libpython:
+        sys.exit(1)
+    
+def main(version, trace_code=False):
+    global inferior_python_version 
+    
+    inferior_python_version = version
+    
     if trace_code:
         tracer = trace.Trace(count=False, trace=True, outfile=sys.stderr,
                             ignoredirs=[sys.prefix, sys.exec_prefix])
-        tracer.runfunc(_main)
+        tracer.runfunc(runtests)
     else:
-        _main()
-
-main()
+        runtests()

Cython/Debugger/Tests/test_libpython_in_gdb.py

+# -*- coding: UTF-8 -*-
+
+"""
+Test libpython.py. This is already partly tested by test_libcython_in_gdb and 
+Lib/test/test_gdb.py in the Python source. These tests are run in gdb and
+called from test_libcython_in_gdb.main()
+"""
+
+import os
+import sys
+
+import gdb
+
+from Cython.Debugger import libcython
+from Cython.Debugger import libpython
+
+import test_libcython_in_gdb
+from test_libcython_in_gdb import _debug, inferior_python_version
+
+
+class TestPrettyPrinters(test_libcython_in_gdb.DebugTestCase):
+    """
+    Test whether types of Python objects are correctly inferred and that
+    the right libpython.PySomeTypeObjectPtr classes are instantiated.
+    
+    Also test whether values are appropriately formatted (don't be too 
+    laborious as Lib/test/test_gdb.py already covers this extensively).
+    
+    Don't take care of decreffing newly allocated objects as a new 
+    interpreter is started for every test anyway.
+    """
+    
+    def setUp(self):
+        super(TestPrettyPrinters, self).setUp()
+        self.break_and_run('b = c = d = 0')
+    
+    def get_pyobject(self, code):
+        value = gdb.parse_and_eval(code)
+        assert libpython.pointervalue(value) != 0
+        return value
+    
+    def pyobject_fromcode(self, code, gdbvar=None):
+        if gdbvar is not None:
+            d = {'varname':gdbvar, 'code':code}
+            gdb.execute('set $%(varname)s = %(code)s' % d)
+            code = '$' + gdbvar
+            
+        return libpython.PyObjectPtr.from_pyobject_ptr(self.get_pyobject(code))
+    
+    def get_repr(self, pyobject):
+        return pyobject.get_truncated_repr(libpython.MAX_OUTPUT_LEN)
+    
+    def alloc_bytestring(self, string, gdbvar=None):
+        if inferior_python_version < (3, 0):
+            funcname = 'PyString_FromString'
+        else:
+            funcname = 'PyBytes_FromString'
+        
+        assert '"' not in string
+        
+        # ensure double quotes
+        code = '(PyObject *) %s("%s")' % (funcname, string)
+        return self.pyobject_fromcode(code, gdbvar=gdbvar)
+        
+    def alloc_unicodestring(self, string, gdbvar=None):
+        self.alloc_bytestring(string.encode('UTF-8'), gdbvar='_temp')
+        
+        postfix = libpython.get_inferior_unicode_postfix()
+        funcname = 'PyUnicode%s_FromEncodedObject' % (postfix,)
+        
+        return self.pyobject_fromcode(
+            '(PyObject *) %s($_temp, "UTF-8", "strict")' % funcname, 
+            gdbvar=gdbvar)
+        
+    def test_bytestring(self):
+        bytestring = self.alloc_bytestring("spam")
+        
+        if inferior_python_version < (3, 0):
+            bytestring_class = libpython.PyStringObjectPtr
+            expected = repr("spam")
+        else:
+            bytestring_class = libpython.PyBytesObjectPtr
+            expected = "b'spam'"
+        
+        self.assertEqual(type(bytestring), bytestring_class)
+        self.assertEqual(self.get_repr(bytestring), expected)
+    
+    def test_unicode(self):
+        unicode_string = self.alloc_unicodestring(u"spam ἄλφα")
+        
+        expected = "'spam ἄλφα'"
+        if inferior_python_version < (3, 0):
+            expected = 'u' + expected
+        
+        self.assertEqual(type(unicode_string), libpython.PyUnicodeObjectPtr)
+        self.assertEqual(self.get_repr(unicode_string), expected)
+    
+    def test_int(self):
+        if inferior_python_version < (3, 0):
+            intval = self.pyobject_fromcode('PyInt_FromLong(100)')
+            self.assertEqual(type(intval), libpython.PyIntObjectPtr)
+            self.assertEqual(self.get_repr(intval), '100')
+    
+    def test_long(self):
+        longval = self.pyobject_fromcode('PyLong_FromLong(200)', 
+                                         gdbvar='longval')
+        assert gdb.parse_and_eval('$longval->ob_type == &PyLong_Type')
+        
+        self.assertEqual(type(longval), libpython.PyLongObjectPtr)
+        self.assertEqual(self.get_repr(longval), '200')
+        
+    def test_frame_type(self):
+        frame = self.pyobject_fromcode('PyEval_GetFrame()')
+        
+        self.assertEqual(type(frame), libpython.PyFrameObjectPtr)
+
+    
\ No newline at end of file

Cython/Debugger/libpython.py

@@ -55,6 +55,7 @@ import locale
 import atexit
 import warnings
 import tempfile
+import textwrap
 import itertools
 
 import gdb
@@ -69,12 +70,11 @@ if sys.version_info[0] < 3:
 
 # Look up the gdb.Type for some standard types:
 _type_char_ptr = gdb.lookup_type('char').pointer() # char*
-_type_unsigned_char_ptr = gdb.lookup_type('unsigned char').pointer() # unsigned char*
+_type_unsigned_char_ptr = gdb.lookup_type('unsigned char').pointer()
 _type_void_ptr = gdb.lookup_type('void').pointer() # void*
 
 SIZEOF_VOID_P = _type_void_ptr.sizeof
 
-
 Py_TPFLAGS_HEAPTYPE = (1L << 9)
 
 Py_TPFLAGS_INT_SUBCLASS      = (1L << 23)
@@ -88,8 +88,7 @@ Py_TPFLAGS_DICT_SUBCLASS     = (1L << 29)
 Py_TPFLAGS_BASE_EXC_SUBCLASS = (1L << 30)
 Py_TPFLAGS_TYPE_SUBCLASS     = (1L << 31)
 
-
-MAX_OUTPUT_LEN=1024
+MAX_OUTPUT_LEN = 1024
 
 hexdigits = "0123456789abcdef"
 
@@ -158,6 +157,17 @@ class TruncatedStringIO(object):
     def getvalue(self):
         return self._val
 
+
+# pretty printer lookup
+all_pretty_typenames = set()
+
+class PrettyPrinterTrackerMeta(type):
+    
+    def __init__(self, name, bases, dict):
+        super(PrettyPrinterTrackerMeta, self).__init__(name, bases, dict)
+        all_pretty_typenames.add(self._typename)
+
+
 class PyObjectPtr(object):
     """
     Class wrapping a gdb.Value that's a either a (PyObject*) within the
@@ -169,8 +179,11 @@ class PyObjectPtr(object):
     Note that at every stage the underlying pointer could be NULL, point
     to corrupt data, etc; this is the debugger, after all.
     """
+    
+    __metaclass__ = PrettyPrinterTrackerMeta
+    
     _typename = 'PyObject'
-
+    
     def __init__(self, gdbval, cast_to=None):
         if cast_to:
             self._gdbval = gdbval.cast(cast_to)
@@ -355,7 +368,7 @@ class PyObjectPtr(object):
                     }
         if tp_name in name_map:
             return name_map[tp_name]
-
+        
         if tp_flags & Py_TPFLAGS_HEAPTYPE:
             return HeapTypeObjectPtr
 
@@ -408,6 +421,7 @@ class PyObjectPtr(object):
     def as_address(self):
         return long(self._gdbval)
 
+
 class PyVarObjectPtr(PyObjectPtr):
     _typename = 'PyVarObject'
 
@@ -472,7 +486,7 @@ def _PyObject_VAR_SIZE(typeobj, nitems):
 
 class HeapTypeObjectPtr(PyObjectPtr):
     _typename = 'PyObject'
-
+    
     def get_attr_dict(self):
         '''
         Get the PyDictObject ptr representing the attribute dictionary
@@ -550,7 +564,7 @@ class PyBaseExceptionObjectPtr(PyObjectPtr):
     within the process being debugged.
     """
     _typename = 'PyBaseExceptionObject'
-
+    
     def proxyval(self, visited):
         # Guard against infinite loops:
         if self.as_address() in visited:
@@ -697,7 +711,7 @@ class PyDictObjectPtr(PyObjectPtr):
 
 class PyInstanceObjectPtr(PyObjectPtr):
     _typename = 'PyInstanceObject'
-
+    
     def proxyval(self, visited):
         # Guard against infinite loops:
         if self.as_address() in visited:
@@ -742,7 +756,7 @@ class PyIntObjectPtr(PyObjectPtr):
 
 class PyListObjectPtr(PyObjectPtr):
     _typename = 'PyListObject'
-
+    
     def __getitem__(self, i):
         # Get the gdb.Value for the (PyObject*) with the given index:
         field_ob_item = self.field('ob_item')
@@ -775,7 +789,7 @@ class PyListObjectPtr(PyObjectPtr):
 
 class PyLongObjectPtr(PyObjectPtr):
     _typename = 'PyLongObject'
-
+    
     def proxyval(self, visited):
         '''
         Python's Include/longobjrep.h has this declaration:
@@ -793,7 +807,7 @@ class PyLongObjectPtr(PyObjectPtr):
         where SHIFT can be either:
             #define PyLong_SHIFT        30
             #define PyLong_SHIFT        15
-        '''
+        '''        
         ob_size = long(self.field('ob_size'))
         if ob_size == 0:
             return 0L
@@ -823,9 +837,12 @@ class PyBoolObjectPtr(PyLongObjectPtr):
     Class wrapping a gdb.Value that's a PyBoolObject* i.e. one of the two
     <bool> instances (Py_True/Py_False) within the process being debugged.
     """
+    _typename = 'PyBoolObject'
     
     def proxyval(self, visited):
-        return bool(PyLongObjectPtr.proxyval(self, visited))
+        castto = gdb.lookup_type('PyLongObject').pointer()
+        self._gdbval = self._gdbval.cast(castto)
+        return bool(PyLongObjectPtr(self._gdbval).proxyval(visited))
 
 
 class PyNoneStructPtr(PyObjectPtr):
@@ -1033,7 +1050,7 @@ class PySetObjectPtr(PyObjectPtr):
 
 class PyBytesObjectPtr(PyObjectPtr):
     _typename = 'PyBytesObject'
-
+    
     def __str__(self):
         field_ob_size = self.field('ob_size')
         field_ob_sval = self.field('ob_sval')
@@ -1043,7 +1060,7 @@ class PyBytesObjectPtr(PyObjectPtr):
     def proxyval(self, visited):
         return str(self)
 
-    def write_repr(self, out, visited):
+    def write_repr(self, out, visited, py3=True):
         # Write this out as a Python 3 bytes literal, i.e. with a "b" prefix
 
         # Get a PyStringObject* within the Python 2 gdb process:
@@ -1054,7 +1071,10 @@ class PyBytesObjectPtr(PyObjectPtr):
         quote = "'"
         if "'" in proxy and not '"' in proxy:
             quote = '"'
-        out.write('b')
+        
+        if py3:
+            out.write('b')
+
         out.write(quote)
         for byte in proxy:
             if byte == quote or byte == '\\':
@@ -1077,6 +1097,8 @@ class PyBytesObjectPtr(PyObjectPtr):
 class PyStringObjectPtr(PyBytesObjectPtr):
     _typename = 'PyStringObject'
 
+    def write_repr(self, out, visited):
+        return super(PyStringObjectPtr, self).write_repr(out, visited, py3=False)
 
 class PyTupleObjectPtr(PyObjectPtr):
     _typename = 'PyTupleObject'
@@ -1184,13 +1206,20 @@ class PyUnicodeObjectPtr(PyObjectPtr):
         return result
 
     def write_repr(self, out, visited):
-        # Write this out as a Python 3 str literal, i.e. without a "u" prefix
-
         # Get a PyUnicodeObject* within the Python 2 gdb process:
         proxy = self.proxyval(visited)
 
         # Transliteration of Python 3's Object/unicodeobject.c:unicode_repr
         # to Python 2:
+        try:
+            gdb.parse_and_eval('PyString_Type')
+        except RuntimeError:
+            # Python 3, don't write 'u' as prefix
+            pass
+        else:
+            # Python 2, write the 'u'
+            out.write('u')
+        
         if "'" in proxy and '"' not in proxy:
             quote = '"'
         else:
@@ -1292,8 +1321,6 @@ class PyUnicodeObjectPtr(PyObjectPtr):
         out.write(quote)
 
 
-
-
 def int_from_int(gdbval):
     return int(str(gdbval))
 
@@ -1324,16 +1351,11 @@ class PyObjectPtrPrinter:
             proxyval = pyop.proxyval(set())
             return stringify(proxyval)
 
-
 def pretty_printer_lookup(gdbval):
     type = gdbval.type.unqualified()
     if type.code == gdb.TYPE_CODE_PTR:
         type = type.target().unqualified()
-        # do this every time to allow new subclasses to "register"
-        # alternatively, we could use a metaclass to register all the typenames
-        classes = [PyObjectPtr]
-        classes.extend(PyObjectPtr.__subclasses__())
-        if str(type) in [cls._typename for cls in classes]:
+        if str(type) in all_pretty_typenames:
             return PyObjectPtrPrinter(gdbval)
 
 """
@@ -1364,8 +1386,6 @@ def register (obj):
 
 register (gdb.current_objfile ())
 
-
-
 # Unfortunately, the exact API exposed by the gdb module varies somewhat
 # from build to build
 # See http://bugs.python.org/issue8279?#msg102276
@@ -1868,11 +1888,8 @@ class GenericCodeStepper(gdb.Command):
         Keep all breakpoints around and simply disable/enable them each time
         we are stepping. We need this because if you set and delete a 
         breakpoint, gdb will not repeat your command (this is due to 'delete').
-        Why? I'm buggered if I know. To further annoy us, we can't use the 
-        breakpoint API because there's no option to make breakpoint setting 
-        silent.
-        So now! We may have an insane amount of breakpoints to list when the
-        user does 'info breakpoints' :(
+        We also can't use the breakpoint API because there's no option to make 
+        breakpoint setting silent.
         
         This method must be called whenever the list of functions we should
         step into changes. It can be called on any GenericCodeStepper instance.
@@ -1890,10 +1907,11 @@ class GenericCodeStepper(gdb.Command):
                 except RuntimeError:
                     # gdb.Breakpoint does take an 'internal' argument, use it
                     # and hide output
-                    result = gdb.execute(
-                        "python bp = gdb.Breakpoint(%r, gdb.BP_BREAKPOINT, internal=True); "
-                               "print bp.number", 
-                        to_string=True)
+                    result = gdb.execute(textwrap.dedent("""\
+                        python bp = gdb.Breakpoint(%r, gdb.BP_BREAKPOINT, \
+                                                   internal=True); \
+                               print bp.number""", 
+                        to_string=True))
                     
                     breakpoint = int(result)
 
@@ -2177,6 +2195,19 @@ def pointervalue(gdbval):
         
     return pointer
 
+def get_inferior_unicode_postfix():
+    try:
+        gdb.parse_and_eval('PyUnicode_FromEncodedObject')
+    except RuntimeError:
+        try: 
+            gdb.parse_and_eval('PyUnicodeUCS2_FromEncodedObject')
+        except RuntimeError:
+            return 'UCS4'
+        else:
+            return 'UCS2'
+    else:
+        return ''
+        
 class PythonCodeExecutor(object):
         
     def malloc(self, size):
@@ -2197,16 +2228,14 @@ class PythonCodeExecutor(object):
     def alloc_pystring(self, string):
         stringp = self.alloc_string(string)
         PyString_FromStringAndSize = 'PyString_FromStringAndSize'
+        
         try:
             gdb.parse_and_eval(PyString_FromStringAndSize)
         except RuntimeError:
-            try:
-                gdb.parse_and_eval('PyUnicode_FromStringAndSize')
-            except RuntimeError:
-                PyString_FromStringAndSize = 'PyUnicodeUCS2_FromStringAndSize'
-            else:
-                PyString_FromStringAndSize = 'PyUnicode_FromStringAndSize'
-            
+            # Python 3
+            PyString_FromStringAndSize = ('PyUnicode%s_FromStringAndSize' % 
+                                               (get_inferior_unicode_postfix,))
+
         try:
             result = gdb.parse_and_eval(
                 '(PyObject *) %s((char *) %d, (size_t) %d)' % (
@@ -2259,7 +2288,7 @@ class PythonCodeExecutor(object):
         
         code = """
             PyRun_String(
-                (PyObject *) %(code)d,
+                (char *) %(code)d,
                 (int) %(start)d,
                 (PyObject *) %(globals)s,
                 (PyObject *) %(locals)d)

runtests.py

@@ -644,8 +644,8 @@ class CythonUnitTestCase(CythonCompileTestCase):
         except Exception:
             pass
 
-# TODO: Re-enable once they're more robust.
-include_debugger = sys.version_info[:2] >= (2, 5) and False
+
+include_debugger = sys.version_info[:2] > (2, 5)
 
 def collect_unittests(path, module_prefix, suite, selectors):
     def file_matches(filename):

setup.py

@@ -71,7 +71,7 @@ else:
 setuptools_extra_args = {}
 
 # tells whether to include cygdb (the script and the Cython.Debugger package
-include_debugger = sys.version_info[:2] >= (2, 5)
+include_debugger = sys.version_info[:2] > (2, 5)
 
 if 'setuptools' in sys.modules:
     setuptools_extra_args['zip_safe'] = False