Merge remote branch 'upstream/master'

.hgtags

@@ -33,3 +33,8 @@ ef9d2c680684d0df7d81f529cda29e9e1741f575 cython-0.10.1
 59c67af0674bd93c5fd8958e08c76a9dab9aae37 sage-cythonizes
 0000000000000000000000000000000000000000 sage-cythonizes
 478f57be445d18fe294db849d7ad317fe7d7658f 0.14.alpha0
+31b531a6c45b2c34ae5a1af8a2c09f152adea60d 0.14.beta1
+7fa84cb6d3d75eb3d015aeeb60bf8b642171fe93 0.14.beta2
+7fa84cb6d3d75eb3d015aeeb60bf8b642171fe93 0.14.beta2
+8412b39fbc3eb709a543e2f1e95c0c8881ea9ed4 0.14.beta2
+a6b9f0a6d02d23fc3d3a9d0587867faa3afb2fcd 0.14.rc0

Cython/Build/Dependencies.py

 from glob import glob
 import re, os, sys
+from cython import set
 
-try:
-    set
-except NameError:
-    # Python 2.3
-    from sets import Set as set
 
 from distutils.extension import Extension
 
@@ -56,8 +52,8 @@ distutils_settings = {
     'runtime_library_dirs': transitive_list,
     'include_dirs':         transitive_list,
     'extra_objects':        list,
-    'extra_compile_args':   list,
-    'extra_link_args':      list,
+    'extra_compile_args':   transitive_list,
+    'extra_link_args':      transitive_list,
     'export_symbols':       list,
     'depends':              transitive_list,
     'language':             transitive_str,
@@ -97,7 +93,7 @@ class DistutilsInfo(object):
         elif exn is not None:
             for key in distutils_settings:
                 if key in ('name', 'sources'):
-                    pass
+                    continue
                 value = getattr(exn, key, None)
                 if value:
                     self.values[key] = value
@@ -168,7 +164,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
 
         # Try to close the quote.
         elif in_quote:
-            if code[q-1] == '\\':
+            if code[q-1] == '\\' and not raw:
                 k = 2
                 while q >= k and code[q-k] == '\\':
                     k += 1
@@ -177,7 +173,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                     continue
             if code[q:q+len(in_quote)] == in_quote:
                 counter += 1
-                label = "%s%s" % (prefix, counter)
+                label = "%s%s_" % (prefix, counter)
                 literals[label] = code[start+len(in_quote):q]
                 new_code.append("%s%s%s" % (in_quote, label, in_quote))
                 q += len(in_quote)
@@ -193,7 +189,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                 end = None
             new_code.append(code[start:hash_mark+1])
             counter += 1
-            label = "%s%s" % (prefix, counter)
+            label = "%s%s_" % (prefix, counter)
             literals[label] = code[hash_mark+1:end]
             new_code.append(label)
             if end is None:
@@ -208,11 +204,11 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                 in_quote = code[q]*3
             else:
                 in_quote = code[q]
-            end = q
-            while end>0 and code[end-1] in 'rRbBuU':
-                if code[end-1] in 'rR':
+            end = marker = q
+            while marker > 0 and code[marker-1] in 'rRbBuU':
+                if code[marker-1] in 'rR':
                     raw = True
-                end -= 1
+                marker -= 1
             new_code.append(code[start:end])
             start = q
             q += len(in_quote)
@@ -314,8 +310,8 @@ class DependencyTree(object):
             self_pxd = []
         a = self.cimports(filename)
         b = filter(None, [self.find_pxd(m, filename) for m in self.cimports(filename)])
-        if len(a) != len(b):
-            print(filename)
+        if len(a) - int('cython' in a) != len(b):
+            print("missing cimport", filename)
             print("\n\t".join(a))
             print("\n\t".join(b))
         return tuple(self_pxd + filter(None, [self.find_pxd(m, filename) for m in self.cimports(filename)]))
@@ -390,9 +386,14 @@ def create_dependency_tree(ctx=None):
     return _dep_tree
 
 # This may be useful for advanced users?
-def create_extension_list(patterns, ctx=None, aliases=None):
+def create_extension_list(patterns, exclude=[], ctx=None, aliases=None):
     seen = set()
     deps = create_dependency_tree(ctx)
+    to_exclude = set()
+    if not isinstance(exclude, list):
+        exclude = [exclude]
+    for pattern in exclude:
+        to_exclude.update(glob(pattern))
     if not isinstance(patterns, list):
         patterns = [patterns]
     module_list = []
@@ -416,27 +417,39 @@ def create_extension_list(patterns, ctx=None, aliases=None):
         else:
             raise TypeError(pattern)
         for file in glob(filepattern):
+            if file in to_exclude:
+                continue
             pkg = deps.package(file)
             if '*' in name:
                 module_name = deps.fully_qualifeid_name(file)
             else:
                 module_name = name
             if module_name not in seen:
+                kwds = deps.distutils_info(file, aliases, base).values
+                if base is not None:
+                    for key, value in base.values.items():
+                        if key not in kwds:
+                            kwds[key] = value
                 module_list.append(exn_type(
                         name=module_name,
                         sources=[file],
-                        **deps.distutils_info(file, aliases, base).values))
+                        **kwds))
+                m = module_list[-1]
                 seen.add(name)
     return module_list
 
 # This is the user-exposed entry point.
-def cythonize(module_list, nthreads=0, aliases=None, **options):
+def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **options):
     if 'include_path' not in options:
         options['include_path'] = ['.']
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
     ctx = c_options.create_context()
-    module_list = create_extension_list(module_list, ctx=ctx, aliases=aliases)
+    module_list = create_extension_list(
+        module_list,
+        exclude=exclude,
+        ctx=ctx,
+        aliases=aliases)
     deps = create_dependency_tree(ctx)
     to_compile = []
     for m in module_list:
@@ -463,7 +476,11 @@ def cythonize(module_list, nthreads=0, aliases=None, **options):
                     dep_timestamp, dep = deps.newest_dependency(source)
                     priority = 2 - (dep in deps.immediate_dependencies(source))
                 if c_timestamp < dep_timestamp:
-                    print("Compiling %s because it depends on %s" % (source, dep))
+                    if not quiet:
+                        if source == dep:
+                            print("Compiling %s because it changed." % source)
+                        else:
+                            print("Compiling %s because it depends on %s." % (source, dep))
                     to_compile.append((priority, source, c_file, options))
                 new_sources.append(c_file)
             else:

Cython/Build/Inline.py

-#no doctest
-print "Warning: Using prototype cython.inline code..."
-
 import tempfile
 import sys, os, re, inspect
+from cython import set
 
 try:
     import hashlib
@@ -51,7 +49,14 @@ def unbound_symbols(code, context=None):
             unbound.append(name)
     return unbound
 
-def get_type(arg, context=None):
+def unsafe_type(arg, context=None):
+    py_type = type(arg)
+    if py_type is int:
+        return 'long'
+    else:
+        return safe_type(arg, context)
+
+def safe_type(arg, context=None):
     py_type = type(arg)
     if py_type in [list, tuple, dict, str]:
         return py_type.__name__
@@ -61,8 +66,6 @@ def get_type(arg, context=None):
         return 'double'
     elif py_type is bool:
         return 'bint'
-    elif py_type is int:
-        return 'long'
     elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray):
         return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim)
     else:
@@ -77,15 +80,19 @@ def get_type(arg, context=None):
         return 'object'
 
 def cython_inline(code, 
-                  types='aggressive',
+                  get_type=unsafe_type,
                   lib_dir=os.path.expanduser('~/.cython/inline'),
                   cython_include_dirs=['.'],
+                  force=False,
+                  quiet=False,
                   locals=None,
                   globals=None,
                   **kwds):
+    if get_type is None:
+        get_type = lambda x: 'object'
     code, literals = strip_string_literals(code)
     code = strip_common_indent(code)
-    ctx = Context(include_dirs, default_options)
+    ctx = Context(cython_include_dirs, default_options)
     if locals is None:
         locals = inspect.currentframe().f_back.f_back.f_locals
     if globals is None:
@@ -99,10 +106,11 @@ def cython_inline(code,
             elif symbol in globals:
                 kwds[symbol] = globals[symbol]
             else:
-                print "Couldn't find ", symbol
+                print("Couldn't find ", symbol)
     except AssertionError:
+        if not quiet:
             # Parsing from strings not fully supported (e.g. cimports).
-        print "Could not parse code as a string (to extract unbound symbols)."
+            print("Could not parse code as a string (to extract unbound symbols).")
     arg_names = kwds.keys()
     arg_names.sort()
     arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
@@ -113,8 +121,12 @@ def cython_inline(code,
             os.makedirs(lib_dir)
         if lib_dir not in sys.path:
             sys.path.append(lib_dir)
+        if force:
+            raise ImportError
+        else:
             __import__(module_name)
     except ImportError:
+        cflags = []
         c_include_dirs = []
         cimports = []
         qualified = re.compile(r'([.\w]+)[.]')
@@ -126,6 +138,7 @@ def cython_inline(code,
                 if m.groups()[0] == 'numpy':
                     import numpy
                     c_include_dirs.append(numpy.get_include())
+                    cflags.append('-Wno-unused')
         module_body, func_body = extract_func_code(code)
         params = ', '.join(['%s %s' % a for a in arg_sigs])
         module_code = """
@@ -141,10 +154,11 @@ def __invoke(%(params)s):
         extension = Extension(
             name = module_name,
             sources = [pyx_file],
-            include_dirs = c_include_dirs)
+            include_dirs = c_include_dirs,
+            extra_compile_args = cflags)
         build_extension = build_ext(Distribution())
         build_extension.finalize_options()
-        build_extension.extensions = cythonize([extension], ctx=ctx)
+        build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
         build_extension.build_temp = os.path.dirname(pyx_file)
         build_extension.build_lib  = lib_dir
         build_extension.run()

Cython/Build/Tests/TestInline.py

+from Cython.Shadow import inline
+from Cython.Build.Inline import safe_type
+from Cython.TestUtils import CythonTest
+
+try:
+    import numpy
+    has_numpy = True
+except:
+    has_numpy = False
+
+test_kwds = dict(force=True, quiet=True)
+
+global_value = 100
+
+class TestStripLiterals(CythonTest):
+
+    def test_simple(self):
+        self.assertEquals(inline("return 1+2", **test_kwds), 3)
+
+    def test_types(self):
+        self.assertEquals(inline("""
+            cimport cython
+            return cython.typeof(a), cython.typeof(b)
+        """, a=1.0, b=[], **test_kwds), ('double', 'list object'))
+
+    def test_locals(self):
+        a = 1
+        b = 2
+        self.assertEquals(inline("return a+b", **test_kwds), 3)
+
+    def test_globals(self):
+        self.assertEquals(inline("return global_value + 1", **test_kwds), global_value + 1)
+
+    if has_numpy:
+    
+        def test_numpy(self):
+            import numpy
+            a = numpy.ndarray((10, 20))
+            a[0,0] = 10
+            self.assertEquals(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]')
+            self.assertEquals(inline("return a[0,0]", a=a, **test_kwds), 10.0)

Cython/Build/Tests/TestStripLiterals.py

+from Cython.Build.Dependencies import strip_string_literals
+
+from Cython.TestUtils import CythonTest
+
+class TestStripLiterals(CythonTest):
+
+    def t(self, before, expected):
+        actual, literals = strip_string_literals(before, prefix="_L")
+        self.assertEquals(expected, actual)
+        for key, value in literals.items():
+            actual = actual.replace(key, value)
+        self.assertEquals(before, actual)
+
+    def test_empty(self):
+        self.t("", "")
+
+    def test_single_quote(self):
+        self.t("'x'", "'_L1_'")
+
+    def test_double_quote(self):
+        self.t('"x"', '"_L1_"')
+
+    def test_nested_quotes(self):
+        self.t(""" '"' "'" """, """ '_L1_' "_L2_" """)
+
+    def test_triple_quote(self):
+        self.t(" '''a\n''' ", " '''_L1_''' ")
+
+    def test_backslash(self):
+        self.t(r"'a\'b'", "'_L1_'")
+        self.t(r"'a\\'", "'_L1_'")
+        self.t(r"'a\\\'b'", "'_L1_'")
+
+    def test_unicode(self):
+        self.t("u'abc'", "u'_L1_'")
+
+    def test_raw(self):
+        self.t(r"r'abc\'", "r'_L1_'")
+
+    def test_raw_unicode(self):
+        self.t(r"ru'abc\'", "ru'_L1_'")
+
+    def test_comment(self):
+        self.t("abc # foo", "abc #_L1_")
+
+    def test_comment_and_quote(self):
+        self.t("abc # 'x'", "abc #_L1_")
+        self.t("'abc#'", "'_L1_'")
+
+    def test_include(self):
+        self.t("include 'a.pxi' # something here",
+               "include '_L1_' #_L2_")
+
+    def test_extern(self):
+        self.t("cdef extern from 'a.h': # comment",
+               "cdef extern from '_L1_': #_L2_")
+

Cython/Compiler/CmdLine.py

@@ -2,6 +2,7 @@
 #   Cython - Command Line Parsing
 #
 
+import os
 import sys
 import Options
 
@@ -27,6 +28,7 @@ Options:
                                  Level indicates aggressiveness, default 0 releases nothing. 
   -w, --working <directory>      Sets the working directory for Cython (the directory modules 
                                  are searched from)
+  --gdb                          Output debug information for cygdb
 
   -D, --no-docstrings            Remove docstrings.
   -a, --annotate                 Produce a colorized HTML version of the source.
@@ -114,6 +116,9 @@ def parse_command_line(args):
                 Options.convert_range = True
             elif option == "--line-directives":
                 options.emit_linenums = True
+            elif option == "--gdb":
+                options.gdb_debug = True
+                options.output_dir = os.curdir
             elif option == '-2':
                 options.language_level = 2
             elif option == '-3':

Cython/Compiler/Code.py

@@ -917,6 +917,14 @@ class CCodeWriter(object):
         return self.buffer.getvalue()
 
     def write(self, s):
+        # also put invalid markers (lineno 0), to indicate that those lines 
+        # have no Cython source code correspondence
+        if self.marker is None:
+            cython_lineno = self.last_marker_line
+        else:
+            cython_lineno = self.marker[0]
+        
+        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
         self.buffer.write(s)
 
     def insertion_point(self):
@@ -1001,6 +1009,7 @@ class CCodeWriter(object):
             self.emit_marker()
         if self.emit_linenums and self.last_marker_line != 0:
             self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
+        
         if code:
             if safe:
                 self.put_safe(code)

Cython/Compiler/Errors.py

@@ -97,6 +97,10 @@ class CompilerCrash(CompileError):
             message += u'%s: %s' % (cause.__class__.__name__, cause)
         CompileError.__init__(self, pos, message)
 
+class NoElementTreeInstalledException(PyrexError):
+    """raised when the user enabled options.gdb_debug but no ElementTree 
+    implementation was found
+    """
 
 listing_file = None
 num_errors = 0

Cython/Compiler/ExprNodes.py

@@ -4187,12 +4187,13 @@ class ScopedExprNode(ExprNode):
                 code.put_var_declaration(entry)
                 if entry.type.is_pyobject and entry.used:
                     py_entries.append(entry)
-                    code.put_init_var_to_py_none(entry)
         if not py_entries:
             # no local Python references => no cleanup required
             generate_inner_evaluation_code(code)
             code.putln('} /* exit inner scope */')
             return
+        for entry in py_entries:
+            code.put_init_var_to_py_none(entry)
 
         # must free all local Python references at each exit point
         old_loop_labels = tuple(code.new_loop_labels())

Cython/Compiler/Main.py

@@ -13,7 +13,9 @@ except NameError:
     # Python 2.3
     from sets import Set as set
 
+import itertools
 from time import time
+
 import Code
 import Errors
 import Parsing
@@ -85,6 +87,8 @@ class Context(object):
 
         self.set_language_level(language_level)
         
+        self.gdb_debug_outputwriter = None
+
     def set_language_level(self, level):
         self.language_level = level
         if level >= 3:
@@ -180,13 +184,22 @@ class Context(object):
             from Cython.TestUtils import TreeAssertVisitor
             test_support.append(TreeAssertVisitor())
 
-        return ([
-                create_parse(self),
-            ] + self.create_pipeline(pxd=False, py=py) + test_support + [
-                inject_pxd_code,
-                abort_on_errors,
-                generate_pyx_code,
-            ])
+        if options.gdb_debug:
+            from Cython.Debugger import DebugWriter
+            from ParseTreeTransforms import DebugTransform
+            self.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
+                options.output_dir)
+            debug_transform = [DebugTransform(self, options, result)]
+        else:
+            debug_transform = []
+            
+        return list(itertools.chain(
+            [create_parse(self)],
+            self.create_pipeline(pxd=False, py=py),
+            test_support,
+            [inject_pxd_code, abort_on_errors],
+            debug_transform,
+            [generate_pyx_code]))
 
     def create_pxd_pipeline(self, scope, module_name):
         def parse_pxd(source_desc):
@@ -706,6 +719,7 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
+    context = options.create_context()
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()
@@ -809,4 +823,5 @@ default_options = dict(
     evaluate_tree_assertions = False,
     emit_linenums = False,
     language_level = 2,
+    gdb_debug = False,
 )

Cython/Compiler/ModuleNode.py

@@ -297,12 +297,34 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         
         f = open_new_file(result.c_file)
         rootwriter.copyto(f)
+        if options.gdb_debug:
+            self._serialize_lineno_map(env, rootwriter)
         f.close()
         result.c_file_generated = 1
         if Options.annotate or options.annotate:
             self.annotate(rootwriter)
             rootwriter.save_annotation(result.main_source_file, result.c_file)
     
+    def _serialize_lineno_map(self, env, ccodewriter):
+        tb = env.context.gdb_debug_outputwriter
+        markers = ccodewriter.buffer.allmarkers()
+        
+        d = {}
+        for c_lineno, cython_lineno in enumerate(markers): 
+            if cython_lineno > 0:
+                d.setdefault(cython_lineno, []).append(c_lineno + 1)
+        
+        tb.start('LineNumberMapping')
+        for cython_lineno, c_linenos in sorted(d.iteritems()):
+                attrs = {
+                    'c_linenos': ' '.join(map(str, c_linenos)),
+                    'cython_lineno': str(cython_lineno),
+                }
+                tb.start('LineNumber', attrs)
+                tb.end('LineNumber')
+        tb.end('LineNumberMapping')
+        tb.serialize()
+        
     def find_referenced_modules(self, env, module_list, modules_seen):
         if env not in modules_seen:
             modules_seen[env] = 1

Cython/Compiler/ParseTreeTransforms.py

@@ -1728,3 +1728,136 @@ class TransformBuiltinMethods(EnvTransform):
         
         self.visitchildren(node)
         return node
+
+
+class DebugTransform(CythonTransform):
+    """
+    Create debug information and all functions' visibility to extern in order
+    to enable debugging.
+    """
+    
+    def __init__(self, context, options, result):
+        super(DebugTransform, self).__init__(context)
+        self.visited = cython.set()
+        # our treebuilder and debug output writer 
+        # (see Cython.Debugger.debug_output.CythonDebugWriter)
+        self.tb = self.context.gdb_debug_outputwriter
+        #self.c_output_file = options.output_file 
+        self.c_output_file = result.c_file
+        
+        # tells visit_NameNode whether it should register step-into functions
+        self.register_stepinto = False
+        
+    def visit_ModuleNode(self, node):
+        self.tb.module_name = node.full_module_name
+        attrs = dict(
+            module_name=node.full_module_name,
+            filename=node.pos[0].filename,
+            c_filename=self.c_output_file)
+        
+        self.tb.start('Module', attrs)
+        
+        # serialize functions
+        self.tb.start('Functions')
+        self.visitchildren(node)
+        self.tb.end('Functions')
+        
+        # 2.3 compatibility. Serialize global variables
+        self.tb.start('Globals')
+        entries = {}
+
+        for k, v in node.scope.entries.iteritems():
+            if (v.qualified_name not in self.visited and not
+                v.name.startswith('__pyx_') and not 
+                v.type.is_cfunction and not
+                v.type.is_extension_type):
+                entries[k]= v
+                
+        self.serialize_local_variables(entries)
+        self.tb.end('Globals')
+        # self.tb.end('Module') # end Module after the line number mapping in
+        # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
+        return node
+    
+    def visit_FuncDefNode(self, node):
+        self.visited.add(node.local_scope.qualified_name)
+        # node.entry.visibility = 'extern'
+        if node.py_func is None:
+            pf_cname = ''
+        else:
+            pf_cname = node.py_func.entry.func_cname
+            
+        attrs = dict(
+            name=node.entry.name,
+            cname=node.entry.func_cname,
+            pf_cname=pf_cname,
+            qualified_name=node.local_scope.qualified_name,
+            lineno=str(node.pos[1]))
+        
+        self.tb.start('Function', attrs=attrs)
+        
+        self.tb.start('Locals')
+        self.serialize_local_variables(node.local_scope.entries)
+        self.tb.end('Locals')
+
+        self.tb.start('Arguments')
+        for arg in node.local_scope.arg_entries:
+            self.tb.start(arg.name)
+            self.tb.end(arg.name)
+        self.tb.end('Arguments')
+
+        self.tb.start('StepIntoFunctions')
+        self.register_stepinto = True
+        self.visitchildren(node)
+        self.register_stepinto = False
+        self.tb.end('StepIntoFunctions')
+        self.tb.end('Function')
+
+        return node
+
+    def visit_NameNode(self, node):
+        if (self.register_stepinto and 
+            node.type.is_cfunction and 
+            getattr(node, 'is_called', False) and
+            node.entry.func_cname is not None):
+            # don't check node.entry.in_cinclude, as 'cdef extern: ...' 
+            # declared functions are not 'in_cinclude'. 
+            # This means we will list called 'cdef' functions as 
+            # "step into functions", but this is not an issue as they will be 
+            # recognized as Cython functions anyway.
+            attrs = dict(name=node.entry.func_cname)
+            self.tb.start('StepIntoFunction', attrs=attrs)
+            self.tb.end('StepIntoFunction')
+        
+        self.visitchildren(node)
+        return node
+    
+    def serialize_local_variables(self, entries):
+        for entry in entries.values():
+            if entry.type.is_pyobject:
+                vartype = 'PythonObject'
+            else:
+                vartype = 'CObject'
+            
+            cname = entry.cname
+            # if entry.type.is_extension_type:
+                # cname = entry.type.typeptr_cname
+            
+            if not entry.pos:
+                # this happens for variables that are not in the user's code,
+                # e.g. for the global __builtins__, __doc__, etc. We can just
+                # set the lineno to 0 for those.
+                lineno = '0'
+            else:
+                lineno = str(entry.pos[1])
+                
+            attrs = dict(
+                name=entry.name,
+                cname=cname,
+                qualified_name=entry.qualified_name,
+                type=vartype,
+                lineno=lineno)
+            
+            self.tb.start('LocalVar', attrs)
+            self.tb.end('LocalVar')
+        

Cython/Compiler/Tests/TestParseTreeTransforms.py

+import os
+
+from Cython.Compiler import CmdLine
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.Nodes import *
 
+
 class TestNormalizeTree(TransformTest):
     def test_parserbehaviour_is_what_we_coded_for(self):
         t = self.fragment(u"if x: y").root
@@ -140,6 +144,81 @@ class TestWithTransform(object): # (TransformTest): # Disabled!
         """, t)
                           
 
+# TODO: Re-enable once they're more robust.
+if sys.version_info[:2] >= (2, 5) and False:
+    from Cython.Debugger import DebugWriter
+    from Cython.Debugger.Tests.TestLibCython import DebuggerTestCase
+else:
+    # skip test, don't let it inherit unittest.TestCase
+    DebuggerTestCase = object
+
+class TestDebugTransform(DebuggerTestCase):
+    
+    def elem_hasattrs(self, elem, attrs):
+        # we shall supporteth python 2.3 !
+        return all([attr in elem.attrib for attr in attrs])
+    
+    def test_debug_info(self):
+        try:
+            assert os.path.exists(self.debug_dest)
+            
+            t = DebugWriter.etree.parse(self.debug_dest)
+            # the xpath of the standard ElementTree is primitive, don't use
+            # anything fancy
+            L = list(t.find('/Module/Globals'))
+            # assertTrue is retarded, use the normal assert statement
+            assert L
+            xml_globals = dict(
+                            [(e.attrib['name'], e.attrib['type']) for e in L])
+            self.assertEqual(len(L), len(xml_globals))
+            
+            L = list(t.find('/Module/Functions'))
+            assert L
+            xml_funcs = dict([(e.attrib['qualified_name'], e) for e in L])
+            self.assertEqual(len(L), len(xml_funcs))
+            
+            # test globals
+            self.assertEqual('CObject', xml_globals.get('c_var'))
+            self.assertEqual('PythonObject', xml_globals.get('python_var'))
+            
+            # test functions
+            funcnames = 'codefile.spam', 'codefile.ham', 'codefile.eggs'
+            required_xml_attrs = 'name', 'cname', 'qualified_name'
+            assert all([f in xml_funcs for f in funcnames])
+            spam, ham, eggs = [xml_funcs[funcname] for funcname in funcnames]
+            
+            self.assertEqual(spam.attrib['name'], 'spam')
+            self.assertNotEqual('spam', spam.attrib['cname'])
+            assert self.elem_hasattrs(spam, required_xml_attrs)
+
+            # test locals of functions
+            spam_locals = list(spam.find('Locals'))
+            assert spam_locals
+            spam_locals.sort(key=lambda e: e.attrib['name'])
+            names = [e.attrib['name'] for e in spam_locals]
+            self.assertEqual(list('abcd'), names)
+            assert self.elem_hasattrs(spam_locals[0], required_xml_attrs)
+            
+            # test arguments of functions
+            spam_arguments = list(spam.find('Arguments'))
+            assert spam_arguments
+            self.assertEqual(1, len(list(spam_arguments)))
+            
+            # test step-into functions
+            step_into = spam.find('StepIntoFunctions')
+            spam_stepinto = [x.attrib['name'] for x in step_into]
+            assert spam_stepinto
+            self.assertEqual(2, len(spam_stepinto))
+            assert 'puts' in spam_stepinto
+            assert 'some_c_function' in spam_stepinto
+        except:
+            print open(self.debug_dest).read()
+            raise
+            
+
+
+    
+
 if __name__ == "__main__":
     import unittest
     unittest.main()

Cython/Debugger/Cygdb.py

+#!/usr/bin/env python
+
+"""
+The Cython debugger
+
+The current directory should contain a directory named 'cython_debug', or a
+path to the cython project directory should be given (the parent directory of
+cython_debug).
+
+Additional gdb args can be provided only if a path to the project directory is 
+given.
+"""
+
+import os
+import sys
+import glob
+import tempfile
+import textwrap
+import subprocess
+
+usage = "Usage: cygdb [PATH [GDB_ARGUMENTS]]"
+
+def make_command_file(path_to_debug_info, prefix_code='', no_import=False):
+    if not no_import:
+        pattern = os.path.join(path_to_debug_info, 
+                               'cython_debug',
+                               'cython_debug_info_*')
+        debug_files = glob.glob(pattern)
+
+        if not debug_files:
+            sys.exit('%s.\nNo debug files were found in %s. Aborting.' % (
+                                   usage, os.path.abspath(path_to_debug_info)))
+    
+    fd, tempfilename = tempfile.mkstemp()
+    f = os.fdopen(fd, 'w')
+    f.write(prefix_code)
+    f.write('set breakpoint pending on\n')
+    f.write("set print pretty on\n")
+    f.write('python from Cython.Debugger import libcython, libpython\n')
+    
+    if no_import:
+        # don't do this, this overrides file command in .gdbinit
+        # f.write("file %s\n" % sys.executable)
+        pass
+    else:
+        path = os.path.join(path_to_debug_info, "cython_debug", "interpreter")
+        interpreter = open(path).read()
+        f.write("file %s\n" % interpreter)
+        f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
+        f.write(textwrap.dedent('''\
+            python
+            import sys
+            try:
+                gdb.lookup_type('PyModuleObject')
+            except RuntimeError:
+                sys.stderr.write(
+                    'Python was not compiled with debug symbols (or it was '
+                    'stripped). Some functionality may not work (properly).\\n')
+            end
+        '''))
+    
+    f.close()
+    
+    return tempfilename
+
+def main(path_to_debug_info=None, gdb_argv=None, no_import=False):
+    """
+    Start the Cython debugger. This tells gdb to import the Cython and Python
+    extensions (libcython.py and libpython.py) and it enables gdb's pending 
+    breakpoints.
+    
+    path_to_debug_info is the path to the Cython build directory
+    gdb_argv is the list of options to gdb
+    no_import tells cygdb whether it should import debug information
+    """
+    if path_to_debug_info is None:
+        if len(sys.argv) > 1:
+            path_to_debug_info = sys.argv[1]
+        else:
+            path_to_debug_info = os.curdir
+    
+    if gdb_argv is None:
+        gdb_argv = sys.argv[2:]
+    
+    if path_to_debug_info == '--':
+        no_import = True
+    
+    tempfilename = make_command_file(path_to_debug_info, no_import=no_import)
+    p = subprocess.Popen(['gdb', '-command', tempfilename] + gdb_argv)
+    while True:
+        try:
+            p.wait()
+        except KeyboardInterrupt:
+            pass
+        else:
+            break
+    os.remove(tempfilename)

Cython/Debugger/DebugWriter.py

+from __future__ import with_statement
+
+import os
+import sys
+import errno
+
+try:
+  from lxml import etree
+  have_lxml = True
+except ImportError:
+    have_lxml = False
+    try:
+        # Python 2.5
+        from xml.etree import cElementTree as etree
+    except ImportError:
+        try:
+            # Python 2.5
+            from xml.etree import ElementTree as etree
+        except ImportError:
+            try:
+                # normal cElementTree install
+                import cElementTree as etree
+            except ImportError:
+                try:
+                    # normal ElementTree install
+                    import elementtree.ElementTree as etree
+                except ImportError:
+                    etree = None
+
+from Cython.Compiler import Errors
+
+
+class CythonDebugWriter(object):
+    """
+    Class to output debugging information for cygdb
+    
+    It writes debug information to cython_debug/cython_debug_info_<modulename>
+    in the build directory.
+    """
+    
+    def __init__(self, output_dir):
+        if etree is None:
+            raise Errors.NoElementTreeInstalledException()
+        
+        self.output_dir = os.path.join(output_dir, 'cython_debug')
+        self.tb = etree.TreeBuilder()
+        # set by Cython.Compiler.ParseTreeTransforms.DebugTransform
+        self.module_name = None
+        self.start('cython_debug', attrs=dict(version='1.0'))
+    
+    def start(self, name, attrs=None):
+        self.tb.start(name, attrs or {})
+    
+    def end(self, name):
+        self.tb.end(name)
+
+    def serialize(self):
+        self.tb.end('Module')
+        self.tb.end('cython_debug')
+        xml_root_element = self.tb.close()
+
+        try:
+            os.makedirs(self.output_dir)
+        except OSError, e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        et = etree.ElementTree(xml_root_element)
+        kw = {}
+        if have_lxml:
+            kw['pretty_print'] = True
+        
+        fn = "cython_debug_info_" + self.module_name
+        et.write(os.path.join(self.output_dir, fn), encoding="UTF-8", **kw)
+        
+        interpreter_path = os.path.join(self.output_dir, 'interpreter')
+        with open(interpreter_path, 'w') as f:
+            f.write(sys.executable)

Cython/Debugger/Tests/TestLibCython.py

+from __future__ import with_statement
+
+import os
+import re
+import sys
+import uuid
+import shutil
+import warnings
+import textwrap
+import unittest
+import tempfile
+import subprocess
+import distutils.core
+from distutils import sysconfig
+from distutils import ccompiler
+
+import runtests
+import Cython.Distutils.extension
+from Cython.Debugger import Cygdb as cygdb
+
+root = os.path.dirname(os.path.abspath(__file__))
+codefile = os.path.join(root, 'codefile')
+cfuncs_file = os.path.join(root, 'cfuncs.c')
+with open(codefile) as f:
+    source_to_lineno = dict((line.strip(), i + 1) for i, line in enumerate(f))
+
+class DebuggerTestCase(unittest.TestCase):
+    
+    def setUp(self):
+        """
+        Run gdb and have cygdb import the debug information from the code
+        defined in TestParseTreeTransforms's setUp method
+        """
+        self.tempdir = tempfile.mkdtemp()
+        self.destfile = os.path.join(self.tempdir, 'codefile.pyx')
+        self.debug_dest = os.path.join(self.tempdir, 
+                                      'cython_debug', 
+                                      'cython_debug_info_codefile')
+        self.cfuncs_destfile = os.path.join(self.tempdir, 'cfuncs')
+        
+        self.cwd = os.getcwd()
+        os.chdir(self.tempdir)
+        
+        shutil.copy(codefile, self.destfile)
+        shutil.copy(cfuncs_file, self.cfuncs_destfile + '.c')
+        
+        compiler = ccompiler.new_compiler()
+        compiler.compile(['cfuncs.c'], debug=True, extra_postargs=['-fPIC'])
+        
+        opts = dict(
+            test_directory=self.tempdir,
+            module='codefile',
+        )
+        
+        cython_compile_testcase = runtests.CythonCompileTestCase(
+            workdir=self.tempdir,
+            # we clean up everything (not only compiled files)
+            cleanup_workdir=False,
+            **opts
+        )
+        
+        cython_compile_testcase.run_cython(
+            targetdir=self.tempdir,
+            incdir=None,
+            annotate=False,
+            extra_compile_options={
+                'gdb_debug':True,
+                'output_dir':self.tempdir,
+            },
+            **opts
+        )
+        
+        cython_compile_testcase.run_distutils(
+            incdir=None,
+            workdir=self.tempdir,
+            extra_extension_args={'extra_objects':['cfuncs.o']},
+            **opts
+        )
+        
+        # ext = Cython.Distutils.extension.Extension(
+            # 'codefile',
+            # ['codefile.pyx'], 
+            # pyrex_gdb=True,
+            # extra_objects=['cfuncs.o'])
+        # 
+        # distutils.core.setup(
+            # script_args=['build_ext', '--inplace'],
+            # ext_modules=[ext],
+            # cmdclass=dict(build_ext=Cython.Distutils.build_ext)
+        # )
+        
+    def tearDown(self):
+        os.chdir(self.cwd)
+        shutil.rmtree(self.tempdir)
+
+
+class GdbDebuggerTestCase(DebuggerTestCase):
+    def setUp(self):
+        super(GdbDebuggerTestCase, self).setUp()
+        
+        prefix_code = textwrap.dedent('''\
+            python
+                
+            import os
+            import sys
+            import traceback
+            
+            def excepthook(type, value, tb):
+                traceback.print_exception(type, value, tb)
+                os._exit(1)
+            
+            sys.excepthook = excepthook
+            
+            # Have tracebacks end up on sys.stderr (gdb replaces sys.stderr
+            # with an object that calls gdb.write())
+            sys.stderr = sys.__stderr__
+            
+            end
+            ''')
+        
+        code = textwrap.dedent('''\
+            python
+            
+            from Cython.Debugger.Tests import test_libcython_in_gdb
+            test_libcython_in_gdb.main(version=%r)
+            
+            end
+            ''' % (sys.version_info[:2],))
+        
+        self.gdb_command_file = cygdb.make_command_file(self.tempdir, 
+                                                        prefix_code)
+        
+        with open(self.gdb_command_file, 'a') as f:
+            f.write(code)
+        
+        args = ['gdb', '-batch', '-x', self.gdb_command_file, '-n', '--args',
+                sys.executable, '-c', 'import codefile']
+        
+        paths = []
+        path = os.environ.get('PYTHONPATH')
+        if path:
+            paths.append(path)
+        paths.append(os.path.dirname(os.path.dirname(
+            os.path.abspath(Cython.__file__))))
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(paths))
+        
+        try:
+            p = subprocess.Popen(['gdb', '-v'], stdout=subprocess.PIPE)
+            have_gdb = True
+        except OSError:
+            # gdb was not installed
+            have_gdb = False
+        else:
+            gdb_version = p.stdout.read().decode('ascii')
+            p.wait()
+            p.stdout.close()
+        
+        if have_gdb:
+            # Based on Lib/test/test_gdb.py
+            regex = "^GNU gdb [^\d]*(\d+)\.(\d+)"
+            gdb_version_number = re.search(regex, gdb_version).groups()
+
+        # Be Python 3 compatible
+        if not have_gdb or list(map(int, gdb_version_number)) < [7, 2]:
+            self.p = None
+            warnings.warn('Skipping gdb tests, need gdb >= 7.2')
+        else:
+            self.p = subprocess.Popen(
+                args,
+                stdout=open(os.devnull, 'w'),
+                stderr=subprocess.PIPE,
+                env=env)
+        
+    def tearDown(self):
+        super(GdbDebuggerTestCase, self).tearDown()
+        if self.p:
+            self.p.stderr.close()
+            self.p.wait()
+        os.remove(self.gdb_command_file)
+        
+   
+class TestAll(GdbDebuggerTestCase):
+    
+    def test_all(self):
+        if self.p is None:
+            return
+            
+        out, err = self.p.communicate()
+        border = '*' * 30
+        start = '%s   v INSIDE GDB v   %s' % (border, border)
+        end   = '%s   ^ INSIDE GDB ^   %s' % (border, border)
+        errmsg = '\n%s\n%s%s' % (start, err.decode('UTF-8'), end)
+        self.assertEquals(0, self.p.wait(), errmsg)
+        sys.stderr.write(err)
+
+if __name__ == '__main__':
+    unittest.main()

Cython/Debugger/Tests/cfuncs.c

+void
+some_c_function(void)
+{
+    int a, b, c;
+    
+    a = 1;
+    b = 2;
+}

Cython/Debugger/Tests/codefile

+cdef extern from "stdio.h":
+    int puts(char *s)
+
+cdef extern:
+    void some_c_function()
+
+import os
+
+cdef int c_var = 12
+python_var = 13
+
+def spam(a=0):
+    cdef:
+        int b, c
+    
+    b = c = d = 0
+    
+    b = 1
+    c = 2
+    int(10)
+    puts("spam")
+    os.path.join("foo", "bar")
+    some_c_function()
+    
+cdef ham():
+    pass
+    
+cpdef eggs():
+    pass
+
+cdef class SomeClass(object):
+    def spam(self):
+        pass
+
+spam()
+print "bye!"

Cython/Debugger/Tests/test_libpython_in_gdb.py

+# -*- coding: UTF-8 -*-
+
+"""
+Test libpython.py. This is already partly tested by test_libcython_in_gdb and 
+Lib/test/test_gdb.py in the Python source. These tests are run in gdb and
+called from test_libcython_in_gdb.main()
+"""
+
+import os
+import sys
+
+import gdb
+
+from Cython.Debugger import libcython
+from Cython.Debugger import libpython
+
+import test_libcython_in_gdb
+from test_libcython_in_gdb import _debug, inferior_python_version
+
+
+class TestPrettyPrinters(test_libcython_in_gdb.DebugTestCase):
+    """
+    Test whether types of Python objects are correctly inferred and that
+    the right libpython.PySomeTypeObjectPtr classes are instantiated.
+    
+    Also test whether values are appropriately formatted (don't be too 
+    laborious as Lib/test/test_gdb.py already covers this extensively).
+    
+    Don't take care of decreffing newly allocated objects as a new 
+    interpreter is started for every test anyway.
+    """
+    
+    def setUp(self):
+        super(TestPrettyPrinters, self).setUp()
+        self.break_and_run('b = c = d = 0')
+    
+    def get_pyobject(self, code):
+        value = gdb.parse_and_eval(code)
+        assert libpython.pointervalue(value) != 0
+        return value
+    
+    def pyobject_fromcode(self, code, gdbvar=None):
+        if gdbvar is not None:
+            d = {'varname':gdbvar, 'code':code}
+            gdb.execute('set $%(varname)s = %(code)s' % d)
+            code = '$' + gdbvar
+            
+        return libpython.PyObjectPtr.from_pyobject_ptr(self.get_pyobject(code))
+    
+    def get_repr(self, pyobject):
+        return pyobject.get_truncated_repr(libpython.MAX_OUTPUT_LEN)
+    
+    def alloc_bytestring(self, string, gdbvar=None):
+        if inferior_python_version < (3, 0):
+            funcname = 'PyString_FromString'
+        else:
+            funcname = 'PyBytes_FromString'
+        
+        assert '"' not in string
+        
+        # ensure double quotes
+        code = '(PyObject *) %s("%s")' % (funcname, string)
+        return self.pyobject_fromcode(code, gdbvar=gdbvar)
+        
+    def alloc_unicodestring(self, string, gdbvar=None):
+        self.alloc_bytestring(string.encode('UTF-8'), gdbvar='_temp')
+        
+        postfix = libpython.get_inferior_unicode_postfix()
+        funcname = 'PyUnicode%s_FromEncodedObject' % (postfix,)
+        
+        return self.pyobject_fromcode(
+            '(PyObject *) %s($_temp, "UTF-8", "strict")' % funcname, 
+            gdbvar=gdbvar)
+        
+    def test_bytestring(self):
+        bytestring = self.alloc_bytestring("spam")
+        
+        if inferior_python_version < (3, 0):
+            bytestring_class = libpython.PyStringObjectPtr
+            expected = repr("spam")
+        else:
+            bytestring_class = libpython.PyBytesObjectPtr
+            expected = "b'spam'"
+        
+        self.assertEqual(type(bytestring), bytestring_class)
+        self.assertEqual(self.get_repr(bytestring), expected)
+    
+    def test_unicode(self):
+        unicode_string = self.alloc_unicodestring(u"spam ἄλφα")
+        
+        expected = "'spam ἄλφα'"
+        if inferior_python_version < (3, 0):
+            expected = 'u' + expected
+        
+        self.assertEqual(type(unicode_string), libpython.PyUnicodeObjectPtr)
+        self.assertEqual(self.get_repr(unicode_string), expected)
+    
+    def test_int(self):
+        if inferior_python_version < (3, 0):
+            intval = self.pyobject_fromcode('PyInt_FromLong(100)')
+            self.assertEqual(type(intval), libpython.PyIntObjectPtr)
+            self.assertEqual(self.get_repr(intval), '100')
+    
+    def test_long(self):
+        longval = self.pyobject_fromcode('PyLong_FromLong(200)', 
+                                         gdbvar='longval')
+        assert gdb.parse_and_eval('$longval->ob_type == &PyLong_Type')
+        
+        self.assertEqual(type(longval), libpython.PyLongObjectPtr)
+        self.assertEqual(self.get_repr(longval), '200')
+        
+    def test_frame_type(self):
+        frame = self.pyobject_fromcode('PyEval_GetFrame()')
+        
+        self.assertEqual(type(frame), libpython.PyFrameObjectPtr)
+
+    

Cython/Distutils/build_ext.py

@@ -17,11 +17,55 @@ from distutils.dep_util import newer, newer_group
 from distutils import log
 from distutils.dir_util import mkpath
 from distutils.command import build_ext as _build_ext
+from distutils import sysconfig
 
 extension_name_re = _build_ext.extension_name_re
 
 show_compilers = _build_ext.show_compilers
 
+class Optimization(object):
+    def __init__(self):
+        self.flags = (
+            'OPT', 
+            'CFLAGS',
+            'CPPFLAGS',
+            'EXTRA_CFLAGS', 
+            'BASECFLAGS',
+            'PY_CFLAGS',
+        )
+        self.state = sysconfig.get_config_vars(*self.flags)
+        self.config_vars = sysconfig.get_config_vars()
+        
+        
+    def disable_optimization(self):
+        "disable optimization for the C or C++ compiler"
+        badoptions = ('-O1', '-O2', '-O3')
+        
+        for flag, option in zip(self.flags, self.state):
+            if option is not None:
+                L = [opt for opt in option.split() if opt not in badoptions]
+                self.config_vars[flag] = ' '.join(L)
+    
+    def restore_state(self):
+        "restore the original state"
+        for flag, option in zip(self.flags, self.state):
+            if option is not None:
+                self.config_vars[flag] = option
+
+
+optimization = Optimization()
+
+try:
+    any
+except NameError:
+    def any(it):
+        for x in it:
+            if x:
+                return True
+        
+        return False
+        
+
 class build_ext(_build_ext.build_ext):
 
     description = "build C/C++ and Cython extensions (compile/link to build directory)"
@@ -47,10 +91,13 @@ class build_ext(_build_ext.build_ext):
             "generate .pxi file for public declarations"),
         ('pyrex-directives=', None,
             "compiler directive overrides"),
+        ('pyrex-gdb', None,
+         "generate debug information for cygdb"),
         ])
 
     boolean_options.extend([
-        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 'pyrex-c-in-temp'
+        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 
+        'pyrex-c-in-temp', 'pyrex-gdb',
     ])
 
     def initialize_options(self):
@@ -62,6 +109,7 @@ class build_ext(_build_ext.build_ext):
         self.pyrex_directives = None
         self.pyrex_c_in_temp = 0
         self.pyrex_gen_pxi = 0
+        self.pyrex_gdb = False
 
     def finalize_options (self):
         _build_ext.build_ext.finalize_options(self)
@@ -74,9 +122,21 @@ class build_ext(_build_ext.build_ext):
             self.pyrex_directives = {}
     # finalize_options ()
     
+    def run(self):
+        # We have one shot at this before build_ext initializes the compiler.
+        # If --pyrex-gdb is in effect as a command line option or as option
+        # of any Extension module, disable optimization for the C or C++
+        # compiler.
+        if (self.pyrex_gdb or any([getattr(ext, 'pyrex_gdb', False) 
+                                       for ext in self.extensions])):
+            optimization.disable_optimization()
+        
+        _build_ext.build_ext.run(self)
+    
     def build_extensions(self):
         # First, sanity-check the 'extensions' list
         self.check_extensions_list(self.extensions)
+        
         for ext in self.extensions:
             ext.sources = self.cython_sources(ext.sources, ext)
             self.build_extension(ext)
@@ -128,7 +188,7 @@ class build_ext(_build_ext.build_ext):
         cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
                 (extension.language and extension.language.lower() == 'c++')
         pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0)
-
+        pyrex_gdb = self.pyrex_gdb or getattr(extension, 'pyrex_gdb', False)
         # Set up the include_path for the Cython compiler:
         #    1.    Start with the command line option.
         #    2.    Add in any (unique) paths from the extension
@@ -201,6 +261,10 @@ class build_ext(_build_ext.build_ext):
             if rebuild:
                 log.info("cythoning %s to %s", source, target)
                 self.mkpath(os.path.dirname(target))
+                if self.inplace:
+                    output_dir = os.curdir
+                else:
+                    output_dir = self.build_lib
                 options = CompilationOptions(pyrex_default_options, 
                     use_listing_file = create_listing,
                     include_path = includes,
@@ -208,7 +272,9 @@ class build_ext(_build_ext.build_ext):
                     output_file = target,
                     cplus = cplus,
                     emit_linenums = line_directives,
-                    generate_pxi = pyrex_gen_pxi)
+                    generate_pxi = pyrex_gen_pxi,
+                    output_dir = output_dir,
+                    gdb_debug = pyrex_gdb)
                 result = cython_compile(source, options=options,
                                         full_module_name=module_name)
             else:

Cython/Distutils/extension.py

@@ -31,6 +31,8 @@ class Extension(_Extension.Extension):
         put generated C files in temp directory.
     pyrex_gen_pxi : boolean
         generate .pxi file for public declarations
+    pyrex_gdb : boolean
+        generate Cython debug information for this extension for cygdb
     """
 
     # When adding arguments to this constructor, be sure to update
@@ -56,6 +58,7 @@ class Extension(_Extension.Extension):
             pyrex_cplus = 0,
             pyrex_c_in_temp = 0,
             pyrex_gen_pxi = 0,
+            pyrex_gdb = False,
             **kw):
 
         _Extension.Extension.__init__(self, name, sources,
@@ -81,6 +84,7 @@ class Extension(_Extension.Extension):
         self.pyrex_cplus = pyrex_cplus
         self.pyrex_c_in_temp = pyrex_c_in_temp
         self.pyrex_gen_pxi = pyrex_gen_pxi
+        self.pyrex_gdb = pyrex_gdb
 
 # class Extension
 

Cython/Shadow.py

@@ -188,6 +188,7 @@ except NameError: # Py3
 py_float = float
 py_complex = complex
 
+
 try:
     # Python 3
     from builtins import set, frozenset

Cython/StringIOTree.py

@@ -11,6 +11,7 @@ class StringIOTree(object):
             stream = StringIO()
         self.stream = stream
         self.write = stream.write
+        self.markers = []
 
     def getvalue(self):
         content = [x.getvalue() for x in self.prepended_children]
@@ -31,6 +32,8 @@ class StringIOTree(object):
         # itself is empty -- this makes it ready for insertion
         if self.stream.tell():
             self.prepended_children.append(StringIOTree(self.stream))
+            self.prepended_children[-1].markers = self.markers
+            self.markers = []
             self.stream = StringIO()
             self.write = self.stream.write
 
@@ -59,6 +62,11 @@ class StringIOTree(object):
         self.prepended_children.append(other)
         return other
 
+    def allmarkers(self):
+        children = self.prepended_children
+        return [m for c in children for m in c.allmarkers()] + self.markers
+        
+
 __doc__ = r"""
 Implements a buffer with insertion points. When you know you need to
 "get back" to a place and write more later, simply call insertion_point()

Cython/Tests/TestStringIOTree.py

+import unittest
+
+from Cython import StringIOTree as stringtree
+
+code = """
+cdef int spam                   # line 1
+
+cdef ham():
+    a = 1
+    b = 2
+    c = 3
+    d = 4
+
+def eggs():
+    pass
+
+cpdef bacon():
+    print spam
+    print 'scotch'
+    print 'tea?'
+    print 'or coffee?'          # line 16
+"""
+
+linemap = dict(enumerate(code.splitlines()))
+
+class TestStringIOTree(unittest.TestCase):
+    
+    def setUp(self):
+        self.tree = stringtree.StringIOTree()
+    
+    def test_markers(self):
+        assert not self.tree.allmarkers()
+    
+    def test_insertion(self):
+        self.write_lines((1, 2, 3))
+        line_4_to_6_insertion_point = self.tree.insertion_point()
+        self.write_lines((7, 8))
+        line_9_to_13_insertion_point = self.tree.insertion_point()
+        self.write_lines((14, 15, 16))
+    
+        line_4_insertion_point = line_4_to_6_insertion_point.insertion_point()
+        self.write_lines((5, 6), tree=line_4_to_6_insertion_point)
+        
+        line_9_to_12_insertion_point = (
+            line_9_to_13_insertion_point.insertion_point())
+        self.write_line(13, tree=line_9_to_13_insertion_point)
+        
+        self.write_line(4, tree=line_4_insertion_point)
+        self.write_line(9, tree=line_9_to_12_insertion_point)
+        line_10_insertion_point = line_9_to_12_insertion_point.insertion_point()
+        self.write_line(11, tree=line_9_to_12_insertion_point)
+        self.write_line(10, tree=line_10_insertion_point)
+        self.write_line(12, tree=line_9_to_12_insertion_point)
+        
+        self.assertEqual(self.tree.allmarkers(), range(1, 17))
+        self.assertEqual(code.strip(), self.tree.getvalue().strip())
+        
+        
+    def write_lines(self, linenos, tree=None):
+        for lineno in linenos:
+            self.write_line(lineno, tree=tree)
+    
+    def write_line(self, lineno, tree=None):
+        if tree is None:
+            tree = self.tree
+        tree.markers.append(lineno)
+        tree.write(linemap[lineno] + '\n')

Cython/__init__.py

-__version__ = "0.14.alpha0"
+__version__ = "0.14.rc0"
 
 # Void cython.* directives (for case insensitive operating systems). 
 from Cython.Shadow import *

Demos/numeric_demo.pyx

-#
-#  This example demonstrates how to access the internals
-#  of a Numeric array object.
-#
-
-cdef extern from "Numeric/arrayobject.h":
-
-  struct PyArray_Descr:
-    int type_num, elsize
-    char type
-
-  ctypedef class Numeric.ArrayType [object PyArrayObject]:
-    cdef char *data
-    cdef int nd
-    cdef int *dimensions, *strides
-    cdef object base
-    cdef PyArray_Descr *descr
-    cdef int flags
-
-def print_2d_array(ArrayType a):
-  print "Type:", chr(a.descr.type)
-  if chr(a.descr.type) <> "f":
-    raise TypeError("Float array required")
-  if a.nd <> 2:
-    raise ValueError("2 dimensional array required")
-  cdef int nrows, ncols
-  cdef float *elems, x
-  nrows = a.dimensions[0]
-  ncols = a.dimensions[1]
-  elems = <float *>a.data
-  hyphen = "-"
-  divider = ("+" + 10 * hyphen) * ncols + "+"
-  print divider
-  for row in range(nrows):
-    for col in range(ncols):
-      x = elems[row * ncols + col]
-      print "| %8f" % x,
-    print "|"
-    print divider

Demos/numpy_demo.pyx

+cimport numpy
+import numpy
+
+def sum_of_squares(numpy.ndarray[double, ndim=1] arr):
+    cdef long N = arr.shape[0]
+    cdef double ss = 0
+    for i in range(N):
+        ss += arr[i]**2
+    return ss

Demos/setup.py

-import glob
+# Run as:
+#    python setup.py build_ext --inplace
 
 from distutils.core import setup
 from distutils.extension import Extension
-from Cython.Distutils import build_ext
+from Cython.Build import cythonize
 
+ext_modules = cythonize("*.pyx", exclude="numpy_*.pyx")
+
+# Only compile the following if numpy is installed.
 try:
     from numpy.distutils.misc_util import get_numpy_include_dirs
-    numpy_include_dirs = get_numpy_include_dirs()
-except:
-    numpy_include_dirs = []
-
-ext_modules=[ 
-    Extension("primes",       ["primes.pyx"]),
-    Extension("spam",         ["spam.pyx"]),
-]
-
-for file in glob.glob("*.pyx"):
-    if file != "numeric_demo.pyx":
-        ext_modules.append(Extension(file[:-4], [file], include_dirs = numpy_include_dirs))
+    numpy_demo = Extension("*",
+                           ["numpy_*.pyx"],
+                           include_dirs=get_numpy_include_dirs())
+    ext_modules.extend(cythonize(numpy_demo))
+except ImportError:
+    pass
 
 setup(
   name = 'Demos',
-  cmdclass = {'build_ext': build_ext},
   ext_modules = ext_modules,
 )

Demos/spam.pyx

@@ -5,7 +5,7 @@
 cdef class Spam:
   cdef public int amount
 
-  def __new__(self):
+  def __cinit__(self):
     self.amount = 0
 
   def __dealloc__(self):

MANIFEST.in

@@ -5,6 +5,7 @@ include setup.py
 include setupegg.py
 include bin/*
 include cython.py
+include cygdb.py
 recursive-include Cython *.pyx *.pxd
 include Doc/*
 
@@ -24,5 +25,8 @@ include runtests.py
 include Cython/Mac/Makefile
 include Cython/Mac/_Filemodule_patched.c
 
+include Cython/Debugger/Tests/cfuncs.c
+include Cython/Debugger/Tests/codefile
+
 recursive-include pyximport *.py
 include pyximport/PKG-INFO pyximport/README

bin/cygdb

+#!/usr/bin/env python
+
+import sys
+
+from Cython.Debugger import Cygdb as cygdb
+
+if __name__ == '__main__':
+    cygdb.main()

cygdb.py

+#!/usr/bin/env python
+
+import sys
+
+from Cython.Debugger import Cygdb as cygdb
+
+if __name__ == '__main__':
+    cygdb.main()

runtests.py

@@ -344,15 +344,27 @@ class CythonCompileTestCase(unittest.TestCase):
         else:
             return geterrors()
 
-    def run_cython(self, test_directory, module, targetdir, incdir, annotate):
+    def run_cython(self, test_directory, module, targetdir, incdir, annotate,
+                   extra_compile_options=None):
         include_dirs = INCLUDE_DIRS[:]
         if incdir:
             include_dirs.append(incdir)
         source = self.find_module_source_file(
             os.path.join(test_directory, module + '.pyx'))
         target = os.path.join(targetdir, self.build_target_filename(module))
+        
+        if extra_compile_options is None:
+            extra_compile_options = {}
+        
+        try:
+            CompilationOptions
+        except NameError:
+            from Cython.Compiler.Main import CompilationOptions
+            from Cython.Compiler.Main import compile as cython_compile
+            from Cython.Compiler.Main import default_options
+        
         options = CompilationOptions(
-            pyrex_default_options,
+            default_options,
             include_path = include_dirs,
             output_file = target,
             annotate = annotate,
@@ -361,11 +373,13 @@ class CythonCompileTestCase(unittest.TestCase):
             language_level = self.language_level,
             generate_pxi = False,
             evaluate_tree_assertions = True,
+            **extra_compile_options
             )
         cython_compile(source, options=options,
                        full_module_name=module)
 
-    def run_distutils(self, test_directory, module, workdir, incdir):
+    def run_distutils(self, test_directory, module, workdir, incdir, 
+                      extra_extension_args=None):
         cwd = os.getcwd()
         os.chdir(workdir)
         try:
@@ -379,11 +393,16 @@ class CythonCompileTestCase(unittest.TestCase):
                 if match(module):
                     ext_include_dirs += get_additional_include_dirs()
             self.copy_related_files(test_directory, workdir, module)
+            
+            if extra_extension_args is None:
+                extra_extension_args = {}
+            
             extension = Extension(
                 module,
                 sources = self.find_source_files(workdir, module),
                 include_dirs = ext_include_dirs,
                 extra_compile_args = CFLAGS,
+                **extra_extension_args
                 )
             if self.language == 'cpp':
                 extension.language = 'c++'
@@ -626,6 +645,13 @@ class CythonUnitTestCase(CythonCompileTestCase):
         except Exception:
             pass
 
+
+try:
+    import gdb
+    include_debugger = sys.version_info[:2] > (2, 5)
+except:
+    include_debugger = False
+
 def collect_unittests(path, module_prefix, suite, selectors):
     def file_matches(filename):
         return filename.startswith("Test") and filename.endswith(".py")
@@ -635,7 +661,11 @@ def collect_unittests(path, module_prefix, suite, selectors):
 
     loader = unittest.TestLoader()
     
+    if include_debugger:
         skipped_dirs = []
+    else:
+        cython_dir = os.path.dirname(os.path.abspath(__file__))
+        skipped_dirs = [os.path.join(cython_dir, 'Cython', 'Debugger')]
 
     for dirpath, dirnames, filenames in os.walk(path):
         if dirpath != path and "__init__.py" not in filenames:
@@ -660,22 +690,32 @@ def collect_unittests(path, module_prefix, suite, selectors):
                         module = getattr(module, x)
                     suite.addTests([loader.loadTestsFromModule(module)])
 
+
+
 def collect_doctests(path, module_prefix, suite, selectors):
     def package_matches(dirname):
+        if dirname == 'Debugger' and not include_debugger:
+            return False
         return dirname not in ("Mac", "Distutils", "Plex")
     def file_matches(filename):
-        return (filename.endswith(".py") and not ('~' in filename
-                or '#' in filename or filename.startswith('.')))
+        filename, ext = os.path.splitext(filename)
+        blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb', 
+                     'TestLibCython']
+        return (ext == '.py' and not
+                '~' in filename and not
+                '#' in filename and not
+                filename.startswith('.') and not
+                filename in blacklist)
     import doctest, types
     for dirpath, dirnames, filenames in os.walk(path):
-        parentname = os.path.split(dirpath)[-1]
-        if package_matches(parentname):
+        for dir in list(dirnames):
+            if not package_matches(dir):
+                dirnames.remove(dir)
         for f in filenames:
             if file_matches(f):
                 if not f.endswith('.py'): continue
                 filepath = os.path.join(dirpath, f)
                 if os.path.getsize(filepath) == 0: continue
-                    if 'no doctest' in open(filepath).next(): continue
                 filepath = filepath[:-len(".py")]
                 modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
                 if not [ 1 for match in selectors if match(modulename) ]:

setup.py

@@ -70,6 +70,9 @@ else:
 # specific to setup
 setuptools_extra_args = {}
 
+# tells whether to include cygdb (the script and the Cython.Debugger package
+include_debugger = sys.version_info[:2] > (2, 5)
+
 if 'setuptools' in sys.modules:
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
@@ -81,8 +84,12 @@ if 'setuptools' in sys.modules:
 else:
     if os.name == "posix":
         scripts = ["bin/cython"]
+        if include_debugger:
+            scripts.append('bin/cygdb')
     else:
         scripts = ["cython.py"]
+        if include_debugger:
+            scripts.append('cygdb.py')
 
 def compile_cython_modules(profile=False, compile_more=False, cython_with_refnanny=False):
     source_root = os.path.abspath(os.path.dirname(__file__))
@@ -249,6 +256,22 @@ setup_args.update(setuptools_extra_args)
 
 from Cython import __version__ as version
 
+packages = [
+    'Cython',
+    'Cython.Build',
+    'Cython.Compiler',
+    'Cython.Runtime',
+    'Cython.Distutils',
+    'Cython.Plex',
+    'Cython.Tests',
+    'Cython.Build.Tests',
+    'Cython.Compiler.Tests',
+]
+
+if include_debugger:
+    packages.append('Cython.Debugger')
+    packages.append('Cython.Debugger.Tests')
+
 setup(
   name = 'Cython',
   version = version,
@@ -289,17 +312,7 @@ setup(
   ],
 
   scripts = scripts,
-  packages=[
-    'Cython',
-    'Cython.Build',
-    'Cython.Compiler',
-    'Cython.Runtime',
-    'Cython.Distutils',
-    'Cython.Plex',
-
-    'Cython.Tests',
-    'Cython.Compiler.Tests',
-    ],
+  packages=packages,
 
   # pyximport
   py_modules = ["pyximport/__init__",