Merge branch 'master' into static

Conflicts:
	CHANGES.rst
	Cython/Compiler/Nodes.py

CHANGES.rst

@@ -48,6 +48,16 @@ Features added
 Optimizations
 -------------
 
+* The "and"/"or" operators try to avoid unnecessary coercions of their
+  arguments.  They now evaluate the truth value of each argument
+  independently and only coerce the final result of the whole expression
+  to the target type (e.g. the type on the left side of an assignment).
+  This also avoids reference counting overhead for Python values during
+  evaluation and generally improves the code flow in the generated C code.
+
+* Cascaded assignments (a = b = ...) try to minimise the number of
+  type coercions.
+
 * The Python expression "2 ** N" is optimised into bit shifting.
   See http://bugs.python.org/issue21420
 

Cython/Compiler/ExprNodes.py

@@ -5322,9 +5322,16 @@ class AttributeNode(ExprNode):
                 else:
                     # Create a temporary entry describing the C method
                     # as an ordinary function.
-                    ubcm_entry = Symtab.Entry(entry.name,
-                        "%s->%s" % (type.vtabptr_cname, entry.cname),
-                        entry.type)
+                    if entry.func_cname:
+                        cname = entry.func_cname
+                        # Fix self type.
+                        ctype = copy.copy(entry.type)
+                        ctype.args = ctype.args[:]
+                        ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None)
+                    else:
+                        cname = "%s->%s" % (type.vtabptr_cname, entry.cname)
+                        ctype = entry.type
+                    ubcm_entry = Symtab.Entry(entry.name, cname, ctype)
                     ubcm_entry.is_cfunction = 1
                     ubcm_entry.func_cname = entry.func_cname
                     ubcm_entry.is_unbound_cmethod = 1
@@ -9622,12 +9629,6 @@ class PowNode(NumBinopNode):
         return super(PowNode, self).py_operation_function(code)
 
 
-# Note: This class is temporarily "shut down" into an ineffective temp
-# allocation mode.
-#
-# More sophisticated temp reuse was going on before, one could have a
-# look at adding this again after /all/ classes are converted to the
-# new temp scheme. (The temp juggling cannot work otherwise).
 class BoolBinopNode(ExprNode):
     #  Short-circuiting boolean operation.
     #
@@ -9636,6 +9637,9 @@ class BoolBinopNode(ExprNode):
     #  operand2     ExprNode
 
     subexprs = ['operand1', 'operand2']
+    operator = None
+    operand1 = None
+    operand2 = None
 
     def infer_type(self, env):
         type1 = self.operand1.infer_type(env)
@@ -9649,49 +9653,48 @@ class BoolBinopNode(ExprNode):
             return self.operand1.may_be_none() or self.operand2.may_be_none()
 
     def calculate_constant_result(self):
+        operand1 = self.operand1.constant_result
+        operand2 = self.operand2.constant_result
         if self.operator == 'and':
-            self.constant_result = \
-                self.operand1.constant_result and \
-                self.operand2.constant_result
+            self.constant_result = operand1 and operand2
         else:
-            self.constant_result = \
-                self.operand1.constant_result or \
-                self.operand2.constant_result
+            self.constant_result = operand1 or operand2
 
     def compile_time_value(self, denv):
+        operand1 = self.operand1.compile_time_value(denv)
+        operand2 = self.operand2.compile_time_value(denv)
         if self.operator == 'and':
-            return self.operand1.compile_time_value(denv) \
-                and self.operand2.compile_time_value(denv)
+            return operand1 and operand2
         else:
-            return self.operand1.compile_time_value(denv) \
-                or self.operand2.compile_time_value(denv)
+            return operand1 or operand2
 
     def coerce_to_boolean(self, env):
         return BoolBinopNode.from_node(
             self,
             operator=self.operator,
-            operand1=self.operand1.coerce_to_boolean(env),
-            operand2=self.operand2.coerce_to_boolean(env),
+            operand1=self.operand1.coerce_to_boolean(env).coerce_to_simple(env),
+            operand2=self.operand2.coerce_to_boolean(env).coerce_to_simple(env),
             type=PyrexTypes.c_bint_type,
             is_temp=self.is_temp)
 
+    def coerce_to(self, dst_type, env):
+        if dst_type is PyrexTypes.c_bint_type:
+            return self.coerce_to_boolean(env)
+        return GenericBoolBinopNode.from_node(
+            self, env=env, type=dst_type,
+            operator=self.operator, operand1=self.operand1, operand2=self.operand2)
+
+    def is_ephemeral(self):
+        return self.operand1.is_ephemeral() or self.operand2.is_ephemeral()
+
     def analyse_types(self, env):
+        # Note: we do not do any coercion here as we most likely do not know the final type anyway.
+        # We even accept to set self.type to ErrorType if both operands do not have a spanning type.
+        # The coercion to the final type and to a "simple" value is left to coerce_to().
         self.operand1 = self.operand1.analyse_types(env)
         self.operand2 = self.operand2.analyse_types(env)
         self.type = PyrexTypes.independent_spanning_type(
             self.operand1.type, self.operand2.type)
-        # note: self.type might be ErrorType, but we allow this here
-        # in order to support eventual coercion to boolean
-        if not self.type.is_pyobject and not self.type.is_error:
-            if self.operand1.is_ephemeral() or self.operand2.is_ephemeral():
-                error(self.pos, "Unsafe C derivative of temporary Python reference used in and/or expression")
-        self.operand1 = self.operand1.coerce_to(self.type, env)
-        self.operand2 = self.operand2.coerce_to(self.type, env)
-
-        # For what we're about to do, it's vital that
-        # both operands be temp nodes.
-        self.operand1 = self.operand1.coerce_to_simple(env)
-        self.operand2 = self.operand2.coerce_to_simple(env)
         self.is_temp = 1
         return self
 
@@ -9740,8 +9743,8 @@ class BoolBinopNode(ExprNode):
     def generate_operand1_test(self, code):
         #  Generate code to test the truth of the first operand.
         if self.type.is_pyobject:
-            test_result = code.funcstate.allocate_temp(PyrexTypes.c_bint_type,
-                                                       manage_ref=False)
+            test_result = code.funcstate.allocate_temp(
+                PyrexTypes.c_bint_type, manage_ref=False)
             code.putln(
                 "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
                     test_result,
@@ -9752,6 +9755,148 @@ class BoolBinopNode(ExprNode):
         return (test_result, self.type.is_pyobject)
 
 
+class BoolBinopResultNode(ExprNode):
+    """
+    Intermediate result of a short-circuiting and/or expression.
+    Tests the result for 'truthiness' and takes care of coercing the final result
+    of the overall expression to the target type.
+
+    Note that this node provides the same code generation method as
+    GenericBoolBinopNode to simplify expression nesting.
+
+    arg     ExprNode    the argument to test
+    value   ExprNode    the coerced result value node
+    """
+
+    subexprs = ['arg', 'value']
+    is_temp = True
+    arg = None
+    value = None
+
+    def __init__(self, arg, result_type, env):
+        # using 'arg' multiple times, so it must be a simple/temp value
+        arg = arg.coerce_to_simple(env)
+        # wrap in ProxyNode, in case a transform wants to replace self.arg later
+        arg = ProxyNode(arg)
+        super(BoolBinopResultNode, self).__init__(
+            arg.pos, arg=arg, type=result_type,
+            value=CloneNode(arg).coerce_to(result_type, env))
+
+    def coerce_to_boolean(self, env):
+        # coercing to simple boolean case after being instantiated => replace by simple coerced result
+        return self.arg.arg.coerce_to_boolean(env)
+
+    def generate_operand_test(self, code):
+        #  Generate code to test the truth of the first operand.
+        if self.arg.type.is_pyobject:
+            test_result = code.funcstate.allocate_temp(
+                PyrexTypes.c_bint_type, manage_ref=False)
+            code.putln(
+                "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
+                    test_result,
+                    self.arg.py_result(),
+                    code.error_goto_if_neg(test_result, self.pos)))
+        else:
+            test_result = self.arg.result()
+        return (test_result, self.arg.type.is_pyobject)
+
+    def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label):
+        code.mark_pos(self.pos)
+
+        # x => x
+        # x and ... or ... => next 'and' / 'or'
+        # False ... or x => next 'or'
+        # True and x => next 'and'
+        # True or x => True (operand)
+
+        self.arg.generate_evaluation_code(code)
+        if and_label or or_label:
+            test_result, uses_temp = self.generate_operand_test(code)
+            sense = '!' if or_label else ''
+            code.putln("if (%s%s) {" % (sense, test_result))
+            if uses_temp:
+                code.funcstate.release_temp(test_result)
+            self.arg.generate_disposal_code(code)
+
+            if or_label:
+                # value is false => short-circuit to next 'or'
+                code.put_goto(or_label)
+                code.putln("} else {")
+            if and_label:
+                # value is true => go to next 'and'
+                code.put_goto(and_label)
+                if not or_label:
+                    code.putln("} else {")
+
+        if not and_label or not or_label:
+            # if no next 'and' or 'or', we provide the result
+            self.value.generate_evaluation_code(code)
+            self.value.make_owned_reference(code)
+            code.putln("%s = %s;" % (final_result_temp, self.value.result()))
+            self.value.generate_post_assignment_code(code)
+            self.arg.generate_disposal_code(code)
+            self.value.free_temps(code)
+            if and_label or or_label:
+                code.put_goto(end_label)
+
+        if and_label or or_label:
+            code.putln("}")
+        self.arg.free_temps(code)
+
+
+class GenericBoolBinopNode(BoolBinopNode):
+    """
+    BoolBinopNode with arbitrary non-bool result type.
+
+    Note that this node provides the same code generation method as
+    BoolBinopResultNode to simplify expression nesting.
+
+    operator  string                                     "and"/"or"
+    operand1  GenericBoolBinopNode/BoolBinopResultNode   left operand
+    operand2  GenericBoolBinopNode/BoolBinopResultNode   right operand
+    """
+    subexprs = ['operand1', 'operand2']
+    is_temp = True
+
+    def __init__(self, pos, env, type, operator, operand1, operand2, **kwargs):
+        super(GenericBoolBinopNode, self).__init__(
+            pos, operator=operator, type=type,
+            operand1=self._wrap_operand(operand1, type, env),
+            operand2=self._wrap_operand(operand2, type, env),
+            **kwargs)
+
+    def _wrap_operand(self, operand, result_type, env):
+        if isinstance(operand, (GenericBoolBinopNode, BoolBinopResultNode)):
+            return operand
+        if isinstance(operand, BoolBinopNode):
+            return operand.coerce_to(result_type, env)
+        else:
+            return BoolBinopResultNode(operand, result_type, env)
+
+    def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label):
+        code.mark_pos(self.pos)
+
+        outer_labels = (and_label, or_label)
+        if self.operator == 'and':
+            my_label = and_label = code.new_label('next_and')
+        else:
+            my_label = or_label = code.new_label('next_or')
+        self.operand1.generate_bool_evaluation_code(code, final_result_temp, and_label, or_label, end_label)
+
+        and_label, or_label = outer_labels
+
+        code.put_label(my_label)
+        self.operand2.generate_bool_evaluation_code(code, final_result_temp, and_label, or_label, end_label)
+
+    def generate_evaluation_code(self, code):
+        self.allocate_temp_result(code)
+        or_label = and_label = None
+        end_label = code.new_label('bool_binop_done')
+        self.generate_bool_evaluation_code(code, self.result(), and_label, or_label, end_label)
+        if code.label_used(end_label):
+            code.put_label(end_label)
+
+
 class CondExprNode(ExprNode):
     #  Short-circuiting conditional expression.
     #

Cython/Compiler/Nodes.py

@@ -2303,7 +2303,6 @@ class CFuncDefNode(FuncDefNode):
                            "private types")
 
     def call_self_node(self, omit_optional_args=0, is_module_scope=0):
-        # OLD - DELETE
         from . import ExprNodes
         args = self.type.args
         if omit_optional_args:
@@ -2315,34 +2314,17 @@ class CFuncDefNode(FuncDefNode):
             skip_dispatch = Options.lookup_module_cpdef
         elif self.type.is_static_method:
             class_entry = self.entry.scope.parent_type.entry
-            self_arg = ExprNodes.NameNode(self.pos, name=class_entry.name)
-            self_arg.entry = class_entry
-            cfunc = ExprNodes.AttributeNode(self.pos, obj=self_arg, attribute=self.entry.name)
-            call_arg_names = arg_names
+            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
+            class_node.entry = class_entry
+            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
             # Calling static c(p)def methods on an instance disallowed.
             # TODO(robertwb): Support by passing self to check for override?
             skip_dispatch = True
         else:
-            self_arg = ExprNodes.NameNode(self.pos, name=arg_names[0])
-            cfunc = ExprNodes.AttributeNode(self.pos, obj=self_arg, attribute=self.entry.name)
-            call_arg_names = arg_names[1:]
-            skip_dispatch = False
-        c_call = ExprNodes.SimpleCallNode(self.pos, function=cfunc, args=[ExprNodes.NameNode(self.pos, name=n) for n in call_arg_names], wrapper_call=skip_dispatch)
-        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)
-
-    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
-        from . import ExprNodes
-        args = self.type.args
-        if omit_optional_args:
-            args = args[:len(args) - self.type.optional_arg_count]
-        arg_names = [arg.name for arg in args]
-        # The @cname decorator may mutate this later.
-        func_cname = LazyStr(lambda: self.entry.func_cname)
-        cfunc = ExprNodes.PythonCapiFunctionNode(self.pos, self.entry.name, func_cname, self.type)
-        # The entry is inspected due to self.type.is_overridable, but it
-        # has the wrong self type.
-        cfunc.entry = copy.copy(self.entry)
-        cfunc.entry.type = self.type
+            type_entry = self.type.args[0].type.entry
+            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
+            type_arg.entry = type_entry
+            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
         skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
         c_call = ExprNodes.SimpleCallNode(
             self.pos,
@@ -2439,8 +2421,8 @@ class CFuncDefNode(FuncDefNode):
     def generate_argument_parsing_code(self, env, code):
         i = 0
         used = 0
-        if self.type.optional_arg_count:
         scope = self.local_scope
+        if self.type.optional_arg_count:
             code.putln('if (%s) {' % Naming.optional_args_cname)
             for arg in self.args:
                 if arg.default:
@@ -2461,6 +2443,16 @@ class CFuncDefNode(FuncDefNode):
                 code.putln('}')
             code.putln('}')
 
+        # Move arguments into closure if required
+        def put_into_closure(entry):
+            if entry.in_closure and not arg.default:
+                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
+                code.put_var_incref(entry)
+                code.put_var_giveref(entry)
+        for arg in self.args:
+            put_into_closure(scope.lookup_here(arg.name))
+
+
     def generate_argument_conversion_code(self, code):
         pass
 

Cython/Compiler/Optimize.py

@@ -816,6 +816,8 @@ class SwitchTransform(Visitor.CythonTransform):
             if isinstance(cond, (ExprNodes.CoerceToTempNode,
                                  ExprNodes.CoerceToBooleanNode)):
                 cond = cond.arg
+            elif isinstance(cond, ExprNodes.BoolBinopResultNode):
+                cond = cond.arg.arg
             elif isinstance(cond, UtilNodes.EvalWithTempExprNode):
                 # this is what we get from the FlattenInListTransform
                 cond = cond.subexpression
@@ -860,7 +862,7 @@ class SwitchTransform(Visitor.CythonTransform):
                     elif getattr(cond.operand1, 'entry', None) \
                              and cond.operand1.entry.is_const:
                         return not_in, cond.operand2, [cond.operand1]
-        elif isinstance(cond, ExprNodes.BoolBinopNode):
+        elif isinstance(cond, (ExprNodes.BoolBinopNode, ExprNodes.GenericBoolBinopNode)):
             if cond.operator == 'or' or (allow_not_in and cond.operator == 'and'):
                 allow_not_in = (cond.operator == 'and')
                 not_in_1, t1, c1 = self.extract_conditions(cond.operand1, allow_not_in)

Cython/Compiler/ParseTreeTransforms.py

@@ -2266,8 +2266,8 @@ class MarkClosureVisitor(CythonTransform):
 
     def visit_CFuncDefNode(self, node):
         self.visit_FuncDefNode(node)
-        if node.needs_closure:
-            error(node.pos, "closures inside cdef functions not yet supported")
+        if node.needs_closure and node.overridable:
+            error(node.pos, "closures inside cpdef functions not yet supported")
         return node
 
     def visit_LambdaNode(self, node):
@@ -2406,6 +2406,9 @@ class CreateClosureClasses(CythonTransform):
         return node
 
     def visit_CFuncDefNode(self, node):
+        if not node.overridable:
+            return self.visit_FuncDefNode(node)
+        else:
             self.visitchildren(node)
             return node
 

cythonize.py

+#!/usr/bin/env python
+
+#
+#   Cython -- enhanced main program
+#
+
+if __name__ == '__main__':
+    from Cython.Build.Cythonize import main
+    main()

runtests.py

@@ -115,6 +115,77 @@ def unpatch_inspect_isfunction():
     else:
         inspect.isfunction = orig_isfunction
 
+def def_to_cdef(source):
+    '''
+    Converts the module-level def methods into cdef methods, i.e.
+
+        @decorator
+        def foo([args]):
+            """
+            [tests]
+            """
+            [body]
+
+    becomes
+
+        def foo([args]):
+            """
+            [tests]
+            """
+            return foo_c([args])
+
+        cdef foo_c([args]):
+            [body]
+    '''
+    output = []
+    skip = False
+    def_node = re.compile(r'def (\w+)\(([^()*]*)\):').match
+    lines = iter(source.split('\n'))
+    for line in lines:
+        if not line.strip():
+            output.append(line)
+            continue
+
+        if skip:
+            if line[0] != ' ':
+                skip = False
+            else:
+                continue
+
+        if line[0] == '@':
+            skip = True
+            continue
+
+        m = def_node(line)
+        if m:
+            name = m.group(1)
+            args = m.group(2)
+            if args:
+                args_no_types = ", ".join(arg.split()[-1] for arg in args.split(','))
+            else:
+                args_no_types = ""
+            output.append("def %s(%s):" % (name, args_no_types))
+            line = next(lines)
+            if '"""' in line:
+                has_docstring = True
+                output.append(line)
+                for line in lines:
+                    output.append(line)
+                    if '"""' in line:
+                        break
+            else:
+                has_docstring = False
+            output.append("    return %s_c(%s)" % (name, args_no_types))
+            output.append('')
+            output.append("cdef %s_c(%s):" % (name, args))
+            if not has_docstring:
+                output.append(line)
+
+        else:
+            output.append(line)
+
+    return '\n'.join(output)
+
 def update_linetrace_extension(ext):
     ext.define_macros.append(('CYTHON_TRACE', 1))
     return ext
@@ -331,7 +402,7 @@ def parse_tags(filepath):
                 if tag == 'tags':
                     tag = 'tag'
                     print("WARNING: test tags use the 'tag' directive, not 'tags' (%s)" % filepath)
-                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils'):
+                if tag not in ('mode', 'tag', 'ticket', 'cython', 'distutils', 'preparse'):
                     print("WARNING: unknown test directive '%s' found (%s)" % (tag, filepath))
                 values = values.split(',')
                 tags[tag].extend(filter(None, [value.strip() for value in values]))
@@ -532,19 +603,25 @@ class TestBuilder(object):
         elif 'no-cpp' in tags['tag'] and 'cpp' in self.languages:
             languages = list(languages)
             languages.remove('cpp')
+
+        preparse_list = tags.get('preparse', ['id'])
         tests = [ self.build_test(test_class, path, workdir, module, tags,
-                                  language, expect_errors, warning_errors)
-                  for language in languages ]
+                                  language, expect_errors, warning_errors, preparse)
+                  for language in languages
+                  for preparse in preparse_list ]
         return tests
 
     def build_test(self, test_class, path, workdir, module, tags,
-                   language, expect_errors, warning_errors):
+                   language, expect_errors, warning_errors, preparse):
         language_workdir = os.path.join(workdir, language)
         if not os.path.exists(language_workdir):
             os.makedirs(language_workdir)
         workdir = os.path.join(language_workdir, module)
+        if preparse != 'id':
+            workdir += '_%s' % str(preparse)
         return test_class(path, workdir, module, tags,
                           language=language,
+                          preparse=preparse,
                           expect_errors=expect_errors,
                           annotate=self.annotate,
                           cleanup_workdir=self.cleanup_workdir,
@@ -556,7 +633,7 @@ class TestBuilder(object):
                           warning_errors=warning_errors)
 
 class CythonCompileTestCase(unittest.TestCase):
-    def __init__(self, test_directory, workdir, module, tags, language='c',
+    def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id',
                  expect_errors=False, annotate=False, cleanup_workdir=True,
                  cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False,
                  fork=True, language_level=2, warning_errors=False):
@@ -565,6 +642,8 @@ class CythonCompileTestCase(unittest.TestCase):
         self.workdir = workdir
         self.module = module
         self.language = language
+        self.preparse = preparse
+        self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse)
         self.expect_errors = expect_errors
         self.annotate = annotate
         self.cleanup_workdir = cleanup_workdir
@@ -577,7 +656,7 @@ class CythonCompileTestCase(unittest.TestCase):
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
-        return "compiling (%s) %s" % (self.language, self.module)
+        return "compiling (%s) %s" % (self.language, self.name)
 
     def setUp(self):
         from Cython.Compiler import Options
@@ -660,6 +739,11 @@ class CythonCompileTestCase(unittest.TestCase):
                 if is_related(filename)]
 
     def copy_files(self, test_directory, target_directory, file_list):
+        if self.preparse and self.preparse != 'id':
+            preparse_func = globals()[self.preparse]
+            def copy(src, dest):
+                open(dest, 'w').write(preparse_func(open(src).read()))
+        else:
             # use symlink on Unix, copy on Windows
             try:
                 copy = os.symlink
@@ -707,6 +791,12 @@ class CythonCompileTestCase(unittest.TestCase):
             include_dirs.append(incdir)
         source = self.find_module_source_file(
             os.path.join(test_directory, module + '.pyx'))
+        if self.preparse == 'id':
+            source = self.find_module_source_file(
+                os.path.join(test_directory, module + '.pyx'))
+        else:
+            self.copy_files(test_directory, targetdir, [module + '.pyx'])
+            source = os.path.join(targetdir, module + '.pyx')
         target = os.path.join(targetdir, self.build_target_filename(module))
 
         if extra_compile_options is None:
@@ -903,7 +993,7 @@ class CythonRunTestCase(CythonCompileTestCase):
         if self.cython_only:
             return CythonCompileTestCase.shortDescription(self)
         else:
-            return "compiling (%s) and running %s" % (self.language, self.module)
+            return "compiling (%s) and running %s" % (self.language, self.name)
 
     def run(self, result=None):
         if result is None:
@@ -1105,7 +1195,7 @@ class PartialTestResult(_TextTestResult):
 
 class CythonUnitTestCase(CythonRunTestCase):
     def shortDescription(self):
-        return "compiling (%s) tests in %s" % (self.language, self.module)
+        return "compiling (%s) tests in %s" % (self.language, self.name)
 
     def run_tests(self, result, ext_so_path):
         module = import_ext(self.module, ext_so_path)

setup.py

@@ -74,14 +74,15 @@ if 'setuptools' in sys.modules:
     setuptools_extra_args['entry_points'] = {
         'console_scripts': [
             'cython = Cython.Compiler.Main:setuptools_main',
+            'cythonize = Cython.Build.Cythonize:main'
         ]
     }
     scripts = []
 else:
     if os.name == "posix":
-        scripts = ["bin/cython"]
+        scripts = ["bin/cython", 'bin/cythonize']
     else:
-        scripts = ["cython.py"]
+        scripts = ["cython.py", "cythonize.py"]
 
 if include_debugger:
     if 'setuptools' in sys.modules:

tests/bugs.txt

@@ -6,7 +6,6 @@ unsignedbehaviour_T184
 missing_baseclass_in_predecl_T262
 cfunc_call_tuple_args_T408
 cpp_structs
-closure_inside_cdef_T554
 genexpr_iterable_lookup_T600
 generator_expressions_in_class
 for_from_pyvar_loop_T601

tests/errors/e_cdef_closure.pyx

 # mode: error
 
-cdef cdef_yield():
-    def inner():
-        pass
-
 cpdef cpdef_yield():
     def inner():
         pass
 
 _ERRORS = u"""
-3:5: closures inside cdef functions not yet supported
-7:6: closures inside cdef functions not yet supported
+3:6: closures inside cpdef functions not yet supported
 """

tests/run/closure_self.pyx

@@ -38,3 +38,19 @@ cdef class SelfInClosure(object):
         def nested():
             return self.x, t.x
         return nested
+
+    def call_closure_method_cdef_attr_c(self, Test t):
+        """
+        >>> o = SelfInClosure()
+        >>> o.call_closure_method_cdef_attr_c(Test())()
+        (1, 2)
+        """
+        return self.closure_method_cdef_attr_c(t)
+
+    cdef closure_method_cdef_attr_c(self, Test t):
+        t.x = 2
+        self._t = t
+        self.x = 1
+        def nested():
+            return self.x, t.x
+        return nested

tests/run/closure_tests_1.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_1.pyx
 #

tests/run/closure_tests_2.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_2.pyx
 #

tests/run/closure_tests_3.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_3.pyx
 #

tests/run/closure_tests_4.pyx

 # mode: run
 # tag: closures
+# preparse: id
+# preparse: def_to_cdef
 #
 # closure_tests_4.pyx
 #

tests/run/closures_T82.pyx

 # mode: run
 # tag: closures
 # ticket: 82
+# preparse: id
+# preparse: def_to_cdef
 
 cimport cython
 

tests/run/constant_folding.py

@@ -390,8 +390,8 @@ def combined():
     '//IntNode[@value = "4"]',
     '//IntNode[@value = "5"]',
     '//IntNode[@value = "7"]',
-    '//BoolBinopNode//PrimaryCmpNode',
-    '//BoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
+    '//GenericBoolBinopNode//PrimaryCmpNode',
+    '//GenericBoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
     '//PrimaryCmpNode[.//IntNode[@value = "2"] and .//IntNode[@value = "4"]]',
     '//PrimaryCmpNode[.//IntNode[@value = "5"] and .//IntNode[@value = "7"]]',
 )
@@ -423,11 +423,11 @@ def cascaded_cmp_with_partial_constants(a, b):
     '//IntNode[@value = "4"]',
     '//IntNode[@value = "5"]',
     '//IntNode[@value = "7"]',
-    '//BoolBinopNode',
-    '//SingleAssignmentNode//BoolBinopNode',
-    '//SingleAssignmentNode//BoolBinopNode//NameNode[@name = "a"]',
-    '//SingleAssignmentNode//BoolBinopNode//NameNode[@name = "b"]',
-    '//BoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
+    '//GenericBoolBinopNode',
+    '//SingleAssignmentNode//GenericBoolBinopNode',
+    '//SingleAssignmentNode//GenericBoolBinopNode//NameNode[@name = "a"]',
+    '//SingleAssignmentNode//GenericBoolBinopNode//NameNode[@name = "b"]',
+    '//GenericBoolBinopNode[.//PrimaryCmpNode//IntNode[@value = "4"] and .//PrimaryCmpNode//IntNode[@value = "5"]]',
     '//BoolNode[@value = False]',
 )
 @cython.test_fail_if_path_exists(

tests/run/notinop.pyx

@@ -83,7 +83,7 @@ def m_tuple(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//BoolBinopNode", "//GenericBoolBinopNode", "//PrimaryCmpNode")
 def m_set(int a):
     """
     >>> m_set(2)
@@ -97,7 +97,7 @@ def m_set(int a):
 cdef bytes bytes_string = b'abcdefg'
 
 @cython.test_assert_path_exists("//PrimaryCmpNode")
-@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode", "//GenericBoolBinopNode")
 def m_bytes(char a):
     """
     >>> m_bytes(ord('f'))
@@ -109,7 +109,7 @@ def m_bytes(char a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//BoolBinopNode", "//GenericBoolBinopNode", "//PrimaryCmpNode")
 def m_bytes_literal(char a):
     """
     >>> m_bytes_literal(ord('f'))
@@ -127,7 +127,7 @@ cdef unicode klingon_character = u'\uF8D2'
 py_klingon_character = klingon_character
 
 @cython.test_assert_path_exists("//PrimaryCmpNode")
-@cython.test_fail_if_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_fail_if_path_exists("//SwitchStatNode", "//GenericBoolBinopNode", "//BoolBinopNode")
 def m_unicode(Py_UNICODE a, unicode unicode_string):
     """
     >>> m_unicode(ord('f'), py_unicode_string)
@@ -147,7 +147,7 @@ def m_unicode(Py_UNICODE a, unicode unicode_string):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def m_unicode_literal(Py_UNICODE a):
     """
     >>> m_unicode_literal(ord('f'))
@@ -160,7 +160,7 @@ def m_unicode_literal(Py_UNICODE a):
     cdef int result = a not in u'abcdefg\u1234\uF8D2'
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_in_or_notin(int a):
     """
@@ -174,7 +174,7 @@ def m_tuple_in_or_notin(int a):
     cdef int result = a not in (1,2,3,4) or a in (3,4)
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_notin_or_notin(int a):
     """
@@ -189,7 +189,7 @@ def m_tuple_notin_or_notin(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def m_tuple_notin_and_notin(int a):
     """
     >>> m_tuple_notin_and_notin(2)
@@ -202,7 +202,7 @@ def m_tuple_notin_and_notin(int a):
     cdef int result = a not in (1,2,3,4) and a not in (6,7)
     return result
 
-@cython.test_assert_path_exists("//SwitchStatNode", "//BoolBinopNode")
+@cython.test_assert_path_exists("//SwitchStatNode", "//GenericBoolBinopNode")
 @cython.test_fail_if_path_exists("//PrimaryCmpNode")
 def m_tuple_notin_and_notin_overlap(int a):
     """
@@ -217,7 +217,7 @@ def m_tuple_notin_and_notin_overlap(int a):
     return result
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_int(int a):
     """
     >>> conditional_int(1)
@@ -230,7 +230,7 @@ def conditional_int(int a):
     return 1 if a not in (1,2,3,4) else 2
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_object(int a):
     """
     >>> conditional_object(1)
@@ -243,7 +243,7 @@ def conditional_object(int a):
     return 1 if a not in (1,2,3,4) else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_bytes(char a):
     """
     >>> conditional_bytes(ord('a'))
@@ -256,7 +256,7 @@ def conditional_bytes(char a):
     return 1 if a not in b'abc' else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_unicode(Py_UNICODE a):
     """
     >>> conditional_unicode(ord('a'))
@@ -269,7 +269,7 @@ def conditional_unicode(Py_UNICODE a):
     return 1 if a not in u'abc' else '2'
 
 @cython.test_assert_path_exists("//SwitchStatNode")
-@cython.test_fail_if_path_exists("//BoolBinopNode", "//PrimaryCmpNode")
+@cython.test_fail_if_path_exists("//GenericBoolBinopNode", "//BoolBinopNode", "//PrimaryCmpNode")
 def conditional_none(int a):
     """
     >>> conditional_none(1)