Add support for generator expressions, currently breaking inlining.

Cython/Compiler/ExprNodes.py

@@ -4343,15 +4343,18 @@ class DictComprehensionAppendNode(ComprehensionAppendNode):
         self.value_expr.annotate(code)
 
 
-class GeneratorExpressionNode(ScopedExprNode):
-    # A generator expression, e.g.  (i for i in range(10))
-    #
-    # Result is a generator.
+class InlinedGeneratorExpressionNode(ScopedExprNode):
+    # An inlined generator expression for which the result is
+    # calculated inside of the loop.  This will only be created by
+    # transforms when replacing builtin calls on generator
+    # expressions.
     #
-    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
+    # result_node    ResultRefNode    the reference to the result value temp
+    # orig_func      String           the name of the builtin function this node replaces
 
     child_attrs = ["loop"]
-
+    loop_analysed = False
     type = py_object_type
 
     def analyse_scoped_declarations(self, env):
@@ -4362,30 +4365,12 @@ class GeneratorExpressionNode(ScopedExprNode):
             self.loop.analyse_expressions(env)
         self.is_temp = True
 
-    def analyse_scoped_expressions(self, env):
-        if self.has_local_scope:
-            self.loop.analyse_expressions(env)
-
     def may_be_none(self):
         return False
 
     def annotate(self, code):
         self.loop.annotate(code)
 
-
-class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
-    # An inlined generator expression for which the result is
-    # calculated inside of the loop.  This will only be created by
-    # transforms when replacing builtin calls on generator
-    # expressions.
-    #
-    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
-    # result_node    ResultRefNode    the reference to the result value temp
-    # orig_func      String           the name of the builtin function this node replaces
-
-    child_attrs = ["loop"]
-    loop_analysed = False
-
     def infer_type(self, env):
         return self.result_node.infer_type(env)
 
@@ -4398,7 +4383,8 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
 
     def analyse_scoped_expressions(self, env):
         self.loop_analysed = True
-        GeneratorExpressionNode.analyse_scoped_expressions(self, env)
+        if self.has_local_scope:
+            self.loop.analyse_expressions(env)
 
     def coerce_to(self, dst_type, env):
         if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
@@ -4409,7 +4395,7 @@ class InlinedGeneratorExpressionNode(GeneratorExpressionNode):
             # assignments.
             self.result_node.type = self.type = dst_type
             return self
-        return GeneratorExpressionNode.coerce_to(self, dst_type, env)
+        return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
 
     def generate_result_code(self, code):
         self.result_node.result_code = self.result()
@@ -4957,6 +4943,35 @@ class LambdaNode(InnerFunctionNode):
         env.add_lambda_def(self.def_node)
 
 
+class GeneratorExpressionNode(LambdaNode):
+    # A generator expression, e.g.  (i for i in range(10))
+    #
+    # Result is a generator.
+    #
+    # loop      ForStatNode   the for-loop, containing a YieldExprNode
+    # def_node  DefNode       the underlying generator 'def' node
+
+    child_attrs = ["loop", "def_node"]
+    name = StringEncoding.EncodedString('<genexpr>')
+    binding = False
+
+    def analyse_declarations(self, env):
+        # XXX: dirty hack to disable assignment synthesis
+        self.def_node.needs_assignment_synthesis = lambda *args, **kwargs: False
+        self.def_node.analyse_declarations(env)
+        #super(GeneratorExpressionNode, self).analyse_declarations(env)
+        env.add_lambda_def(self.def_node)
+
+    def generate_result_code(self, code):
+        code.putln(
+            '%s = %s(%s, NULL); %s' % (
+                self.result(),
+                self.def_node.entry.func_cname,
+                self.self_result_code(),
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+
 class YieldExprNode(ExprNode):
     # Yield expression node
     #

Cython/Compiler/Optimize.py

@@ -1167,11 +1167,12 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
             self.yield_nodes = []
 
         visit_Node = Visitor.TreeVisitor.visitchildren
-        def visit_YieldExprNode(self, node):
+        # XXX: disable inlining while it's not back supported
+        def __visit_YieldExprNode(self, node):
             self.yield_nodes.append(node)
             self.visitchildren(node)
 
-        def visit_ExprStatNode(self, node):
+        def __visit_ExprStatNode(self, node):
             self.visitchildren(node)
             if node.expr in self.yield_nodes:
                 self.yield_stat_nodes[node.expr] = node

Cython/Compiler/ParseTreeTransforms.py

@@ -183,6 +183,7 @@ class PostParse(ScopeTrackingTransform):
 
     def visit_ModuleNode(self, node):
         self.lambda_counter = 1
+        self.genexpr_counter = 1
         return super(PostParse, self).visit_ModuleNode(node)
 
     def visit_LambdaNode(self, node):
@@ -201,6 +202,20 @@ class PostParse(ScopeTrackingTransform):
         self.visitchildren(node)
         return node
 
+    def visit_GeneratorExpressionNode(self, node):
+        # unpack a generator expression into the corresponding DefNode
+        genexpr_id = self.genexpr_counter
+        self.genexpr_counter += 1
+        node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
+
+        node.def_node = Nodes.DefNode(node.pos, name=node.genexpr_name,
+                                      doc=None,
+                                      args=[], star_arg=None,
+                                      starstar_arg=None,
+                                      body=node.loop)
+        self.visitchildren(node)
+        return node
+
     # cdef variables
     def handle_bufferdefaults(self, decl):
         if not isinstance(decl.default, ExprNodes.DictNode):

tests/bugs.txt

@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
 compile.cpp_operators
 cpp_templated_ctypedef
 cpp_structs
-genexpr_T491
 with_statement_module_level_T536
 function_as_method_T494
 closure_inside_cdef_T554