Merged recent changes to the compiler package, adding "yield" support and

fixing bugs.

lib/python/RestrictedPython/RestrictionMutator.py

@@ -87,7 +87,7 @@ RestrictionMutator modifies a tree produced by
 compiler.transformer.Transformer, restricting and enhancing the
 code in various ways before sending it to pycodegen.
 '''
-__version__='$Revision: 1.5 $'[11:-2]
+__version__='$Revision: 1.6 $'[11:-2]
 
 from compiler import ast
 from compiler.transformer import parse
@@ -313,6 +313,9 @@ class RestrictionMutator:
     def visitExec(self, node, walker):
         self.error(node, 'Exec statements are not allowed.')
 
+    def visitYield(self, node, walker):
+        self.error(node, 'Yield statements are not allowed.')
+
     def visitClass(self, node, walker):
         # Should classes be allowed at all??
         self.checkName(node, node.name)

lib/python/RestrictedPython/compiler/ast.py

@@ -40,6 +40,8 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
+    def getChildNodes(self):
+        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):
@@ -188,7 +190,7 @@ class Sub(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Sub(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Sub((%s, %s))" % (repr(self.left), repr(self.right))
 
 class ListCompIf(Node):
     nodes["listcompif"] = "ListCompIf"
@@ -207,7 +209,7 @@ class Div(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Div(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Div((%s, %s))" % (repr(self.left), repr(self.right))
 
 class Discard(Node):
     nodes["discard"] = "Discard"
@@ -235,7 +237,7 @@ class RightShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "RightShift(%s, %s)" % (repr(self.left), repr(self.right))
+        return "RightShift((%s, %s))" % (repr(self.left), repr(self.right))
 
 class Continue(Node):
     nodes["continue"] = "Continue"
@@ -275,7 +277,7 @@ class LeftShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "LeftShift(%s, %s)" % (repr(self.left), repr(self.right))
+        return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right))
 
 class Mul(Node):
     nodes["mul"] = "Mul"
@@ -285,7 +287,16 @@ class Mul(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mul(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Mul((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Yield(Node):
+    nodes["yield"] = "Yield"
+    def __init__(self, value):
+        self.value = value
+    def _getChildren(self):
+        return self.value,
+    def __repr__(self):
+        return "Yield(%s)" % (repr(self.value),)
 
 class List(Node):
     nodes["list"] = "List"
@@ -354,7 +365,7 @@ class Mod(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mod(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Mod((%s, %s))" % (repr(self.left), repr(self.right))
 
 class Class(Node):
     nodes["class"] = "Class"
@@ -452,16 +463,16 @@ class Power(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Power(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Power((%s, %s))" % (repr(self.left), repr(self.right))
 
-class Import(Node):
-    nodes["import"] = "Import"
-    def __init__(self, names):
-        self.names = names
+class Ellipsis(Node):
+    nodes["ellipsis"] = "Ellipsis"
+    def __init__(self, ):
+        pass
     def _getChildren(self):
-        return self.names,
+        return ()
     def __repr__(self):
-        return "Import(%s)" % (repr(self.names),)
+        return "Ellipsis()"
 
 class Return(Node):
     nodes["return"] = "Return"
@@ -480,7 +491,7 @@ class Add(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Add(%s, %s)" % (repr(self.left), repr(self.right))
+        return "Add((%s, %s))" % (repr(self.left), repr(self.right))
 
 class Function(Node):
     nodes["function"] = "Function"
@@ -525,14 +536,14 @@ class Subscript(Node):
     def __repr__(self):
         return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs))
 
-class Ellipsis(Node):
-    nodes["ellipsis"] = "Ellipsis"
-    def __init__(self, ):
-        pass
+class Import(Node):
+    nodes["import"] = "Import"
+    def __init__(self, names):
+        self.names = names
     def _getChildren(self):
-        return ()
+        return self.names,
     def __repr__(self):
-        return "Ellipsis()"
+        return "Import(%s)" % (repr(self.names),)
 
 class Print(Node):
     nodes["print"] = "Print"

lib/python/RestrictedPython/compiler/astgen.py

@@ -81,6 +81,8 @@ class NodeInfo:
         print >> buf, "    def __repr__(self):"
         if self.argnames:
             fmt = COMMA.join(["%s"] * self.nargs)
+            if '(' in self.args:
+                fmt = '(%s)' % fmt
             vals = ["repr(self.%s)" % name for name in self.argnames]
             vals = COMMA.join(vals)
             if self.nargs == 1:
@@ -171,6 +173,8 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
+    def getChildNodes(self):
+        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):

lib/python/RestrictedPython/compiler/future.py

@@ -16,19 +16,14 @@ def is_future(stmt):
 
 class FutureParser:
 
-    features = ("nested_scopes",)
+    features = ("nested_scopes", "generators", "division")
     
     def __init__(self):
         self.found = {} # set
 
     def visitModule(self, node):
-        if node.doc is None:
-            off = 0
-        else:
-            off = 1
-
         stmt = node.node
-        for s in stmt.nodes[off:]:
+        for s in stmt.nodes:
             if not self.check_stmt(s):
                 break
 

lib/python/RestrictedPython/compiler/transformer.py

@@ -153,8 +153,12 @@ class Transformer:
 
     def file_input(self, nodelist):
         doc = self.get_docstring(nodelist, symbol.file_input)
+        if doc is not None:
+            i = 1
+        else:
+            i = 0
         stmts = []
-        for node in nodelist:
+        for node in nodelist[i:]:
             if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
                 self.com_append_stmt(stmts, node)
         return Module(doc, Stmt(stmts))
@@ -340,6 +344,11 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
+    def yield_stmt(self, nodelist):
+        n = Yield(self.com_node(nodelist[1]))
+        n.lineno = nodelist[0][2]
+        return n
+
     def raise_stmt(self, nodelist):
         # raise: [test [',' test [',' test]]]
         if len(nodelist) > 5:
@@ -1245,6 +1254,7 @@ _legal_node_types = [
     symbol.continue_stmt,
     symbol.return_stmt,
     symbol.raise_stmt,
+    #symbol.yield_stmt,
     symbol.import_stmt,
     symbol.global_stmt,
     symbol.exec_stmt,
@@ -1271,6 +1281,9 @@ _legal_node_types = [
     symbol.atom,
     ]
 
+if hasattr(symbol, 'yield_stmt'):
+    _legal_node_types.append(symbol.yield_stmt)
+
 _assign_types = [
     symbol.test,
     symbol.and_test,