Support for parse tree transformations.

Cython/Compiler/CmdLine.py

@@ -4,6 +4,7 @@
 
 import sys
 import Options
+import Transform
 
 usage = """\
 Cython (http://cython.org) is a compiler for code written in the
@@ -36,11 +37,36 @@ Options:
 #  -+, --cplus      Use C++ compiler for compiling and linking
 #  Additional .o files to link may be supplied when using -X."""
 
+#The following options are very experimental and is used for plugging in code
+#into different transform stages.
+#  -T phase:factory At the phase given, hand off the tree to the transform returned
+#                   when calling factory without arguments. Factory should be fully
+#                   specified (ie Module.SubModule.factory) and the containing module
+#                   will be imported. This option can be repeated to add more transforms,
+#                   transforms for the same phase will be used in the order they are given.
+
 def bad_usage():
     print >>sys.stderr, usage
     sys.exit(1)
 
 def parse_command_line(args):
+
+    def parse_add_transform(transforms, param):
+        def import_symbol(fqn):
+            modsplitpt = fqn.rfind(".")
+            if modsplitpt == -1: bad_usage()
+            modulename = fqn[:modsplitpt]
+            symbolname = fqn[modsplitpt+1:]
+            module = __import__(modulename, fromlist=[symbolname], level=0)
+            return getattr(module, symbolname)
+    
+        stagename, factoryname = param.split(":")
+        if not stagename in Transform.PHASES:
+            bad_usage()
+        factory = import_symbol(factoryname)
+        transform = factory()
+        transforms[stagename].append(transform)
+    
     from Cython.Compiler.Main import \
         CompilationOptions, default_options
 
@@ -93,6 +119,9 @@ def parse_command_line(args):
                 Options.annotate = True
             elif option == "--convert-range":
                 Options.convert_range = True
+            elif option.startswith("-T"):
+                parse_add_transform(options.transforms, get_param(option))
+                # Note: this can occur multiple times, each time appends
             else:
                 bad_usage()
         else:

Cython/Compiler/ExprNodes.py

@@ -31,6 +31,7 @@ class ExprNode(Node):
     #                            Cached result of subexpr_nodes()
     
     result_ctype = None
+    type = None
 
     #  The Analyse Expressions phase for expressions is split
     #  into two sub-phases:
@@ -165,6 +166,14 @@ class ExprNode(Node):
     saved_subexpr_nodes = None
     is_temp = 0
 
+    def get_child_attrs(self):
+        """Automatically provide the contents of subexprs as children, unless child_attr
+        has been declared. See Nodes.Node.get_child_accessors."""
+        if self.child_attrs != None:
+            return self.child_attr
+        elif self.subexprs != None:
+            return self.subexprs
+        
     def not_implemented(self, method_name):
         print_call_chain(method_name, "not implemented") ###
         raise InternalError(

Cython/Compiler/Main.py

@@ -18,6 +18,7 @@ from Symtab import BuiltinScope, ModuleScope
 import Code
 from Cython.Utils import replace_suffix
 from Cython import Utils
+import Transform
 
 verbose = 0
 
@@ -236,6 +237,7 @@ class CompilationOptions:
     include_path      [string]  Directories to search for include files
     output_file       string    Name of generated .c file
     generate_pxi      boolean   Generate .pxi file for public declarations
+    transforms        Transform.TransformSet Transforms to use on the parse tree
     
     Following options are experimental and only used on MacOSX:
     
@@ -342,7 +344,8 @@ default_options = dict(
     obj_only = 1,
     cplus = 0,
     output_file = None,
-    generate_pxi = 0)
+    generate_pxi = 0,
+    transforms = Transform.TransformSet())
     
 if sys.platform == "mac":
     from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError

Cython/Compiler/ModuleNode.py

@@ -26,6 +26,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  referenced_modules   [ModuleScope]
     #  module_temp_cname    string
     #  full_module_name     string
+
+    children_attrs = ["body"]
     
     def analyse_declarations(self, env):
         if Options.embed_pos_in_docstring:
@@ -46,7 +48,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if self.has_imported_c_functions():
             self.module_temp_cname = env.allocate_temp_pyobject()
             env.release_temp(self.module_temp_cname)
-        self.generate_c_code(env, result)
+        self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
     
@@ -199,7 +201,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             i_code.putln("pass")
         i_code.dedent()
     
-    def generate_c_code(self, env, result):
+    def generate_c_code(self, env, options, result):
         modules = self.referenced_modules
         if Options.annotate:
             code = Annotate.AnnotationCCodeWriter(StringIO())
@@ -216,7 +218,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_interned_name_decls(env, code)
         self.generate_py_string_decls(env, code)
         self.generate_cached_builtins_decls(env, code)
-        self.body.generate_function_definitions(env, code)
+        self.body.generate_function_definitions(env, code, options.transforms)
         code.mark_pos(None)
         self.generate_interned_name_table(env, code)
         self.generate_py_string_table(env, code)

Cython/Compiler/Transform.py

+#
+#   Tree transform framework
+#
+import Nodes
+import ExprNodes
+
+class Transform(object):
+    #  parent_stack [Node]       A stack providing information about where in the tree
+    #                            we currently are. Nodes here should be considered
+    #                            read-only.
+
+    # Transforms for the parse tree should usually extend this class for convenience.
+    # The caller of a transform will only first call initialize and then process_node on
+    # the root node, the rest are utility functions and conventions.
+    
+    # Transformations usually happens by recursively filtering through the stream.
+    # process_node is always expected to return a new node, however it is ok to simply
+    # return the input node untouched. Returning None will remove the node from the
+    # parent.
+    
+    def __init__(self):
+        self.parent_stack = []
+    
+    def initialize(self, phase, **options):
+        pass
+
+    def process_children(self, node):
+        """For all children of node, either process_list (if isinstance(node, list))
+        or process_node (otherwise) is called."""
+        if node == None: return
+        
+        self.parent_stack.append(node)
+        for childacc in node.get_child_accessors():
+            child = childacc.get()
+            if isinstance(child, list):
+                newchild = self.process_list(child, childacc.name())
+                if not isinstance(newchild, list): raise Exception("Cannot replace list with non-list!")
+            else:
+                newchild = self.process_node(child, childacc.name())
+                if newchild is not None and not isinstance(newchild, Nodes.Node):
+                    raise Exception("Cannot replace Node with non-Node!")
+            childacc.set(newchild)
+        self.parent_stack.pop()
+
+    def process_list(self, l, name):
+        """Calls process_node on all the items in l, using the name one gets when appending
+        [idx] to the name. Each item in l is transformed in-place by the item process_node
+        returns, then l is returned."""
+        # Comment: If moving to a copying strategy, it might makes sense to return a
+        # new list instead.
+        for idx in xrange(len(l)):
+            l[idx] = self.process_node(l[idx], "%s[%d]" % (name, idx))
+        return l
+
+    def process_node(self, node, name):
+        """Override this method to process nodes. name specifies which kind of relation the
+        parent has with child. This method should always return the node which the parent
+        should use for this relation, which can either be the same node, None to remove
+        the node, or a different node."""
+        raise InternalError("Not implemented")
+
+class PrintTree(Transform):
+    """Prints a representation of the tree to standard output.
+    Subclass and override repr_of to provide more information
+    about nodes. """
+    def __init__(self):
+        Transform.__init__(self)
+        self._indent = ""
+
+    def indent(self):
+        self._indent += "  "
+    def unindent(self):
+        self._indent = self._indent[:-2]
+
+    def initialize(self, phase, **options):
+        print("Parse tree dump at phase '%s'" % phase)
+
+    # Don't do anything about process_list, the defaults gives
+    # nice-looking name[idx] nodes which will visually appear
+    # under the parent-node, not displaying the list itself in
+    # the hierarchy.
+    
+    def process_node(self, node, name):
+        print("%s- %s: %s" % (self._indent, name, self.repr_of(node)))
+        self.indent()
+        self.process_children(node)
+        self.unindent()
+        return node
+
+    def repr_of(self, node):
+        if node is None:
+            return "(none)"
+        else:
+            result = node.__class__.__name__
+            if isinstance(node, ExprNodes.ExprNode):
+                t = node.type
+                result += "(type=%s)" % repr(t)
+            return result
+
+
+PHASES = [
+    'before_analyse_function', # run in FuncDefNode.generate_function_definitions
+    'after_analyse_function'   # run in FuncDefNode.generate_function_definitions
+]
+
+class TransformSet(dict):
+    def __init__(self):
+        self.update([(name, []) for name in PHASES])
+    def run(self, name, node, **options):
+        assert name in self
+        for transform in self[name]:
+            transform.initialize(phase=name, **options)
+            transform.process_node(node, "(root)")
+
+