Forking CCodeWriter done (and used for module header generation)

Cython/Compiler/Annotate.py

@@ -17,32 +17,34 @@ special_chars = [(u'<', u'\xF0', u'&lt;'),
 
 class AnnotationCCodeWriter(CCodeWriter):
 
-    def __init__(self, f):
-        CCodeWriter.__init__(self, self)
-        self.buffer = StringIO()
-        self.real_f = f
+    def __init__(self, create_from=None, buffer=None):
+        CCodeWriter.__init__(self, create_from, buffer)
+        self.annotation_buffer = StringIO()
+        if create_from is None:
             self.annotations = []
             self.last_pos = None
             self.code = {}
+        else:
+            # When forking, keep references to the same database
+            self.annotation_buffer = create_from.annotation_buffer
+            self.annotations = create_from.annotations
+            self.code = create_from.code
 
-    def getvalue(self):
-        return self.real_f.getvalue()
+    def create_new(self, create_from, buffer):
+        return AnnotationCCodeWriter(create_from, buffer)
 
     def write(self, s):
-        self.real_f.write(s)
-        self.buffer.write(s)
+        CCodeWriter.write(self, s)
+        self.annotation_buffer.write(s)
         
     def mark_pos(self, pos):
 #        if pos is not None:
 #            CCodeWriter.mark_pos(self, pos)
 #        return
         if self.last_pos:
-            try:
-                code = self.code[self.last_pos[1]]
-            except KeyError:
-                code = ""
-            self.code[self.last_pos[1]] = code + self.buffer.getvalue()
-        self.buffer = StringIO()
+            code = self.code.get(self.last_pos[1], "")
+            self.code[self.last_pos[1]] = code + self.annotation_buffer.getvalue()
+        self.annotation_buffer = StringIO()
         self.last_pos = pos
 
     def annotate(self, pos, item):

Cython/Compiler/Code.py

@@ -11,18 +11,7 @@ from TypeSlots import method_coexist
 from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
 
-
-class CFunctionScope:
-    """
-    Used by CCodeWriters to keep track of state within a
-    C function. This means:
-    - labels
-    - temporary variables
-
-    When a code writer forks, it inherits the same scope.
-    """
-
-class CCodeWriter:
+class CCodeWriter(object):
     """
     Utility class to output C code. Each codewriter is forkable (see
     StringIOTree).
@@ -40,7 +29,9 @@ class CCodeWriter:
     - utility code: Same story as with labels and temps; use enter_implementation
       and exit_implementation.
     - marker: Only kept in last fork.
-    - filename_table, filename_list: Decision to be made.
+    - filename_table, filename_list, input_file_contents: All forks share
+      the same instances simultaneously.
+    - 
     """ 
     
     # f                file            output file
@@ -68,12 +59,12 @@ class CCodeWriter:
     def __init__(self, create_from=None, buffer=None):
         if buffer is None: buffer = StringIOTree()
         self.buffer = buffer
-        self._write = self.buffer.write
+        self.marker = None
+        self.last_marker_line = 0
         if create_from is None:
+            # Root CCodeWriter
             self.level = 0
             self.bol = 1
-            self.marker = None
-            self.last_marker_line = 0
             self.filename_table = {}
             self.filename_list = []
             self.exc_vars = None
@@ -84,16 +75,29 @@ class CCodeWriter:
             c = create_from
             self.level = c.level
             self.bol = c.bol
+            self.in_cfunc = c.in_cfunc
+            # Note: NOT copying but sharing instance
+            self.filename_table = c.filename_table
+            self.filename_list = []
+            self.input_file_contents = c.input_file_contents
             # Leave other state alone
 
-    def create_fork_spinoff(self, buffer):
-        result = CCodeWriter
+    def create_new(self, create_from, buffer):
+        # polymorphic constructor -- very slightly more versatile
+        # than using __class__
+        return CCodeWriter(create_from, buffer)
 
     def copyto(self, f):
         self.buffer.copyto(f)
 
+    def getvalue(self):
+        return self.buffer.getvalue()
+
+    def write(self, s):
+        self.buffer.write(s)
+        
     def fork(self):
-        other = CCodeWriter(create_from=self, buffer=self.buffer.fork())
+        other = self.create_new(create_from=self, buffer=self.buffer.fork())
         # If we need to do something with our own state on fork, do it here
         return other
 
@@ -122,13 +126,13 @@ class CCodeWriter:
             self.emit_marker()
         if code:
             self.put(code)
-        self._write("\n");
+        self.write("\n");
         self.bol = 1
     
     def emit_marker(self):
-        self._write("\n");
+        self.write("\n");
         self.indent()
-        self._write("/* %s */\n" % self.marker[1])
+        self.write("/* %s */\n" % self.marker[1])
         self.last_marker_line = self.marker[0]
         self.marker = None
 
@@ -140,7 +144,7 @@ class CCodeWriter:
             self.level -= 1
         if self.bol:
             self.indent()
-        self._write(code)
+        self.write(code)
         self.bol = 0
         if dl > 0:
             self.level += dl
@@ -162,7 +166,7 @@ class CCodeWriter:
         self.putln("}")
     
     def indent(self):
-        self._write("  " * self.level)
+        self.write("  " * self.level)
 
     def get_py_version_hex(self, pyversion):
         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

Cython/Compiler/ModuleNode.py

@@ -212,7 +212,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             h_code.putln("")
             h_code.putln("#endif")
             
-            h_code.copy_to(open_new_file(result.api_file))
+            h_code.copyto(open_new_file(result.api_file))
     
     def generate_cclass_header_code(self, type, h_code):
         h_code.putln("%s DL_IMPORT(PyTypeObject) %s;" % (
@@ -239,8 +239,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code = Annotate.AnnotationCCodeWriter()
         else:
             code = Code.CCodeWriter()
-        code.h = Code.CCodeWriter()
-        self.generate_module_preamble(env, modules, code.h)
+        h_code = code.fork()
+        self.generate_module_preamble(env, modules, h_code)
 
         code.putln("")
         code.putln("/* Implementation of %s */" % env.qualified_name)
@@ -262,13 +262,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.mark_pos(None)
         self.generate_module_cleanup_func(env, code)
         self.generate_filename_table(code)
-        self.generate_utility_functions(env, code)
+        self.generate_utility_functions(env, code, h_code)
 
-        self.generate_declarations_for_modules(env, modules, code.h)
+        self.generate_declarations_for_modules(env, modules, h_code)
+        h_code.write('\n')
 
         f = open_new_file(result.c_file)
-        code.h.copyto(f)
-        f.write("\n")
         code.copyto(f)
         f.close()
         result.c_file_generated = 1
@@ -1947,7 +1946,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "%s = &%s;" % (
                     type.typeptr_cname, type.typeobj_cname))
     
-    def generate_utility_functions(self, env, code):
+    def generate_utility_functions(self, env, code, h_code):
         code.putln("")
         code.putln("/* Runtime support code */")
         code.putln("")
@@ -1956,7 +1955,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             (Naming.filetable_cname, Naming.filenames_cname))
         code.putln("}")
         for utility_code in env.utility_code_used:
-            code.h.put(utility_code[0])
+            h_code.put(utility_code[0])
             code.put(utility_code[1])
         code.put(PyrexTypes.type_conversion_functions)
         code.putln("")

Cython/Compiler/Nodes.py

@@ -967,10 +967,10 @@ class FuncDefNode(StatNode, BlockNode):
             code.putln("return %s;" % Naming.retval_cname)
         code.putln("}")
         # ----- Python version
+        code.exit_cfunc_scope()
         if self.py_func:
             self.py_func.generate_function_definitions(env, code, transforms)
         self.generate_optarg_wrapper_function(env, code)
-        code.exit_cfunc_scope()
         
     def put_stararg_decrefs(self, code):
         pass