major cleanup refactoring of the utility code loading support,

simple support for loading requirements (at least from the same utility code type and file),
caching support

Cython/Compiler/Buffer.py

@@ -748,8 +748,11 @@ def get_type_information_cname(code, dtype, maxdepth=None):
                         ), safe=True)
     return name
 
-def load_buffer_utility(util_code_name, **kwargs):
-    return UtilityCode.load(util_code_name, "Buffer.c", **kwargs)
+def load_buffer_utility(util_code_name, context=None, **kwargs):
+    if context is None:
+        return UtilityCode.load(util_code_name, "Buffer.c", **kwargs)
+    else:
+        return TempitaUtilityCode.load(util_code_name, "Buffer.c", context=context, **kwargs)
 
 context = dict(max_dims=str(Options.buffer_max_dims))
 buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare",

Cython/Compiler/Code.py

@@ -7,7 +7,7 @@ import cython
 cython.declare(os=object, re=object, operator=object,
                Naming=object, Options=object, StringEncoding=object,
                Utils=object, SourceDescriptor=object, StringIOTree=object,
-               DebugFlags=object, none_or_sub=object, basestring=object)
+               DebugFlags=object, basestring=object)
 
 import os
 import re
@@ -48,13 +48,31 @@ def get_utility_dir():
     return os.path.join(Cython_dir, "Utility")
 
 class UtilityCodeBase(object):
+    """
+    Support for loading utility code from a file.
+
+    Code sections in the file can be specified as follows:
+
+        ##### MyUtility.proto #####
+
+        ##### MyUtility #####
+        #@requires: MyOtherUtility
+
+    for prototypes and implementation respectively.  For non-python or
+    -cython files backslashes should be used instead.  5 to 30 comment
+    characters may be used on either side.
+
+    If the @cname decorator is not used and this is a CythonUtilityCode,
+    one should pass in the 'name' keyword argument to be used for name
+    mangling of such entries.
+    """
 
     is_cython_utility = False
 
     _utility_cache = {}
 
     @classmethod
-    def _add_utility(cls, utility, type, lines, begin_lineno):
+    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
         if utility is None:
             return
 
@@ -65,6 +83,10 @@ class UtilityCodeBase(object):
             utility[0] = code
         else:
             utility[1] = code
+        if tags:
+            all_tags = utility[2]
+            for name, values in tags.iteritems():
+                all_tags.setdefault(name, set()).update(values)
 
     @classmethod
     def load_utilities_from_file(cls, path):
@@ -80,7 +102,10 @@ class UtilityCodeBase(object):
         else:
             comment = '/'
             replace_comments = re.compile(r'^\s*//.*|^\s*/\*[^*]*\*/').sub
-        match_header = re.compile(r'%s{5,30}\s*((\w|\.)+)\s*%s{5,30}' % (comment, comment)).match
+        match_special = re.compile(
+            (r'^%(C)s{5,30}\s*((?:\w|\.)+)\s*%(C)s{5,30}|'
+             r'^%(C)s+@(requires)\s*:\s*((?:\w|\.)+)' # add more tag names here at need
+                ) % {'C':comment}).match
 
         f = Utils.open_source_file(filename, encoding='UTF-8')
         try:
@@ -90,24 +115,30 @@ class UtilityCodeBase(object):
 
         utilities = {}
         lines = []
+        tags = {}
         utility = type = None
         begin_lineno = 0
 
         for lineno, line in enumerate(all_lines):
-            m = match_header(line)
+            m = match_special(line)
             if m:
-                cls._add_utility(utility, type, lines, begin_lineno)
-
-                begin_lineno = lineno + 1
-                lines = []
-
-                name = m.group(1)
-                if name.endswith(".proto"):
-                    name = name[:-6]
-                    type = 'Proto'
+                if m.group(1):
+                    name = m.group(1)
+                    cls._add_utility(utility, type, lines, begin_lineno, tags)
+
+                    begin_lineno = lineno + 1
+                    del lines[:]
+                    tags.clear()
+
+                    if name.endswith(".proto"):
+                        name = name[:-6]
+                        type = 'Proto'
+                    else:
+                        type = 'Code'
+                    utility = utilities.setdefault(name, [None, None, {}])
                 else:
-                    type = 'Code'
-                utility = utilities.setdefault(name, [None, None])
+                    tags.setdefault(m.group(2), set()).add(m.group(3))
+                    lines.append('') # keep line number correct
             else:
                 lines.append(replace_comments('', line).rstrip())
 
@@ -115,55 +146,18 @@ class UtilityCodeBase(object):
             raise ValueError("Empty utility code file")
 
         # Don't forget to add the last utility code
-        cls._add_utility(utility, type, lines, begin_lineno)
+        cls._add_utility(utility, type, lines, begin_lineno, tags)
 
         cls._utility_cache[path] = utilities
         return utilities
 
     @classmethod
-    def load(cls, util_code_name, from_file=None, context=None, **kwargs):
-        """
-        Load a utility code from a file specified by from_file (relative to
-        Cython/Utility) and name util_code_name. If from_file is not given,
-        load it from the file util_code_name.*. There should be only one file
-        matched by this pattern.
-
-        Utilities in the file can be specified as follows:
-
-            ##### MyUtility.proto #####
-            ##### MyUtility #####
-
-        for prototypes and implementation respectively. For non-python or
-        -cython files /-es should be used instead. 5 to 30 pound signs may be
-        used on either side.
-
-        If context is given, the utility is considered a tempita template.
-        The context dict (which may be empty) will be unpacked to form
-        all the variables in the template.
-
-        If the @cname decorator is not used and this is a CythonUtilityCode,
-        one should pass in the 'name' keyword argument to be used for name
-        mangling of such entries.
+    def load(cls, util_code_name, from_file=None, **kwargs):
         """
-        proto, impl = cls.load_as_string(util_code_name, from_file, context)
-
-        if proto is not None:
-            kwargs['proto'] = proto
-        if impl is not None:
-            kwargs['impl'] = impl
-
-        if 'name' not in kwargs:
-            kwargs['name'] = util_code_name
-
-        if 'file' not in kwargs and from_file:
-            kwargs['file'] = from_file
-
-        return cls(**kwargs)
-
-    @classmethod
-    def load_as_string(cls, util_code_name, from_file=None, context=None):
-        """
-        Load a utility code as a string. Returns (proto, implementation)
+        Load utility code from a file specified by from_file (relative to
+        Cython/Utility) and name util_code_name.  If from_file is not given,
+        load it from the file util_code_name.*.  There should be only one
+        file matched by this pattern.
         """
         if from_file is None:
             utility_dir = get_utility_dir()
@@ -178,29 +172,56 @@ class UtilityCodeBase(object):
             from_file = files[0]
 
         utilities = cls.load_utilities_from_file(from_file)
+        proto, impl, tags = utilities[util_code_name]
+
+        if tags:
+            orig_kwargs = kwargs.copy()
+            for name, values in tags.iteritems():
+                if name in kwargs:
+                    continue
+                # only pass lists when we have to: most argument expect one value or None
+                if name == 'requires':
+                    values = [ cls.load(dep, from_file, **orig_kwargs) for dep in values ]
+                elif not values:
+                    values = None
+                elif len(values) == 1:
+                    values = values[0]
+                kwargs[name] = values
 
-        proto, impl = utilities[util_code_name]
-        if context is not None:
-            proto = sub_tempita(proto, context, from_file, util_code_name)
-            impl = sub_tempita(impl, context, from_file, util_code_name)
+        if proto is not None:
+            kwargs['proto'] = proto
+        if impl is not None:
+            kwargs['impl'] = impl
 
-        if cls.is_cython_utility:
-            # Remember line numbers
-            return proto, impl
+        if 'name' not in kwargs:
+            kwargs['name'] = util_code_name
 
-        return proto and proto.lstrip(), impl and impl.lstrip()
+        if 'file' not in kwargs and from_file:
+            kwargs['file'] = from_file
 
-    def none_or_sub(self, s, context, tempita):
+        return cls(**kwargs)
+
+    @classmethod
+    def load_cached(cls, utility_code_name, from_file=None, _cache={}):
         """
-        Format a string in this utility code with context. If None, do nothing.
+        Calls .load(), but using a per-type cache based on utility name and file name.
         """
-        if s is None:
-            return None
-
-        if tempita:
-            return sub_tempita(s, context, self.file, self.name)
+        key = (cls, from_file, utility_code_name)
+        try:
+            return _cache[key]
+        except KeyError:
+            pass
+        code = _cache[key] = cls.load(utility_code_name, from_file)
+        return code
 
-        return s % context
+    @classmethod
+    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
+        """
+        Load a utility code as a string. Returns (proto, implementation)
+        """
+        util = cls.load(util_code_name, from_file, **kwargs)
+        proto, impl = util.proto, util.impl
+        return proto and proto.lstrip(), impl and impl.lstrip()
 
     def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
         """
@@ -214,20 +235,6 @@ class UtilityCodeBase(object):
         return "<%s(%s)" % (type(self).__name__, self.name)
 
 
-def sub_tempita(s, context, file, name):
-    "Run tempita on string s with given context."
-    if not s:
-        return None
-
-    if file:
-        context['__name'] = "%s:%s" % (file, name)
-    elif name:
-        context['__name'] = name
-
-    from Cython.Tempita import sub
-    return sub(s, **context)
-
-
 class UtilityCode(UtilityCodeBase):
     """
     Stores utility code to add during code generation.
@@ -261,15 +268,36 @@ class UtilityCode(UtilityCodeBase):
         self.name = name
         self.file = file
 
+    def __hash__(self):
+        return hash((self.proto, self.impl))
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if not isinstance(other, type(self)):
+            return False
+
+        self_proto = getattr(self, 'proto', None)
+        other_proto = getattr(other, 'proto', None)
+        return (self_proto, self.impl) == (other_proto, other.impl)
+
     def get_tree(self):
         pass
 
-    def specialize(self, pyrex_type=None, tempita=False, **data):
+    def none_or_sub(self, s, context):
+        """
+        Format a string in this utility code with context. If None, do nothing.
+        """
+        if s is None:
+            return None
+        return s % context
+
+    def specialize(self, pyrex_type=None, **data):
         # Dicts aren't hashable...
         if pyrex_type is not None:
             data['type'] = pyrex_type.declaration_code('')
             data['type_name'] = pyrex_type.specialization_name()
-        key = data.items(); key.sort(); key = tuple(key)
+        key = tuple(sorted(data.iteritems()))
         try:
             return self._cache[key]
         except KeyError:
@@ -279,10 +307,10 @@ class UtilityCode(UtilityCodeBase):
                 requires = [r.specialize(data) for r in self.requires]
 
             s = self._cache[key] = UtilityCode(
-                    self.none_or_sub(self.proto, data, tempita),
-                    self.none_or_sub(self.impl, data, tempita),
-                    self.none_or_sub(self.init, data, tempita),
-                    self.none_or_sub(self.cleanup, data, tempita),
+                    self.none_or_sub(self.proto, data),
+                    self.none_or_sub(self.impl, data),
+                    self.none_or_sub(self.init, data),
+                    self.none_or_sub(self.cleanup, data),
                     requires,
                     self.proto_block)
 
@@ -311,21 +339,33 @@ class UtilityCode(UtilityCodeBase):
                 self.cleanup(writer, output.module_pos)
 
 
-class ContentHashingUtilityCode(UtilityCode):
-    "UtilityCode that hashes and compares based on self.proto and self.impl"
+def sub_tempita(s, context, file, name):
+    "Run tempita on string s with given context."
+    if not s:
+        return None
 
-    def __hash__(self):
-        return hash((self.proto, self.impl))
+    if file:
+        context['__name'] = "%s:%s" % (file, name)
+    elif name:
+        context['__name'] = name
 
-    def __eq__(self, other):
-        if self is other:
-            return True
-        if not isinstance(other, type(self)):
-            return False
+    from Cython.Tempita import sub
+    return sub(s, **context)
 
-        self_proto = getattr(self, 'proto', None)
-        other_proto = getattr(other, 'proto', None)
-        return (self_proto, self.impl) == (other_proto, other.impl)
+class TempitaUtilityCode(UtilityCode):
+    def __init__(self, name=None, proto=None, impl=None, file=None, context=None, **kwargs):
+        proto = sub_tempita(proto, context, file, name)
+        impl = sub_tempita(impl, context, file, name)
+        super(TempitaUtilityCode, self).__init__(
+            proto, impl, name=name, file=file, **kwargs)
+
+    def none_or_sub(self, s, context):
+        """
+        Format a string in this utility code with context. If None, do nothing.
+        """
+        if s is None:
+            return None
+        return sub_tempita(s, context, self.file, self.name)
 
 
 class LazyUtilityCode(UtilityCodeBase):

Cython/Compiler/ExprNodes.py

@@ -4,7 +4,8 @@
 
 import cython
 cython.declare(error=object, warning=object, warn_once=object, InternalError=object,
-               CompileError=object, UtilityCode=object, StringEncoding=object, operator=object,
+               CompileError=object, UtilityCode=object, TempitaUtilityCode=object,
+               StringEncoding=object, operator=object,
                Naming=object, Nodes=object, PyrexTypes=object, py_object_type=object,
                list_type=object, tuple_type=object, set_type=object, dict_type=object, \
                unicode_type=object, str_type=object, bytes_type=object, type_type=object,
@@ -15,7 +16,7 @@ import operator
 
 from Errors import error, warning, warn_once, InternalError, CompileError
 from Errors import hold_errors, release_errors, held_errors, report_error
-from Code import UtilityCode
+from Code import UtilityCode, TempitaUtilityCode
 import StringEncoding
 import Naming
 import Nodes
@@ -9967,9 +9968,9 @@ proto="""
         (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x)))
 """)
 
-binding_cfunc_utility_code = UtilityCode.load("CythonFunction",
-                                              context=vars(Naming))
-fused_function_utility_code = UtilityCode.load(
+binding_cfunc_utility_code = TempitaUtilityCode.load(
+    "CythonFunction", context=vars(Naming))
+fused_function_utility_code = TempitaUtilityCode.load(
         "FusedFunction",
         "CythonFunction.c",
         context=vars(Naming),

Cython/Compiler/MemoryView.py

@@ -2,7 +2,7 @@ from Errors import CompileError, error
 import ExprNodes
 from ExprNodes import IntNode, NameNode, AttributeNode
 import Options
-from Code import UtilityCode
+from Code import UtilityCode, TempitaUtilityCode
 from UtilityCode import CythonUtilityCode
 import Buffer
 import PyrexTypes
@@ -931,8 +931,11 @@ def load_memview_cy_utility(util_code_name, context=None, **kwargs):
                                   context=context, **kwargs)
 
 def load_memview_c_utility(util_code_name, context=None, **kwargs):
-    return UtilityCode.load(util_code_name, "MemoryView_C.c",
-                            context=context, **kwargs)
+    if context is None:
+        return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
+    else:
+        return TempitaUtilityCode.load(util_code_name, "MemoryView_C.c",
+                                       context=context, **kwargs)
 
 def use_cython_array_utility_code(env):
     env.global_scope().context.cython_scope.lookup('array_cwrapper').used = True

Cython/Compiler/Optimize.py

@@ -15,7 +15,7 @@ import Symtab
 import Options
 import Naming
 
-from Code import UtilityCode, ContentHashingUtilityCode
+from Code import UtilityCode
 from StringEncoding import EncodedString, BytesLiteral
 from Errors import error
 from ParseTreeTransforms import SkipDeclarations
@@ -32,11 +32,8 @@ try:
 except ImportError:
     basestring = str # Python 3
 
-_utility_cache = {}
 def load_c_utility(name):
-    if name not in _utility_cache:
-      _utility_cache[name] = ContentHashingUtilityCode.load(name, "Optimize.c")
-    return _utility_cache[name]
+    return UtilityCode.load_cached(name, "Optimize.c")
 
 class FakePythonEnv(object):
     "A fake environment for creating type test nodes etc."

Cython/Compiler/PyrexTypes.py

@@ -2,7 +2,7 @@
 #   Cython/Python language types
 #
 
-from Code import UtilityCode, LazyUtilityCode, ContentHashingUtilityCode
+from Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode
 import StringEncoding
 import Naming
 import copy
@@ -610,14 +610,14 @@ class MemoryViewSliceType(PyrexType):
         return cname + '.memview'
 
     def create_from_py_utility_code(self, env):
-        import MemoryView, Buffer, Code
+        import MemoryView, Buffer
 
         # We don't have 'code', so use a LazyUtilityCode with a callback.
         def lazy_utility_callback(code):
             context['dtype_typeinfo'] = Buffer.get_type_information_cname(
                                                           code, self.dtype)
-            return ContentHashingUtilityCode.load(
-                        "ObjectToMemviewSlice", "MemoryView_C.c", context)
+            return TempitaUtilityCode.load(
+                        "ObjectToMemviewSlice", "MemoryView_C.c", context=context)
 
         env.use_utility_code(Buffer.acquire_utility_code)
         env.use_utility_code(MemoryView.memviewslice_init_code)
@@ -693,7 +693,7 @@ class MemoryViewSliceType(PyrexType):
                 error_condition = error_condition,
             )
 
-        utility = ContentHashingUtilityCode.load(
+        utility = TempitaUtilityCode.load(
                         utility_name, "MemoryView_C.c", context=context)
         env.use_utility_code(utility)
         return get_function, set_function
@@ -2796,8 +2796,8 @@ class CStructOrUnionType(CType):
                 funcname = self.from_py_function,
                 init = '%s 0 %s' % ('{' * nesting_depth, '}' * nesting_depth)
             )
-            self._convert_from_py_code = ContentHashingUtilityCode.load(
-                      "FromPyStructUtility", "TypeConversion.c", context)
+            self._convert_from_py_code = TempitaUtilityCode.load(
+                      "FromPyStructUtility", "TypeConversion.c", context=context)
 
         env.use_utility_code(self._convert_from_py_code)
         return True

Cython/Compiler/Tests/TestUtilityLoad.py

@@ -12,10 +12,8 @@ class TestUtilityLoader(unittest.TestCase):
     """
 
     expected = "test {{loader}} prototype", "test {{loader}} impl"
-    expected_tempita = (expected[0].replace('{{loader}}', 'Loader'),
-                        expected[1].replace('{{loader}}', 'Loader'))
 
-    required = "I am a dependency proto", "I am a dependency impl"
+    required = "req {{loader}} proto", "req {{loader}} impl"
 
     context = dict(loader='Loader')
 
@@ -30,24 +28,55 @@ class TestUtilityLoader(unittest.TestCase):
         got = strip_2tup(self.cls.load_as_string(self.name, self.filename))
         self.assertEquals(got, self.expected)
 
-        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
-        self.assertEquals(got, self.expected_tempita)
-
     def test_load(self):
         utility = self.cls.load(self.name)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEquals(got, self.expected)
 
-        # Not implemented yet
-        #required, = utility.requires
-        #self.assertEquals((required.proto, required.impl), self.required)
+        required, = utility.requires
+        got = strip_2tup((required.proto, required.impl))
+        self.assertEquals(got, self.required)
 
         utility = self.cls.load(self.name, from_file=self.filename)
         got = strip_2tup((utility.proto, utility.impl))
         self.assertEquals(got, self.expected)
 
+        utility = self.cls.load_cached(self.name, from_file=self.filename)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected)
+
 
-class TestCythonUtilityLoader(TestUtilityLoader):
+class TestTempitaUtilityLoader(TestUtilityLoader):
+    """
+    Test loading UtilityCodes with Tempita substitution
+    """
+    expected_tempita = (TestUtilityLoader.expected[0].replace('{{loader}}', 'Loader'),
+                        TestUtilityLoader.expected[1].replace('{{loader}}', 'Loader'))
+
+    required_tempita = (TestUtilityLoader.required[0].replace('{{loader}}', 'Loader'),
+                        TestUtilityLoader.required[1].replace('{{loader}}', 'Loader'))
+
+    cls = Code.TempitaUtilityCode
+
+    def test_load_as_string(self):
+        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
+        self.assertEquals(got, self.expected_tempita)
+
+    def test_load(self):
+        utility = self.cls.load(self.name, context=self.context)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected_tempita)
+
+        required, = utility.requires
+        got = strip_2tup((required.proto, required.impl))
+        self.assertEquals(got, self.required_tempita)
+
+        utility = self.cls.load(self.name, from_file=self.filename, context=self.context)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected_tempita)
+
+
+class TestCythonUtilityLoader(TestTempitaUtilityLoader):
     """
     Test loading CythonUtilityCodes
     """
@@ -56,7 +85,8 @@ class TestCythonUtilityLoader(TestUtilityLoader):
     expected = None, "test {{cy_loader}} impl"
     expected_tempita = None, "test CyLoader impl"
 
-    required = None, "I am a Cython dependency impl"
+    required = None, "req {{cy_loader}} impl"
+    required_tempita = None, "req CyLoader impl"
 
     context = dict(cy_loader='CyLoader')
 
@@ -65,4 +95,7 @@ class TestCythonUtilityLoader(TestUtilityLoader):
     cls = UtilityCode.CythonUtilityCode
 
     # Small hack to pass our tests above
-    cls.proto = None
\ No newline at end of file
+    cls.proto = None
+
+    test_load = TestUtilityLoader.test_load
+    test_load_tempita = TestTempitaUtilityLoader.test_load

Cython/Compiler/UtilityCode.py

@@ -66,13 +66,15 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     is_cython_utility = True
 
     def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
-                 file=None, from_scope=None):
+                 file=None, from_scope=None, context=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
         #    while the generated node trees can be altered in the compilation of a
         #    single file.
         # Hence, delay any processing until later.
+        if context is not None:
+            impl = Code.sub_tempita(impl, context, file, name)
         self.impl = impl
         self.name = name
         self.file = file
@@ -128,6 +130,14 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
+    @classmethod
+    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
+        """
+        Load a utility code as a string. Returns (proto, implementation)
+        """
+        util = cls.load(util_code_name, from_file, **kwargs)
+        return util.proto, util.impl # keep line numbers => no lstrip()
+
     def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only

Cython/Utility/TestCyUtilityLoader.pyx

 ########## TestCyUtilityLoader ##########
+#@requires: OtherUtility
+
 test {{cy_loader}} impl
+
+
+########## OtherUtility ##########
+req {{cy_loader}} impl

Cython/Utility/TestUtilityLoader.c

@@ -2,4 +2,11 @@
 test {{loader}} prototype
 
 ////////// TestUtilityLoader //////////
+//@requires: OtherUtility
 test {{loader}} impl
+
+////////// OtherUtility.proto //////////
+req {{loader}} proto
+
+////////// OtherUtility //////////
+req {{loader}} impl