Support cimports in pure Python code. (GH-4190)

Uses a special package prefix "cython.cimports", followed by the actual .pxd module/package names.
Also allows "import cython.cimports.libc.math as libc_math" etc., rather than only "from ... import ...".
All we need to guard against is that the import does not overwrite the name "cython".

Cython/Compiler/ExprNodes.py

@@ -6990,6 +6990,7 @@ class AttributeNode(ExprNode):
                     or entry.is_type or entry.is_const):
                 return self.as_name_node(env, entry, target)
             if self.is_cimported_module_without_shadow(env):
+                # TODO: search for submodule
                 error(self.pos, "cimported module has no attribute '%s'" % self.attribute)
                 return self
         return None

Cython/Compiler/ParseTreeTransforms.py

@@ -775,22 +775,33 @@ class InterpretCompilerDirectives(CythonTransform):
         return result
 
     def visit_CImportStatNode(self, node):
-        if node.module_name == u"cython":
+        module_name = node.module_name
+        if module_name == u"cython.cimports":
+            error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.")
+        if module_name.startswith(u"cython.cimports."):
+            if node.as_name and node.as_name != u'cython':
+                node.module_name = module_name[len(u"cython.cimports."):]
+                return node
+            error(node.pos,
+                  "Python cimports must use 'from cython.cimports... import ...'"
+                  " or 'import ... as ...', not just 'import ...'")
+
+        if module_name == u"cython":
             self.cython_module_names.add(node.as_name or u"cython")
-        elif node.module_name.startswith(u"cython."):
-            if node.module_name.startswith(u"cython.parallel."):
+        elif module_name.startswith(u"cython."):
+            if module_name.startswith(u"cython.parallel."):
                 error(node.pos, node.module_name + " is not a module")
-            if node.module_name == u"cython.parallel":
+            if module_name == u"cython.parallel":
                 if node.as_name and node.as_name != u"cython":
-                    self.parallel_directives[node.as_name] = node.module_name
+                    self.parallel_directives[node.as_name] = module_name
                 else:
                     self.cython_module_names.add(u"cython")
                     self.parallel_directives[
-                                    u"cython.parallel"] = node.module_name
+                                    u"cython.parallel"] = module_name
                 self.module_scope.use_utility_code(
                     UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
             elif node.as_name:
-                self.directive_names[node.as_name] = node.module_name[7:]
+                self.directive_names[node.as_name] = module_name[7:]
             else:
                 self.cython_module_names.add(u"cython")
             # if this cimport was a compiler directive, we don't
@@ -799,9 +810,14 @@ class InterpretCompilerDirectives(CythonTransform):
         return node
 
     def visit_FromCImportStatNode(self, node):
-        if not node.relative_level and (
-                node.module_name == u"cython" or node.module_name.startswith(u"cython.")):
-            submodule = (node.module_name + u".")[7:]
+        module_name = node.module_name
+        if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."):
+            # only supported for convenience
+            return self._create_cimport_from_import(
+                node.pos, module_name, node.relative_level, node.imported_names)
+        elif not node.relative_level and (
+                module_name == u"cython" or module_name.startswith(u"cython.")):
+            submodule = (module_name + u".")[7:]
             newimp = []
 
             for pos, name, as_name, kind in node.imported_names:
@@ -827,9 +843,17 @@ class InterpretCompilerDirectives(CythonTransform):
         return node
 
     def visit_FromImportStatNode(self, node):
-        if (node.module.module_name.value == u"cython") or \
-               node.module.module_name.value.startswith(u"cython."):
-            submodule = (node.module.module_name.value + u".")[7:]
+        import_node = node.module
+        module_name = import_node.module_name.value
+        if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."):
+            imported_names = []
+            for name, name_node in node.items:
+                imported_names.append(
+                    (name_node.pos, name, None if name == name_node.name else name_node.name, None))
+            return self._create_cimport_from_import(
+                node.pos, module_name, import_node.level, imported_names)
+        elif module_name == u"cython" or module_name.startswith(u"cython."):
+            submodule = (module_name + u".")[7:]
             newimp = []
             for name, name_node in node.items:
                 full_name = submodule + name
@@ -845,20 +869,35 @@ class InterpretCompilerDirectives(CythonTransform):
             node.items = newimp
         return node
 
+    def _create_cimport_from_import(self, node_pos, module_name, level, imported_names):
+        if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."):
+            module_name = EncodedString(module_name[len(u"cython.cimports."):])  # may be empty
+
+        if module_name:
+            # from cython.cimports.a.b import x, y, z  =>  from a.b cimport x, y, z
+            return Nodes.FromCImportStatNode(
+                node_pos, module_name=module_name,
+                relative_level=level,
+                imported_names=imported_names)
+        else:
+            # from cython.cimports import x, y, z  =>  cimport x; cimport y; cimport z
+            return [
+                Nodes.CImportStatNode(
+                    pos,
+                    module_name=dotted_name,
+                    as_name=as_name,
+                    is_absolute=level == 0)
+                for pos, dotted_name, as_name, _ in imported_names
+            ]
+
     def visit_SingleAssignmentNode(self, node):
         if isinstance(node.rhs, ExprNodes.ImportNode):
             module_name = node.rhs.module_name.value
-            is_parallel = (module_name + u".").startswith(u"cython.parallel.")
-
-            if module_name != u"cython" and not is_parallel:
+            is_special_module = (module_name + u".").startswith((u"cython.parallel.", u"cython.cimports."))
+            if module_name != u"cython" and not is_special_module:
                 return node
 
-            module_name = node.rhs.module_name.value
-            as_name = node.lhs.name
-
-            node = Nodes.CImportStatNode(node.pos,
-                                         module_name = module_name,
-                                         as_name = as_name)
+            node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name)
             node = self.visit_CImportStatNode(node)
         else:
             self.visitchildren(node)

Cython/Shadow.py

@@ -525,6 +525,26 @@ class CythonDotParallel(object):
     # def threadsavailable(self):
         # return 1
 
-import sys
+
+class CythonCImports(object):
+    """
+    Simplistic module mock to make cimports sort-of work in Python code.
+    """
+    def __init__(self, module):
+        self.__path__ = []
+        self.__file__ = None
+        self.__name__ = module
+        self.__package__ = module
+
+    def __getattr__(self, item):
+        if item.startswith('__') and item.endswith('__'):
+            raise AttributeError(item)
+        return __import__(item)
+
+
+import math, sys
 sys.modules['cython.parallel'] = CythonDotParallel()
-del sys
+sys.modules['cython.cimports'] = CythonCImports('cython.cimports')
+sys.modules['cython.cimports.libc'] = CythonCImports('cython.cimports.libc')
+sys.modules['cython.cimports.libc.math'] = math
+del math, sys

docs/src/tutorial/pure.rst

@@ -209,6 +209,27 @@ Here is an example of a :keyword:`cdef` function::
         return a == b
 
 
+cimports
+^^^^^^^^
+
+The special ``cython.cimports`` package name gives access to cimports
+in code that uses Python syntax.  Note that this does not mean that C
+libraries become available to Python code.  It only means that you can
+tell Cython what cimports you want to use, without requiring special
+syntax.  Running such code in plain Python will fail.
+
+::
+
+    from cython.cimports.libc import math
+
+    print(math.ceil(5.5))
+
+Since such code must necessarily refer to the non-existing
+``cython.cimports`` 'package', the plain cimport form
+``cimport cython.cimports...`` is not available.
+You must use the form ``from cython.cimports...``.
+
+
 Further Cython functions and declarations
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

tests/errors/e_pure_cimports.pyx

+# mode: error
+# tag: pure, import, cimport
+
+import cython.cimportsy     # FIXME: not currently an error?
+
+import cython.cimports
+import cython.cimports.libc
+import cython.cimports as cim
+
+cimport cython.cimports
+cimport cython.cimports.libc
+cimport cython.cimports as cim
+import cython.cimports.libc as cython
+
+
+# ok
+import cython.cimports.libc as libc
+from cython.cimports import libc
+from cython.cimports cimport libc
+
+
+_ERRORS = """
+6:7: Cannot cimport the 'cython.cimports' package directly, only submodules.
+7:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...'
+8:7: Cannot cimport the 'cython.cimports' package directly, only submodules.
+10:8: Cannot cimport the 'cython.cimports' package directly, only submodules.
+11:8: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...'
+12:8: Cannot cimport the 'cython.cimports' package directly, only submodules.
+# The following is not an accurate error message, but it's difficult to distinguish this case. And it's rare.
+13:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...'
+"""

tests/run/pure_py_cimports.py

+# mode: run
+# tag: pure, import, cimport
+
+from cython.cimports.libc import math
+from cython.cimports.libc.math import ceil
+
+
+def libc_math_ceil(x):
+    """
+    >>> libc_math_ceil(1.5)
+    [2, 2]
+    """
+    return [int(n) for n in [ceil(x), math.ceil(x)]]

tests/run/pure_pyx_cimports.pyx

+# mode: run
+# tag: pure, import, cimport
+
+cimport cython.cimports.libc.math as libc_math1
+
+from cython.cimports.libc import math as libc_math2
+from cython.cimports.libc.math import ceil as math_ceil
+
+#from cython.cimports cimport libc    # FIXME: currently crashes during analysis when submodule cannot be found
+from cython.cimports.libc cimport math
+from cython.cimports.libc.math cimport ceil
+
+
+def libc_math_ceil(x):
+    """
+    >>> libc_math_ceil(1.5)
+    [2, 2, 2, 2, 2]
+    """
+    return [int(n) for n in [ceil(x), math.ceil(x), libc_math1.ceil(x), libc_math2.ceil(x), math_ceil(x)]]