Use cythonize() in pyximport (GH-4339)

Closes https://github.com/cython/cython/issues/2304

Cython/Build/Dependencies.py

@@ -1034,7 +1034,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                 # setup for out of place build directory if enabled
                 if build_dir:
                     if os.path.isabs(c_file):
-                        warnings.warn("build_dir has no effect for absolute source paths")
+                        c_file = os.path.splitdrive(c_file)[1]
+                        c_file = c_file.split(os.sep, 1)[1]
                     c_file = os.path.join(build_dir, c_file)
                     dir = os.path.dirname(c_file)
                     safe_makedirs_once(dir)

Cython/Distutils/build_ext.py

@@ -12,14 +12,32 @@ else:
 
 
 class new_build_ext(_build_ext, object):
-    def finalize_options(self):
-        if self.distribution.ext_modules:
-            nthreads = getattr(self, 'parallel', None)  # -j option in Py3.5+
-            nthreads = int(nthreads) if nthreads else None
+
+    user_options = _build_ext.user_options[:]
+    boolean_options = _build_ext.boolean_options[:]
+
+    user_options.extend([
+        ('cython-c-in-temp', None,
+             "put generated C files in temp directory"),
+        ])
+
+    boolean_options.extend([
+        'cython-c-in-temp'
+    ])
+
+    def initialize_options(self):
+        _build_ext.initialize_options(self)
+        self.cython_c_in_temp = 0
+
+    def build_extension(self, ext):
         from Cython.Build.Dependencies import cythonize
-            self.distribution.ext_modules[:] = cythonize(
-                self.distribution.ext_modules, nthreads=nthreads, force=self.force)
-        super(new_build_ext, self).finalize_options()
+        if self.cython_c_in_temp:
+            build_dir = self.build_temp
+        else:
+            build_dir = None
+        new_ext = cythonize(ext,force=self.force, quiet=self.verbose == 0, build_dir=build_dir)[0]
+        ext.sources = new_ext.sources
+        super(new_build_ext, self).build_extension(ext)
 
 # This will become new_build_ext in the future.
 from .old_build_ext import old_build_ext as build_ext

pyximport/pyxbuild.py

@@ -10,7 +10,7 @@ from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
 from distutils.extension import Extension
 from distutils.util import grok_environment_error
 try:
-    from Cython.Distutils.old_build_ext import old_build_ext as build_ext
+    from Cython.Distutils.build_ext import new_build_ext as build_ext
     HAS_CYTHON = True
 except ImportError:
     HAS_CYTHON = False
@@ -53,6 +53,9 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
         quiet = "--verbose"
     else:
         quiet = "--quiet"
+    if build_in_temp:
+        args = [quiet, "build_ext", '--cython-c-in-temp']
+    else:
         args = [quiet, "build_ext"]
     if force_rebuild:
         args.append("--force")
@@ -65,8 +68,6 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil
             elif 'set_initial_path' not in ext.cython_directives:
                 ext.cython_directives['set_initial_path'] = 'SOURCEFILE'
 
-    if HAS_CYTHON and build_in_temp:
-        args.append("--pyrex-c-in-temp")
     sargs = setup_args.copy()
     sargs.update({
         "script_name": None,

pyximport/pyximport.py

@@ -351,11 +351,12 @@ class PyImporter(PyxImporter):
         self.uncompilable_modules = {}
         self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
                                 'distutils']
+        self.blocked_packages = ['Cython.', 'distutils.']
 
     def find_module(self, fullname, package_path=None):
         if fullname in sys.modules:
             return None
-        if fullname.startswith('Cython.'):
+        if any([fullname.startswith(pkg) for pkg in self.blocked_packages]):
             return None
         if fullname in self.blocked_modules:
             # prevent infinite recursion

tests/pyximport/pyximport_pyimport_only.srctree

+
+PYTHON -c "import pyimport_test; pyimport_test.test()"
+
+######## pyimport_test.py ########
+
+import os.path
+import pyximport
+
+pyximport.install(pyximport=False, pyimport=True,
+                  build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP"))
+
+def test():
+    import mymodule
+    assert mymodule.test_string == "TEST"
+    assert not mymodule.__file__.rstrip('oc').endswith('.py'), mymodule.__file__
+
+######## mymodule.py ########
+
+test_string = "TEST"

tests/pyximport/pyximport_pyxbld.srctree

+
+PYTHON -c "import basic_test; basic_test.test()"
+
+######## basic_test.py ########
+
+import os.path
+import pyximport
+
+pyximport.install(build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP"))
+
+def test():
+    import mymodule
+    assert mymodule.test_string == "TEST"
+    assert mymodule.header_value == 5
+    assert not mymodule.__file__.rstrip('oc').endswith('.py'), mymodule.__file__
+
+######## mymodule.pyxbld ########
+
+def make_ext(modname, pyxfilename):
+    from distutils.extension import Extension
+    return Extension(name = modname,
+                     sources=[pyxfilename],
+                     include_dirs=['./headers'] )
+
+######## mymodule.pyx ########
+
+cdef extern from "myheader.h":
+    int test_value
+
+header_value = test_value
+test_string = "TEST"
+
+######## headers/myheader.h ########
+
+static int test_value = 5;