applied pyximport patch from ticket 312

pyximport/pyxbuild.py

@@ -6,7 +6,6 @@ out_fname = pyx_to_dll("foo.pyx")
 import os
 import sys
 
-import distutils
 from distutils.dist import Distribution
 from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
 from distutils.extension import Extension
@@ -16,11 +15,13 @@ try:
     HAS_CYTHON = True
 except ImportError:
     HAS_CYTHON = False
-import shutil
 
 DEBUG = 0
+
+_reloads={}
+
 def pyx_to_dll(filename, ext = None, force_rebuild = 0,
-               build_in_temp=False, pyxbuild_dir=None):
+               build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False):
     """Compile a PYX file to a DLL and return the name of the generated .so 
        or .dll ."""
     assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
@@ -37,7 +38,8 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
     if not pyxbuild_dir:
         pyxbuild_dir = os.path.join(path, "_pyxbld")
 
-    if DEBUG:
+    script_args=setup_args.get("script_args",[])
+    if DEBUG or "--verbose" in script_args:
         quiet = "--verbose"
     else:
         quiet = "--quiet"
@@ -46,7 +48,11 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
         args.append("--force")
     if HAS_CYTHON and build_in_temp:
         args.append("--pyrex-c-in-temp")
-    dist = Distribution({"script_name": None, "script_args": args})
+    sargs = setup_args.copy()
+    sargs.update(
+        {"script_name": None,
+         "script_args": args + script_args} )
+    dist = Distribution(sargs)
     if not dist.ext_modules:
         dist.ext_modules = []
     dist.ext_modules.append(ext)
@@ -60,6 +66,10 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
     except ValueError: pass
     dist.parse_config_files(config_files)
 
+    cfgfiles = dist.find_config_files()
+    try: cfgfiles.remove('setup.cfg')
+    except ValueError: pass
+    dist.parse_config_files(cfgfiles)
     try:
         ok = dist.parse_command_line()
     except DistutilsArgError:
@@ -73,7 +83,39 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
 
     try:
         dist.run_commands()
-        return dist.get_command_obj("build_ext").get_outputs()[0]
+        obj_build_ext = dist.get_command_obj("build_ext")
+        so_path = obj_build_ext.get_outputs()[0]
+        if obj_build_ext.inplace:
+            # Python distutils get_outputs()[ returns a wrong so_path 
+            # when --inplace ; see http://bugs.python.org/issue5977
+            # workaround:
+            so_path = os.path.join(os.path.dirname(filename),
+                                   os.path.basename(so_path))
+        if reload_support:
+            org_path = so_path
+            timestamp = os.path.getmtime(org_path)
+            global _reloads
+            last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) )
+            if last_timestamp == timestamp:
+                so_path = last_path
+            else:
+                basename = os.path.basename(org_path)
+                while count < 100:
+                    count += 1
+                    r_path = os.path.join(obj_build_ext.build_lib,
+                                          basename + '.reload%s'%count)
+                    try:
+                        import shutil # late import / reload_support is: debugging
+                        shutil.copy2(org_path, r_path)
+                        so_path = r_path
+                    except IOError:
+                        continue
+                    break
+                else:
+                    # used up all 100 slots 
+                    raise ImportError("reload count for %s reached maximum"%org_path)
+                _reloads[org_path]=(timestamp, so_path, count)
+        return so_path
     except KeyboardInterrupt:
         sys.exit(1)
     except (IOError, os.error):
@@ -83,15 +125,6 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
         if DEBUG:
             sys.stderr.write(error + "\n")
         raise
-        else:
-            raise RuntimeError(error)
-
-    except (DistutilsError, CCompilerError):
-        if DEBUG:
-            raise
-        else:
-            exc = sys.exc_info()[1]
-            raise RuntimeError(repr(exc))
 
 if __name__=="__main__":
     pyx_to_dll("dummy.pyx")

pyximport/pyximport.py

@@ -14,6 +14,22 @@ For instance on the Mac with a non-system Python 2.3, you could create
 sitecustomize.py with only those two lines at
 /usr/local/lib/python2.3/site-packages/sitecustomize.py .
 
+A custom distutils.core.Extension instance and setup() args
+(Distribution) for for the build can be defined by a <modulename>.pyxbld
+file like:
+
+# examplemod.pyxbdl
+def make_ext(modname, pyxfilename):
+    from distutils.extension import Extension
+    return Extension(name = modname,
+                     sources=[pyxfilename, 'hello.c'],
+                     include_dirs=['/myinclude'] )
+def make_setup_args():
+    return dict(script_args=["--compiler=mingw32"])
+
+Extra dependencies can be defined by a <modulename>.pyxdep .
+See README.
+
 Since Cython 0.11, the :mod:`pyximport` module also has experimental
 compilation support for normal Python modules.  This allows you to
 automatically run Cython on every .pyx and .py module that Python
@@ -30,18 +46,11 @@ the documentation.
 
 This code is based on the Py2.3+ import protocol as described in PEP 302.
 """
+
 import sys
 import os
 import glob
 import imp
-import pyxbuild
-from distutils.dep_util import newer
-from distutils.extension import Extension
-
-try:
-    import hashlib
-except ImportError:
-    import md5 as hashlib
 
 mod_name = "pyximport"
 
@@ -64,30 +73,43 @@ def _load_pyrex(name, filename):
     "Load a pyrex file given a name and filename."
 
 def get_distutils_extension(modname, pyxfilename):
-    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()  
+#    try:
+#        import hashlib
+#    except ImportError:
+#        import md5 as hashlib
+#    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()  
 #    modname = modname + extra
-    extension_mod = handle_special_build(modname, pyxfilename)
+    extension_mod,setup_args = handle_special_build(modname, pyxfilename)
     if not extension_mod:
+        from distutils.extension import Extension
         extension_mod = Extension(name = modname, sources=[pyxfilename])
-    return extension_mod
+    return extension_mod,setup_args
 
 def handle_special_build(modname, pyxfilename):
     special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
-
-    if not os.path.exists(special_build): 
     ext = None
-    else:
-        globls = {}
-        locs = {}
+    setup_args={}
+    if os.path.exists(special_build): 
+        # globls = {}
+        # locs = {}
         # execfile(special_build, globls, locs)
         # ext = locs["make_ext"](modname, pyxfilename)
         mod = imp.load_source("XXXX", special_build, open(special_build))
-        ext = mod.make_ext(modname, pyxfilename)
+        make_ext = getattr(mod,'make_ext',None)
+        if make_ext:
+            ext = make_ext(modname, pyxfilename)
             assert ext and ext.sources, ("make_ext in %s did not return Extension" 
                                          % special_build)
+        make_setup_args = getattr(mod,'make_setup_args',None)
+        if make_setup_args:
+            setup_args = make_setup_args()
+            assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" 
+                                         % special_build)
+        assert set or setup_args, ("neither make_ext nor make_setup_args %s" 
+                                         % special_build)
         ext.sources = [os.path.join(os.path.dirname(special_build), source) 
                        for source in ext.sources]
-    return ext
+    return ext, setup_args
 
 def handle_dependencies(pyxfilename):
     dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
@@ -110,12 +132,13 @@ def handle_dependencies(pyxfilename):
             files.extend(glob.glob(fullpath))
 
         # only for unit testing to see we did the right thing
-        _test_files[:] = []
+        _test_files[:] = []  #$pycheck_no
 
         # if any file that the pyxfile depends upon is newer than
         # the pyx file, 'touch' the pyx file so that distutils will
         # be tricked into rebuilding it.
         for file in files:
+            from distutils.dep_util import newer
             if newer(file, pyxfilename):
                 print("Rebuilding because of ", file)
                 filetime = os.path.getmtime(file)
@@ -127,14 +150,21 @@ def build_module(name, pyxfilename, pyxbuild_dir=None):
         "Path does not exist: %s" % pyxfilename)
     handle_dependencies(pyxfilename)
 
-    extension_mod = get_distutils_extension(name, pyxfilename)
+    extension_mod,setup_args = get_distutils_extension(name, pyxfilename)
+    build_in_temp=pyxargs.build_in_temp
+    sargs=pyxargs.setup_args.copy()
+    sargs.update(setup_args)
+    build_in_temp=sargs.pop('build_in_temp',build_in_temp)
 
+    import pyxbuild
     so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
-                                  build_in_temp=True,
-                                  pyxbuild_dir=pyxbuild_dir)
+                                  build_in_temp=build_in_temp,
+                                  pyxbuild_dir=pyxbuild_dir,
+                                  setup_args=sargs,
+                                  reload_support=pyxargs.reload_support)
     assert os.path.exists(so_path), "Cannot find: %s" % so_path
     
-    junkpath = os.path.join(os.path.dirname(so_path), name+"_*")
+    junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ?
     junkstuff = glob.glob(junkpath)
     for path in junkstuff:
         if path!=so_path:
@@ -151,7 +181,9 @@ def load_module(name, pyxfilename, pyxbuild_dir=None):
         mod = imp.load_dynamic(name, so_path)
         assert mod.__file__ == so_path, (mod.__file__, so_path)
     except Exception, e:
-        raise ImportError("Building module failed: %s" % e)
+        import traceback
+        raise ImportError("Building module failed: %s" %
+                          traceback.format_exception_only(*sys.exc_info()[:2])),None,sys.exc_info()[2]
     return mod
 
 
@@ -165,16 +197,33 @@ class PyxImporter(object):
         self.pyxbuild_dir = pyxbuild_dir
 
     def find_module(self, fullname, package_path=None):
-        if fullname in sys.modules:
+        if fullname in sys.modules  and  not pyxargs.reload_support:
+            return None  # only here when reload() 
+        try:
+            fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
+            if fp: fp.close()  # Python should offer a Default-Loader to avoid this double find/open!
+            if ty!=imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
                 return None
-        if DEBUG_IMPORT:
-            print("SEARCHING", fullname, package_path)
-        if '.' in fullname:
+
+            # find .pyx fast, when .so/.pyd exist --inplace
+            pyxpath = os.path.splitext(pathname)[0]+self.extension
+            if os.path.isfile(pyxpath):
+                return PyxLoader(fullname, pyxpath,
+                                 pyxbuild_dir=self.pyxbuild_dir)
+            
+            # .so/.pyd's on PATH should not be remote from .pyx's
+            # think no need to implement PyxArgs.importer_search_remote here?
+                
+        except ImportError:
+            pass
+
+        # searching sys.path ...
+                
+        #if DEBUG_IMPORT:  print "SEARCHING", fullname, package_path
+        if '.' in fullname: # only when package_path anyway?
             mod_parts = fullname.split('.')
-            package = '.'.join(mod_parts[:-1])
             module_name = mod_parts[-1]
         else:
-            package = None
             module_name = fullname
         pyx_module_name = module_name + self.extension
         # this may work, but it returns the file content, not its path
@@ -187,24 +236,15 @@ class PyxImporter(object):
             paths = sys.path
         join_path = os.path.join
         is_file = os.path.isfile
-        is_dir = os.path.isdir
+        #is_dir = os.path.isdir
+        sep = os.path.sep
         for path in paths:
-            if not is_dir(path):
             if not path:
                 path = os.getcwd()
-                else:
-                    continue
-            for filename in os.listdir(path):
-                if filename == pyx_module_name:
-                    return PyxLoader(fullname, join_path(path, filename),
-                                     pyxbuild_dir=self.pyxbuild_dir)
-                elif filename == module_name:
-                    package_path = join_path(path, filename)
-                    init_path = join_path(package_path,
-                                          '__init__' + self.extension)
-                    if is_file(init_path):
-                        return PyxLoader(fullname, package_path, init_path,
+            if is_file(path+sep+pyx_module_name):
+                return PyxLoader(fullname, join_path(path, pyx_module_name),
                                  pyxbuild_dir=self.pyxbuild_dir)
+                
         # not found, normal package, not a .pyx file, none of our business
         return None
 
@@ -289,7 +329,16 @@ class PyxLoader(object):
         return module
 
 
-def install(pyximport=True, pyimport=False, build_dir=None):
+#install args
+class PyxArgs(object):
+    build_dir=True
+    build_in_temp=True
+    setup_args={}   #None
+
+##pyxargs=None   
+    
+def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
+            setup_args={}, reload_support=False ):
     """Main entry point. Call this to install the .pyx import hook in
     your meta-path for a single Python process.  If you want it to be
     installed whenever you use Python, add it to your sitecustomize
@@ -303,10 +352,32 @@ def install(pyximport=True, pyimport=False, build_dir=None):
     By default, compiled modules will end up in a ``.pyxbld``
     directory in the user's home directory.  Passing a different path
     as ``build_dir`` will override this.
+
+    ``build_in_temp=False`` will produce the C files locally. Working
+    with complex dependencies and debugging becomes more easy. This
+    can principally interfere with existing files of the same name.
+    build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
+    by a dict item of 'build_in_temp'
+
+    ``setup_args``: dict of arguments for Distribution - see
+    distutils.core.setup() . They are extended/overriden by those of
+    <modulename>.pyxbld/make_setup_args()
+
+    ``reload_support``:  Enables support for dynamic
+    reload(<pyxmodulename>), e.g. after a change in the Cython code.
+    Additional files <so_path>.reloadNN may arise on that account, when
+    the previously loaded module file cannot be overwritten.
     """
     if not build_dir:
         build_dir = os.path.expanduser('~/.pyxbld')
         
+    global pyxargs
+    pyxargs = PyxArgs()  #$pycheck_no
+    pyxargs.build_dir = build_dir
+    pyxargs.build_in_temp = build_in_temp
+    pyxargs.setup_args = (setup_args or {}).copy()
+    pyxargs.reload_support = reload_support
+
     has_py_importer = False
     has_pyx_importer = False
     for importer in sys.meta_path: