Commit b16f44a4 authored by Stefan Behnel's avatar Stefan Behnel

applied pyximport patch from ticket 312

parent a7e5883e
...@@ -6,7 +6,6 @@ out_fname = pyx_to_dll("foo.pyx") ...@@ -6,7 +6,6 @@ out_fname = pyx_to_dll("foo.pyx")
import os import os
import sys import sys
import distutils
from distutils.dist import Distribution from distutils.dist import Distribution
from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError from distutils.errors import DistutilsArgError, DistutilsError, CCompilerError
from distutils.extension import Extension from distutils.extension import Extension
...@@ -16,11 +15,13 @@ try: ...@@ -16,11 +15,13 @@ try:
HAS_CYTHON = True HAS_CYTHON = True
except ImportError: except ImportError:
HAS_CYTHON = False HAS_CYTHON = False
import shutil
DEBUG = 0 DEBUG = 0
_reloads={}
def pyx_to_dll(filename, ext = None, force_rebuild = 0, def pyx_to_dll(filename, ext = None, force_rebuild = 0,
build_in_temp=False, pyxbuild_dir=None): build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False):
"""Compile a PYX file to a DLL and return the name of the generated .so """Compile a PYX file to a DLL and return the name of the generated .so
or .dll .""" or .dll ."""
assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename) assert os.path.exists(filename), "Could not find %s" % os.path.abspath(filename)
...@@ -37,7 +38,8 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0, ...@@ -37,7 +38,8 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
if not pyxbuild_dir: if not pyxbuild_dir:
pyxbuild_dir = os.path.join(path, "_pyxbld") pyxbuild_dir = os.path.join(path, "_pyxbld")
if DEBUG: script_args=setup_args.get("script_args",[])
if DEBUG or "--verbose" in script_args:
quiet = "--verbose" quiet = "--verbose"
else: else:
quiet = "--quiet" quiet = "--quiet"
...@@ -46,7 +48,11 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0, ...@@ -46,7 +48,11 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
args.append("--force") args.append("--force")
if HAS_CYTHON and build_in_temp: if HAS_CYTHON and build_in_temp:
args.append("--pyrex-c-in-temp") args.append("--pyrex-c-in-temp")
dist = Distribution({"script_name": None, "script_args": args}) sargs = setup_args.copy()
sargs.update(
{"script_name": None,
"script_args": args + script_args} )
dist = Distribution(sargs)
if not dist.ext_modules: if not dist.ext_modules:
dist.ext_modules = [] dist.ext_modules = []
dist.ext_modules.append(ext) dist.ext_modules.append(ext)
...@@ -60,6 +66,10 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0, ...@@ -60,6 +66,10 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
except ValueError: pass except ValueError: pass
dist.parse_config_files(config_files) dist.parse_config_files(config_files)
cfgfiles = dist.find_config_files()
try: cfgfiles.remove('setup.cfg')
except ValueError: pass
dist.parse_config_files(cfgfiles)
try: try:
ok = dist.parse_command_line() ok = dist.parse_command_line()
except DistutilsArgError: except DistutilsArgError:
...@@ -73,7 +83,39 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0, ...@@ -73,7 +83,39 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
try: try:
dist.run_commands() dist.run_commands()
return dist.get_command_obj("build_ext").get_outputs()[0] obj_build_ext = dist.get_command_obj("build_ext")
so_path = obj_build_ext.get_outputs()[0]
if obj_build_ext.inplace:
# Python distutils get_outputs()[ returns a wrong so_path
# when --inplace ; see http://bugs.python.org/issue5977
# workaround:
so_path = os.path.join(os.path.dirname(filename),
os.path.basename(so_path))
if reload_support:
org_path = so_path
timestamp = os.path.getmtime(org_path)
global _reloads
last_timestamp, last_path, count = _reloads.get(org_path, (None,None,0) )
if last_timestamp == timestamp:
so_path = last_path
else:
basename = os.path.basename(org_path)
while count < 100:
count += 1
r_path = os.path.join(obj_build_ext.build_lib,
basename + '.reload%s'%count)
try:
import shutil # late import / reload_support is: debugging
shutil.copy2(org_path, r_path)
so_path = r_path
except IOError:
continue
break
else:
# used up all 100 slots
raise ImportError("reload count for %s reached maximum"%org_path)
_reloads[org_path]=(timestamp, so_path, count)
return so_path
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit(1) sys.exit(1)
except (IOError, os.error): except (IOError, os.error):
...@@ -83,15 +125,6 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0, ...@@ -83,15 +125,6 @@ def pyx_to_dll(filename, ext = None, force_rebuild = 0,
if DEBUG: if DEBUG:
sys.stderr.write(error + "\n") sys.stderr.write(error + "\n")
raise raise
else:
raise RuntimeError(error)
except (DistutilsError, CCompilerError):
if DEBUG:
raise
else:
exc = sys.exc_info()[1]
raise RuntimeError(repr(exc))
if __name__=="__main__": if __name__=="__main__":
pyx_to_dll("dummy.pyx") pyx_to_dll("dummy.pyx")
......
...@@ -14,6 +14,22 @@ For instance on the Mac with a non-system Python 2.3, you could create ...@@ -14,6 +14,22 @@ For instance on the Mac with a non-system Python 2.3, you could create
sitecustomize.py with only those two lines at sitecustomize.py with only those two lines at
/usr/local/lib/python2.3/site-packages/sitecustomize.py . /usr/local/lib/python2.3/site-packages/sitecustomize.py .
A custom distutils.core.Extension instance and setup() args
(Distribution) for for the build can be defined by a <modulename>.pyxbld
file like:
# examplemod.pyxbdl
def make_ext(modname, pyxfilename):
from distutils.extension import Extension
return Extension(name = modname,
sources=[pyxfilename, 'hello.c'],
include_dirs=['/myinclude'] )
def make_setup_args():
return dict(script_args=["--compiler=mingw32"])
Extra dependencies can be defined by a <modulename>.pyxdep .
See README.
Since Cython 0.11, the :mod:`pyximport` module also has experimental Since Cython 0.11, the :mod:`pyximport` module also has experimental
compilation support for normal Python modules. This allows you to compilation support for normal Python modules. This allows you to
automatically run Cython on every .pyx and .py module that Python automatically run Cython on every .pyx and .py module that Python
...@@ -30,18 +46,11 @@ the documentation. ...@@ -30,18 +46,11 @@ the documentation.
This code is based on the Py2.3+ import protocol as described in PEP 302. This code is based on the Py2.3+ import protocol as described in PEP 302.
""" """
import sys import sys
import os import os
import glob import glob
import imp import imp
import pyxbuild
from distutils.dep_util import newer
from distutils.extension import Extension
try:
import hashlib
except ImportError:
import md5 as hashlib
mod_name = "pyximport" mod_name = "pyximport"
...@@ -64,30 +73,43 @@ def _load_pyrex(name, filename): ...@@ -64,30 +73,43 @@ def _load_pyrex(name, filename):
"Load a pyrex file given a name and filename." "Load a pyrex file given a name and filename."
def get_distutils_extension(modname, pyxfilename): def get_distutils_extension(modname, pyxfilename):
extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() # try:
# import hashlib
# except ImportError:
# import md5 as hashlib
# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()
# modname = modname + extra # modname = modname + extra
extension_mod = handle_special_build(modname, pyxfilename) extension_mod,setup_args = handle_special_build(modname, pyxfilename)
if not extension_mod: if not extension_mod:
from distutils.extension import Extension
extension_mod = Extension(name = modname, sources=[pyxfilename]) extension_mod = Extension(name = modname, sources=[pyxfilename])
return extension_mod return extension_mod,setup_args
def handle_special_build(modname, pyxfilename): def handle_special_build(modname, pyxfilename):
special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
if not os.path.exists(special_build):
ext = None ext = None
else: setup_args={}
globls = {} if os.path.exists(special_build):
locs = {} # globls = {}
# locs = {}
# execfile(special_build, globls, locs) # execfile(special_build, globls, locs)
# ext = locs["make_ext"](modname, pyxfilename) # ext = locs["make_ext"](modname, pyxfilename)
mod = imp.load_source("XXXX", special_build, open(special_build)) mod = imp.load_source("XXXX", special_build, open(special_build))
ext = mod.make_ext(modname, pyxfilename) make_ext = getattr(mod,'make_ext',None)
if make_ext:
ext = make_ext(modname, pyxfilename)
assert ext and ext.sources, ("make_ext in %s did not return Extension" assert ext and ext.sources, ("make_ext in %s did not return Extension"
% special_build) % special_build)
make_setup_args = getattr(mod,'make_setup_args',None)
if make_setup_args:
setup_args = make_setup_args()
assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict"
% special_build)
assert set or setup_args, ("neither make_ext nor make_setup_args %s"
% special_build)
ext.sources = [os.path.join(os.path.dirname(special_build), source) ext.sources = [os.path.join(os.path.dirname(special_build), source)
for source in ext.sources] for source in ext.sources]
return ext return ext, setup_args
def handle_dependencies(pyxfilename): def handle_dependencies(pyxfilename):
dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
...@@ -110,12 +132,13 @@ def handle_dependencies(pyxfilename): ...@@ -110,12 +132,13 @@ def handle_dependencies(pyxfilename):
files.extend(glob.glob(fullpath)) files.extend(glob.glob(fullpath))
# only for unit testing to see we did the right thing # only for unit testing to see we did the right thing
_test_files[:] = [] _test_files[:] = [] #$pycheck_no
# if any file that the pyxfile depends upon is newer than # if any file that the pyxfile depends upon is newer than
# the pyx file, 'touch' the pyx file so that distutils will # the pyx file, 'touch' the pyx file so that distutils will
# be tricked into rebuilding it. # be tricked into rebuilding it.
for file in files: for file in files:
from distutils.dep_util import newer
if newer(file, pyxfilename): if newer(file, pyxfilename):
print("Rebuilding because of ", file) print("Rebuilding because of ", file)
filetime = os.path.getmtime(file) filetime = os.path.getmtime(file)
...@@ -127,14 +150,21 @@ def build_module(name, pyxfilename, pyxbuild_dir=None): ...@@ -127,14 +150,21 @@ def build_module(name, pyxfilename, pyxbuild_dir=None):
"Path does not exist: %s" % pyxfilename) "Path does not exist: %s" % pyxfilename)
handle_dependencies(pyxfilename) handle_dependencies(pyxfilename)
extension_mod = get_distutils_extension(name, pyxfilename) extension_mod,setup_args = get_distutils_extension(name, pyxfilename)
build_in_temp=pyxargs.build_in_temp
sargs=pyxargs.setup_args.copy()
sargs.update(setup_args)
build_in_temp=sargs.pop('build_in_temp',build_in_temp)
import pyxbuild
so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod, so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
build_in_temp=True, build_in_temp=build_in_temp,
pyxbuild_dir=pyxbuild_dir) pyxbuild_dir=pyxbuild_dir,
setup_args=sargs,
reload_support=pyxargs.reload_support)
assert os.path.exists(so_path), "Cannot find: %s" % so_path assert os.path.exists(so_path), "Cannot find: %s" % so_path
junkpath = os.path.join(os.path.dirname(so_path), name+"_*") junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ?
junkstuff = glob.glob(junkpath) junkstuff = glob.glob(junkpath)
for path in junkstuff: for path in junkstuff:
if path!=so_path: if path!=so_path:
...@@ -151,7 +181,9 @@ def load_module(name, pyxfilename, pyxbuild_dir=None): ...@@ -151,7 +181,9 @@ def load_module(name, pyxfilename, pyxbuild_dir=None):
mod = imp.load_dynamic(name, so_path) mod = imp.load_dynamic(name, so_path)
assert mod.__file__ == so_path, (mod.__file__, so_path) assert mod.__file__ == so_path, (mod.__file__, so_path)
except Exception, e: except Exception, e:
raise ImportError("Building module failed: %s" % e) import traceback
raise ImportError("Building module failed: %s" %
traceback.format_exception_only(*sys.exc_info()[:2])),None,sys.exc_info()[2]
return mod return mod
...@@ -165,16 +197,33 @@ class PyxImporter(object): ...@@ -165,16 +197,33 @@ class PyxImporter(object):
self.pyxbuild_dir = pyxbuild_dir self.pyxbuild_dir = pyxbuild_dir
def find_module(self, fullname, package_path=None): def find_module(self, fullname, package_path=None):
if fullname in sys.modules: if fullname in sys.modules and not pyxargs.reload_support:
return None # only here when reload()
try:
fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open!
if ty!=imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
return None return None
if DEBUG_IMPORT:
print("SEARCHING", fullname, package_path) # find .pyx fast, when .so/.pyd exist --inplace
if '.' in fullname: pyxpath = os.path.splitext(pathname)[0]+self.extension
if os.path.isfile(pyxpath):
return PyxLoader(fullname, pyxpath,
pyxbuild_dir=self.pyxbuild_dir)
# .so/.pyd's on PATH should not be remote from .pyx's
# think no need to implement PyxArgs.importer_search_remote here?
except ImportError:
pass
# searching sys.path ...
#if DEBUG_IMPORT: print "SEARCHING", fullname, package_path
if '.' in fullname: # only when package_path anyway?
mod_parts = fullname.split('.') mod_parts = fullname.split('.')
package = '.'.join(mod_parts[:-1])
module_name = mod_parts[-1] module_name = mod_parts[-1]
else: else:
package = None
module_name = fullname module_name = fullname
pyx_module_name = module_name + self.extension pyx_module_name = module_name + self.extension
# this may work, but it returns the file content, not its path # this may work, but it returns the file content, not its path
...@@ -187,24 +236,15 @@ class PyxImporter(object): ...@@ -187,24 +236,15 @@ class PyxImporter(object):
paths = sys.path paths = sys.path
join_path = os.path.join join_path = os.path.join
is_file = os.path.isfile is_file = os.path.isfile
is_dir = os.path.isdir #is_dir = os.path.isdir
sep = os.path.sep
for path in paths: for path in paths:
if not is_dir(path):
if not path: if not path:
path = os.getcwd() path = os.getcwd()
else: if is_file(path+sep+pyx_module_name):
continue return PyxLoader(fullname, join_path(path, pyx_module_name),
for filename in os.listdir(path):
if filename == pyx_module_name:
return PyxLoader(fullname, join_path(path, filename),
pyxbuild_dir=self.pyxbuild_dir)
elif filename == module_name:
package_path = join_path(path, filename)
init_path = join_path(package_path,
'__init__' + self.extension)
if is_file(init_path):
return PyxLoader(fullname, package_path, init_path,
pyxbuild_dir=self.pyxbuild_dir) pyxbuild_dir=self.pyxbuild_dir)
# not found, normal package, not a .pyx file, none of our business # not found, normal package, not a .pyx file, none of our business
return None return None
...@@ -289,7 +329,16 @@ class PyxLoader(object): ...@@ -289,7 +329,16 @@ class PyxLoader(object):
return module return module
def install(pyximport=True, pyimport=False, build_dir=None): #install args
class PyxArgs(object):
build_dir=True
build_in_temp=True
setup_args={} #None
##pyxargs=None
def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
setup_args={}, reload_support=False ):
"""Main entry point. Call this to install the .pyx import hook in """Main entry point. Call this to install the .pyx import hook in
your meta-path for a single Python process. If you want it to be your meta-path for a single Python process. If you want it to be
installed whenever you use Python, add it to your sitecustomize installed whenever you use Python, add it to your sitecustomize
...@@ -303,10 +352,32 @@ def install(pyximport=True, pyimport=False, build_dir=None): ...@@ -303,10 +352,32 @@ def install(pyximport=True, pyimport=False, build_dir=None):
By default, compiled modules will end up in a ``.pyxbld`` By default, compiled modules will end up in a ``.pyxbld``
directory in the user's home directory. Passing a different path directory in the user's home directory. Passing a different path
as ``build_dir`` will override this. as ``build_dir`` will override this.
``build_in_temp=False`` will produce the C files locally. Working
with complex dependencies and debugging becomes more easy. This
can principally interfere with existing files of the same name.
build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
by a dict item of 'build_in_temp'
``setup_args``: dict of arguments for Distribution - see
distutils.core.setup() . They are extended/overriden by those of
<modulename>.pyxbld/make_setup_args()
``reload_support``: Enables support for dynamic
reload(<pyxmodulename>), e.g. after a change in the Cython code.
Additional files <so_path>.reloadNN may arise on that account, when
the previously loaded module file cannot be overwritten.
""" """
if not build_dir: if not build_dir:
build_dir = os.path.expanduser('~/.pyxbld') build_dir = os.path.expanduser('~/.pyxbld')
global pyxargs
pyxargs = PyxArgs() #$pycheck_no
pyxargs.build_dir = build_dir
pyxargs.build_in_temp = build_in_temp
pyxargs.setup_args = (setup_args or {}).copy()
pyxargs.reload_support = reload_support
has_py_importer = False has_py_importer = False
has_pyx_importer = False has_pyx_importer = False
for importer in sys.meta_path: for importer in sys.meta_path:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment