Commit 9ebcb0ef authored by Stefan Behnel's avatar Stefan Behnel

Merge branch 'master' into 0.19.x

Conflicts:
	CHANGES.rst
parents 4dde32bf 96fde4d1
...@@ -26,6 +26,9 @@ Features added ...@@ -26,6 +26,9 @@ Features added
* Constant folding for boolean expressions (and/or) was improved. * Constant folding for boolean expressions (and/or) was improved.
* Added a build_dir option to cythonize() which allows one to place
the generated .c files outside the source tree.
Bugs fixed Bugs fixed
---------- ----------
......
...@@ -18,13 +18,28 @@ try: ...@@ -18,13 +18,28 @@ try:
except ImportError: except ImportError:
import md5 as hashlib import md5 as hashlib
try:
from os.path import relpath as _relpath
except ImportError:
# Py<2.6
def _relpath(path, start=os.path.curdir):
if not path:
raise ValueError("no path specified")
start_list = os.path.abspath(start).split(os.path.sep)
path_list = os.path.abspath(path).split(os.path.sep)
i = len(os.path.commonprefix([start_list, path_list]))
rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
if not rel_list:
return os.path.curdir
return os.path.join(*rel_list)
from distutils.extension import Extension from distutils.extension import Extension
from Cython import Utils from Cython import Utils
from Cython.Utils import cached_function, cached_method, path_exists from Cython.Utils import cached_function, cached_method, path_exists, find_root_package_dir
from Cython.Compiler.Main import Context, CompilationOptions, default_options from Cython.Compiler.Main import Context, CompilationOptions, default_options
join_path = cached_function(os.path.join) join_path = cached_function(os.path.join)
if sys.version_info[0] < 3: if sys.version_info[0] < 3:
...@@ -208,7 +223,7 @@ def strip_string_literals(code, prefix='__Pyx_L'): ...@@ -208,7 +223,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
in_quote = False in_quote = False
hash_mark = single_q = double_q = -1 hash_mark = single_q = double_q = -1
code_len = len(code) code_len = len(code)
while True: while True:
if hash_mark < q: if hash_mark < q:
hash_mark = code.find('#', q) hash_mark = code.find('#', q)
...@@ -374,7 +389,7 @@ class DependencyTree(object): ...@@ -374,7 +389,7 @@ class DependencyTree(object):
elif not self.quiet: elif not self.quiet:
print("Unable to locate '%s' referenced from '%s'" % (filename, include)) print("Unable to locate '%s' referenced from '%s'" % (filename, include))
return all return all
@cached_method @cached_method
def cimports_and_externs(self, filename): def cimports_and_externs(self, filename):
# This is really ugly. Nested cimports are resolved with respect to the # This is really ugly. Nested cimports are resolved with respect to the
...@@ -598,7 +613,7 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=Fa ...@@ -598,7 +613,7 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=Fa
sources.append(source) sources.append(source)
del kwds['sources'] del kwds['sources']
if 'depends' in kwds: if 'depends' in kwds:
depends = resolve_depends(kwds['depends'], kwds.get('include_dirs') or []) depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
if template is not None: if template is not None:
# Always include everything from the template. # Always include everything from the template.
depends = list(set(template.depends).union(set(depends))) depends = list(set(template.depends).union(set(depends)))
...@@ -640,6 +655,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -640,6 +655,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
c_options = CompilationOptions(**options) c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context() ctx = c_options.create_context()
options = c_options
module_list = create_extension_list( module_list = create_extension_list(
module_list, module_list,
exclude=exclude, exclude=exclude,
...@@ -648,9 +664,23 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -648,9 +664,23 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
exclude_failures=exclude_failures, exclude_failures=exclude_failures,
aliases=aliases) aliases=aliases)
deps = create_dependency_tree(ctx, quiet=quiet) deps = create_dependency_tree(ctx, quiet=quiet)
build_dir = getattr(options, 'build_dir', None)
modules_by_cfile = {} modules_by_cfile = {}
to_compile = [] to_compile = []
for m in module_list: for m in module_list:
if build_dir:
root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
def copy_to_build_dir(filepath, root=root):
filepath = os.path.abspath(filepath)
if os.path.realpath(filepath).startswith(root):
mod_dir = os.path.join(
build_dir, os.path.dirname(_relpath(filepath)))
if not os.path.isdir(mod_dir):
os.makedirs(mod_dir)
shutil.copy(filepath, mod_dir)
for dep in m.depends:
copy_to_build_dir(dep)
new_sources = [] new_sources = []
for source in m.sources: for source in m.sources:
base, ext = os.path.splitext(source) base, ext = os.path.splitext(source)
...@@ -661,11 +691,19 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -661,11 +691,19 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
else: else:
c_file = base + '.c' c_file = base + '.c'
options = c_options options = c_options
# setup for out of place build directory if enabled
if build_dir:
c_file = os.path.join(build_dir, c_file)
dir = os.path.dirname(c_file)
if not os.path.isdir(dir):
os.makedirs(dir)
if os.path.exists(c_file): if os.path.exists(c_file):
c_timestamp = os.path.getmtime(c_file) c_timestamp = os.path.getmtime(c_file)
else: else:
c_timestamp = -1 c_timestamp = -1
# Priority goes first to modified files, second to direct # Priority goes first to modified files, second to direct
# dependents, and finally to indirect dependents. # dependents, and finally to indirect dependents.
if c_timestamp < deps.timestamp(source): if c_timestamp < deps.timestamp(source):
...@@ -694,6 +732,8 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -694,6 +732,8 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
modules_by_cfile[c_file].append(m) modules_by_cfile[c_file].append(m)
else: else:
new_sources.append(source) new_sources.append(source)
if build_dir:
copy_to_build_dir(source)
m.sources = new_sources m.sources = new_sources
if hasattr(options, 'cache'): if hasattr(options, 'cache'):
if not os.path.exists(options.cache): if not os.path.exists(options.cache):
......
...@@ -56,7 +56,7 @@ cdef extern from "Python.h": ...@@ -56,7 +56,7 @@ cdef extern from "Python.h":
void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices)
# ?? # ??
int PyBuffer_SizeFromFormat(char *) # actually const char Py_ssize_t PyBuffer_SizeFromFormat(char *) # actually const char
# Return the implied ~Py_buffer.itemsize from the struct-stype # Return the implied ~Py_buffer.itemsize from the struct-stype
# ~Py_buffer.format # ~Py_buffer.format
...@@ -90,7 +90,7 @@ cdef extern from "Python.h": ...@@ -90,7 +90,7 @@ cdef extern from "Python.h":
void PyBuffer_FillContiguousStrides(int ndims, void PyBuffer_FillContiguousStrides(int ndims,
Py_ssize_t *shape, Py_ssize_t *shape,
Py_ssize_t *strides, Py_ssize_t *strides,
int itemsize, Py_ssize_t itemsize,
char fort) char fort)
# Fill the strides array with byte-strides of a contiguous # Fill the strides array with byte-strides of a contiguous
# (Fortran-style if fort is 'F' or C-style otherwise) array of the # (Fortran-style if fort is 'F' or C-style otherwise) array of the
......
...@@ -84,7 +84,7 @@ cdef extern from "Python.h": ...@@ -84,7 +84,7 @@ cdef extern from "Python.h":
# value, return 1, otherwise return 0. On error, return -1. This # value, return 1, otherwise return 0. On error, return -1. This
# is equivalent to the Python expression "value in o". # is equivalent to the Python expression "value in o".
int PySequence_Index(object o, object value) except -1 Py_ssize_t PySequence_Index(object o, object value) except -1
# Return the first index i for which o[i] == value. On error, # Return the first index i for which o[i] == value. On error,
# return -1. This is equivalent to the Python expression # return -1. This is equivalent to the Python expression
# "o.index(value)". # "o.index(value)".
...@@ -126,7 +126,7 @@ cdef extern from "Python.h": ...@@ -126,7 +126,7 @@ cdef extern from "Python.h":
# PySequence_Check(o) is true and without adjustment for negative # PySequence_Check(o) is true and without adjustment for negative
# indices. # indices.
int PySequence_Fast_GET_SIZE(object o) Py_ssize_t PySequence_Fast_GET_SIZE(object o)
# Returns the length of o, assuming that o was returned by # Returns the length of o, assuming that o was returned by
# PySequence_Fast() and that o is not NULL. The size can also be # PySequence_Fast() and that o is not NULL. The size can also be
# gotten by calling PySequence_Size() on o, but # gotten by calling PySequence_Size() on o, but
......
...@@ -66,12 +66,12 @@ cdef extern from "Python.h": ...@@ -66,12 +66,12 @@ cdef extern from "Python.h":
# The following functions and macros are available for instances # The following functions and macros are available for instances
# of set or frozenset or instances of their subtypes. # of set or frozenset or instances of their subtypes.
int PySet_Size(object anyset) except -1 Py_ssize_t PySet_Size(object anyset) except -1
# Return the length of a set or frozenset object. Equivalent to # Return the length of a set or frozenset object. Equivalent to
# "len(anyset)". Raises a PyExc_SystemError if anyset is not a # "len(anyset)". Raises a PyExc_SystemError if anyset is not a
# set, frozenset, or an instance of a subtype. # set, frozenset, or an instance of a subtype.
int PySet_GET_SIZE(object anyset) Py_ssize_t PySet_GET_SIZE(object anyset)
# Macro form of PySet_Size() without error checking. # Macro form of PySet_Size() without error checking.
bint PySet_Contains(object anyset, object key) except -1 bint PySet_Contains(object anyset, object key) except -1
......
...@@ -24,10 +24,10 @@ cdef extern from "Python.h": ...@@ -24,10 +24,10 @@ cdef extern from "Python.h":
# pointing to Python objects. "PyTuple_Pack(2, a, b)" is # pointing to Python objects. "PyTuple_Pack(2, a, b)" is
# equivalent to "Py_BuildValue("(OO)", a, b)". # equivalent to "Py_BuildValue("(OO)", a, b)".
int PyTuple_Size(object p) except -1 Py_ssize_t PyTuple_Size(object p) except -1
# Take a pointer to a tuple object, and return the size of that tuple. # Take a pointer to a tuple object, and return the size of that tuple.
int PyTuple_GET_SIZE(object p) Py_ssize_t PyTuple_GET_SIZE(object p)
# Return the size of the tuple p, which must be non-NULL and point # Return the size of the tuple p, which must be non-NULL and point
# to a tuple; no error checking is performed. # to a tuple; no error checking is performed.
......
...@@ -377,7 +377,8 @@ cdef extern from *: ...@@ -377,7 +377,8 @@ cdef extern from *:
# consumed is not NULL, PyUnicode_DecodeMBCSStateful() will not # consumed is not NULL, PyUnicode_DecodeMBCSStateful() will not
# decode trailing lead byte and the number of bytes that have been # decode trailing lead byte and the number of bytes that have been
# decoded will be stored in consumed. New in version 2.5. # decoded will be stored in consumed. New in version 2.5.
object PyUnicode_DecodeMBCSStateful(char *s, int size, char *errors, int *consumed) # NOTE: Python 2.x uses 'int' values for 'size' and 'consumed' (changed in 3.0)
object PyUnicode_DecodeMBCSStateful(char *s, Py_ssize_t size, char *errors, Py_ssize_t *consumed)
# Encode the Py_UNICODE buffer of the given size using MBCS and # Encode the Py_UNICODE buffer of the given size using MBCS and
# return a Python string object. Return NULL if an exception was # return a Python string object. Return NULL if an exception was
......
...@@ -117,12 +117,12 @@ def search_include_directories(dirs, qualified_name, suffix, pos, ...@@ -117,12 +117,12 @@ def search_include_directories(dirs, qualified_name, suffix, pos,
@cached_function @cached_function
def find_root_package_dir(file_path): def find_root_package_dir(file_path):
dir = os.path.dirname(file_path) dir = os.path.dirname(file_path)
while is_package_dir(dir): if file_path == dir:
parent = os.path.dirname(dir) return dir
if parent == dir: elif is_package_dir(dir):
break return find_root_package_dir(dir)
dir = parent else:
return dir return dir
@cached_function @cached_function
def check_package_dir(dir, package_names): def check_package_dir(dir, package_names):
......
PYTHON setup.py build_ext --inplace
PYTHON -c "import a"
PYTHON -c "import pkg.b"
PYTHON check_paths.py
######## setup.py ########
# TODO: Better interface...
from Cython.Build.Dependencies import cythonize
from distutils.core import setup
setup(
ext_modules = (cythonize("*.pyx", build_dir="scratchA") +
cythonize("pkg/*.pyx", build_dir="scratchB")),
)
######## a.pyx ########
cdef extern from "helper.h":
int value1
cdef extern from "subdir/helper.h":
int value2
cdef extern from "pkg/pkg_helper.h":
int value3
assert value1 == 100
assert value2 == 200
assert value3 == 300
######## helper.h ########
int value1 = 100;
######## subdir/helper.h ########
int value2 = 200;
######## pkg/__init__.py ########
######## pkg/b.pyx ########
cdef extern from "pkg_helper.h":
int value3
cdef extern from "subdir/pkg_helper.h":
int value4
assert value3 == 300
assert value4 == 400
######## pkg/pkg_helper.h ########
int value3 = 300;
######## pkg/subdir/pkg_helper.h ########
int value4 = 400;
######## check_paths.py ########
import os
assert os.path.exists("scratchA/a.c")
assert os.path.exists("scratchA/helper.h")
assert os.path.exists("scratchA/subdir/helper.h")
assert os.path.exists("scratchA/pkg/pkg_helper.h")
assert not os.path.exists("a.c")
assert os.path.exists("scratchB/pkg/b.c")
assert os.path.exists("scratchB/pkg/pkg_helper.h")
assert os.path.exists("scratchB/pkg/subdir/pkg_helper.h")
assert not os.path.exists("b.c")
assert not os.path.exists("pkg/b.c")
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment