Dependencies.py

from __future__ import absolute_import, print_function

import cython
from .. import __version__

import re, os, sys, time
from glob import iglob

try:
    import gzip
    gzip_open = gzip.open
    gzip_ext = '.gz'
except ImportError:
    gzip_open = open
    gzip_ext = ''
import shutil
import subprocess

try:
    import hashlib
except ImportError:
    import md5 as hashlib

try:
    from io import open as io_open
except ImportError:
    from codecs import open as io_open

try:
    from os.path import relpath as _relpath
except ImportError:
    # Py<2.6
    def _relpath(path, start=os.path.curdir):
        if not path:
            raise ValueError("no path specified")
        start_list = os.path.abspath(start).split(os.path.sep)
        path_list = os.path.abspath(path).split(os.path.sep)
        i = len(os.path.commonprefix([start_list, path_list]))
        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
        if not rel_list:
            return os.path.curdir
        return os.path.join(*rel_list)


from distutils.extension import Extension

from .. import Utils
from ..Utils import (cached_function, cached_method, path_exists,
    safe_makedirs, copy_file_to_dir_if_newer, is_package_dir)
from ..Compiler.Main import Context, CompilationOptions, default_options

join_path = cached_function(os.path.join)
copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
safe_makedirs_once = cached_function(safe_makedirs)

if sys.version_info[0] < 3:
    # stupid Py2 distutils enforces str type in list of sources
    _fs_encoding = sys.getfilesystemencoding()
    if _fs_encoding is None:
        _fs_encoding = sys.getdefaultencoding()
    def encode_filename_in_py2(filename):
        if not isinstance(filename, bytes):
            return filename.encode(_fs_encoding)
        return filename
else:
    def encode_filename_in_py2(filename):
        return filename
    basestring = str


def extended_iglob(pattern):
    if '{' in pattern:
        m = re.match('(.*){([^}]+)}(.*)', pattern)
        if m:
            before, switch, after = m.groups()
            for case in switch.split(','):
                for path in extended_iglob(before + case + after):
                    yield path
            return
    if '**/' in pattern:
        seen = set()
        first, rest = pattern.split('**/', 1)
        if first:
            first = iglob(first+'/')
        else:
            first = ['']
        for root in first:
            for path in extended_iglob(join_path(root, rest)):
                if path not in seen:
                    seen.add(path)
                    yield path
            for path in extended_iglob(join_path(root, '*', '**/' + rest)):
                if path not in seen:
                    seen.add(path)
                    yield path
    else:
        for path in iglob(pattern):
            yield path


def nonempty(it, error_msg="expected non-empty iterator"):
    empty = True
    for value in it:
        empty = False
        yield value
    if empty:
        raise ValueError(error_msg)


@cached_function
def file_hash(filename):
    path = os.path.normpath(filename.encode("UTF-8"))
    m = hashlib.md5(str(len(path)) + ":")
    m.update(path)
    f = open(filename, 'rb')
    try:
        data = f.read(65000)
        while data:
            m.update(data)
            data = f.read(65000)
    finally:
        f.close()
    return m.hexdigest()


def parse_list(s):
    """
    >>> parse_list("")
    []
    >>> parse_list("a")
    ['a']
    >>> parse_list("a b c")
    ['a', 'b', 'c']
    >>> parse_list("[a, b, c]")
    ['a', 'b', 'c']
    >>> parse_list('a " " b')
    ['a', ' ', 'b']
    >>> parse_list('[a, ",a", "a,", ",", ]')
    ['a', ',a', 'a,', ',']
    """
    if len(s) >= 2 and s[0] == '[' and s[-1] == ']':
        s = s[1:-1]
        delimiter = ','
    else:
        delimiter = ' '
    s, literals = strip_string_literals(s)
    def unquote(literal):
        literal = literal.strip()
        if literal[0] in "'\"":
            return literals[literal[1:-1]]
        else:
            return literal
    return [unquote(item) for item in s.split(delimiter) if item.strip()]


transitive_str = object()
transitive_list = object()

distutils_settings = {
    'name':                 str,
    'sources':              list,
    'define_macros':        list,
    'undef_macros':         list,
    'libraries':            transitive_list,
    'library_dirs':         transitive_list,
    'runtime_library_dirs': transitive_list,
    'include_dirs':         transitive_list,
    'extra_objects':        list,
    'extra_compile_args':   transitive_list,
    'extra_link_args':      transitive_list,
    'export_symbols':       list,
    'depends':              transitive_list,
    'language':             transitive_str,
}


@cython.locals(start=cython.Py_ssize_t, end=cython.Py_ssize_t)
def line_iter(source):
    if isinstance(source, basestring):
        start = 0
        while True:
            end = source.find('\n', start)
            if end == -1:
                yield source[start:]
                return
            yield source[start:end]
            start = end+1
    else:
        for line in source:
            yield line


class DistutilsInfo(object):

    def __init__(self, source=None, exn=None):
        self.values = {}
        if source is not None:
            for line in line_iter(source):
                line = line.strip()
                if line != '' and line[0] != '#':
                    break
                line = line[1:].strip()
                if line[:10] == 'distutils:':
                    line = line[10:]
                    ix = line.index('=')
                    key = str(line[:ix].strip())
                    value = line[ix+1:].strip()
                    type = distutils_settings[key]
                    if type in (list, transitive_list):
                        value = parse_list(value)
                        if key == 'define_macros':
                            value = [tuple(macro.split('=')) for macro in value]
                    self.values[key] = value
        elif exn is not None:
            for key in distutils_settings:
                if key in ('name', 'sources'):
                    continue
                value = getattr(exn, key, None)
                if value:
                    self.values[key] = value

    def merge(self, other):
        if other is None:
            return self
        for key, value in other.values.items():
            type = distutils_settings[key]
            if type is transitive_str and key not in self.values:
                self.values[key] = value
            elif type is transitive_list:
                if key in self.values:
                    # Change a *copy* of the list (Trac #845)
                    all = self.values[key][:]
                    for v in value:
                        if v not in all:
                            all.append(v)
                    value = all
                self.values[key] = value
        return self

    def subs(self, aliases):
        if aliases is None:
            return self
        resolved = DistutilsInfo()
        for key, value in self.values.items():
            type = distutils_settings[key]
            if type in [list, transitive_list]:
                new_value_list = []
                for v in value:
                    if v in aliases:
                        v = aliases[v]
                    if isinstance(v, list):
                        new_value_list += v
                    else:
                        new_value_list.append(v)
                value = new_value_list
            else:
                if value in aliases:
                    value = aliases[value]
            resolved.values[key] = value
        return resolved

    def apply(self, extension):
        for key, value in self.values.items():
            type = distutils_settings[key]
            if type in [list, transitive_list]:
                value = getattr(extension, key) + list(value)
            setattr(extension, key, value)


@cython.locals(start=cython.Py_ssize_t, q=cython.Py_ssize_t,
               single_q=cython.Py_ssize_t, double_q=cython.Py_ssize_t,
               hash_mark=cython.Py_ssize_t, end=cython.Py_ssize_t,
               k=cython.Py_ssize_t, counter=cython.Py_ssize_t, quote_len=cython.Py_ssize_t)
def strip_string_literals(code, prefix='__Pyx_L'):
    """
    Normalizes every string literal to be of the form '__Pyx_Lxxx',
    returning the normalized code and a mapping of labels to
    string literals.
    """
    new_code = []
    literals = {}
    counter = 0
    start = q = 0
    in_quote = False
    hash_mark = single_q = double_q = -1
    code_len = len(code)
    quote_type = quote_len = None

    while True:
        if hash_mark < q:
            hash_mark = code.find('#', q)
        if single_q < q:
            single_q = code.find("'", q)
        if double_q < q:
            double_q = code.find('"', q)
        q = min(single_q, double_q)
        if q == -1:
            q = max(single_q, double_q)

        # We're done.
        if q == -1 and hash_mark == -1:
            new_code.append(code[start:])
            break

        # Try to close the quote.
        elif in_quote:
            if code[q-1] == u'\\':
                k = 2
                while q >= k and code[q-k] == u'\\':
                    k += 1
                if k % 2 == 0:
                    q += 1
                    continue
            if code[q] == quote_type and (
                    quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])):
                counter += 1
                label = "%s%s_" % (prefix, counter)
                literals[label] = code[start+quote_len:q]
                full_quote = code[q:q+quote_len]
                new_code.append(full_quote)
                new_code.append(label)
                new_code.append(full_quote)
                q += quote_len
                in_quote = False
                start = q
            else:
                q += 1

        # Process comment.
        elif -1 != hash_mark and (hash_mark < q or q == -1):
            new_code.append(code[start:hash_mark+1])
            end = code.find('\n', hash_mark)
            counter += 1
            label = "%s%s_" % (prefix, counter)
            if end == -1:
                end_or_none = None
            else:
                end_or_none = end
            literals[label] = code[hash_mark+1:end_or_none]
            new_code.append(label)
            if end == -1:
                break
            start = q = end

        # Open the quote.
        else:
            if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]):
                quote_len = 3
            else:
                quote_len = 1
            in_quote = True
            quote_type = code[q]
            new_code.append(code[start:q])
            start = q
            q += quote_len

    return "".join(new_code), literals


dependency_regex = re.compile(r"(?:^from +([0-9a-zA-Z_.]+) +cimport)|"
                              r"(?:^cimport +([0-9a-zA-Z_.]+(?: *, *[0-9a-zA-Z_.]+)*))|"
                              r"(?:^cdef +extern +from +['\"]([^'\"]+)['\"])|"
                              r"(?:^include +['\"]([^'\"]+)['\"])", re.M)


def normalize_existing(base_path, rel_paths):
    return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths)))


@cached_function
def normalize_existing0(base_dir, rel_paths):
    normalized = []
    for rel in rel_paths:
        path = join_path(base_dir, rel)
        if path_exists(path):
            normalized.append(os.path.normpath(path))
        else:
            normalized.append(rel)
    return normalized


def resolve_depends(depends, include_dirs):
    include_dirs = tuple(include_dirs)
    resolved = []
    for depend in depends:
        path = resolve_depend(depend, include_dirs)
        if path is not None:
            resolved.append(path)
    return resolved


@cached_function
def resolve_depend(depend, include_dirs):
    if depend[0] == '<' and depend[-1] == '>':
        return None
    for dir in include_dirs:
        path = join_path(dir, depend)
        if path_exists(path):
            return os.path.normpath(path)
    return None


@cached_function
def package(filename):
    dir = os.path.dirname(os.path.abspath(str(filename)))
    if dir != filename and is_package_dir(dir):
        return package(dir) + (os.path.basename(dir),)
    else:
        return ()


@cached_function
def fully_qualified_name(filename):
    module = os.path.splitext(os.path.basename(filename))[0]
    return '.'.join(package(filename) + (module,))


@cached_function
def parse_dependencies(source_filename):
    # Actual parsing is way too slow, so we use regular expressions.
    # The only catch is that we must strip comments and string
    # literals ahead of time.
    fh = Utils.open_source_file(source_filename, error_handling='ignore')
    try:
        source = fh.read()
    finally:
        fh.close()
    distutils_info = DistutilsInfo(source)
    source, literals = strip_string_literals(source)
    source = source.replace('\\\n', ' ').replace('\t', ' ')

    # TODO: pure mode
    cimports = []
    includes = []
    externs  = []
    for m in dependency_regex.finditer(source):
        cimport_from, cimport_list, extern, include = m.groups()
        if cimport_from:
            cimports.append(cimport_from)
        elif cimport_list:
            cimports.extend(x.strip() for x in cimport_list.split(","))
        elif extern:
            externs.append(literals[extern])
        else:
            includes.append(literals[include])
    return cimports, includes, externs, distutils_info


class DependencyTree(object):

    def __init__(self, context, quiet=False):
        self.context = context
        self.quiet = quiet
        self._transitive_cache = {}

    def parse_dependencies(self, source_filename):
        if path_exists(source_filename):
            source_filename = os.path.normpath(source_filename)
        return parse_dependencies(source_filename)

    @cached_method
    def included_files(self, filename):
        # This is messy because included files are textually included, resolving
        # cimports (but not includes) relative to the including file.
        all = set()
        for include in self.parse_dependencies(filename)[1]:
            include_path = join_path(os.path.dirname(filename), include)
            if not path_exists(include_path):
                include_path = self.context.find_include_file(include, None)
            if include_path:
                if '.' + os.path.sep in include_path:
                    include_path = os.path.normpath(include_path)
                all.add(include_path)
                all.update(self.included_files(include_path))
            elif not self.quiet:
                print("Unable to locate '%s' referenced from '%s'" % (filename, include))
        return all

    @cached_method
    def cimports_and_externs(self, filename):
        # This is really ugly. Nested cimports are resolved with respect to the
        # includer, but includes are resolved with respect to the includee.
        cimports, includes, externs = self.parse_dependencies(filename)[:3]
        cimports = set(cimports)
        externs = set(externs)
        for include in self.included_files(filename):
            included_cimports, included_externs = self.cimports_and_externs(include)
            cimports.update(included_cimports)
            externs.update(included_externs)
        return tuple(cimports), normalize_existing(filename, externs)

    def cimports(self, filename):
        return self.cimports_and_externs(filename)[0]

    def package(self, filename):
        return package(filename)

    def fully_qualified_name(self, filename):
        return fully_qualified_name(filename)

    @cached_method
    def find_pxd(self, module, filename=None):
        is_relative = module[0] == '.'
        if is_relative and not filename:
            raise NotImplementedError("New relative imports.")
        if filename is not None:
            module_path = module.split('.')
            if is_relative:
                module_path.pop(0)  # just explicitly relative
            package_path = list(self.package(filename))
            while module_path and not module_path[0]:
                try:
                    package_path.pop()
                except IndexError:
                    return None   # FIXME: error?
                module_path.pop(0)
            relative = '.'.join(package_path + module_path)
            pxd = self.context.find_pxd_file(relative, None)
            if pxd:
                return pxd
        if is_relative:
            return None   # FIXME: error?
        return self.context.find_pxd_file(module, None)

    @cached_method
    def cimported_files(self, filename):
        if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'):
            pxd_list = [filename[:-4] + '.pxd']
        else:
            pxd_list = []
        for module in self.cimports(filename):
            if module[:7] == 'cython.' or module == 'cython':
                continue
            pxd_file = self.find_pxd(module, filename)
            if pxd_file is not None:
                pxd_list.append(pxd_file)
            elif not self.quiet:
                print("missing cimport in module '%s': %s" % (module, filename))
        return tuple(pxd_list)

    @cached_method
    def immediate_dependencies(self, filename):
        all = set([filename])
        all.update(self.cimported_files(filename))
        all.update(self.included_files(filename))
        return all

    def all_dependencies(self, filename):
        return self.transitive_merge(filename, self.immediate_dependencies, set.union)

    @cached_method
    def timestamp(self, filename):
        return os.path.getmtime(filename)

    def extract_timestamp(self, filename):
        return self.timestamp(filename), filename

    def newest_dependency(self, filename):
        return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)])

    def transitive_fingerprint(self, filename, extra=None):
        try:
            m = hashlib.md5(__version__)
            m.update(file_hash(filename))
            for x in sorted(self.all_dependencies(filename)):
                if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'):
                    m.update(file_hash(x))
            if extra is not None:
                m.update(str(extra))
            return m.hexdigest()
        except IOError:
            return None

    def distutils_info0(self, filename):
        info = self.parse_dependencies(filename)[3]
        externs = self.cimports_and_externs(filename)[1]
        if externs:
            if 'depends' in info.values:
                info.values['depends'] = list(set(info.values['depends']).union(externs))
            else:
                info.values['depends'] = list(externs)
        return info

    def distutils_info(self, filename, aliases=None, base=None):
        return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge)
            .subs(aliases)
            .merge(base))

    def transitive_merge(self, node, extract, merge):
        try:
            seen = self._transitive_cache[extract, merge]
        except KeyError:
            seen = self._transitive_cache[extract, merge] = {}
        return self.transitive_merge_helper(
            node, extract, merge, seen, {}, self.cimported_files)[0]

    def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing):
        if node in seen:
            return seen[node], None
        deps = extract(node)
        if node in stack:
            return deps, node
        try:
            stack[node] = len(stack)
            loop = None
            for next in outgoing(node):
                sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing)
                if sub_loop is not None:
                    if loop is not None and stack[loop] < stack[sub_loop]:
                        pass
                    else:
                        loop = sub_loop
                deps = merge(deps, sub_deps)
            if loop == node:
                loop = None
            if loop is None:
                seen[node] = deps
            return deps, loop
        finally:
            del stack[node]


_dep_tree = None

def create_dependency_tree(ctx=None, quiet=False):
    global _dep_tree
    if _dep_tree is None:
        if ctx is None:
            ctx = Context(["."], CompilationOptions(default_options))
        _dep_tree = DependencyTree(ctx, quiet=quiet)
    return _dep_tree


# This may be useful for advanced users?
def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None,
                          exclude_failures=False):
    if language is not None:
        print('Please put "# distutils: langauge=%s" in your .pyx or .pxd file(s)' % language)
    if exclude is None:
        exclude = []
    if not isinstance(patterns, (list, tuple)):
        patterns = [patterns]
    explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)])
    seen = set()
    deps = create_dependency_tree(ctx, quiet=quiet)
    to_exclude = set()
    if not isinstance(exclude, list):
        exclude = [exclude]
    for pattern in exclude:
        to_exclude.update(map(os.path.abspath, extended_iglob(pattern)))

    module_list = []
    module_metadata = {}

    # workaround for setuptools
    if 'setuptools' in sys.modules:
        Extension_distutils = sys.modules['setuptools.extension']._Extension
        Extension_setuptools = sys.modules['setuptools'].Extension
    else:
        # dummy class, in case we do not have setuptools
        Extension_distutils = Extension
        class Extension_setuptools(Extension): pass

    for pattern in patterns:
        if isinstance(pattern, str):
            filepattern = pattern
            template = None
            name = '*'
            base = None
            exn_type = Extension
            ext_language = language
        elif isinstance(pattern, (Extension_distutils, Extension_setuptools)):
            for filepattern in pattern.sources:
                if os.path.splitext(filepattern)[1] in ('.py', '.pyx'):
                    break
            else:
                # ignore non-cython modules
                module_list.append(pattern)
                continue
            template = pattern
            name = template.name
            base = DistutilsInfo(exn=template)
            exn_type = template.__class__
            ext_language = None  # do not override whatever the Extension says
        else:
            msg = str("pattern is not of type str nor subclass of Extension (%s)"
                      " but of type %s and class %s" % (repr(Extension),
                                                        type(pattern),
                                                        pattern.__class__))
            raise TypeError(msg)

        for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern):
            if os.path.abspath(file) in to_exclude:
                continue
            pkg = deps.package(file)
            if '*' in name:
                module_name = deps.fully_qualified_name(file)
                if module_name in explicit_modules:
                    continue
            else:
                module_name = name

            if module_name not in seen:
                try:
                    kwds = deps.distutils_info(file, aliases, base).values
                except Exception:
                    if exclude_failures:
                        continue
                    raise
                if base is not None:
                    for key, value in base.values.items():
                        if key not in kwds:
                            kwds[key] = value

                sources = [file]
                if template is not None:
                    sources += [m for m in template.sources if m != filepattern]
                if 'sources' in kwds:
                    # allow users to add .c files etc.
                    for source in kwds['sources']:
                        source = encode_filename_in_py2(source)
                        if source not in sources:
                            sources.append(source)
                    extra_sources = kwds['sources']
                    del kwds['sources']
                else:
                    extra_sources = None
                if 'depends' in kwds:
                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + ["."])
                    if template is not None:
                        # Always include everything from the template.
                        depends = set(template.depends).union(depends)
                    # Sort depends to make the metadata dump in the
                    # Cython-generated C code predictable.
                    kwds['depends'] = sorted(depends)

                if ext_language and 'language' not in kwds:
                    kwds['language'] = ext_language

                module_list.append(exn_type(
                        name=module_name,
                        sources=sources,
                        **kwds))
                if extra_sources:
                    kwds['sources'] = extra_sources
                module_metadata[module_name] = {'distutils': kwds}
                m = module_list[-1]
                seen.add(name)
    return module_list, module_metadata


# This is the user-exposed entry point.
def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None,
              exclude_failures=False, **options):
    """
    Compile a set of source modules into C/C++ files and return a list of distutils
    Extension objects for them.

    As module list, pass either a glob pattern, a list of glob patterns or a list of
    Extension objects.  The latter allows you to configure the extensions separately
    through the normal distutils options.

    When using glob patterns, you can exclude certain module names explicitly
    by passing them into the 'exclude' option.

    To globally enable C++ mode, you can pass language='c++'.  Otherwise, this
    will be determined at a per-file level based on compiler directives.  This
    affects only modules found based on file names.  Extension instances passed
    into cythonize() will not be changed.

    For parallel compilation, set the 'nthreads' option to the number of
    concurrent builds.

    For a broad 'try to compile' mode that ignores compilation failures and
    simply excludes the failed extensions, pass 'exclude_failures=True'. Note
    that this only really makes sense for compiling .py files which can also
    be used without compilation.

    Additional compilation options can be passed as keyword arguments.
    """
    if exclude is None:
        exclude = []
    if 'include_path' not in options:
        options['include_path'] = ['.']
    if 'common_utility_include_dir' in options:
        if options.get('cache'):
            raise NotImplementedError("common_utility_include_dir does not yet work with caching")
        safe_makedirs(options['common_utility_include_dir'])
    c_options = CompilationOptions(**options)
    cpp_options = CompilationOptions(**options); cpp_options.cplus = True
    ctx = c_options.create_context()
    options = c_options
    module_list, module_metadata = create_extension_list(
        module_list,
        exclude=exclude,
        ctx=ctx,
        quiet=quiet,
        exclude_failures=exclude_failures,
        language=language,
        aliases=aliases)
    deps = create_dependency_tree(ctx, quiet=quiet)
    build_dir = getattr(options, 'build_dir', None)

    modules_by_cfile = {}
    to_compile = []
    for m in module_list:
        if build_dir:
            root = os.getcwd()  # distutil extension depends are relative to cwd
            def copy_to_build_dir(filepath, root=root):
                filepath_abs = os.path.abspath(filepath)
                if os.path.isabs(filepath):
                    filepath = filepath_abs
                if filepath_abs.startswith(root):
                    mod_dir = join_path(build_dir,
                            os.path.dirname(_relpath(filepath, root)))
                    copy_once_if_newer(filepath_abs, mod_dir)
            for dep in m.depends:
                copy_to_build_dir(dep)

        new_sources = []
        for source in m.sources:
            base, ext = os.path.splitext(source)
            if ext in ('.pyx', '.py'):
                if m.language == 'c++':
                    c_file = base + '.cpp'
                    options = cpp_options
                else:
                    c_file = base + '.c'
                    options = c_options

                # setup for out of place build directory if enabled
                if build_dir:
                    c_file = os.path.join(build_dir, c_file)
                    dir = os.path.dirname(c_file)
                    safe_makedirs_once(dir)

                if os.path.exists(c_file):
                    c_timestamp = os.path.getmtime(c_file)
                else:
                    c_timestamp = -1

                # Priority goes first to modified files, second to direct
                # dependents, and finally to indirect dependents.
                if c_timestamp < deps.timestamp(source):
                    dep_timestamp, dep = deps.timestamp(source), source
                    priority = 0
                else:
                    dep_timestamp, dep = deps.newest_dependency(source)
                    priority = 2 - (dep in deps.immediate_dependencies(source))
                if force or c_timestamp < dep_timestamp:
                    if not quiet:
                        if source == dep:
                            print("Compiling %s because it changed." % source)
                        else:
                            print("Compiling %s because it depends on %s." % (source, dep))
                    if not force and hasattr(options, 'cache'):
                        extra = m.language
                        fingerprint = deps.transitive_fingerprint(source, extra)
                    else:
                        fingerprint = None
                    to_compile.append((priority, source, c_file, fingerprint, quiet,
                                       options, not exclude_failures, module_metadata.get(m.name)))
                new_sources.append(c_file)
                if c_file not in modules_by_cfile:
                    modules_by_cfile[c_file] = [m]
                else:
                    modules_by_cfile[c_file].append(m)
            else:
                new_sources.append(source)
                if build_dir:
                    copy_to_build_dir(source)
        m.sources = new_sources

    if hasattr(options, 'cache'):
        if not os.path.exists(options.cache):
            os.makedirs(options.cache)
    to_compile.sort()
    # Drop "priority" component of "to_compile" entries and add a
    # simple progress indicator.
    N = len(to_compile)
    progress_fmt = "[{0:%d}/{1}] " % len(str(N))
    for i in range(N):
        progress = progress_fmt.format(i+1, N)
        to_compile[i] = to_compile[i][1:] + (progress,)

    if N <= 1:
        nthreads = 0
    if nthreads:
        # Requires multiprocessing (or Python >= 2.6)
        try:
            import multiprocessing
            pool = multiprocessing.Pool(
                nthreads, initializer=_init_multiprocessing_helper)
        except (ImportError, OSError):
            print("multiprocessing required for parallel cythonization")
            nthreads = 0
        else:
            # This is a bit more involved than it should be, because KeyboardInterrupts
            # break the multiprocessing workers when using a normal pool.map().
            # See, for example:
            # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt
            try:
                result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1)
                pool.close()
                while not result.ready():
                    try:
                        result.get(99999)  # seconds
                    except multiprocessing.TimeoutError:
                        pass
            except KeyboardInterrupt:
                pool.terminate()
                raise
            pool.join()
    if not nthreads:
        for args in to_compile:
            cythonize_one(*args)

    if exclude_failures:
        failed_modules = set()
        for c_file, modules in modules_by_cfile.items():
            if not os.path.exists(c_file):
                failed_modules.update(modules)
            elif os.path.getsize(c_file) < 200:
                f = io_open(c_file, 'r', encoding='iso8859-1')
                try:
                    if f.read(len('#error ')) == '#error ':
                        # dead compilation result
                        failed_modules.update(modules)
                finally:
                    f.close()
        if failed_modules:
            for module in failed_modules:
                module_list.remove(module)
            print("Failed compilations: %s" % ', '.join(sorted([
                module.name for module in failed_modules])))

    if hasattr(options, 'cache'):
        cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
    # cythonize() is often followed by the (non-Python-buffered)
    # compiler output, flush now to avoid interleaving output.
    sys.stdout.flush()
    return module_list


if os.environ.get('XML_RESULTS'):
    compile_result_dir = os.environ['XML_RESULTS']
    def record_results(func):
        def with_record(*args):
            t = time.time()
            success = True
            try:
                try:
                    func(*args)
                except:
                    success = False
            finally:
                t = time.time() - t
                module = fully_qualified_name(args[0])
                name = "cythonize." + module
                failures = 1 - success
                if success:
                    failure_item = ""
                else:
                    failure_item = "failure"
                output = open(os.path.join(compile_result_dir, name + ".xml"), "w")
                output.write("""
                    <?xml version="1.0" ?>
                    <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s">
                    <testcase classname="%(name)s" name="cythonize">
                    %(failure_item)s
                    </testcase>
                    </testsuite>
                """.strip() % locals())
                output.close()
        return with_record
else:
    def record_results(func):
        return func


# TODO: Share context? Issue: pyx processing leaks into pxd module
@record_results
def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True, embedded_metadata=None, progress=""):
    from ..Compiler.Main import compile, default_options
    from ..Compiler.Errors import CompileError, PyrexError

    if fingerprint:
        if not os.path.exists(options.cache):
            try:
                os.mkdir(options.cache)
            except:
                if not os.path.exists(options.cache):
                    raise
        # Cython-generated c files are highly compressible.
        # (E.g. a compression ratio of about 10 for Sage).
        fingerprint_file = join_path(
            options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
        if os.path.exists(fingerprint_file):
            if not quiet:
                print("%sFound compiled %s in cache" % (progress, pyx_file))
            os.utime(fingerprint_file, None)
            g = gzip_open(fingerprint_file, 'rb')
            try:
                f = open(c_file, 'wb')
                try:
                    shutil.copyfileobj(g, f)
                finally:
                    f.close()
            finally:
                g.close()
            return
    if not quiet:
        print("%sCythonizing %s" % (progress, pyx_file))
    if options is None:
        options = CompilationOptions(default_options)
    options.output_file = c_file
    options.embedded_metadata = embedded_metadata

    any_failures = 0
    try:
        result = compile([pyx_file], options)
        if result.num_errors > 0:
            any_failures = 1
    except (EnvironmentError, PyrexError) as e:
        sys.stderr.write('%s\n' % e)
        any_failures = 1
        # XXX
        import traceback
        traceback.print_exc()
    except Exception:
        if raise_on_failure:
            raise
        import traceback
        traceback.print_exc()
        any_failures = 1
    if any_failures:
        if raise_on_failure:
            raise CompileError(None, pyx_file)
        elif os.path.exists(c_file):
            os.remove(c_file)
    elif fingerprint:
        f = open(c_file, 'rb')
        try:
            g = gzip_open(fingerprint_file, 'wb')
            try:
                shutil.copyfileobj(f, g)
            finally:
                g.close()
        finally:
            f.close()


def cythonize_one_helper(m):
    import traceback
    try:
        return cythonize_one(*m)
    except Exception:
        traceback.print_exc()
        raise


def _init_multiprocessing_helper():
    # KeyboardInterrupt kills workers, so don't let them get it
    import signal
    signal.signal(signal.SIGINT, signal.SIG_IGN)


def cleanup_cache(cache, target_size, ratio=.85):
    try:
        p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE)
        res = p.wait()
        if res == 0:
            total_size = 1024 * int(p.stdout.read().strip().split()[0])
            if total_size < target_size:
                return
    except (OSError, ValueError):
        pass
    total_size = 0
    all = []
    for file in os.listdir(cache):
        path = join_path(cache, file)
        s = os.stat(path)
        total_size += s.st_size
        all.append((s.st_atime, s.st_size, path))
    if total_size > target_size:
        for time, size, file in reversed(sorted(all)):
            os.unlink(file)
            total_size -= size
            if total_size < target_size * ratio:
                break