cystdlib.py 4.18 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
"""
Highly experimental script that compiles the CPython standard library using Cython.

Execute the script either in the CPython 'Lib' directory or pass the
option '--current-python' to compile the standard library of the running
Python interpreter.

Pass '--parallel' to get a parallel build.

Usage example::

    $ python cystdlib.py --current-python build_ext -i
"""

import sys
16 17 18 19 20 21 22 23
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options

# improve Python compatibility by allowing some broken code
Options.error_on_unknown_names = False

excludes = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
24 25 26 27 28
broken = [
    'idlelib/MultiCall.py',
    'email/utils.py',
    'multiprocessing/reduction.py',
    'multiprocessing/util.py',
29
    'threading.py',      # interrupt handling
30 31
    'lib2to3/fixes/fix_sys_exc.py',
    'traceback.py',
32
]
33

34
default_directives = dict(
35
    auto_cpdef=False,   # enable when it's safe, see long list of failures below
36
    binding=True,
37 38
    set_initial_path='SOURCEFILE')

39
special_directives = [
40
    (['pkgutil.py',
41
      'decimal.py',
42 43 44 45 46 47
      'datetime.py',
      'optparse.py',
      'sndhdr.py',
      'opcode.py',
      'ntpath.py',
      'urllib/request.py',
48
      'plat-*/TYPES.py',
49
      'plat-*/IN.py',
50
      'tkinter/_fix.py',
51
      'lib2to3/refactor.py',
52
      'webbrowser.py',
53
      'shutil.py',
54
      'multiprocessing/forking.py',
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
      'xml/sax/expatreader.py',
      'xmlrpc/client.py',
      'pydoc.py',
      'xml/etree/ElementTree.py',
      'posixpath.py',
      'inspect.py',
      'ctypes/util.py',
      'urllib/parse.py',
      'warnings.py',
      'tempfile.py',
      'trace.py',
      'heapq.py',
      'pickletools.py',
      'multiprocessing/connection.py',
      'hashlib.py',
      'getopt.py',
71 72
      'os.py',
      'types.py',
73
     ], dict(auto_cpdef=False)),
74
]
75
del special_directives[:]  # currently unused
76

77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
def build_extensions(includes='**/*.py',
                     excludes=excludes+broken,
                     special_directives=special_directives,
                     parallel=None):
    if isinstance(includes, str):
        includes = [includes]

    all_groups = (special_directives or []) + [(includes, {})]
    extensions = []
    for modules, directives in all_groups:
        exclude_now = excludes[:]
        for other_modules, _ in special_directives:
            if other_modules != modules:
                exclude_now.extend(other_modules)

        d = dict(default_directives)
        d.update(directives)

        extensions.extend(
            cythonize(modules,
                exclude=exclude_now,
                exclude_failures=True,
                language_level=pyver,
                compiler_directives=d,
                nthreads=parallel,
                ))
    return extensions

def build(extensions):
    try:
        setup(name = 'stuff', ext_modules = extensions)
        return extensions, True
    except:
        import traceback
111
        print('error building extensions %s' % ([ext.name for ext in extensions],))
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
        traceback.print_exc()
        return extensions, False

def _build(args):
    sys_args, ext = args
    sys.argv[1:] = sys_args
    return build([ext])


if __name__ == '__main__':
    import sys
    pyver = sys.version_info[0]
    try:
        sys.argv.remove('--current-python')
    except ValueError:
        pass
    else:
        # assume that the stdlib is where the "os" module lives
        import os
        os.chdir(os.path.dirname(os.__file__))

    try:
        sys.argv.remove('--parallel')
        import multiprocessing
        parallel_compiles = multiprocessing.cpu_count() * 2
        print("Building in %d parallel processes" % parallel_compiles)
    except (ValueError, ImportError):
        parallel_compiles = None
140

141 142 143 144 145 146 147 148 149
    extensions = build_extensions(parallel=parallel_compiles)
    if parallel_compiles:
        pool = multiprocessing.Pool(parallel_compiles)
        sys_args = sys.argv[1:]
        results = pool.map(_build, [ (sys_args, ext) for ext in extensions ])
        pool.close()
        pool.join()
        for ext, result in results:
            if not result:
150
                print("building extension %s failed" % (ext[0].name,))
151 152
    else:
        build(extensions)