Add documentation to new build tools.

Clean up argument parsing.

Makefile

@@ -117,6 +117,7 @@ clean:
 	rm build/*
 	rm src/*.bc
 	make -C packages clean
+	make -C six clean
 	echo "The Emsdk and CPython are not cleaned. cd into those directories to do so."
 
 

packages/Makefile

@@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..)
 include ../Makefile.envs
 
 all:
-	../tools/buildall . --output=../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
+	../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
 
 clean:
 	rm -rf */build

six/Makefile

@@ -14,7 +14,6 @@ URL=https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c
 all: $(BUILD)/__init__.py
 
 
-
 clean:
 	-rm -fr downloads
 	-rm -fr $(SRC)

six/README.md

+Six is a special case package, since it's so commonly used, we want to include
+it in the main Python package.

tools/buildall

 #!/usr/bin/env python3
 
+"""
+Build all of the packages in a given directory.
+"""
+
 import argparse
 import json
 import os
@@ -10,9 +14,11 @@ import common
 import buildpkg
 
 
-def build_package(pkgname, reqs, dependencies, packagesdir, args):
+def build_package(pkgname, dependencies, packagesdir, args):
+    reqs = dependencies[pkgname]
+    # Make sure all of the package's requirements are built first
     for req in reqs:
-        build_package(req, dependencies[req], dependencies, packagesdir, args)
+        build_package(req, dependencies, packagesdir, args)
     if not os.path.isfile(
             os.path.join(packagesdir, pkgname, 'build', '.packaged')):
         print("BUILDING PACKAGE: " + pkgname)
@@ -27,8 +33,8 @@ def build_package(pkgname, reqs, dependencies, packagesdir, args):
 
 
 def build_packages(packagesdir, args):
-    # We have to build the packages in order, so first we build a dependency
-    # tree
+    # We have to build the packages in the correct order (dependencies first),
+    # so first load in all of the package metadata and build a dependency map.
     dependencies = {}
     for pkgdir in os.listdir(packagesdir):
         pkgdir = os.path.join(packagesdir, pkgdir)
@@ -39,8 +45,8 @@ def build_packages(packagesdir, args):
             reqs = pkg.get('requirements', {}).get('run', [])
             dependencies[name] = reqs
 
-    for pkgname, reqs in dependencies.items():
-        build_package(pkgname, reqs, dependencies, packagesdir, args)
+    for pkgname in dependencies.keys():
+        build_package(pkgname, dependencies, packagesdir, args)
 
     # This is done last so the main Makefile can use it as a completion token
     with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd:
@@ -48,17 +54,26 @@ def build_packages(packagesdir, args):
 
 
 def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('dir', type=str, nargs=1)
-    parser.add_argument('--cflags', type=str, nargs=1, default=[''])
+    parser = argparse.ArgumentParser(
+        "Build all of the packages in a given directory")
+    parser.add_argument(
+        'dir', type=str, nargs=1,
+        help='Input directory containing a tree of package definitions')
+    parser.add_argument(
+        'output', type=str, nargs=1,
+        help='Output directory in which to put all built packages')
     parser.add_argument(
-        '--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD])
+        '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
+        help='Extra compiling flags')
     parser.add_argument(
-        '--host', type=str, nargs=1, default=[common.HOSTPYTHON])
+        '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
+        help='Extra linking flags')
     parser.add_argument(
-        '--target', type=str, nargs=1, default=[common.TARGETPYTHON])
+        '--host', type=str, nargs='?', default=[common.HOSTPYTHON],
+        help='The path to the host Python installation')
     parser.add_argument(
-        '--output', '-o', type=str, nargs=1)
+        '--target', type=str, nargs='?', default=[common.TARGETPYTHON],
+        help='The path to the target Python installation')
     return parser.parse_args()
 
 

tools/buildpkg.py

 #!/usr/bin/env python3
 
+"""
+Builds a Pyodide package.
+"""
+
 import argparse
 import hashlib
 import os
@@ -13,7 +17,13 @@ import common
 ROOTDIR = os.path.abspath(os.path.dirname(__file__))
 
 
-def do_checksum(path, checksum):
+def check_checksum(path, pkg):
+    """
+    Checks that a tarball matches the checksum in the package metadata.
+    """
+    if 'md5' not in pkg['source']:
+        return
+    checksum = pkg['source']['md5']
     CHUNK_SIZE = 1 << 16
     h = hashlib.md5()
     with open(path, 'rb') as fd:
@@ -33,7 +43,7 @@ def download_and_extract(buildpath, packagedir, pkg, args):
         subprocess.run([
             'wget', '-q', '-O', tarballpath, pkg['source']['url']
         ], check=True)
-        do_checksum(tarballpath, pkg['source']['md5'])
+        check_checksum(tarballpath, pkg)
     srcpath = os.path.join(buildpath, packagedir)
     if not os.path.isdir(srcpath):
         shutil.unpack_archive(tarballpath, buildpath)
@@ -44,6 +54,7 @@ def patch(path, srcpath, pkg, args):
     if os.path.isfile(os.path.join(srcpath, '.patched')):
         return
 
+    # Apply all of the patches
     orig_dir = os.getcwd()
     pkgdir = os.path.abspath(os.path.dirname(path))
     os.chdir(srcpath)
@@ -55,6 +66,7 @@ def patch(path, srcpath, pkg, args):
     finally:
         os.chdir(orig_dir)
 
+    # Add any extra files
     for src, dst in pkg['source'].get('extras', []):
         shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst))
 
@@ -63,8 +75,10 @@ def patch(path, srcpath, pkg, args):
 
 
 def get_libdir(srcpath, args):
+    # Get the name of the build/lib.XXX directory that distutils wrote its
+    # output to
     slug = subprocess.check_output([
-        os.path.join(args.host[0], 'bin', 'python3'),
+        os.path.join(args.host, 'bin', 'python3'),
         '-c',
         'import sysconfig, sys; '
         'print("{}-{}.{}".format('
@@ -87,15 +101,16 @@ def compile(path, srcpath, pkg, args):
     os.chdir(srcpath)
     try:
         subprocess.run([
+            os.path.join(args.host, 'bin', 'python3'),
             os.path.join(ROOTDIR, 'pywasmcross'),
             '--cflags',
-            args.cflags[0] + ' ' +
+            args.cflags + ' ' +
             pkg.get('build', {}).get('cflags', ''),
             '--ldflags',
-            args.ldflags[0] + ' ' +
+            args.ldflags + ' ' +
             pkg.get('build', {}).get('ldflags', ''),
-            '--host', args.host[0],
-            '--target', args.target[0]], check=True)
+            '--host', args.host,
+            '--target', args.target], check=True)
     finally:
         os.chdir(orig_dir)
 
@@ -159,16 +174,22 @@ def build_package(path, args):
 
 
 def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('package', type=str, nargs=1)
+    parser = argparse.ArgumentParser('Build a pyodide package.')
+    parser.add_argument(
+        'package', type=str, nargs=1,
+        help="Path to meta.yaml package description")
     parser.add_argument(
-        '--cflags', type=str, nargs=1, default=[''])
+        '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
+        help='Extra compiling flags')
     parser.add_argument(
-        '--ldflags', type=str, nargs=1, default=[common.DEFAULT_LD])
+        '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
+        help='Extra linking flags')
     parser.add_argument(
-        '--host', type=str, nargs=1, default=[common.HOSTPYTHON])
+        '--host', type=str, nargs='?', default=[common.HOSTPYTHON],
+        help='The path to the host Python installation')
     parser.add_argument(
-        '--target', type=str, nargs=1, default=[common.TARGETPYTHON])
+        '--target', type=str, nargs='?', default=[common.TARGETPYTHON],
+        help='The path to the target Python installation')
     return parser.parse_args()
 
 

tools/common.py

@@ -7,7 +7,8 @@ HOSTPYTHON = os.path.abspath(
     os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host'))
 TARGETPYTHON = os.path.abspath(
     os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4'))
-DEFAULT_LD = ' '.join([
+DEFAULTCFLAGS = ''
+DEFAULTLDFLAGS = ' '.join([
     '-O3',
     '-s', "BINARYEN_METHOD='native-wasm'",
     '-Werror',

tools/pywasmcross

 #!/usr/bin/env python3
 
+"""Helper for cross-compiling distutils-based Python extensions.
+
+distutils has never had a proper cross-compilation story. This is a hack, which
+miraculously works, to get around that.
+
+The gist is:
+
+- Compile the package natively, replacing calls to the compiler and linker with
+  wrappers that store the arguments in a log, and then delegate along to the
+  real native compiler and linker.
+
+- Remove all of the native build products.
+
+- Play back the log, replacing the native compiler with emscripten and
+  adjusting include paths and flags as necessary for cross-compiling to
+  emscripten. This overwrites the results from the original native compilation.
+
+While this results in more work than strictly necessary (it builds a native
+version of the package, even though we then throw it away), it seems to be the
+only reliable way to automatically build a package that interleaves
+configuration with build.
+"""
+
+
 import argparse
 import importlib.machinery
 import json
@@ -9,26 +33,44 @@ import subprocess
 import sys
 
 
+import common
+
+
 ROOTDIR = os.path.abspath(os.path.dirname(__file__))
 symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
 
 
 def collect_args(basename):
+    """
+    This is called when this script is called through a symlink that looks like
+    a compiler or linker.
+
+    It writes the arguments to the build.log, and then delegates to the real
+    native compiler or linker.
+    """
+    # Remove the symlink compiler from the PATH, so we can delegate to the
+    # native compiler
     env = dict(os.environ)
     path = env['PATH']
     while ROOTDIR + ':' in path:
         path = path.replace(ROOTDIR + ':', '')
-
     env['PATH'] = path
+
     with open('build.log', 'a') as fd:
         json.dump([basename] + sys.argv[1:], fd)
         fd.write('\n')
 
-    sys.exit(subprocess.run([basename] + sys.argv[1:],
-                            env=env).returncode)
+    sys.exit(
+        subprocess.run(
+            [basename] + sys.argv[1:],
+            env=env).returncode)
 
 
 def make_symlinks(env):
+    """
+    Makes sure all of the symlinks that make this script look like a compiler
+    exist.
+    """
     exec_path = os.path.abspath(__file__)
     for symlink in symlinks:
         symlink_path = os.path.join(ROOTDIR, symlink)
@@ -47,9 +89,9 @@ def capture_compile(args):
     env['PATH'] = ROOTDIR + ':' + os.environ['PATH']
 
     result = subprocess.run(
-            [os.path.join(args.host[0], 'bin', 'python3'),
-             'setup.py',
-             'install'], env=env)
+        [os.path.join(args.host[0], 'bin', 'python3'),
+         'setup.py',
+         'install'], env=env)
     if result.returncode != 0:
         if os.path.exists('build.log'):
             os.remove('build.log')
@@ -57,7 +99,8 @@ def capture_compile(args):
 
 
 def handle_command(line, args):
-    # This is a special case to skip the compilation tests in numpy
+    # This is a special case to skip the compilation tests in numpy that aren't
+    # actually part of the build
     for arg in line:
         if r'/file.c' in arg or '_configtest' in arg:
             return
@@ -65,37 +108,35 @@ def handle_command(line, args):
             return
 
     if line[0] == 'ar':
-        line[0] = 'emar'
+        new_args = ['emar']
     elif line[0] == 'c++':
-        line[0] = 'em++'
+        new_args = ['em++']
     else:
-        line[0] = 'emcc'
+        new_args = ['emcc']
         # distutils doesn't use the c++ compiler when compiling c++ <sigh>
-        for arg in line:
-            if arg.endswith('.cpp'):
-                line[0] = 'em++'
-                break
+        if any(arg.endswith('.cpp') for arg in line):
+            new_args = ['em++']
     shared = '-shared' in line
 
-    new_args = [line[0]]
     if shared:
         new_args.extend(args.ldflags[0].split())
-    elif line[0] in ('emcc', 'em++'):
+    elif new_args[0] in ('emcc', 'em++'):
         new_args.extend(args.cflags[0].split())
 
-    skip_next = False
+    # Go through and adjust arguments
     for arg in line[1:]:
-        if skip_next:
-            skip_next = False
-            continue
         if arg.startswith('-I'):
+            # Don't include any system directories
+            if arg[2:].startswith('/usr'):
+                continue
             if (os.path.abspath(arg[2:]).startswith(args.host[0]) and
                     'site-packages' not in arg):
                 arg = arg.replace('-I' + args.host[0], '-I' + args.target[0])
-            if arg[2:].startswith('/usr'):
-                continue
+        # Don't include any system directories
         if arg.startswith('-L/usr'):
             continue
+        # The native build is possibly multithreaded, but the emscripten one
+        # definitely isn't
         arg = re.sub(r'/python([0-9]\.[0-9]+)m', r'/python\1', arg)
         if arg.endswith('.o'):
             arg = arg[:-2] + '.bc'
@@ -110,6 +151,7 @@ def handle_command(line, args):
     if result.returncode != 0:
         sys.exit(result.returncode)
 
+    # Emscripten .so files shouldn't have the native platform slug
     if shared:
         renamed = output[:-5] + '.so'
         for ext in importlib.machinery.EXTENSION_SUFFIXES:
@@ -122,6 +164,8 @@ def handle_command(line, args):
 
 
 def replay_compile(args):
+    # If pure Python, there will be no build.log file, which is fine -- just do
+    # nothing
     if os.path.isfile('build.log'):
         with open('build.log', 'r') as fd:
             for line in fd:
@@ -146,11 +190,21 @@ def build_wrap(args):
 
 
 def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--cflags', type=str, nargs=1, default=[''])
-    parser.add_argument('--ldflags', type=str, nargs=1, default=[''])
-    parser.add_argument('--host', type=str, nargs=1)
-    parser.add_argument('--target', type=str, nargs=1)
+    parser = argparse.ArgumentParser(
+        'Cross compile a Python distutils package. '
+        'Run from the root directory of the package\'s source')
+    parser.add_argument(
+        '--cflags', type=str, nargs='?', default=[common.DEFAULTCFLAGS],
+        help='Extra compiling flags')
+    parser.add_argument(
+        '--ldflags', type=str, nargs='?', default=[common.DEFAULTLDFLAGS],
+        help='Extra linking flags')
+    parser.add_argument(
+        '--host', type=str, nargs='?', default=[common.HOSTPYTHON],
+        help='The path to the host Python installation')
+    parser.add_argument(
+        '--target', type=str, nargs='?', default=[common.TARGETPYTHON],
+        help='The path to the target Python installation')
     args = parser.parse_args()
     return args