Merge pull request #231 from rth/skip-c-extension-build-target

Fix Skip building C extensions for the target env

docs/new_packages.md

@@ -90,6 +90,17 @@ source tree (the expanded tarball).
 
 ### `build`
 
+#### `build/skip_host`
+
+Skip building C extensions for the host environment. Default: `True`.
+
+Setting this to `False` will result in ~2x slower builds for packages that
+include C extensions. It should only be needed when a package is a build
+time dependency for other packages. For instance, numpy is imported during
+installation of matplotlib, importing numpy also imports included C extensions,
+therefore it is built both for host and target.
+
+
 #### `build/cflags`
 
 Extra arguments to pass to the compiler when building for WebAssembly.

packages/numpy/meta.yaml

@@ -18,6 +18,7 @@ source:
     - patches/fix-install-with-skip-build.patch
 
 build:
+  skip_host: False
   cflags: -include math.h -I../../config
 
 

pyodide_build/buildpkg.py

@@ -82,6 +82,10 @@ def compile(path, srcpath, pkg, args):
 
     orig_dir = Path.cwd()
     os.chdir(srcpath)
+    env = dict(os.environ)
+    if pkg.get('build', {}).get('skip_host', True):
+        env['SKIP_HOST'] = ''
+
     try:
         subprocess.run([
             str(Path(args.host) / 'bin' / 'python3'),
@@ -93,7 +97,7 @@ def compile(path, srcpath, pkg, args):
             args.ldflags + ' ' +
             pkg.get('build', {}).get('ldflags', ''),
             '--host', args.host,
-            '--target', args.target], check=True)
+            '--target', args.target], env=env, check=True)
     finally:
         os.chdir(orig_dir)
 

pyodide_build/pywasmcross.py

@@ -60,10 +60,37 @@ def collect_args(basename):
         path = path.replace(str(ROOTDIR) + ':', '')
     env['PATH'] = path
 
+    skip_host = 'SKIP_HOST' in os.environ
+
+    # Skip compilations of C/Fortran extensions for the target environement.
+    # We still need to generate the output files for distutils to continue
+    # the build.
+    # TODO: This may need slight tuning for new projects. In particular,
+    #       currently ar is not skipped, so a known failure would happen when
+    #       we create some object files (that are empty as gcc is skipped), on
+    #       which we run the actual ar command.
+    skip = False
+    if (basename in ['gcc', 'cc', 'c++', 'gfortran', 'ld']
+            and '-o' in sys.argv[1:]
+            # do not skip numpy as it is needed as build time
+            # dependency by other packages (e.g. matplotlib)
+            and skip_host):
+        out_idx = sys.argv.index('-o')
+        if (out_idx + 1) < len(sys.argv):
+            # get the index of the output file path
+            out_idx += 1
+            with open(sys.argv[out_idx], 'wb') as fh:
+                fh.write(b'')
+            skip = True
+
     with open('build.log', 'a') as fd:
+        # TODO: store skip status in the build.log
         json.dump([basename] + sys.argv[1:], fd)
         fd.write('\n')
 
+    if skip:
+        sys.exit(0)
+
     sys.exit(
         subprocess.run(
             [basename] + sys.argv[1:],

test/test_common.py

@@ -27,6 +27,18 @@ UNSUPPORTED_PACKAGES = {'chrome': ['pandas'],
                         'firefox': []}
 
 
+@pytest.mark.parametrize('name', registered_packages())
+def test_parse_package(name):
+    # check that we can parse the meta.yaml
+    meta = parse_package(PKG_DIR / name / 'meta.yaml')
+
+    skip_host = meta.get('build', {}).get('skip_host', True)
+    if name == 'numpy':
+        assert skip_host is False
+    elif name == 'pandas':
+        assert skip_host is True
+
+
 @pytest.mark.parametrize('name', registered_packages())
 def test_import(name, selenium_standalone):
     # check that we can parse the meta.yaml