Merge pull request #107 from rth/pyodide-package

Refactoring common functions under pyodide_build

.circleci/config.yml

@@ -34,16 +34,6 @@ jobs:
 
             pip install pytest pytest-xdist pytest-instafail selenium PyYAML
 
-            # Get recent version of Firefox and geckodriver
-            wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US
-            tar jxf firefox.tar.bz2
-            wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz
-            tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox
-
-            # Get recent version of chromedriver
-            wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
-            unzip chromedriver_linux64.zip
-            mv chromedriver firefox
 
       - run:
           name: lint
@@ -79,7 +69,6 @@ jobs:
           paths:
             - ./build
             - ./pyodide-env
-            - ./firefox
 
       - store_artifacts:
           path: /home/circleci/repo/build/
@@ -97,6 +86,12 @@ jobs:
             # causes Firefox to complain when loading it.  Let's just add the new mime type.
             sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types"
 
+            # Get recent version of Firefox and geckodriver
+            wget -O firefox.tar.bz2 https://download.mozilla.org/\?product\=firefox-latest-ssl\&os\=linux64\&lang\=en-US
+            tar jxf firefox.tar.bz2
+            wget https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz
+            tar zxf geckodriver-v0.21.0-linux64.tar.gz -C firefox
+
             source pyodide-env/bin/activate
             export PATH=$PWD/firefox:$PATH
             pytest test -v -k firefox
@@ -114,8 +109,12 @@ jobs:
             # causes Firefox to complain when loading it.  Let's just add the new mime type.
             sudo bash -c "echo 'application/wasm wasm' >> /etc/mime.types"
 
+            # Get recent version of chromedriver
+            wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip
+            unzip chromedriver_linux64.zip
+            mv chromedriver pyodide-env/bin/
+
             source pyodide-env/bin/activate
-            export PATH=$PWD/firefox:$PATH
             pytest test -v -k chrome
 
   deploy:

.gitignore

@@ -22,3 +22,4 @@ ccache
 
 /six/six-1.11.0
 /lz4/lz4-1.8.3
+*.egg-info/

bin/pyodide

+#!/bin/bash
+
+# get the absolute path to the root directory
+ROOTDIR=$(python -c 'import pathlib, sys; \
+                     print(pathlib.Path(sys.argv[1]).resolve().parents[1])' \
+          "${BASH_SOURCE[0]}")
+
+export PYTHONPATH="${PYTHONPATH}:${ROOTDIR}"
+python -m pyodide_build "$@"

packages/Makefile

@@ -2,7 +2,8 @@ PYODIDE_ROOT=$(abspath ..)
 include ../Makefile.envs
 
 all:
-	../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
+	../bin/pyodide buildall . ../build \
+		--ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
 
 clean:
 	rm -rf ./*/build

pyodide_build/__init__.py

+__version__ = '0.1.0'

pyodide_build/__main__.py

+#!/usr/bin/env python3
+import argparse
+
+from . import buildall
+from . import buildpkg
+from . import pywasmcross
+
+
+def main():
+    main_parser = argparse.ArgumentParser(prog='pyodide')
+    subparsers = main_parser.add_subparsers(help='action')
+
+    for command_name, module in (("buildpkg", buildpkg),
+                                 ("buildall", buildall),
+                                 ("pywasmcross", pywasmcross)):
+        parser = module.make_parser(subparsers.add_parser(command_name))
+        parser.set_defaults(func=module.main)
+
+    args = main_parser.parse_args()
+    # run the selected action
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

tools/buildall → pyodide_build/buildall.py

@@ -10,8 +10,8 @@ from pathlib import Path
 import shutil
 
 
-import common
-import buildpkg
+from . import common
+from . import buildpkg
 
 
 def build_package(pkgname, dependencies, packagesdir, outputdir, args):
@@ -54,9 +54,8 @@ def build_packages(packagesdir, outputdir, args):
         json.dump({'dependencies': dependencies}, fd)
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(
-        "Build all of the packages in a given directory")
+def make_parser(parser):
+    parser.description = "Build all of the packages in a given directory"
     parser.add_argument(
         'dir', type=str, nargs=1,
         help='Input directory containing a tree of package definitions')
@@ -75,7 +74,7 @@ def parse_args():
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
-    return parser.parse_args()
+    return parser
 
 
 def main(args):
@@ -85,5 +84,6 @@ def main(args):
 
 
 if __name__ == '__main__':
-    args = parse_args()
+    parser = make_parser(argparse.ArgumentParser())
+    args = parser.parse_args()
     main(args)

tools/buildpkg.py → pyodide_build/buildpkg.py

@@ -12,10 +12,7 @@ import shutil
 import subprocess
 
 
-import common
-
-
-ROOTDIR = Path(__file__).parent.resolve()
+from . import common
 
 
 def check_checksum(path, pkg):
@@ -88,7 +85,7 @@ def compile(path, srcpath, pkg, args):
     try:
         subprocess.run([
             str(Path(args.host) / 'bin' / 'python3'),
-            str(ROOTDIR / 'pywasmcross'),
+            '-m', 'pyodide_build', 'pywasmcross',
             '--cflags',
             args.cflags + ' ' +
             pkg.get('build', {}).get('cflags', ''),
@@ -124,7 +121,7 @@ def package_files(buildpath, srcpath, pkg, args):
     install_prefix = (srcpath / 'install').resolve()
     subprocess.run([
         'python',
-        Path(ROOTDIR) / 'file_packager.py',
+        common.ROOTDIR / 'file_packager.py',
         name + '.data',
         '--lz4',
         '--preload',
@@ -163,8 +160,8 @@ def build_package(path, args):
         os.chdir(orig_path)
 
 
-def parse_args():
-    parser = argparse.ArgumentParser('Build a pyodide package.')
+def make_parser(parser):
+    parser.description = 'Build a pyodide package.'
     parser.add_argument(
         'package', type=str, nargs=1,
         help="Path to meta.yaml package description")
@@ -180,7 +177,7 @@ def parse_args():
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
-    return parser.parse_args()
+    return parser
 
 
 def main(args):
@@ -189,5 +186,6 @@ def main(args):
 
 
 if __name__ == '__main__':
-    args = parse_args()
+    parser = make_parser(argparse.ArgumentParser())
+    args = parser.parse_args()
     main(args)

tools/common.py → pyodide_build/common.py

 from pathlib import Path
 
 
-ROOTDIR = Path(__file__).parent.resolve()
+ROOTDIR = Path(__file__).parents[1].resolve() / 'tools'
 HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host'
 TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0'
 DEFAULTCFLAGS = ''

tools/pywasmcross → pyodide_build/pywasmcross.py

@@ -34,10 +34,12 @@ import subprocess
 import sys
 
 
-import common
+# absolute import is necessary as this file will be symlinked
+# under tools
+from pyodide_build import common
 
 
-ROOTDIR = Path(__file__).parent.resolve()
+ROOTDIR = common.ROOTDIR
 symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
 
 
@@ -186,7 +188,7 @@ def clean_out_native_artifacts():
                 path.unlink()
 
 
-def install_for_distribution():
+def install_for_distribution(args):
     subprocess.check_call(
         [Path(args.host) / 'bin' / 'python3',
          'setup.py',
@@ -202,33 +204,46 @@ def build_wrap(args):
         capture_compile(args)
     clean_out_native_artifacts()
     replay_compile(args)
-    install_for_distribution()
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        'Cross compile a Python distutils package. '
-        'Run from the root directory of the package\'s source')
-    parser.add_argument(
-        '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
-        help='Extra compiling flags')
-    parser.add_argument(
-        '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS,
-        help='Extra linking flags')
-    parser.add_argument(
-        '--host', type=str, nargs='?', default=common.HOSTPYTHON,
-        help='The path to the host Python installation')
-    parser.add_argument(
-        '--target', type=str, nargs='?', default=common.TARGETPYTHON,
-        help='The path to the target Python installation')
-    args = parser.parse_args()
-    return args
+    install_for_distribution(args)
 
 
-if __name__ == '__main__':
+def make_parser(parser):
+    basename = Path(sys.argv[0]).name
+    if basename in symlinks:
+        # skip parsing of all arguments
+        parser._actions = []
+    else:
+        parser.description = (
+            'Cross compile a Python distutils package. '
+            'Run from the root directory of the package\'s source')
+        parser.add_argument(
+            '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
+            help='Extra compiling flags')
+        parser.add_argument(
+            '--ldflags', type=str, nargs='?', default=common.DEFAULTLDFLAGS,
+            help='Extra linking flags')
+        parser.add_argument(
+            '--host', type=str, nargs='?', default=common.HOSTPYTHON,
+            help='The path to the host Python installation')
+        parser.add_argument(
+            '--target', type=str, nargs='?', default=common.TARGETPYTHON,
+            help='The path to the target Python installation')
+    return parser
+
+
+def main(args):
     basename = Path(sys.argv[0]).name
     if basename in symlinks:
         collect_args(basename)
     else:
-        args = parse_args()
         build_wrap(args)
+
+
+if __name__ == '__main__':
+    basename = Path(sys.argv[0]).name
+    if basename in symlinks:
+        main(None)
+    else:
+        parser = make_parser(argparse.ArgumentParser())
+        args = parser.parse_args()
+        main(args)

test/make_test_list.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 
 TEST_DIR = (Path(__file__).parent
-            / "cpython/build/3.6.4/host/lib/python3.7/test")
+            / "cpython/build/3.7.0/host/lib/python3.7/test")
 
 
 def collect_tests(base_dir):

test/test_common.py

 import pytest
 import os
 from pathlib import Path
-import sys
+from pyodide_build.common import parse_package
 
 BASE_DIR = Path(__file__).parent.parent
 PKG_DIR = BASE_DIR / 'packages'
 
-# TODO: remove once we have a proper Python package for common functions
-sys.path.append(str(BASE_DIR / 'tools'))
-import common  # noqa
-
 
 def registered_packages():
     """Returns a list of registred package names"""
@@ -23,7 +19,7 @@ def registered_packages_meta():
     for each registed package
     """
     packages = registered_packages
-    return {name: common.parse_package(PKG_DIR / name / 'meta.yaml')
+    return {name: parse_package(PKG_DIR / name / 'meta.yaml')
             for name in packages}
 
 
@@ -34,7 +30,7 @@ UNSUPPORTED_PACKAGES = {'chrome': ['pandas'],
 @pytest.mark.parametrize('name', registered_packages())
 def test_import(name, selenium_standalone):
     # check that we can parse the meta.yaml
-    meta = common.parse_package(PKG_DIR / name / 'meta.yaml')
+    meta = parse_package(PKG_DIR / name / 'meta.yaml')
 
     if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
         pytest.xfail(