Converting tools scripts to a python package

.gitignore

@@ -22,3 +22,4 @@ ccache
 
 /six/six-1.11.0
 /lz4/lz4-1.8.3
+*.egg-info/

Makefile

@@ -204,6 +204,8 @@ ccache/em++:
 
 $(CPYTHONLIB): emsdk/emsdk/.complete ccache/emcc ccache/em++
 	make -C $(CPYTHONROOT)
+	python -m pip install -e .
+	"$(HOSTPYTHONROOT)/bin/python3" -m pip install -e .
 
 
 $(LZ4LIB):

packages/Makefile

@@ -2,7 +2,7 @@ PYODIDE_ROOT=$(abspath ..)
 include ../Makefile.envs
 
 all:
-	../tools/buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
+	pyodide buildall . ../build --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
 
 clean:
 	rm -rf ./*/build

pyodide_build/__init__.py

+__version__ = '0.1.0'

tools/buildall → pyodide_build/buildall.py

@@ -10,8 +10,8 @@ from pathlib import Path
 import shutil
 
 
-import common
-import buildpkg
+from . import common
+from . import buildpkg
 
 
 def build_package(pkgname, dependencies, packagesdir, outputdir, args):
@@ -54,9 +54,8 @@ def build_packages(packagesdir, outputdir, args):
         json.dump({'dependencies': dependencies}, fd)
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(
-        "Build all of the packages in a given directory")
+def make_parser(parser):
+    parser.description = "Build all of the packages in a given directory"
     parser.add_argument(
         'dir', type=str, nargs=1,
         help='Input directory containing a tree of package definitions')
@@ -75,7 +74,7 @@ def parse_args():
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
-    return parser.parse_args()
+    return parser
 
 
 def main(args):
@@ -85,5 +84,6 @@ def main(args):
 
 
 if __name__ == '__main__':
-    args = parse_args()
+    parser = make_parser(argparse.ArgumentParser())
+    args = parser.parse_args()
     main(args)

tools/buildpkg.py → pyodide_build/buildpkg.py

@@ -12,10 +12,7 @@ import shutil
 import subprocess
 
 
-import common
-
-
-ROOTDIR = Path(__file__).parent.resolve()
+from . import common
 
 
 def check_checksum(path, pkg):
@@ -88,7 +85,8 @@ def compile(path, srcpath, pkg, args):
     try:
         subprocess.run([
             str(Path(args.host) / 'bin' / 'python3'),
-            str(ROOTDIR / 'pywasmcross'),
+            '-m', 'pyodide_build',
+            'pywasmcross',
             '--cflags',
             args.cflags + ' ' +
             pkg.get('build', {}).get('cflags', ''),
@@ -124,7 +122,7 @@ def package_files(buildpath, srcpath, pkg, args):
     install_prefix = (srcpath / 'install').resolve()
     subprocess.run([
         'python',
-        Path(ROOTDIR) / 'file_packager.py',
+        common.TOOLSDIR / 'file_packager.py',
         name + '.data',
         '--lz4',
         '--preload',
@@ -163,8 +161,8 @@ def build_package(path, args):
         os.chdir(orig_path)
 
 
-def parse_args():
-    parser = argparse.ArgumentParser('Build a pyodide package.')
+def make_parser(parser):
+    parser.description = 'Build a pyodide package.'
     parser.add_argument(
         'package', type=str, nargs=1,
         help="Path to meta.yaml package description")
@@ -180,7 +178,7 @@ def parse_args():
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
-    return parser.parse_args()
+    return parser
 
 
 def main(args):
@@ -189,5 +187,6 @@ def main(args):
 
 
 if __name__ == '__main__':
-    args = parse_args()
+    parser = make_parser(argparse.ArgumentParser())
+    args = parser.parse_args()
     main(args)

tools/common.py → pyodide_build/common.py

 from pathlib import Path
 
 
-ROOTDIR = Path(__file__).parent.resolve()
-HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host'
-TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0'
+ROOTDIR = Path(__file__).parents[1].resolve()
+TOOLSDIR = ROOTDIR / 'tools'
+HOSTPYTHON = ROOTDIR / 'cpython' / 'build' / '3.7.0' / 'host'
+TARGETPYTHON = ROOTDIR / 'cpython' / 'installs' / 'python-3.7.0'
 DEFAULTCFLAGS = ''
 DEFAULTLDFLAGS = ' '.join([
     '-O3',

tools/pywasmcross → pyodide_build/pywasmcross.py

@@ -34,10 +34,12 @@ import subprocess
 import sys
 
 
-import common
+# absolute import is necessary as this file will be symlinked
+# under tools
+from pyodide_build import common
 
 
-ROOTDIR = Path(__file__).parent.resolve()
+TOOLSDIR = common.TOOLSDIR
 symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
 
 
@@ -53,8 +55,8 @@ def collect_args(basename):
     # native compiler
     env = dict(os.environ)
     path = env['PATH']
-    while str(ROOTDIR) + ':' in path:
-        path = path.replace(str(ROOTDIR) + ':', '')
+    while str(TOOLSDIR) + ':' in path:
+        path = path.replace(str(TOOLSDIR) + ':', '')
     env['PATH'] = path
 
     with open('build.log', 'a') as fd:
@@ -74,7 +76,7 @@ def make_symlinks(env):
     """
     exec_path = Path(__file__).resolve()
     for symlink in symlinks:
-        symlink_path = ROOTDIR / symlink
+        symlink_path = TOOLSDIR / symlink
         if not symlink_path.exists():
             symlink_path.symlink_to(exec_path)
         if symlink == 'c++':
@@ -87,7 +89,7 @@ def make_symlinks(env):
 def capture_compile(args):
     env = dict(os.environ)
     make_symlinks(env)
-    env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH']
+    env['PATH'] = str(TOOLSDIR) + ':' + os.environ['PATH']
 
     result = subprocess.run(
         [Path(args.host) / 'bin' / 'python3',
@@ -186,7 +188,7 @@ def clean_out_native_artifacts():
                 path.unlink()
 
 
-def install_for_distribution():
+def install_for_distribution(args):
     subprocess.check_call(
         [Path(args.host) / 'bin' / 'python3',
          'setup.py',
@@ -202,11 +204,11 @@ def build_wrap(args):
         capture_compile(args)
     clean_out_native_artifacts()
     replay_compile(args)
-    install_for_distribution()
+    install_for_distribution(args)
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(
+def make_parser(parser):
+    parser.description = (
         'Cross compile a Python distutils package. '
         'Run from the root directory of the package\'s source')
     parser.add_argument(
@@ -221,14 +223,19 @@ def parse_args():
     parser.add_argument(
         '--target', type=str, nargs='?', default=common.TARGETPYTHON,
         help='The path to the target Python installation')
-    args = parser.parse_args()
-    return args
+    parser.add_argument('basename', type=str, nargs='?')
+    return parser
 
 
-if __name__ == '__main__':
+def main(args, unknown=None):
     basename = Path(sys.argv[0]).name
     if basename in symlinks:
         collect_args(basename)
     else:
-        args = parse_args()
         build_wrap(args)
+
+
+if __name__ == '__main__':
+    parser = make_parser(argparse.ArgumentParser())
+    args, unknown = parser.parse_known_args()
+    main(args, unknown)

setup.py

+from setuptools import setup, find_packages
+from pyodide_build import __version__
+
+with open('README.md', 'rt') as fh:
+    LONG_DESCRIPTION = fh.read()
+
+setup(name='pyodide_build',
+      version=__version__,
+      description='pyodide builder',
+      entry_points={
+        'console_scripts': [
+            'pyodide = pyodide_build.__main__:main'
+        ]},
+      url="https://github.com/iodide-project/pyodide",
+      license='MPL',
+      packages=find_packages())