Merge pull request #101 from mdboom/use-pathlib

Use pathlib for path manipulations

benchmark/benchmark.py

 import json
-import os
+from pathlib import Path
 import re
 import subprocess
 import sys
 
-sys.path.insert(0, os.path.abspath(
-     os.path.join(os.path.dirname(__file__), '..', 'test')))
+sys.path.insert(
+    0, str((Path(__file__).resolve().parents[1] / 'test')))
+
 import conftest
 
 
@@ -14,11 +15,12 @@ SKIP = set(['fft', 'hyantes'])
 
 def run_native(hostpython, code):
     output = subprocess.check_output(
-        [os.path.abspath(hostpython), '-c', code],
-        cwd=os.path.dirname(__file__),
+        [hostpython.resolve(), '-c', code],
+        cwd=Path(__file__).resolve().parent,
         env={
             'PYTHONPATH':
-            os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))}
+            str(Path(__file__).resolve().parents[1] / 'src')
+        }
     )
     return float(output.strip().split()[-1])
 
@@ -72,12 +74,12 @@ def parse_numpy_benchmark(filename):
 
 
 def get_numpy_benchmarks():
-    root = '../numpy-benchmarks/benchmarks'
-    for filename in os.listdir(root):
-        name = os.path.splitext(filename)[0]
+    root = Path('../numpy-benchmarks/benchmarks')
+    for filename in root.iterdir():
+        name = filename.name
         if name in SKIP:
             continue
-        content = parse_numpy_benchmark(os.path.join(root, filename))
+        content = parse_numpy_benchmark(filename)
         content += (
             "import numpy as np\n"
             "_ = np.empty(())\n"
@@ -103,6 +105,6 @@ def main(hostpython):
 
 
 if __name__ == '__main__':
-    results = main(sys.argv[-2])
+    results = main(Path(sys.argv[-2]).resolve())
     with open(sys.argv[-1], 'w') as fp:
         json.dump(results, fp)

test/make_test_list.py

@@ -3,21 +3,24 @@ Generate a list of test modules in the CPython distribution.
 """
 
 import os
+from pathlib import Path
 
 tests = []
 
-TEST_DIR = "../cpython/build/3.6.4/host/lib/python3.6/test"
+TEST_DIR = Path("../cpython/build/3.6.4/host/lib/python3.6/test")
+
 for root, dirs, files in os.walk(
         "../cpython/build/3.6.4/host/lib/python3.6/test"):
-    root = os.path.relpath(root, TEST_DIR)
+    root = Path(root).relative_to(TEST_DIR)
     if root == '.':
         root = ''
     else:
         root = '.'.join(root.split('/')) + '.'
 
     for filename in files:
-        if filename.startswith("test_") and filename.endswith(".py"):
-            tests.append(root + os.path.splitext(filename)[0])
+        filename = Path(filename)
+        if str(filename).startswith("test_") and filename.suffix == ".py":
+            tests.append(str(root / filename.stem))
 
 tests.sort()
 with open("python_tests.txt", "w") as fp:

test/test_python.py

 import os
-import pathlib
+from pathlib import Path
 import time
 
 
@@ -295,8 +295,7 @@ def pytest_generate_tests(metafunc):
         test_modules = []
         if 'CIRCLECI' not in os.environ or True:
             with open(
-                    str(pathlib.Path(__file__).parents[0] /
-                        "python_tests.txt")) as fp:
+                    Path(__file__).parent / "python_tests.txt") as fp:
                 for line in fp:
                     line = line.strip()
                     if line.startswith('#'):

tools/buildall

@@ -6,7 +6,7 @@ Build all of the packages in a given directory.
 
 import argparse
 import json
-import os
+from pathlib import Path
 import shutil
 
 
@@ -14,42 +14,39 @@ import common
 import buildpkg
 
 
-def build_package(pkgname, dependencies, packagesdir, args):
+def build_package(pkgname, dependencies, packagesdir, outputdir, args):
     reqs = dependencies[pkgname]
     # Make sure all of the package's requirements are built first
     for req in reqs:
-        build_package(req, dependencies, packagesdir, args)
-    if not os.path.isfile(
-            os.path.join(packagesdir, pkgname, 'build', '.packaged')):
+        build_package(req, dependencies, packagesdir, outputdir, args)
+    if not (packagesdir / pkgname / 'build' / '.packaged').is_file():
         print("BUILDING PACKAGE: " + pkgname)
-        buildpkg.build_package(
-            os.path.join(packagesdir, pkgname, 'meta.yaml'), args)
+        buildpkg.build_package(packagesdir / pkgname / 'meta.yaml', args)
     shutil.copyfile(
-        os.path.join(packagesdir, pkgname, 'build', pkgname + '.data'),
-        os.path.join(args.output[0], pkgname + '.data'))
+        packagesdir / pkgname / 'build' / (pkgname + '.data'),
+        outputdir / (pkgname + '.data'))
     shutil.copyfile(
-        os.path.join(packagesdir, pkgname, 'build', pkgname + '.js'),
-        os.path.join(args.output[0], pkgname + '.js'))
+        packagesdir / pkgname / 'build' / (pkgname + '.js'),
+        outputdir / (pkgname + '.js'))
 
 
-def build_packages(packagesdir, args):
+def build_packages(packagesdir, outputdir, args):
     # We have to build the packages in the correct order (dependencies first),
     # so first load in all of the package metadata and build a dependency map.
     dependencies = {}
-    for pkgdir in os.listdir(packagesdir):
-        pkgdir = os.path.join(packagesdir, pkgdir)
-        pkgpath = os.path.join(pkgdir, 'meta.yaml')
-        if os.path.isdir(pkgdir) and os.path.isfile(pkgpath):
+    for pkgdir in packagesdir.iterdir():
+        pkgpath = pkgdir / 'meta.yaml'
+        if pkgdir.is_dir() and pkgpath.is_file():
             pkg = common.parse_package(pkgpath)
             name = pkg['package']['name']
             reqs = pkg.get('requirements', {}).get('run', [])
             dependencies[name] = reqs
 
     for pkgname in dependencies.keys():
-        build_package(pkgname, dependencies, packagesdir, args)
+        build_package(pkgname, dependencies, packagesdir, outputdir, args)
 
     # This is done last so the main Makefile can use it as a completion token
-    with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd:
+    with open(outputdir / 'packages.json', 'w') as fd:
         json.dump({'dependencies': dependencies}, fd)
 
 
@@ -78,8 +75,9 @@ def parse_args():
 
 
 def main(args):
-    packagesdir = os.path.abspath(args.dir[0])
-    build_packages(packagesdir, args)
+    packagesdir = Path(args.dir[0]).resolve()
+    outputdir = Path(args.output[0]).resolve()
+    build_packages(packagesdir, outputdir, args)
 
 
 if __name__ == '__main__':

tools/buildpkg.py

@@ -7,6 +7,7 @@ Builds a Pyodide package.
 import argparse
 import hashlib
 import os
+from pathlib import Path
 import shutil
 import subprocess
 
@@ -14,7 +15,7 @@ import subprocess
 import common
 
 
-ROOTDIR = os.path.abspath(os.path.dirname(__file__))
+ROOTDIR = Path(__file__).parent.resolve()
 
 
 def check_checksum(path, pkg):
@@ -37,40 +38,39 @@ def check_checksum(path, pkg):
 
 
 def download_and_extract(buildpath, packagedir, pkg, args):
-    tarballpath = os.path.join(
-        buildpath, os.path.basename(pkg['source']['url']))
-    if not os.path.isfile(tarballpath):
+    tarballpath = buildpath / Path(pkg['source']['url']).name
+    if not tarballpath.is_file():
         subprocess.run([
-            'wget', '-q', '-O', tarballpath, pkg['source']['url']
+            'wget', '-q', '-O', str(tarballpath), pkg['source']['url']
         ], check=True)
         check_checksum(tarballpath, pkg)
-    srcpath = os.path.join(buildpath, packagedir)
-    if not os.path.isdir(srcpath):
-        shutil.unpack_archive(tarballpath, buildpath)
+    srcpath = buildpath / packagedir
+    if not srcpath.is_dir():
+        shutil.unpack_archive(str(tarballpath), str(buildpath))
     return srcpath
 
 
 def patch(path, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(srcpath, '.patched')):
+    if (srcpath / '.patched').is_file():
         return
 
     # Apply all of the patches
-    orig_dir = os.getcwd()
-    pkgdir = os.path.abspath(os.path.dirname(path))
+    orig_dir = Path.cwd()
+    pkgdir = path.parent.resolve()
     os.chdir(srcpath)
     try:
         for patch in pkg['source'].get('patches', []):
             subprocess.run([
-                'patch', '-p1', '--binary', '-i', os.path.join(pkgdir, patch)
+                'patch', '-p1', '--binary', '-i', pkgdir / patch
             ], check=True)
     finally:
         os.chdir(orig_dir)
 
     # Add any extra files
     for src, dst in pkg['source'].get('extras', []):
-        shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst))
+        shutil.copyfile(pkgdir / src, srcpath / dst)
 
-    with open(os.path.join(srcpath, '.patched'), 'wb') as fd:
+    with open(srcpath / '.patched', 'wb') as fd:
         fd.write(b'\n')
 
 
@@ -78,31 +78,31 @@ def get_libdir(srcpath, args):
     # Get the name of the build/lib.XXX directory that distutils wrote its
     # output to
     slug = subprocess.check_output([
-        os.path.join(args.host, 'bin', 'python3'),
+        str(Path(args.host) / 'bin' / 'python3'),
         '-c',
         'import sysconfig, sys; '
         'print("{}-{}.{}".format('
         'sysconfig.get_platform(), '
         'sys.version_info[0], '
         'sys.version_info[1]))']).decode('ascii').strip()
-    purelib = os.path.join(srcpath, 'build', 'lib')
-    if os.path.isdir(purelib):
+    purelib = srcpath / 'build' / 'lib'
+    if purelib.is_dir():
         libdir = purelib
     else:
-        libdir = os.path.join(srcpath, 'build', 'lib.' + slug)
+        libdir = srcpath / 'build' / ('lib.' + slug)
     return libdir
 
 
 def compile(path, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(srcpath, '.built')):
+    if (srcpath / '.built').is_file():
         return
 
-    orig_dir = os.getcwd()
+    orig_dir = Path.cwd()
     os.chdir(srcpath)
     try:
         subprocess.run([
-            os.path.join(args.host, 'bin', 'python3'),
-            os.path.join(ROOTDIR, 'pywasmcross'),
+            str(Path(args.host) / 'bin' / 'python3'),
+            str(ROOTDIR / 'pywasmcross'),
             '--cflags',
             args.cflags + ' ' +
             pkg.get('build', {}).get('cflags', ''),
@@ -117,7 +117,7 @@ def compile(path, srcpath, pkg, args):
     post = pkg.get('build', {}).get('post')
     if post is not None:
         libdir = get_libdir(srcpath, args)
-        pkgdir = os.path.abspath(os.path.dirname(path))
+        pkgdir = path.parent.resolve()
         env = {
             'BUILD': libdir,
             'PKGDIR': pkgdir
@@ -125,46 +125,46 @@ def compile(path, srcpath, pkg, args):
         subprocess.run([
             'bash', '-c', post], env=env, check=True)
 
-    with open(os.path.join(srcpath, '.built'), 'wb') as fd:
+    with open(srcpath / '.built', 'wb') as fd:
         fd.write(b'\n')
 
 
 def package_files(buildpath, srcpath, pkg, args):
-    if os.path.isfile(os.path.join(buildpath, '.packaged')):
+    if (buildpath / '.pacakaged').is_file():
         return
 
     name = pkg['package']['name']
     libdir = get_libdir(srcpath, args)
     subprocess.run([
         'python2',
-        os.path.join(os.environ['EMSCRIPTEN'], 'tools', 'file_packager.py'),
-        os.path.join(buildpath, name + '.data'),
+        Path(os.environ['EMSCRIPTEN']) / 'tools' / 'file_packager.py',
+        buildpath / (name + '.data'),
         '--preload',
         '{}@/lib/python3.6/site-packages'.format(libdir),
-        '--js-output={}'.format(os.path.join(buildpath, name + '.js')),
+        '--js-output={}'.format(buildpath / (name + '.js')),
         '--export-name=pyodide',
         '--exclude', '*.wasm.pre',
         '--exclude', '__pycache__',
         '--use-preload-plugins'], check=True)
     subprocess.run([
         'uglifyjs',
-        os.path.join(buildpath, name + '.js'),
+        buildpath / (name + '.js'),
         '-o',
-        os.path.join(buildpath, name + '.js')], check=True)
+        buildpath / (name + '.js')], check=True)
 
-    with open(os.path.join(buildpath, '.packaged'), 'wb') as fd:
+    with open(buildpath / '.packaged', 'wb') as fd:
         fd.write(b'\n')
 
 
 def build_package(path, args):
     pkg = common.parse_package(path)
     packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
-    dirpath = os.path.dirname(path)
-    orig_path = os.getcwd()
+    dirpath = path.parent
+    orig_path = Path.cwd()
     os.chdir(dirpath)
     try:
-        buildpath = os.path.join(dirpath, 'build')
-        if not os.path.exists(buildpath):
+        buildpath = dirpath / 'build'
+        if not buildpath.is_dir():
             os.makedirs(buildpath)
         srcpath = download_and_extract(buildpath, packagedir, pkg, args)
         patch(path, srcpath, pkg, args)
@@ -195,7 +195,7 @@ def parse_args():
 
 
 def main(args):
-    path = os.path.abspath(args.package[0])
+    path = Path(args.package[0]).resolve()
     build_package(path, args)
 
 

tools/common.py

-import os
+from pathlib import Path
 
 
-ROOTDIR = os.path.abspath(os.path.dirname(__file__))
-HOSTPYTHON = os.path.abspath(
-    os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host'))
-TARGETPYTHON = os.path.abspath(
-    os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4'))
+ROOTDIR = Path(__file__).parent.resolve()
+HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host'
+TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.6.4'
 DEFAULTCFLAGS = ''
 DEFAULTLDFLAGS = ' '.join([
     '-O3',

tools/pywasmcross

@@ -28,6 +28,7 @@ import argparse
 import importlib.machinery
 import json
 import os
+from pathlib import Path
 import re
 import subprocess
 import sys
@@ -36,7 +37,7 @@ import sys
 import common
 
 
-ROOTDIR = os.path.abspath(os.path.dirname(__file__))
+ROOTDIR = Path(__file__).parent.resolve()
 symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
 
 
@@ -52,8 +53,8 @@ def collect_args(basename):
     # native compiler
     env = dict(os.environ)
     path = env['PATH']
-    while ROOTDIR + ':' in path:
-        path = path.replace(ROOTDIR + ':', '')
+    while str(ROOTDIR) + ':' in path:
+        path = path.replace(str(ROOTDIR) + ':', '')
     env['PATH'] = path
 
     with open('build.log', 'a') as fd:
@@ -71,11 +72,11 @@ def make_symlinks(env):
     Makes sure all of the symlinks that make this script look like a compiler
     exist.
     """
-    exec_path = os.path.abspath(__file__)
+    exec_path = Path(__file__).resolve()
     for symlink in symlinks:
-        symlink_path = os.path.join(ROOTDIR, symlink)
-        if not os.path.exists(symlink_path):
-            os.symlink(exec_path, symlink_path)
+        symlink_path = ROOTDIR / symlink
+        if not symlink_path.exists():
+            symlink_path.symlink_to(exec_path)
         if symlink == 'c++':
             var = 'CXX'
         else:
@@ -86,15 +87,16 @@ def make_symlinks(env):
 def capture_compile(args):
     env = dict(os.environ)
     make_symlinks(env)
-    env['PATH'] = ROOTDIR + ':' + os.environ['PATH']
+    env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH']
 
     result = subprocess.run(
-        [os.path.join(args.host, 'bin', 'python3'),
+        [Path(args.host) / 'bin' / 'python3',
          'setup.py',
          'install'], env=env)
     if result.returncode != 0:
-        if os.path.exists('build.log'):
-            os.remove('build.log')
+        build_log_path = Path('build.log')
+        if build_log_path.exists():
+            build_log_path.unlink()
         sys.exit(result.returncode)
 
 
@@ -131,7 +133,7 @@ def handle_command(line, args):
             # Don't include any system directories
             if arg[2:].startswith('/usr'):
                 continue
-            if (os.path.abspath(arg[2:]).startswith(args.host) and
+            if (str(Path(arg[2:]).resolve()).startswith(args.host) and
                     'site-packages' not in arg):
                 arg = arg.replace('-I' + args.host, '-I' + args.target)
         # Don't include any system directories
@@ -168,8 +170,9 @@ def handle_command(line, args):
 def replay_compile(args):
     # If pure Python, there will be no build.log file, which is fine -- just do
     # nothing
-    if os.path.isfile('build.log'):
-        with open('build.log', 'r') as fd:
+    build_log_path = Path('build.log')
+    if build_log_path.is_file():
+        with open(build_log_path, 'r') as fd:
             for line in fd:
                 line = json.loads(line)
                 handle_command(line, args)
@@ -178,14 +181,14 @@ def replay_compile(args):
 def clean_out_native_artifacts():
     for root, dirs, files in os.walk('.'):
         for file in files:
-            path = os.path.join(root, file)
-            basename, ext = os.path.splitext(file)
-            if ext in ('.o', '.so', '.a'):
-                os.remove(path)
+            path = Path(root) / file
+            if path.suffix in ('.o', '.so', '.a'):
+                path.unlink()
 
 
 def build_wrap(args):
-    if not os.path.isfile('build.log'):
+    build_log_path = Path('build.log')
+    if not build_log_path.is_file():
         capture_compile(args)
     clean_out_native_artifacts()
     replay_compile(args)
@@ -212,7 +215,7 @@ def parse_args():
 
 
 if __name__ == '__main__':
-    basename = os.path.basename(sys.argv[0])
+    basename = Path(sys.argv[0]).name
     if basename in symlinks:
         collect_args(basename)
     else: