Skip to content

P
pyodide
Commit 2d0b9ee2 authored by Michael Droettboom's avatar Michael Droettboom
Browse files
Options

Use pathlib for path manipulations

parent ef8da067
Hide whitespace changes
Inline Side-by-side
Showing with 101 additions and 97 deletions
benchmark/benchmark.py
View file @ 2d0b9ee2
import json
import os
from pathlib import Path
import re
import subprocess
import sys
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'test')))
sys.path.insert(0, (Path(__file__).parent.parent / 'test').resolve())
import conftest
......@@ -14,11 +15,12 @@ SKIP = set(['fft', 'hyantes'])
def run_native(hostpython, code):
output = subprocess.check_output(
[os.path.abspath(hostpython), '-c', code],
[hostpython.resolve(), '-c', code],
cwd=os.path.dirname(__file__),
env={
'PYTHONPATH':
os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))}
str((Path(__file__).parent.parent / 'src').resolve())
}
)
return float(output.strip().split()[-1])
......@@ -72,12 +74,12 @@ def parse_numpy_benchmark(filename):
def get_numpy_benchmarks():
root = '../numpy-benchmarks/benchmarks'
for filename in os.listdir(root):
name = os.path.splitext(filename)[0]
root = Path('../numpy-benchmarks/benchmarks')
for filename in root.iterdir():
name = filename.name
if name in SKIP:
continue
content = parse_numpy_benchmark(os.path.join(root, filename))
content = parse_numpy_benchmark(root / filename)
content += (
"import numpy as np\n"
"_ = np.empty(())\n"
......
test/make_test_list.py
View file @ 2d0b9ee2
......@@ -3,21 +3,24 @@ Generate a list of test modules in the CPython distribution.
"""
import os
from pathlib import Path
tests = []
TEST_DIR = "../cpython/build/3.6.4/host/lib/python3.6/test"
TEST_DIR = Path("../cpython/build/3.6.4/host/lib/python3.6/test")
for root, dirs, files in os.walk(
"../cpython/build/3.6.4/host/lib/python3.6/test"):
root = os.path.relpath(root, TEST_DIR)
root = Path(root).relative_to(TEST_DIR)
if root == '.':
root = ''
else:
root = '.'.join(root.split('/')) + '.'
for filename in files:
if filename.startswith("test_") and filename.endswith(".py"):
tests.append(root + os.path.splitext(filename)[0])
filename = Path(filename)
if str(filename).startswith("test_") and filename.suffix == ".py":
tests.append(root / filename.stem)
tests.sort()
with open("python_tests.txt", "w") as fp:
......
test/test_python.py
View file @ 2d0b9ee2
import os
import pathlib
from pathlib import Path
import time
......@@ -295,8 +295,7 @@ def pytest_generate_tests(metafunc):
test_modules = []
if 'CIRCLECI' not in os.environ or True:
with open(
str(pathlib.Path(__file__).parents[0] /
"python_tests.txt")) as fp:
Path(__file__).parents[0] / "python_tests.txt") as fp:
for line in fp:
line = line.strip()
if line.startswith('#'):
......
tools/buildall
View file @ 2d0b9ee2
......@@ -6,7 +6,7 @@ Build all of the packages in a given directory.
import argparse
import json
import os
from pathlib import Path
import shutil
......@@ -14,42 +14,40 @@ import common
import buildpkg
def build_package(pkgname, dependencies, packagesdir, args):
def build_package(pkgname, dependencies, packagesdir, outputdir):
reqs = dependencies[pkgname]
# Make sure all of the package's requirements are built first
for req in reqs:
build_package(req, dependencies, packagesdir, args)
if not os.path.isfile(
os.path.join(packagesdir, pkgname, 'build', '.packaged')):
build_package(req, dependencies, packagesdir, outputdir)
if not (packagesdir / pkgname / 'build' / '.packaged').is_file():
print("BUILDING PACKAGE: " + pkgname)
buildpkg.build_package(
os.path.join(packagesdir, pkgname, 'meta.yaml'), args)
buildpkg.build_package(packagesdir / pkgname / 'meta.yaml', args)
shutil.copyfile(
os.path.join(packagesdir, pkgname, 'build', pkgname + '.data'),
os.path.join(args.output[0], pkgname + '.data'))
packagesdir / pkgname / 'build' / (pkgname + '.data'),
outputdir / (pkgname + '.data'))
shutil.copyfile(
os.path.join(packagesdir, pkgname, 'build', pkgname + '.js'),
os.path.join(args.output[0], pkgname + '.js'))
packagesdir / pkgname / 'build' / (pkgname + '.js'),
outputdir / (pkgname + '.js'))
def build_packages(packagesdir, args):
def build_packages(packagesdir, outputdir):
# We have to build the packages in the correct order (dependencies first),
# so first load in all of the package metadata and build a dependency map.
dependencies = {}
for pkgdir in os.listdir(packagesdir):
pkgdir = os.path.join(packagesdir, pkgdir)
pkgpath = os.path.join(pkgdir, 'meta.yaml')
if os.path.isdir(pkgdir) and os.path.isfile(pkgpath):
for pkgdir in packagesdir.iterdir():
pkgdir = packagesdir / pkgdir
pkgpath = pkgdir / 'meta.yaml'
if pkgdir.is_dir() and pkgpath.is_file():
pkg = common.parse_package(pkgpath)
name = pkg['package']['name']
reqs = pkg.get('requirements', {}).get('run', [])
dependencies[name] = reqs
for pkgname in dependencies.keys():
build_package(pkgname, dependencies, packagesdir, args)
build_package(pkgname, dependencies, packagesdir, outputdir)
# This is done last so the main Makefile can use it as a completion token
with open(os.path.join(args.output[0], 'packages.json'), 'w') as fd:
with open(outputdir / 'packages.json', 'w') as fd:
json.dump({'dependencies': dependencies}, fd)
......@@ -78,8 +76,9 @@ def parse_args():
def main(args):
packagesdir = os.path.abspath(args.dir[0])
build_packages(packagesdir, args)
packagesdir = Path(args.dir[0]).resolve()
outputdir = Path(args.output[0]).resolve()
build_packages(packagesdir, outputdir)
if __name__ == '__main__':
......
tools/buildpkg.py
View file @ 2d0b9ee2
......@@ -7,6 +7,7 @@ Builds a Pyodide package.
import argparse
import hashlib
import os
from pathlib import Path
import shutil
import subprocess
......@@ -14,7 +15,7 @@ import subprocess
import common
ROOTDIR = os.path.abspath(os.path.dirname(__file__))
ROOTDIR = Path(__file__).parent.resolve()
def check_checksum(path, pkg):
......@@ -37,40 +38,39 @@ def check_checksum(path, pkg):
def download_and_extract(buildpath, packagedir, pkg, args):
tarballpath = os.path.join(
buildpath, os.path.basename(pkg['source']['url']))
if not os.path.isfile(tarballpath):
tarballpath = buildpath / Path(pkg['source']['url']).name
if not tarballpath.is_file():
subprocess.run([
'wget', '-q', '-O', tarballpath, pkg['source']['url']
'wget', '-q', '-O', str(tarballpath), pkg['source']['url']
], check=True)
check_checksum(tarballpath, pkg)
srcpath = os.path.join(buildpath, packagedir)
if not os.path.isdir(srcpath):
shutil.unpack_archive(tarballpath, buildpath)
srcpath = buildpath / packagedir
if not srcpath.is_dir():
shutil.unpack_archive(str(tarballpath), str(buildpath))
return srcpath
def patch(path, srcpath, pkg, args):
if os.path.isfile(os.path.join(srcpath, '.patched')):
if (srcpath / '.patched').is_file():
return
# Apply all of the patches
orig_dir = os.getcwd()
pkgdir = os.path.abspath(os.path.dirname(path))
orig_dir = Path.cwd()
pkgdir = path.parent.resolve()
os.chdir(srcpath)
try:
for patch in pkg['source'].get('patches', []):
subprocess.run([
'patch', '-p1', '--binary', '-i', os.path.join(pkgdir, patch)
'patch', '-p1', '--binary', '-i', pkgdir / patch
], check=True)
finally:
os.chdir(orig_dir)
# Add any extra files
for src, dst in pkg['source'].get('extras', []):
shutil.copyfile(os.path.join(pkgdir, src), os.path.join(srcpath, dst))
shutil.copyfile(pkgdir / src, srcpath / dst)
with open(os.path.join(srcpath, '.patched'), 'wb') as fd:
with open(srcpath / '.patched', 'wb') as fd:
fd.write(b'\n')
......@@ -78,31 +78,31 @@ def get_libdir(srcpath, args):
# Get the name of the build/lib.XXX directory that distutils wrote its
# output to
slug = subprocess.check_output([
os.path.join(args.host, 'bin', 'python3'),
str(Path(args.host) / 'bin' / 'python3'),
'-c',
'import sysconfig, sys; '
'print("{}-{}.{}".format('
'sysconfig.get_platform(), '
'sys.version_info[0], '
'sys.version_info[1]))']).decode('ascii').strip()
purelib = os.path.join(srcpath, 'build', 'lib')
if os.path.isdir(purelib):
purelib = srcpath / 'build' / 'lib'
if purelib.is_dir():
libdir = purelib
else:
libdir = os.path.join(srcpath, 'build', 'lib.' + slug)
libdir = srcpath / 'build' / ('lib.' + slug)
return libdir
def compile(path, srcpath, pkg, args):
if os.path.isfile(os.path.join(srcpath, '.built')):
if (srcpath / '.built').is_file():
return
orig_dir = os.getcwd()
orig_dir = Path.cwd()
os.chdir(srcpath)
try:
subprocess.run([
os.path.join(args.host, 'bin', 'python3'),
os.path.join(ROOTDIR, 'pywasmcross'),
str(Path(args.host) / 'bin' / 'python3'),
str(ROOTDIR / 'pywasmcross'),
'--cflags',
args.cflags + ' ' +
pkg.get('build', {}).get('cflags', ''),
......@@ -117,7 +117,7 @@ def compile(path, srcpath, pkg, args):
post = pkg.get('build', {}).get('post')
if post is not None:
libdir = get_libdir(srcpath, args)
pkgdir = os.path.abspath(os.path.dirname(path))
pkgdir = path.parent.resolve()
env = {
'BUILD': libdir,
'PKGDIR': pkgdir
......@@ -125,46 +125,46 @@ def compile(path, srcpath, pkg, args):
subprocess.run([
'bash', '-c', post], env=env, check=True)
with open(os.path.join(srcpath, '.built'), 'wb') as fd:
with open(srcpath / '.built', 'wb') as fd:
fd.write(b'\n')
def package_files(buildpath, srcpath, pkg, args):
if os.path.isfile(os.path.join(buildpath, '.packaged')):
if (buildpath / '.pacakaged').is_file():
return
name = pkg['package']['name']
libdir = get_libdir(srcpath, args)
subprocess.run([
'python2',
os.path.join(os.environ['EMSCRIPTEN'], 'tools', 'file_packager.py'),
os.path.join(buildpath, name + '.data'),
Path(os.environ['EMSCRIPTEN']) / 'tools' / 'file_packager.py',
buildpath / (name + '.data'),
'--preload',
'{}@/lib/python3.6/site-packages'.format(libdir),
'--js-output={}'.format(os.path.join(buildpath, name + '.js')),
'--js-output={}'.format(buildpath / (name + '.js')),
'--export-name=pyodide',
'--exclude', '*.wasm.pre',
'--exclude', '__pycache__',
'--use-preload-plugins'], check=True)
subprocess.run([
'uglifyjs',
os.path.join(buildpath, name + '.js'),
buildpath / (name + '.js'),
'-o',
os.path.join(buildpath, name + '.js')], check=True)
buildpath / (name + '.js')], check=True)
with open(os.path.join(buildpath, '.packaged'), 'wb') as fd:
with open(buildpath / '.packaged', 'wb') as fd:
fd.write(b'\n')
def build_package(path, args):
pkg = common.parse_package(path)
packagedir = pkg['package']['name'] + '-' + pkg['package']['version']
dirpath = os.path.dirname(path)
orig_path = os.getcwd()
dirpath = path.parent
orig_path = Path.cwd()
os.chdir(dirpath)
try:
buildpath = os.path.join(dirpath, 'build')
if not os.path.exists(buildpath):
buildpath = dirpath / 'build'
if not buildpath.is_dir():
os.makedirs(buildpath)
srcpath = download_and_extract(buildpath, packagedir, pkg, args)
patch(path, srcpath, pkg, args)
......@@ -195,7 +195,7 @@ def parse_args():
def main(args):
path = os.path.abspath(args.package[0])
path = Path(args.package[0]).resolve()
build_package(path, args)
......
tools/common.py
View file @ 2d0b9ee2
import os
from pathlib import Path
ROOTDIR = os.path.abspath(os.path.dirname(__file__))
HOSTPYTHON = os.path.abspath(
os.path.join(ROOTDIR, '..', 'cpython', 'build', '3.6.4', 'host'))
TARGETPYTHON = os.path.abspath(
os.path.join(ROOTDIR, '..', 'cpython', 'installs', 'python-3.6.4'))
ROOTDIR = Path(__file__).parent.resolve()
HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host'
TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs', 'python-3.6.4'
DEFAULTCFLAGS = ''
DEFAULTLDFLAGS = ' '.join([
'-O3',
......
tools/pywasmcross
View file @ 2d0b9ee2
......@@ -28,6 +28,7 @@ import argparse
import importlib.machinery
import json
import os
from pathlib import Path
import re
import subprocess
import sys
......@@ -36,7 +37,7 @@ import sys
import common
ROOTDIR = os.path.abspath(os.path.dirname(__file__))
ROOTDIR = Path(__file__).parent.resolve()
symlinks = set(['cc', 'c++', 'ld', 'ar', 'gcc'])
......@@ -52,8 +53,8 @@ def collect_args(basename):
# native compiler
env = dict(os.environ)
path = env['PATH']
while ROOTDIR + ':' in path:
path = path.replace(ROOTDIR + ':', '')
while str(ROOTDIR) + ':' in path:
path = path.replace(str(ROOTDIR) + ':', '')
env['PATH'] = path
with open('build.log', 'a') as fd:
......@@ -71,11 +72,11 @@ def make_symlinks(env):
Makes sure all of the symlinks that make this script look like a compiler
exist.
"""
exec_path = os.path.abspath(__file__)
exec_path = Path(__file__).resolve()
for symlink in symlinks:
symlink_path = os.path.join(ROOTDIR, symlink)
if not os.path.exists(symlink_path):
os.symlink(exec_path, symlink_path)
symlink_path = ROOTDIR / symlink
if not symlink_path.exists():
symlink_path.symlink_to(exec_path)
if symlink == 'c++':
var = 'CXX'
else:
......@@ -86,15 +87,16 @@ def make_symlinks(env):
def capture_compile(args):
env = dict(os.environ)
make_symlinks(env)
env['PATH'] = ROOTDIR + ':' + os.environ['PATH']
env['PATH'] = str(ROOTDIR) + ':' + os.environ['PATH']
result = subprocess.run(
[os.path.join(args.host, 'bin', 'python3'),
[Path(args.host) / 'bin' / 'python3',
'setup.py',
'install'], env=env)
if result.returncode != 0:
if os.path.exists('build.log'):
os.remove('build.log')
build_log_path = Path('build.log')
if build_log_path.exists():
build_log_path.unlink()
sys.exit(result.returncode)
......@@ -131,7 +133,7 @@ def handle_command(line, args):
# Don't include any system directories
if arg[2:].startswith('/usr'):
continue
if (os.path.abspath(arg[2:]).startswith(args.host) and
if (str(Path(arg[2:]).resolve()).startswith(args.host) and
'site-packages' not in arg):
arg = arg.replace('-I' + args.host, '-I' + args.target)
# Don't include any system directories
......@@ -168,8 +170,9 @@ def handle_command(line, args):
def replay_compile(args):
# If pure Python, there will be no build.log file, which is fine -- just do
# nothing
if os.path.isfile('build.log'):
with open('build.log', 'r') as fd:
build_log_path = Path('build.log')
if build_log_path.is_file():
with open(build_log_path, 'r') as fd:
for line in fd:
line = json.loads(line)
handle_command(line, args)
......@@ -178,14 +181,14 @@ def replay_compile(args):
def clean_out_native_artifacts():
for root, dirs, files in os.walk('.'):
for file in files:
path = os.path.join(root, file)
basename, ext = os.path.splitext(file)
if ext in ('.o', '.so', '.a'):
os.remove(path)
path = Path(root) / file
if path.suffix in ('.o', '.so', '.a'):
path.unlink()
def build_wrap(args):
if not os.path.isfile('build.log'):
build_log_path = Path('build.log')
if not build_log_path.is_file():
capture_compile(args)
clean_out_native_artifacts()
replay_compile(args)
......@@ -212,7 +215,7 @@ def parse_args():
if __name__ == '__main__':
basename = os.path.basename(sys.argv[0])
basename = Path(sys.argv[0]).name
if basename in symlinks:
collect_args(basename)
else:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7