Skip to content

P
pyodide
Commit 3e1d6348 authored by Roman Yurchak's avatar Roman Yurchak
Browse files
Options

Statically link BLAS/LAPACK

parent 94829842
Hide whitespace changes
Inline Side-by-side
Showing with 147 additions and 295 deletions
.circleci/config.yml
View file @ 3e1d6348
......@@ -23,7 +23,7 @@ jobs:
sudo apt-get install gfortran f2c
# Download BLAS/LAPACK
git clone https://github.com/adrianbg/CLAPACK-WA.git packages/scipy/CLAPACK-WA
git clone https://github.com/rth/CLAPACK-WA.git packages/scipy/CLAPACK-WA
- restore_cache:
keys:
......
Makefile
View file @ 3e1d6348
......@@ -15,6 +15,7 @@ OPTFLAGS=-O3
CFLAGS=$(OPTFLAGS) -g -I$(PYTHONINCLUDE) -Wno-warn-absolute-paths
CXXFLAGS=$(CFLAGS) -std=c++14
# __ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv is in
# EXPORTED_FUNCTIONS to keep the C++ standard library in the core, even though
# there isn't any C++ there, for the sake of loading dynamic modules written in
......@@ -134,6 +135,7 @@ clean:
rm -fr src/*.bc
make -C packages clean
make -C six clean
make -C packages/scipy/CLAPACK-WA cleanall
echo "The Emsdk and CPython are not cleaned. cd into those directories to do so."
......@@ -208,8 +210,14 @@ $(LZ4LIB):
$(SIX_LIBS): $(CPYTHONLIB)
make -C six
$(LAPACK): $(CPYTHONLIB)
emmake make -C packages/scipy/CLAPACK-WA
clapack: $(CPYTHONLIB)
# We build BLAS/LAPACK only for target.
# On host we include -LCLAPACK-WA path which has no effect on host.
# On target it gets rewritten by pywasmcross to the full patch of
# blas_WA.bc, lapack_WA.bc which are linked statically in scipy
# in each module that needs them.
make -C $(LAPACK_DIR)F2CLIBS/libf2c/ arith.h
emmake make -C $(LAPACK_DIR)
$(CLAPACK): $(CPYTHONLIB)
make -C CLAPACK
......@@ -218,6 +226,5 @@ $(CLAPACK): $(CPYTHONLIB)
build/packages.json: $(CPYTHONLIB) $(CLAPACK)
make -C packages
emsdk/emsdk/.complete:
make -C emsdk
packages/scipy/meta.yaml
View file @ 3e1d6348
......@@ -15,13 +15,12 @@ source:
# these patches can be found as commits in
# https://github.com/rth/scipy/tree/0.17.1-pyodide
# on top of the v0.17.1 tag
- patches/disable_modules_with_blas.patch
- patches/fix-blas.patch
- patches/fix-build-gcc5-a80460.patch
- patches/force_malloc.patch
- patches/disable_scipy_stats_mvn.patch
- patches/skip-fortran-fails-to-link.patch
- patches/dummy_threading.patch
- patches/skip-blas-imports.patch
- patches/skip_ellip_harm_2_pyx_ctypes.patch
build:
......
packages/scipy/patches/disable_modules_with_blas.patch deleted 100644 → 0
View file @ 94829842
commit 771537885db80fb664d5671af5048e445b6fa821
Author: Roman Yurchak <rth.yurchak@pm.me>
Date: Mon Oct 8 10:53:11 2018 +0200
Removes files that require LAPACK
diff --git a/scipy/linalg/setup.py b/scipy/linalg/setup.py
index 2c9b9ba22..035b4bb80 100755
--- a/scipy/linalg/setup.py
+++ b/scipy/linalg/setup.py
@@ -16,8 +16,8 @@ def configuration(parent_package='', top_path=None):
lapack_opt = get_info('lapack_opt')
- if not lapack_opt:
- raise NotFoundError('no lapack/blas resources found')
+ # if not lapack_opt:
+ # raise NotFoundError('no lapack/blas resources found')
atlas_version = ([v[3:-3] for k, v in lapack_opt.get('define_macros', [])
if k == 'ATLAS_INFO']+[None])[0]
@@ -29,45 +29,45 @@ def configuration(parent_package='', top_path=None):
sources += get_g77_abi_wrappers(lapack_opt)
sources += get_sgemv_fix(lapack_opt)
- config.add_extension('_fblas',
- sources=sources,
- depends=['fblas_l?.pyf.src'],
- extra_info=lapack_opt
- )
+ #config.add_extension('_fblas',
+ # sources=sources,
+ # depends=['fblas_l?.pyf.src'],
+ # extra_info=lapack_opt
+ # )
# flapack:
- sources = ['flapack.pyf.src']
- sources += get_g77_abi_wrappers(lapack_opt)
- dep_pfx = join('src', 'lapack_deprecations')
- deprecated_lapack_routines = [join(dep_pfx, c + 'gegv.f') for c in 'cdsz']
- sources += deprecated_lapack_routines
-
- config.add_extension('_flapack',
- sources=sources,
- depends=['flapack_user.pyf.src'],
- extra_info=lapack_opt
- )
-
- if atlas_version is not None:
- # cblas:
- config.add_extension('_cblas',
- sources=['cblas.pyf.src'],
- depends=['cblas.pyf.src', 'cblas_l1.pyf.src'],
- extra_info=lapack_opt
- )
-
- # clapack:
- config.add_extension('_clapack',
- sources=['clapack.pyf.src'],
- depends=['clapack.pyf.src'],
- extra_info=lapack_opt
- )
-
- # _flinalg:
- config.add_extension('_flinalg',
- sources=[join('src', 'det.f'), join('src', 'lu.f')],
- extra_info=lapack_opt
- )
+ #sources = ['flapack.pyf.src']
+ #sources += get_g77_abi_wrappers(lapack_opt)
+ #dep_pfx = join('src', 'lapack_deprecations')
+ #deprecated_lapack_routines = [join(dep_pfx, c + 'gegv.f') for c in 'cdsz']
+ #sources += deprecated_lapack_routines
+
+ #config.add_extension('_flapack',
+ # sources=sources,
+ # depends=['flapack_user.pyf.src'],
+ # extra_info=lapack_opt
+ # )
+
+ #if atlas_version is not None:
+ # # cblas:
+ # config.add_extension('_cblas',
+ # sources=['cblas.pyf.src'],
+ # depends=['cblas.pyf.src', 'cblas_l1.pyf.src'],
+ # extra_info=lapack_opt
+ # )
+
+ # # clapack:
+ # config.add_extension('_clapack',
+ # sources=['clapack.pyf.src'],
+ # depends=['clapack.pyf.src'],
+ # extra_info=lapack_opt
+ # )
+
+ ## _flinalg:
+ #config.add_extension('_flinalg',
+ # sources=[join('src', 'det.f'), join('src', 'lu.f')],
+ # extra_info=lapack_opt
+ # )
# _interpolative:
routines_to_split = [
@@ -116,15 +116,15 @@ def configuration(parent_package='', top_path=None):
dirname = os.path.split(os.path.abspath(__file__))[0]
fnames = split_fortran_files(join(dirname, 'src', 'id_dist', 'src'),
routines_to_split)
- fnames = [join('src', 'id_dist', 'src', f) for f in fnames]
- config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
- extra_info=lapack_opt
- )
+ #fnames = [join('src', 'id_dist', 'src', f) for f in fnames]
+ #config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
+ # extra_info=lapack_opt
+ # )
# _calc_lwork:
- config.add_extension('_calc_lwork',
- [join('src', 'calc_lwork.f')],
- extra_info=lapack_opt)
+ #config.add_extension('_calc_lwork',
+ # [join('src', 'calc_lwork.f')],
+ # extra_info=lapack_opt)
# _solve_toeplitz:
config.add_extension('_solve_toeplitz',
@@ -137,27 +137,27 @@ def configuration(parent_package='', top_path=None):
config.add_data_files('cython_blas.pxd')
config.add_data_files('cython_lapack.pxd')
- sources = ['_blas_subroutine_wrappers.f', '_lapack_subroutine_wrappers.f']
- sources += get_g77_abi_wrappers(lapack_opt)
- sources += get_sgemv_fix(lapack_opt)
- includes = numpy_info().get_include_dirs() + [get_python_inc()]
- config.add_library('fwrappers', sources=sources, include_dirs=includes)
-
- config.add_extension('cython_blas',
- sources=['cython_blas.c'],
- depends=['cython_blas.pyx', 'cython_blas.pxd',
- 'fortran_defs.h', '_blas_subroutines.h'],
- include_dirs=['.'],
- libraries=['fwrappers'],
- extra_info=lapack_opt)
-
- config.add_extension('cython_lapack',
- sources=['cython_lapack.c'],
- depends=['cython_lapack.pyx', 'cython_lapack.pxd',
- 'fortran_defs.h', '_lapack_subroutines.h'],
- include_dirs=['.'],
- libraries=['fwrappers'],
- extra_info=lapack_opt)
+ #sources = ['_blas_subroutine_wrappers.f', '_lapack_subroutine_wrappers.f']
+ #sources += get_g77_abi_wrappers(lapack_opt)
+ #sources += get_sgemv_fix(lapack_opt)
+ #includes = numpy_info().get_include_dirs() + [get_python_inc()]
+ #config.add_library('fwrappers', sources=sources, include_dirs=includes)
+
+ #config.add_extension('cython_blas',
+ # sources=['cython_blas.c'],
+ # depends=['cython_blas.pyx', 'cython_blas.pxd',
+ # 'fortran_defs.h', '_blas_subroutines.h'],
+ # include_dirs=['.'],
+ # libraries=['fwrappers'],
+ # extra_info=lapack_opt)
+
+ #config.add_extension('cython_lapack',
+ # sources=['cython_lapack.c'],
+ # depends=['cython_lapack.pyx', 'cython_lapack.pxd',
+ # 'fortran_defs.h', '_lapack_subroutines.h'],
+ # include_dirs=['.'],
+ # libraries=['fwrappers'],
+ # extra_info=lapack_opt)
config.add_extension('_decomp_update',
sources=['_decomp_update.c'])
diff --git a/scipy/sparse/linalg/eigen/arpack/setup.py b/scipy/sparse/linalg/eigen/arpack/setup.py
index a8175a9d5..c63b76dda 100755
--- a/scipy/sparse/linalg/eigen/arpack/setup.py
+++ b/scipy/sparse/linalg/eigen/arpack/setup.py
@@ -13,28 +13,28 @@ def configuration(parent_package='',top_path=None):
lapack_opt = get_info('lapack_opt')
- if not lapack_opt:
- raise NotFoundError('no lapack/blas resources found')
+ #if not lapack_opt:
+ # raise NotFoundError('no lapack/blas resources found')
config = Configuration('arpack', parent_package, top_path)
- arpack_sources = [join('ARPACK','SRC', '*.f')]
- arpack_sources.extend([join('ARPACK','UTIL', '*.f')])
- arpack_sources.extend([join('ARPACK','LAPACK', '*.f')])
+ #arpack_sources = [join('ARPACK','SRC', '*.f')]
+ #arpack_sources.extend([join('ARPACK','UTIL', '*.f')])
+ #arpack_sources.extend([join('ARPACK','LAPACK', '*.f')])
- arpack_sources += get_g77_abi_wrappers(lapack_opt)
+ #arpack_sources += get_g77_abi_wrappers(lapack_opt)
- config.add_library('arpack_scipy', sources=arpack_sources,
- include_dirs=[join('ARPACK', 'SRC')])
+ #config.add_library('arpack_scipy', sources=arpack_sources,
+ # include_dirs=[join('ARPACK', 'SRC')])
- ext_sources = ['arpack.pyf.src']
- ext_sources += get_sgemv_fix(lapack_opt)
- config.add_extension('_arpack',
- sources=ext_sources,
- libraries=['arpack_scipy'],
- extra_info=lapack_opt,
- depends=arpack_sources,
- )
+ #ext_sources = ['arpack.pyf.src']
+ #ext_sources += get_sgemv_fix(lapack_opt)
+ #config.add_extension('_arpack',
+ # sources=ext_sources,
+ # libraries=['arpack_scipy'],
+ # extra_info=lapack_opt,
+ # depends=arpack_sources,
+ # )
config.add_data_dir('tests')
return config
diff --git a/scipy/sparse/linalg/isolve/setup.py b/scipy/sparse/linalg/isolve/setup.py
index becb9237a..17288207e 100755
--- a/scipy/sparse/linalg/isolve/setup.py
+++ b/scipy/sparse/linalg/isolve/setup.py
@@ -13,29 +13,29 @@ def configuration(parent_package='',top_path=None):
lapack_opt = get_info('lapack_opt')
- if not lapack_opt:
- raise NotFoundError('no lapack/blas resources found')
-
- # iterative methods
- methods = ['BiCGREVCOM.f.src',
- 'BiCGSTABREVCOM.f.src',
- 'CGREVCOM.f.src',
- 'CGSREVCOM.f.src',
-# 'ChebyREVCOM.f.src',
- 'GMRESREVCOM.f.src',
-# 'JacobiREVCOM.f.src',
- 'QMRREVCOM.f.src',
-# 'SORREVCOM.f.src'
- ]
-
- Util = ['STOPTEST2.f.src','getbreak.f.src']
- sources = Util + methods + ['_iterative.pyf.src']
- sources = [join('iterative', x) for x in sources]
- sources += get_g77_abi_wrappers(lapack_opt)
-
- config.add_extension('_iterative',
- sources=sources,
- extra_info=lapack_opt)
+ #if not lapack_opt:
+ # raise NotFoundError('no lapack/blas resources found')
+
+ ## iterative methods
+ #methods = ['BiCGREVCOM.f.src',
+ # 'BiCGSTABREVCOM.f.src',
+ # 'CGREVCOM.f.src',
+ # 'CGSREVCOM.f.src',
+# # 'ChebyREVCOM.f.src',
+ # 'GMRESREVCOM.f.src',
+# # 'JacobiREVCOM.f.src',
+ # 'QMRREVCOM.f.src',
+# # 'SORREVCOM.f.src'
+ # ]
+
+ #Util = ['STOPTEST2.f.src','getbreak.f.src']
+ #sources = Util + methods + ['_iterative.pyf.src']
+ #sources = [join('iterative', x) for x in sources]
+ #sources += get_g77_abi_wrappers(lapack_opt)
+
+ #config.add_extension('_iterative',
+ # sources=sources,
+ # extra_info=lapack_opt)
config.add_data_dir('tests')
packages/scipy/patches/fix-blas.patch 0 → 100644
View file @ 3e1d6348
commit 926f2e3cf302f2e04717675bb8e6cb77c1c9ee3d
Author: Roman Yurchak <rth.yurchak@pm.me>
Date: Mon Oct 8 10:53:11 2018 +0200
Partial fixes for BLAS/LAPACK
diff --git a/scipy/linalg/setup.py b/scipy/linalg/setup.py
index 2c9b9ba22..f52f775d6 100755
--- a/scipy/linalg/setup.py
+++ b/scipy/linalg/setup.py
@@ -14,7 +14,14 @@ def configuration(parent_package='', top_path=None):
config = Configuration('linalg', parent_package, top_path)
- lapack_opt = get_info('lapack_opt')
+ # lapack_opt = get_info('lapack_opt')
+ lapack_opt = { # libraries will be auto-generated by pywasmcross
+ 'libraries': [],
+ 'include_dirs': [],
+ 'library_dirs': ['../../CLAPACK-WA/'],
+ 'language': 'c',
+ 'define_macros': [('NO_ATLAS_INFO', 1),
+ ('HAVE_CBLAS', None)]}
if not lapack_opt:
raise NotFoundError('no lapack/blas resources found')
@@ -117,9 +124,22 @@ def configuration(parent_package='', top_path=None):
fnames = split_fortran_files(join(dirname, 'src', 'id_dist', 'src'),
routines_to_split)
fnames = [join('src', 'id_dist', 'src', f) for f in fnames]
- config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
- extra_info=lapack_opt
- )
+ # TODO: The following fails with,
+ # scipy/linalg/src/id_dist/src/idd_sfft.c:114:22: error: conflicting types for 'idd_sffti1__'
+ # /* Subroutine */ int idd_sffti1__(integer *ind, integer *n, doublereal *wsave)
+ # ^
+ # scipy/linalg/src/id_dist/src/idd_sfft.c:72:33: note: previous declaration is here
+ # extern /* Subroutine */ int idd_sffti1__(integer *, integer *,
+ # ^
+ # scipy/linalg/src/id_dist/src/idd_sfft.c:371:22: error: conflicting types for 'idd_sfft1__'
+ # /* Subroutine */ int idd_sfft1__(integer *ind, integer *n, doublereal *v,
+ # ^
+ # scipy/linalg/src/id_dist/src/idd_sfft.c:311:33: note: previous declaration is here
+ # extern /* Subroutine */ int idd_sfft1__(integer *, integer *, doublereal *<Paste>
+ #
+ #config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
+ # extra_info=lapack_opt
+ # )
# _calc_lwork:
config.add_extension('_calc_lwork',
diff --git a/scipy/sparse/linalg/eigen/arpack/setup.py b/scipy/sparse/linalg/eigen/arpack/setup.py
index a8175a9d5..f7af2f0d9 100755
--- a/scipy/sparse/linalg/eigen/arpack/setup.py
+++ b/scipy/sparse/linalg/eigen/arpack/setup.py
@@ -11,7 +11,15 @@ def configuration(parent_package='',top_path=None):
config = Configuration('arpack',parent_package,top_path)
- lapack_opt = get_info('lapack_opt')
+ # lapack_opt = get_info('lapack_opt')
+
+ lapack_opt = { # libraries will be auto-generated by pywasmcross
+ 'libraries': [],
+ 'include_dirs': [],
+ 'library_dirs': ['../../CLAPACK-WA/'],
+ 'language': 'c',
+ 'define_macros': [('NO_ATLAS_INFO', 1),
+ ('HAVE_CBLAS', None)]}
if not lapack_opt:
raise NotFoundError('no lapack/blas resources found')
diff --git a/scipy/sparse/linalg/isolve/setup.py b/scipy/sparse/linalg/isolve/setup.py
index becb9237a..971e85b6b 100755
--- a/scipy/sparse/linalg/isolve/setup.py
+++ b/scipy/sparse/linalg/isolve/setup.py
@@ -11,7 +11,14 @@ def configuration(parent_package='',top_path=None):
config = Configuration('isolve',parent_package,top_path)
- lapack_opt = get_info('lapack_opt')
+ # lapack_opt = get_info('lapack_opt')
+ lapack_opt = { # libraries will be auto-generated by pywasmcross
+ 'libraries': [],
+ 'include_dirs': [],
+ 'library_dirs': ['../../CLAPACK-WA/'],
+ 'language': 'c',
+ 'define_macros': [('NO_ATLAS_INFO', 1),
+ ('HAVE_CBLAS', None)]}
if not lapack_opt:
raise NotFoundError('no lapack/blas resources found')
packages/scipy/patches/fix_mmap.patch 0 → 100644
View file @ 3e1d6348
commit 6d4c53df9fecd236cb569b5f1997908af1daea6b
Author: Roman Yurchak <rth.yurchak@pm.me>
Date: Thu Oct 25 15:28:05 2018 +0200
Fix mmap imports
diff --git a/scipy/io/netcdf.py b/scipy/io/netcdf.py
index 841e97830..f06759015 100644
--- a/scipy/io/netcdf.py
+++ b/scipy/io/netcdf.py
@@ -39,7 +39,6 @@ __all__ = ['netcdf_file']
import warnings
import weakref
from operator import mul
-import mmap as mm
import numpy as np
from numpy.compat import asbytes, asstr
@@ -219,6 +218,8 @@ class netcdf_file(object):
def __init__(self, filename, mode='r', mmap=None, version=1,
maskandscale=False):
"""Initialize netcdf_file from fileobj (str or file-like)."""
+ import mmap as mm
+
if mode not in 'rwa':
raise ValueError("Mode must be either 'r', 'w' or 'a'.")
pyodide_build/pywasmcross.py
View file @ 3e1d6348
......@@ -236,6 +236,8 @@ def handle_command(line, args, dryrun=False):
elif new_args[0] in ('emcc', 'em++'):
new_args.extend(args.cflags.split())
lapack_dir = None
# Go through and adjust arguments
for arg in line[1:]:
if arg.startswith('-I'):
......@@ -257,6 +259,14 @@ def handle_command(line, args, dryrun=False):
elif shared and arg.endswith('.so'):
arg = arg[:-3] + '.wasm'
output = arg
# Fix for scipy to link to the correct BLAS/LAPACK files
if arg.startswith('-L') and 'CLAPACK-WA' in arg:
lapack_dir = arg.replace('-L', '')
for lib_name in ['F2CLIBS/libf2c.bc',
'blas_WA.bc', 'lapack_WA.bc']:
arg = os.path.join(lapack_dir, f"{lib_name}")
new_args.append(arg)
continue
new_args.append(arg)
......
test/packages/test_scipy.py
View file @ 3e1d6348
......@@ -23,17 +23,15 @@ def test_scipy_import(selenium_standalone, request):
""")
# supported modules
for module in ['constants', 'fftpack', 'odr', 'sparse',
'linalg', # heavily patched
'misc', 'ndimage', 'special'
for module in ['cluster', 'constants', 'fftpack', 'odr', 'sparse',
'interpolate',
'linalg',
'misc', 'ndimage', 'spatial', 'special'
]:
selenium.run(f"import scipy.{module}")
# not yet built modules
for module in ['cluster', # needs sparse
'spatial', # needs sparse
'integrate', # needs special
'interpolate', # needs linalg
for module in ['integrate', # needs special
'signal', # needs special
'stats', # need special
'optimize', # needs _odepack
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7