Statically link BLAS/LAPACK

.circleci/config.yml

@@ -23,7 +23,7 @@ jobs:
              sudo apt-get install gfortran f2c
 
              # Download BLAS/LAPACK
-             git clone https://github.com/adrianbg/CLAPACK-WA.git packages/scipy/CLAPACK-WA
+             git clone https://github.com/rth/CLAPACK-WA.git packages/scipy/CLAPACK-WA
 
       - restore_cache:
           keys:

Makefile

@@ -15,6 +15,7 @@ OPTFLAGS=-O3
 CFLAGS=$(OPTFLAGS) -g -I$(PYTHONINCLUDE) -Wno-warn-absolute-paths
 CXXFLAGS=$(CFLAGS) -std=c++14
 
+
 # __ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv is in
 # EXPORTED_FUNCTIONS to keep the C++ standard library in the core, even though
 # there isn't any C++ there, for the sake of loading dynamic modules written in
@@ -134,6 +135,7 @@ clean:
 	rm -fr src/*.bc
 	make -C packages clean
 	make -C six clean
+	make -C packages/scipy/CLAPACK-WA cleanall
 	echo "The Emsdk and CPython are not cleaned. cd into those directories to do so."
 
 
@@ -208,8 +210,14 @@ $(LZ4LIB):
 $(SIX_LIBS): $(CPYTHONLIB)
 	make -C six
 
-$(LAPACK): $(CPYTHONLIB)
-	emmake make -C packages/scipy/CLAPACK-WA
+clapack: $(CPYTHONLIB)
+	# We build BLAS/LAPACK only for target.
+	# On host we include -LCLAPACK-WA path which has no effect on host.
+	# On target it gets rewritten by pywasmcross to the full patch of
+	# blas_WA.bc, lapack_WA.bc which are linked statically in scipy
+	# in each module that needs them.
+	make -C $(LAPACK_DIR)F2CLIBS/libf2c/ arith.h
+	emmake make -C $(LAPACK_DIR)
 
 $(CLAPACK): $(CPYTHONLIB)
 	make -C CLAPACK
@@ -218,6 +226,5 @@ $(CLAPACK): $(CPYTHONLIB)
 build/packages.json: $(CPYTHONLIB) $(CLAPACK)
 	make -C packages
 
-
 emsdk/emsdk/.complete:
 	make -C emsdk

packages/scipy/meta.yaml

@@ -15,13 +15,12 @@ source:
     # these patches can be found as commits in
     #    https://github.com/rth/scipy/tree/0.17.1-pyodide
     # on top of the v0.17.1 tag
-    - patches/disable_modules_with_blas.patch
+    - patches/fix-blas.patch
     - patches/fix-build-gcc5-a80460.patch
     - patches/force_malloc.patch
     - patches/disable_scipy_stats_mvn.patch
     - patches/skip-fortran-fails-to-link.patch
     - patches/dummy_threading.patch
-    - patches/skip-blas-imports.patch
     - patches/skip_ellip_harm_2_pyx_ctypes.patch
 
 build:

packages/scipy/patches/fix-blas.patch

+commit 926f2e3cf302f2e04717675bb8e6cb77c1c9ee3d
+Author: Roman Yurchak <rth.yurchak@pm.me>
+Date:   Mon Oct 8 10:53:11 2018 +0200
+
+    Partial fixes for BLAS/LAPACK
+
+diff --git a/scipy/linalg/setup.py b/scipy/linalg/setup.py
+index 2c9b9ba22..f52f775d6 100755
+--- a/scipy/linalg/setup.py
++++ b/scipy/linalg/setup.py
+@@ -14,7 +14,14 @@ def configuration(parent_package='', top_path=None):
+ 
+     config = Configuration('linalg', parent_package, top_path)
+ 
+-    lapack_opt = get_info('lapack_opt')
++    # lapack_opt = get_info('lapack_opt')
++    lapack_opt = { # libraries will be auto-generated by pywasmcross
++                  'libraries': [],
++                  'include_dirs': [],
++                  'library_dirs': ['../../CLAPACK-WA/'],
++                  'language': 'c',
++                  'define_macros': [('NO_ATLAS_INFO', 1),
++                                    ('HAVE_CBLAS', None)]}
+ 
+     if not lapack_opt:
+         raise NotFoundError('no lapack/blas resources found')
+@@ -117,9 +124,22 @@ def configuration(parent_package='', top_path=None):
+     fnames = split_fortran_files(join(dirname, 'src', 'id_dist', 'src'),
+                                  routines_to_split)
+     fnames = [join('src', 'id_dist', 'src', f) for f in fnames]
+-    config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
+-                         extra_info=lapack_opt
+-                         )
++    # TODO: The following fails with,
++    # scipy/linalg/src/id_dist/src/idd_sfft.c:114:22: error: conflicting types for 'idd_sffti1__'
++    # /* Subroutine */ int idd_sffti1__(integer *ind, integer *n, doublereal *wsave)
++    #                      ^
++    # scipy/linalg/src/id_dist/src/idd_sfft.c:72:33: note: previous declaration is here
++    #     extern /* Subroutine */ int idd_sffti1__(integer *, integer *, 
++    #                                 ^
++    # scipy/linalg/src/id_dist/src/idd_sfft.c:371:22: error: conflicting types for 'idd_sfft1__'
++    # /* Subroutine */ int idd_sfft1__(integer *ind, integer *n, doublereal *v, 
++    #                      ^
++    # scipy/linalg/src/id_dist/src/idd_sfft.c:311:33: note: previous declaration is here
++    #    extern /* Subroutine */ int idd_sfft1__(integer *, integer *, doublereal *<Paste>
++    #
++    #config.add_extension('_interpolative', fnames + ["interpolative.pyf"],
++    #                     extra_info=lapack_opt
++    #                     )
+ 
+     # _calc_lwork:
+     config.add_extension('_calc_lwork',
+diff --git a/scipy/sparse/linalg/eigen/arpack/setup.py b/scipy/sparse/linalg/eigen/arpack/setup.py
+index a8175a9d5..f7af2f0d9 100755
+--- a/scipy/sparse/linalg/eigen/arpack/setup.py
++++ b/scipy/sparse/linalg/eigen/arpack/setup.py
+@@ -11,7 +11,15 @@ def configuration(parent_package='',top_path=None):
+ 
+     config = Configuration('arpack',parent_package,top_path)
+ 
+-    lapack_opt = get_info('lapack_opt')
++    # lapack_opt = get_info('lapack_opt')
++
++    lapack_opt = { # libraries will be auto-generated by pywasmcross
++                  'libraries': [],
++                  'include_dirs': [],
++                  'library_dirs': ['../../CLAPACK-WA/'],
++                  'language': 'c',
++                  'define_macros': [('NO_ATLAS_INFO', 1),
++                                    ('HAVE_CBLAS', None)]}
+ 
+     if not lapack_opt:
+         raise NotFoundError('no lapack/blas resources found')
+diff --git a/scipy/sparse/linalg/isolve/setup.py b/scipy/sparse/linalg/isolve/setup.py
+index becb9237a..971e85b6b 100755
+--- a/scipy/sparse/linalg/isolve/setup.py
++++ b/scipy/sparse/linalg/isolve/setup.py
+@@ -11,7 +11,14 @@ def configuration(parent_package='',top_path=None):
+ 
+     config = Configuration('isolve',parent_package,top_path)
+ 
+-    lapack_opt = get_info('lapack_opt')
++    # lapack_opt = get_info('lapack_opt')
++    lapack_opt = { # libraries will be auto-generated by pywasmcross
++                  'libraries': [],
++                  'include_dirs': [],
++                  'library_dirs': ['../../CLAPACK-WA/'],
++                  'language': 'c',
++                  'define_macros': [('NO_ATLAS_INFO', 1),
++                                    ('HAVE_CBLAS', None)]}
+ 
+     if not lapack_opt:
+         raise NotFoundError('no lapack/blas resources found')

packages/scipy/patches/fix_mmap.patch

+commit 6d4c53df9fecd236cb569b5f1997908af1daea6b
+Author: Roman Yurchak <rth.yurchak@pm.me>
+Date:   Thu Oct 25 15:28:05 2018 +0200
+
+    Fix mmap imports
+
+diff --git a/scipy/io/netcdf.py b/scipy/io/netcdf.py
+index 841e97830..f06759015 100644
+--- a/scipy/io/netcdf.py
++++ b/scipy/io/netcdf.py
+@@ -39,7 +39,6 @@ __all__ = ['netcdf_file']
+ import warnings
+ import weakref
+ from operator import mul
+-import mmap as mm
+ 
+ import numpy as np
+ from numpy.compat import asbytes, asstr
+@@ -219,6 +218,8 @@ class netcdf_file(object):
+     def __init__(self, filename, mode='r', mmap=None, version=1,
+                  maskandscale=False):
+         """Initialize netcdf_file from fileobj (str or file-like)."""
++        import mmap as mm
++
+         if mode not in 'rwa':
+             raise ValueError("Mode must be either 'r', 'w' or 'a'.")
+ 

pyodide_build/pywasmcross.py

@@ -236,6 +236,8 @@ def handle_command(line, args, dryrun=False):
     elif new_args[0] in ('emcc', 'em++'):
         new_args.extend(args.cflags.split())
 
+    lapack_dir = None
+
     # Go through and adjust arguments
     for arg in line[1:]:
         if arg.startswith('-I'):
@@ -257,6 +259,14 @@ def handle_command(line, args, dryrun=False):
         elif shared and arg.endswith('.so'):
             arg = arg[:-3] + '.wasm'
             output = arg
+        # Fix for scipy to link to the correct BLAS/LAPACK files
+        if arg.startswith('-L') and 'CLAPACK-WA' in arg:
+            lapack_dir = arg.replace('-L', '')
+            for lib_name in ['F2CLIBS/libf2c.bc',
+                             'blas_WA.bc', 'lapack_WA.bc']:
+                arg = os.path.join(lapack_dir, f"{lib_name}")
+                new_args.append(arg)
+            continue
 
         new_args.append(arg)
 

test/packages/test_scipy.py

@@ -23,17 +23,15 @@ def test_scipy_import(selenium_standalone, request):
         """)
 
     # supported modules
-    for module in ['constants', 'fftpack', 'odr', 'sparse',
-                   'linalg',  # heavily patched
-                   'misc', 'ndimage', 'special'
+    for module in ['cluster', 'constants', 'fftpack', 'odr', 'sparse',
+                   'interpolate',
+                   'linalg',
+                   'misc', 'ndimage', 'spatial', 'special'
                    ]:
         selenium.run(f"import scipy.{module}")
 
     # not yet built modules
-    for module in ['cluster',  # needs sparse
-                   'spatial',  # needs sparse
-                   'integrate',  # needs special
-                   'interpolate',  # needs linalg
+    for module in ['integrate',  # needs special
                    'signal',  # needs special
                    'stats',  # need special
                    'optimize',  # needs _odepack