Fix build in Docker

.circleci/config.yml

@@ -19,6 +19,8 @@ jobs:
       - run:
            name: dependencies
            command: |
+             sudo apt-get install gfortran f2c
+
              # Download BLAS/LAPACK
              git clone https://github.com/adrianbg/CLAPACK-WA.git packages/scipy/CLAPACK-WA
 

emsdk/patches/num_params.patch

@@ -7,7 +7,7 @@ index 013e9403..d95fc282 100644
  // to match when dynamically linking, and also dynamic linking is why we
  // can't just detect this automatically in the module we see.)
 -static const int NUM_PARAMS = 15;
-+static const int NUM_PARAMS = 37;
++static const int NUM_PARAMS = 45;
 
  // Converts a value to the ABI type of i64.
  static Expression* toABI(Expression* value, Module* module) {

packages/scipy/meta.yaml

@@ -20,10 +20,11 @@ source:
     - patches/fix-build-gcc5-a80460.patch
     - patches/force_malloc.patch
     - patches/disable_scipy_stats_mvn.patch
-    - patches/skip-scipy-special-_ufuncs.patch
+    - patches/skip-fortran-fails-to-link.patch
+    - patches/disable_FuncCastEmulation_num_params.patch
 
 build:
-  cflags: -I/home/rth/src/pyodide/packages/scipy/CLAPACK-WA/INCLUDE -Wno-implicit-function-declaration
+  cflags: -I../../CLAPACK-WA/INCLUDE -Wno-implicit-function-declaration
   cxxflags:
 
 requirements:

packages/scipy/patches/disable_FuncCastEmulation_num_params.patch

+commit dbfb06104aa7f3f36737f394b66a76a29018d5e7
+Author: Roman Yurchak <rth.yurchak@pm.me>
+Date:   Fri Oct 12 10:36:48 2018 +0200
+
+    Disable F2C function that need larger FuncCastEmulation::NUM_PARAMS
+    
+    scipy/interpolate/_fitpack seems to require NUM_PARAMS
+    43 then 45 then 47 (each requiring to rebuild emsdk), so disabling this
+    for now.
+    scipy/ord/_odrpack requires NUM_PARAMS 57
+
+diff --git a/scipy/interpolate/setup.py b/scipy/interpolate/setup.py
+index 2c8279403..9934376e8 100755
+--- a/scipy/interpolate/setup.py
++++ b/scipy/interpolate/setup.py
+@@ -22,18 +22,18 @@ def configuration(parent_package='',top_path=None):
+                          sources=['_ppoly.c'],
+                          **lapack_opt)
+ 
+-    config.add_extension('_fitpack',
+-                         sources=['src/_fitpackmodule.c'],
+-                         libraries=['fitpack'],
+-                         depends=(['src/__fitpack.h','src/multipack.h']
+-                                  + fitpack_src)
+-                         )
+-
+-    config.add_extension('dfitpack',
+-                         sources=['src/fitpack.pyf'],
+-                         libraries=['fitpack'],
+-                         depends=fitpack_src,
+-                         )
++    #config.add_extension('_fitpack',
++    #                     sources=['src/_fitpackmodule.c'],
++    #                     libraries=['fitpack'],
++    #                     depends=(['src/__fitpack.h','src/multipack.h']
++    #                              + fitpack_src)
++    #                     )
++
++    #config.add_extension('dfitpack',
++    #                     sources=['src/fitpack.pyf'],
++    #                     libraries=['fitpack'],
++    #                     depends=fitpack_src,
++    #                     )
+ 
+     #config.add_extension('_interpolate',
+     #                     sources=['src/_interpolate.cpp'],
+diff --git a/scipy/odr/setup.py b/scipy/odr/setup.py
+index 9974dfac6..d7560896f 100644
+--- a/scipy/odr/setup.py
++++ b/scipy/odr/setup.py
+@@ -22,18 +22,18 @@ def configuration(parent_package='', top_path=None):
+         libodr_files.append('d_lpkbls.f')
+ 
+     odrpack_src = [join('odrpack', x) for x in libodr_files]
+-    config.add_library('odrpack', sources=odrpack_src)
++    #config.add_library('odrpack', sources=odrpack_src)
+ 
+     sources = ['__odrpack.c']
+     libraries = ['odrpack'] + blas_info.pop('libraries', [])
+     include_dirs = ['.'] + blas_info.pop('include_dirs', [])
+-    config.add_extension('__odrpack',
+-        sources=sources,
+-        libraries=libraries,
+-        include_dirs=include_dirs,
+-        depends=(['odrpack.h'] + odrpack_src),
+-        **blas_info
+-    )
++    #config.add_extension('__odrpack',
++    #    sources=sources,
++    #    libraries=libraries,
++    #    include_dirs=include_dirs,
++    #    depends=(['odrpack.h'] + odrpack_src),
++    #    **blas_info
++    #)
+ 
+     config.add_data_dir('tests')
+     return config

packages/scipy/patches/force_malloc.patch

+commit d755dab2147732e9213af29f55ab38c38b7ba8d2
+Author: Roman Yurchak <rth.yurchak@pm.me>
+Date:   Mon Oct 8 12:13:55 2018 +0200
+
+    Force malloc
+    
+    This aims to avoid the "invalid call target: $_malloc" issues
+    
+      - fixes scipy/optimize/zeros.c
+      - skips scipy/integrate/dop.pyf
+      - skips scipy/optimize/cobyla
+
+diff --git a/scipy/integrate/setup.py b/scipy/integrate/setup.py
+index 2be454c70..0545dc759 100755
+--- a/scipy/integrate/setup.py
++++ b/scipy/integrate/setup.py
+@@ -67,10 +67,10 @@ def configuration(parent_package='',top_path=None):
+     #                     **lapack_opt)
+ 
+     # dop
+-    config.add_extension('_dop',
+-                         sources=['dop.pyf'],
+-                         libraries=['dop'],
+-                         depends=dop_src)
++    #config.add_extension('_dop',
++    #                     sources=['dop.pyf'],
++    #                     libraries=['dop'],
++    #                     depends=dop_src)
+ 
+     config.add_extension('_test_multivariate',
+                          sources=quadpack_test_src)
+diff --git a/scipy/optimize/__init__.py b/scipy/optimize/__init__.py
+index 17ba78371..5deca7498 100644
+--- a/scipy/optimize/__init__.py
++++ b/scipy/optimize/__init__.py
+@@ -236,7 +236,7 @@ from .minpack import *
+ from .zeros import *
+ from .lbfgsb import fmin_l_bfgs_b, LbfgsInvHessProduct
+ from .tnc import fmin_tnc
+-from .cobyla import fmin_cobyla
++#from .cobyla import fmin_cobyla
+ from .nonlin import *
+ from .slsqp import fmin_slsqp
+ from .nnls import nnls
+diff --git a/scipy/optimize/setup.py b/scipy/optimize/setup.py
+index 7d2b987cb..1861e7563 100755
+--- a/scipy/optimize/setup.py
++++ b/scipy/optimize/setup.py
+@@ -50,11 +50,11 @@ def configuration(parent_package='',top_path=None):
+                          depends=[join('tnc','tnc.h')],
+                          **numpy_nodepr_api)
+ 
+-    config.add_extension('_cobyla',
+-                         sources=[join('cobyla',x) for x in ['cobyla.pyf',
+-                                                             'cobyla2.f',
+-                                                             'trstlp.f']],
+-                         **numpy_nodepr_api)
++    #config.add_extension('_cobyla',
++    #                     sources=[join('cobyla',x) for x in ['cobyla.pyf',
++    #                                                         'cobyla2.f',
++    #                                                         'trstlp.f']],
++    #                     **numpy_nodepr_api)
+ 
+     sources = ['minpack2.pyf', 'dcsrch.f', 'dcstep.f']
+     config.add_extension('minpack2',
 diff --git a/scipy/optimize/zeros.c b/scipy/optimize/zeros.c
 index e557f37cb..d450b3ed4 100644
 --- a/scipy/optimize/zeros.c

packages/scipy/patches/skip-fortran-fails-to-link.patch

+commit 0822e53ae255433469257625e8f292abd13ae562
+Author: Roman Yurchak <rth.yurchak@pm.me>
+Date:   Thu Oct 11 17:10:36 2018 +0200
+
+    Skip fortran code that fails to link
+    
+       scipy.integrate.odepack
+
+diff --git a/scipy/integrate/setup.py b/scipy/integrate/setup.py
+index 9d607af8d..2be454c70 100755
+--- a/scipy/integrate/setup.py
++++ b/scipy/integrate/setup.py
+@@ -27,7 +27,7 @@ def configuration(parent_package='',top_path=None):
+     config.add_library('mach', sources=mach_src,
+                        config_fc={'noopt':(__file__,1)})
+     config.add_library('quadpack', sources=quadpack_src)
+-    config.add_library('odepack', sources=odepack_src)
++    #config.add_library('odepack', sources=odepack_src)
+     config.add_library('dop', sources=dop_src)
+ 
+     # Extensions
+@@ -44,27 +44,27 @@ def configuration(parent_package='',top_path=None):
+ 
+     odepack_opts = lapack_opt.copy()
+     odepack_opts.update(numpy_nodepr_api)
+-    config.add_extension('_odepack',
+-                         sources=['_odepackmodule.c'],
+-                         libraries=odepack_libs,
+-                         depends=(odepack_src + mach_src),
+-                         **odepack_opts)
++    #config.add_extension('_odepack',
++    #                     sources=['_odepackmodule.c'],
++    #                     libraries=odepack_libs,
++    #                     depends=(odepack_src + mach_src),
++    #                     **odepack_opts)
+ 
+     # vode
+-    config.add_extension('vode',
+-                         sources=['vode.pyf'],
+-                         libraries=odepack_libs,
+-                         depends=(odepack_src
+-                                  + mach_src),
+-                         **lapack_opt)
++    #config.add_extension('vode',
++    #                     sources=['vode.pyf'],
++    #                     libraries=odepack_libs,
++    #                     depends=(odepack_src
++    #                              + mach_src),
++    #                     **lapack_opt)
+ 
+     # lsoda
+-    config.add_extension('lsoda',
+-                         sources=['lsoda.pyf'],
+-                         libraries=odepack_libs,
+-                         depends=(odepack_src
+-                                  + mach_src),
+-                         **lapack_opt)
++    #config.add_extension('lsoda',
++    #                     sources=['lsoda.pyf'],
++    #                     libraries=odepack_libs,
++    #                     depends=(odepack_src
++    #                              + mach_src),
++    #                     **lapack_opt)
+ 
+     # dop
+     config.add_extension('_dop',
+@@ -76,11 +76,11 @@ def configuration(parent_package='',top_path=None):
+                          sources=quadpack_test_src)
+ 
+     # Fortran+f2py extension module for testing odeint.
+-    config.add_extension('_test_odeint_banded',
+-                         sources=odeint_banded_test_src,
+-                         libraries=odepack_libs,
+-                         depends=(odepack_src + mach_src),
+-                         **lapack_opt)
++    #config.add_extension('_test_odeint_banded',
++    #                     sources=odeint_banded_test_src,
++    #                     libraries=odepack_libs,
++    #                     depends=(odepack_src + mach_src),
++    #                     **lapack_opt)
+ 
+     config.add_data_dir('tests')
+     return config

packages/scipy/patches/skip-scipy-special-_ufuncs.patch

-commit 066faced67d1d0a8a4a89588194fcbd2faa0a525
-Author: Roman Yurchak <rth.yurchak@pm.me>
-Date:   Wed Oct 10 17:48:17 2018 +0200
-
-    Skip scipy.special._ufuncs
-
-diff --git a/scipy/special/__init__.py b/scipy/special/__init__.py
-index f013b818e..ab4956825 100644
---- a/scipy/special/__init__.py
-+++ b/scipy/special/__init__.py
-@@ -624,7 +624,11 @@ Convenience Functions
- 
- from __future__ import division, print_function, absolute_import
- 
--from ._ufuncs import *
-+try:
-+    from ._ufuncs import *
-+except ImportError:
-+    # TODO
-+    pass
- 
- from .basic import *
- from . import specfun
-diff --git a/scipy/special/_ellip_harm.py b/scipy/special/_ellip_harm.py
-index 6972c2ad1..0558ee6d3 100644
---- a/scipy/special/_ellip_harm.py
-+++ b/scipy/special/_ellip_harm.py
-@@ -3,7 +3,11 @@ from __future__ import division, print_function, absolute_import
- import threading
- import numpy as np
- 
--from ._ufuncs import _ellip_harm
-+try:
-+    from ._ufuncs import _ellip_harm
-+except ImportError:
-+    # TODO
-+    pass
- from ._ellip_harm_2 import _ellipsoid, _ellipsoid_norm
- 
- 
-diff --git a/scipy/special/basic.py b/scipy/special/basic.py
-index bb251e745..f1c788c3d 100644
---- a/scipy/special/basic.py
-+++ b/scipy/special/basic.py
-@@ -11,9 +11,13 @@ from scipy._lib.six import xrange
- from numpy import (pi, asarray, floor, isscalar, iscomplex, real, imag, sqrt,
-                    where, mgrid, sin, place, issubdtype, extract,
-                    less, inexact, nan, zeros, atleast_1d, sinc)
--from ._ufuncs import (ellipkm1, mathieu_a, mathieu_b, iv, jv, gamma, psi, zeta,
--                      hankel1, hankel2, yv, kv, gammaln, ndtri, errprint, poch,
--                      binom)
-+try:
-+    from ._ufuncs import (ellipkm1, mathieu_a, mathieu_b, iv, jv, gamma, psi, zeta,
-+                          hankel1, hankel2, yv, kv, gammaln, ndtri, errprint, poch,
-+                          binom)
-+except ImportError:
-+    # TODO
-+    pass
- from . import specfun
- from . import orthogonal
- 
-diff --git a/scipy/special/lambertw.py b/scipy/special/lambertw.py
-index 4bdac30f9..8a890ce51 100644
---- a/scipy/special/lambertw.py
-+++ b/scipy/special/lambertw.py
-@@ -1,6 +1,10 @@
- from __future__ import division, print_function, absolute_import
- 
--from ._ufuncs import _lambertw
-+try:
-+    from ._ufuncs import _lambertw
-+except ImportError:
-+    # TODO
-+    pass
- 
- 
- def lambertw(z, k=0, tol=1e-8):
-diff --git a/scipy/special/orthogonal.py b/scipy/special/orthogonal.py
-index c2b8dead3..b33ab96ac 100755
---- a/scipy/special/orthogonal.py
-+++ b/scipy/special/orthogonal.py
-@@ -102,8 +102,12 @@ from scipy import linalg
- from scipy.special import airy
- 
- # Local imports.
--from . import _ufuncs as cephes
--_gam = cephes.gamma
-+try:
-+    from . import _ufuncs as cephes
-+    _gam = cephes.gamma
-+except ImportError:
-+    # TODO
-+    pass
- from . import specfun
- 
- __all__ = ['legendre', 'chebyt', 'chebyu', 'chebyc', 'chebys',
-@@ -1657,8 +1661,12 @@ def sh_legendre(n, monic=False):
- # -----------------------------------------------------------------------------
- # Vectorized functions for evaluation
- # -----------------------------------------------------------------------------
--from ._ufuncs import (binom, eval_jacobi, eval_sh_jacobi, eval_gegenbauer,
--                      eval_chebyt, eval_chebyu, eval_chebys, eval_chebyc,
--                      eval_sh_chebyt, eval_sh_chebyu, eval_legendre,
--                      eval_sh_legendre, eval_genlaguerre, eval_laguerre,
--                      eval_hermite, eval_hermitenorm)
-+try:
-+    from ._ufuncs import (binom, eval_jacobi, eval_sh_jacobi, eval_gegenbauer,
-+                          eval_chebyt, eval_chebyu, eval_chebys, eval_chebyc,
-+                          eval_sh_chebyt, eval_sh_chebyu, eval_legendre,
-+                          eval_sh_legendre, eval_genlaguerre, eval_laguerre,
-+                          eval_hermite, eval_hermitenorm)
-+except ImportError:
-+    # TODO
-+    pass

test/conftest.py

@@ -16,7 +16,8 @@ import shutil
 TEST_PATH = pathlib.Path(__file__).parents[0].resolve()
 BUILD_PATH = TEST_PATH / '..' / 'build'
 
-sys.path.append(TEST_PATH / '..')
+sys.path.append(str(TEST_PATH / '..'))
+print(sys.path)
 
 from pyodide_build._fixes import _selenium_is_connectable  # noqa: E402
 import selenium.webdriver.common.utils  # noqa: E402

test/test_scipy.py

+from pathlib import Path
+import subprocess
+import sys
+from textwrap import dedent
+
+import pytest
+
+sys.path.append(str(Path(__file__).parents[1]))
+
+from pyodide_build.common import HOSTPYTHON   # noqa: E402
+
+
+def test_scipy_import(selenium_standalone):
+    from selenium.common.exceptions import JavascriptException
+    selenium = selenium_standalone
+    selenium.load_package("scipy")
+    selenium.run("""
+        import scipy
+        """)
+
+    # supported modules
+    for module in ['constants', 'fftpack']:
+        selenium.run(f"import scipy.{module}")
+
+    # not yet built modules
+    for module in ['cluster',  # needs sparse
+                   'spatial',  # needs sparse
+                   'sparse',
+                   'integrate',  # needs special
+                   'interpolate',  # needs linalg
+                   'linalg',
+                   'misc',   # needs special
+                   'odrpack',
+                   'signal',  # needs special
+                   'ndimage',  # needs special
+                   'stats',  # need special
+                   'optimize',  # needs minpack2
+                   'special']:
+        print(module)
+        with pytest.raises(JavascriptException) as err:
+            selenium.run(f"import scipy.{module}")
+        assert ('ModuleNotFoundError' in str(err.value)
+                or 'ImportError' in str(err.value))
+
+    print(selenium.logs)
+
+
+def test_built_so(selenium_standalone):
+    selenium = selenium_standalone
+    selenium.load_package("scipy")
+
+    cmd = dedent(r"""
+        import scipy as sp
+        import os
+
+        base_dir = os.path.dirname(sp.__file__)
+
+        out = []
+        for (dirpath, dirnames, filenames) in os.walk(base_dir):
+            for path in filenames:
+                if path.endswith('.so'):
+                    rel_path = os.path.relpath(dirpath, base_dir)
+                    out.append(os.path.join(rel_path, path))
+        print("\n".join(out))
+        out
+        """)
+
+    out = subprocess.check_output(
+            [HOSTPYTHON / 'bin' / 'python3', '-c', cmd])
+    modules_host = out.decode('utf-8').split('\n')
+
+    def _get_modules_name(modules):
+        return set([path.split('.')[0] for path in modules if path])
+
+    modules_host = _get_modules_name(modules_host)
+
+    modules_target = selenium.run(cmd)
+    modules_target = _get_modules_name(modules_target)
+
+    print(f'Included modules: {len(modules_target)}')
+    print(f'    {modules_target} ')
+    print(f'\nMissing modules: {len(modules_host.difference(modules_target))}')
+    print(f'     {modules_host.difference(modules_target)}')