Merge pull request #286 from madhur-tandon/master

Check packages against Pyodide ABI version

Makefile

@@ -72,7 +72,7 @@ build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.b
 build/pyodide.asm.data: root/.built
 	( \
 		cd build; \
-		python $(FILEPACKAGER) pyodide.asm.data --lz4 --preload ../root/lib@lib --js-output=pyodide.asm.data.js --use-preload-plugins \
+		python $(FILEPACKAGER) pyodide.asm.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../root/lib@lib --js-output=pyodide.asm.data.js --use-preload-plugins \
   )
 	uglifyjs build/pyodide.asm.data.js -o build/pyodide.asm.data.js
 
@@ -80,11 +80,13 @@ build/pyodide.asm.data: root/.built
 build/pyodide_dev.js: src/pyodide.js
 	cp $< $@
 	sed -i -e "s#{{DEPLOY}}##g" $@
+	sed -i -e "s#{{ABI}}#$(PYODIDE_PACKAGE_ABI)#g" $@
 
 
 build/pyodide.js: src/pyodide.js
 	cp $< $@
 	sed -i -e 's#{{DEPLOY}}#https://iodide.io/pyodide-demo/#g' $@
+	sed -i -e "s#{{ABI}}#$(PYODIDE_PACKAGE_ABI)#g" $@
 
 
 build/python.html: src/python.html
@@ -145,7 +147,7 @@ build/test.data: $(CPYTHONLIB)
 	)
 	( \
 		cd build; \
-		python $(FILEPACKAGER) test.data --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
+		python $(FILEPACKAGER) test.data --abi=$(PYODIDE_PACKAGE_ABI) --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
   )
 	uglifyjs build/test.js -o build/test.js
 

Makefile.envs

@@ -15,6 +15,8 @@ export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3
 export TARGETPYTHONROOT=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)
 export PYTHONINCLUDE=$(PYODIDE_ROOT)/cpython/installs/python-$(PYVERSION)/include/python$(PYMINOR)
 
+export PYODIDE_PACKAGE_ABI=1
+
 export SIDE_LDFLAGS=\
 	-O3 \
   -s "BINARYEN_METHOD='native-wasm'" \

packages/Makefile

@@ -3,7 +3,7 @@ include ../Makefile.envs
 
 all: deps
 	../bin/pyodide buildall . ../build \
-		--ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
+		--package_abi=$(PYODIDE_PACKAGE_ABI) --ldflags="$(SIDE_LDFLAGS)" --host=$(HOSTPYTHONROOT) --target=$(TARGETPYTHONROOT)
 deps:
 	# Install build dependencies
 	$(HOSTPYTHON) -m pip install Cython Tempita

pyodide_build/buildall.py

@@ -69,6 +69,9 @@ def make_parser(parser):
     parser.add_argument(
         'output', type=str, nargs=1,
         help='Output directory in which to put all built packages')
+    parser.add_argument(
+        '--package_abi', type=int, required=True,
+        help='The ABI number for the packages to be built')
     parser.add_argument(
         '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
         help='Extra compiling flags')

pyodide_build/buildpkg.py

@@ -127,6 +127,7 @@ def package_files(buildpath, srcpath, pkg, args):
         'python',
         common.ROOTDIR / 'file_packager.py',
         name + '.data',
+        '--abi={0}'.format(args.package_abi),
         '--lz4',
         '--preload',
         '{}@/'.format(install_prefix),
@@ -169,6 +170,9 @@ def make_parser(parser):
     parser.add_argument(
         'package', type=str, nargs=1,
         help="Path to meta.yaml package description")
+    parser.add_argument(
+        '--package_abi', type=int, required=True,
+        help='The ABI number for the package to be built')
     parser.add_argument(
         '--cflags', type=str, nargs='?', default=common.DEFAULTCFLAGS,
         help='Extra compiling flags')

src/pyodide.js

@@ -159,6 +159,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
             window.pyodide.loadedPackages[package] = toLoad[package];
           }
           delete window.pyodide._module.monitorRunDependencies;
+          window.removeEventListener('error', windowErrorHandler);
           if (!isFirefox) {
             preloadWasm().then(() => {resolve(`Loaded ${packageList}`)});
           } else {
@@ -167,6 +168,17 @@ var languagePluginLoader = new Promise((resolve, reject) => {
         }
       };
 
+      // Add a handler for any exceptions that are thrown in the process of
+      // loading a package
+      var windowErrorHandler = (err) => {
+        delete window.pyodide._module.monitorRunDependencies;
+        window.removeEventListener('error', windowErrorHandler);
+        // Set up a new Promise chain, since this one failed
+        loadPackagePromise = new Promise((resolve) => resolve());
+        reject(err.message);
+      };
+      window.addEventListener('error', windowErrorHandler);
+
       for (let package in toLoad) {
         let script = document.createElement('script');
         let package_uri = toLoad[package];
@@ -179,7 +191,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
           // If the package_uri fails to load, call monitorRunDependencies twice
           // (so packageCounter will still hit 0 and finish loading), and remove
           // the package from toLoad so we don't mark it as loaded.
-          console.log(`Couldn't load package from URL ${script.src}`)
+          console.error(`Couldn't load package from URL ${script.src}`)
           let index = toLoad.indexOf(package);
           if (index !== -1) {
             toLoad.splice(index, 1);
@@ -247,6 +259,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     'repr',
     'runPython',
     'runPythonAsync',
+    'checkABI',
     'version',
   ];
 
@@ -277,6 +290,16 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     return {};
   };
 
+  Module.checkABI = function(ABI_number) {
+    if (ABI_number !== parseInt('{{ABI}}')) {
+      var ABI_mismatch_exception =
+          `ABI numbers differ. Expected {{ABI}}, got ${ABI_number}`;
+      console.error(ABI_mismatch_exception);
+      throw ABI_mismatch_exception;
+    }
+    return true;
+  };
+
   Module.locateFile = (path) => baseURL + path;
   var postRunPromise = new Promise((resolve, reject) => {
     Module.postRun = () => {

test/test_package_loading.py

 import pytest
-
-from pathlib import Path
 import shutil
+import re
+from pathlib import Path
 
 
 @pytest.mark.parametrize('active_server', ['main', 'secondary'])
@@ -108,6 +108,39 @@ def test_load_packages_sequential(selenium_standalone, packages):
     assert selenium.logs.count(f'Loading {packages[1]}') == 1
 
 
+def test_different_ABI(selenium_standalone):
+    url = selenium_standalone.server_hostname
+    port = selenium_standalone.server_port
+
+    build_dir = Path(__file__).parent.parent / 'build'
+
+    original_file = open('build/numpy.js', 'r+')
+    original_contents = original_file.read()
+    original_file.close()
+
+    modified_contents = re.sub(r'checkABI\(\d+\)', 'checkABI(-1)',
+                               original_contents)
+    modified_file = open('build/numpy-broken.js', 'w+')
+    modified_file.write(modified_contents)
+    modified_file.close()
+
+    try:
+        selenium_standalone.load_package(
+            f'http://{url}:{port}/numpy-broken.js'
+        )
+        assert 'ABI numbers differ.' in selenium_standalone.logs
+    finally:
+        (build_dir / 'numpy-broken.js').unlink()
+
+    selenium_standalone.load_package('kiwisolver')
+    selenium_standalone.run('import kiwisolver')
+    assert (
+        selenium_standalone.run('repr(kiwisolver)') ==
+        "<module 'kiwisolver' from "
+        "'/lib/python3.7/site-packages/kiwisolver.so'>"
+    )
+
+
 def test_load_handle_failure(selenium_standalone):
     selenium = selenium_standalone
     selenium.load_package('pytz')

tools/file_packager.py

@@ -144,6 +144,10 @@ for arg in sys.argv[2:]:
     leading = 'preload'
   elif arg == '--embed':
     leading = 'embed'
+  elif arg.startswith('--abi'):
+    if '=' in arg:
+      package_abi = arg.split('=', 1)[1]
+    leading = ''
   elif arg == '--exclude':
     leading = 'exclude'
   elif arg == '--no-force':
@@ -227,6 +231,10 @@ if not from_emcc:
 var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {};
 ''' % {"EXPORT_NAME": export_name}
 
+ret += '''
+Module.checkABI({0});
+'''.format(package_abi)
+
 ret += '''
 if (!Module.expectedDataFileDownloads) {
   Module.expectedDataFileDownloads = 0;