Merge pull request #313 from jstafford/webworkers

Webworker support. Fixes issue #246

Makefile

@@ -57,7 +57,9 @@ all: build/pyodide.asm.js \
 	build/renderedhtml.css \
   build/test.data \
   build/packages.json \
-  build/test.html
+  build/test.html \
+  build/webworker.js \
+  build/webworker_dev.js
 
 
 build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
@@ -112,6 +114,14 @@ build/test.html: src/test.html
 build/renderedhtml.css: src/renderedhtml.less
 	lessc $< $@
 
+build/webworker.js: src/webworker.js
+	cp $< $@
+	sed -i -e 's#{{DEPLOY}}#https://iodide.io/pyodide-demo/#g' $@
+
+build/webworker_dev.js: src/webworker.js
+	cp $< $@
+	sed -i -e "s#{{DEPLOY}}##g" $@
+	sed -i -e "s#pyodide.js#pyodide_dev.js#g" $@
 
 test: all
 	pytest test/ -v

docs/index.md

@@ -3,6 +3,7 @@
 ## Using Pyodide
 
 - [Using Pyodide directly from Javascript](using_pyodide_from_javascript.md)
+- [Using Pyodide from a web worker](using_pyodide_from_webworker.md)
 - [Using Pyodide from Iodide](using_pyodide_from_iodide.md)
 
 ## Developing Pyodide

docs/using_pyodide_from_webworker.md

+# Using Pyodide from a web worker
+
+This document describes how to use pyodide to execute python scripts
+asynchronously in a web worker.
+
+## Startup
+
+Setup your project to serve `webworker.js`. You should also serve
+`pyodide.js`, and all its associated `.asm.js`, `.data`, `.json`, and `.wasm`
+files as well, though this is not strictly required if `pyodide.js` is pointing
+to a site serving current versions of these files.
+
+Update the `webworker.js` sample so that it has as valid URL for `pyodide.js`, and sets
+`self.languagePluginUrl` to the location of the supporting files.
+
+In your application code create a web worker, and add listeners for `onerror`
+and `onmessage`.
+
+Call `postMessage` on your web worker, passing an object with the key `python`
+containing the script to execute as a string. You may pass other keys in the
+data object. By default the web worker assigns these to its global scope so that
+they may be imported from python. The results are returned as the `results` key,
+or if an error was encountered, it is returned in the `error` key.
+
+For example:
+
+```
+var pyodideWorker = new Worker('./webworker.js')
+
+pyodideWorker.onerror = (e) => {
+  console.log(`Error in pyodideWorker at ${e.filename}, Line: ${e.lineno}, ${e.message}`)
+}
+
+pyodideWorker.onmessage = (e) => {
+  const {results, error} = e.data
+  if (results) {
+    console.log('pyodideWorker return results: ', results)
+  } else if (error) {
+    console.log('pyodideWorker error: ', error)
+  }
+}
+
+const data = {
+  A_rank: [0.8, 0.4, 1.2, 3.7, 2.6, 5.8],
+  python:
+    'import statistics\n' +
+    'from js import A_rank\n' +
+    'statistics.stdev(A_rank)'
+}
+
+pyodideWorker.postMessage(data)
+
+```
+
+## Loading packages
+
+Packages referenced from your python script will be automatically downloaded
+the first time they are encountered. Please note that some of the larger
+packages such as `numpy` or `pandas` may take several seconds to load.
+Subsequent uses of these packages will not incur the download overhead of the
+first run, but there is still some time required for the `import` in python
+itself.
+
+If you would like to pre-load some packages, or the automatic package loading
+does not work for you for some reason, you may modify the `webworker.js` source
+to load some specific packages as described in
+[Using Pyodide directly from Javascript](using_pyodide_from_javascript.md).
+
+For example, to always load packages `numpy` and `pytz`, you would insert the
+line `self.pyodide.loadPackage(['numpy', 'pytz']).then(() => {` as shown below:
+```
+self.languagePluginUrl = 'http://localhost:8000/'
+importScripts('./pyodide.js')
+
+var onmessage = function(e) { // eslint-disable-line no-unused-vars
+  languagePluginLoader.then(() => {
+    self.pyodide.loadPackage(['numpy', 'pytz']).then(() => {
+      const data = e.data;
+      const keys = Object.keys(data);
+      for (let key of keys) {
+        if (key !== 'python') {
+          // Keys other than python must be arguments for the python script.
+          // Set them on self, so that `from js import key` works.
+          self[key] = data[key];
+        }
+      }
+      self.pyodide.runPythonAsync(data.python, () => {})
+          .then((results) => { self.postMessage({results}); })
+          .catch((err) => {
+            // if you prefer messages with the error
+            self.postMessage({error : err.message});
+            // if you prefer onerror events
+            // setTimeout(() => { throw err; });
+          });
+    });
+  });
+}
+```
+
+## Caveats
+
+Using a web worker is advantageous because the python code is run in a separate
+thread from your main UI, and hence does not impact your application's
+responsiveness.
+There are some limitations, however.
+At present, Pyodide does not support sharing the Python interpreter and
+packages between multiple web workers or with your main thread.
+Since web workers are each in their own virtual machine, you also cannot share
+globals between a web worker and your main thread.
+Finally, although the web worker is separate from your main thread,
+the web worker is itself single threaded, so only one python script will
+execute at a time.

src/hiwire.c

@@ -183,7 +183,7 @@ EM_JS(void, hiwire_push_object_pair, (int idobj, int idkey, int idval), {
 
 EM_JS(int, hiwire_get_global, (int idname), {
   var jsname = UTF8ToString(idname);
-  return Module.hiwire_new_value(window[jsname]);
+  return Module.hiwire_new_value(self[jsname]);
 });
 
 EM_JS(int, hiwire_get_member_string, (int idobj, int idkey), {

src/pyodide.js

@@ -6,7 +6,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   // This is filled in by the Makefile to be either a local file or the
   // deployed location. TODO: This should be done in a less hacky
   // way.
-  var baseURL = window.languagePluginUrl || '{{DEPLOY}}';
+  var baseURL = self.languagePluginUrl || '{{DEPLOY}}';
   baseURL = baseURL.substr(0, baseURL.lastIndexOf('/')) + '/';
 
   ////////////////////////////////////////////////////////////
@@ -50,7 +50,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       } catch {
         return;
       }
-      for (entry of dirs) {
+      for (let entry of dirs) {
         if (entry.startsWith('.')) {
           continue;
         }
@@ -76,10 +76,27 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   }
   // clang-format on
 
+  function loadScript(url, onload, onerror) {
+    if (self.document) { // browser
+      const script = self.document.createElement('script');
+      script.src = url;
+      script.onload = (e) => { onload(); };
+      script.onerror = (e) => { onerror(); };
+      self.document.head.appendChild(script);
+    } else if (self.importScripts) { // webworker
+      try {
+        self.importScripts(url);
+        onload();
+      } catch {
+        onerror();
+      }
+    }
+  }
+
   let _loadPackage = (names, messageCallback) => {
     // DFS to find all dependencies of the requested packages
-    let packages = window.pyodide._module.packages.dependencies;
-    let loadedPackages = window.pyodide.loadedPackages;
+    let packages = self.pyodide._module.packages.dependencies;
+    let loadedPackages = self.pyodide.loadedPackages;
     let queue = [].concat(names || []);
     let toLoad = new Array();
     while (queue.length) {
@@ -124,7 +141,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       }
     }
 
-    window.pyodide._module.locateFile = (path) => {
+    self.pyodide._module.locateFile = (path) => {
       // handle packages loaded from custom URLs
       let package = path.replace(/\.data$/, "");
       if (package in toLoad) {
@@ -152,14 +169,14 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       // exactly "toLoad * 2" times.
       var packageCounter = Object.keys(toLoad).length * 2;
 
-      window.pyodide._module.monitorRunDependencies = () => {
+      self.pyodide._module.monitorRunDependencies = () => {
         packageCounter--;
         if (packageCounter === 0) {
           for (let package in toLoad) {
-            window.pyodide.loadedPackages[package] = toLoad[package];
+            self.pyodide.loadedPackages[package] = toLoad[package];
           }
-          delete window.pyodide._module.monitorRunDependencies;
-          window.removeEventListener('error', windowErrorHandler);
+          delete self.pyodide._module.monitorRunDependencies;
+          self.removeEventListener('error', windowErrorHandler);
           if (!isFirefox) {
             preloadWasm().then(() => {resolve(`Loaded ${packageList}`)});
           } else {
@@ -171,43 +188,42 @@ var languagePluginLoader = new Promise((resolve, reject) => {
       // Add a handler for any exceptions that are thrown in the process of
       // loading a package
       var windowErrorHandler = (err) => {
-        delete window.pyodide._module.monitorRunDependencies;
-        window.removeEventListener('error', windowErrorHandler);
+        delete self.pyodide._module.monitorRunDependencies;
+        self.removeEventListener('error', windowErrorHandler);
         // Set up a new Promise chain, since this one failed
         loadPackagePromise = new Promise((resolve) => resolve());
         reject(err.message);
       };
-      window.addEventListener('error', windowErrorHandler);
+      self.addEventListener('error', windowErrorHandler);
 
       for (let package in toLoad) {
-        let script = document.createElement('script');
+        let scriptSrc;
         let package_uri = toLoad[package];
         if (package_uri == 'default channel') {
-          script.src = `${baseURL}${package}.js`;
+          scriptSrc = `${baseURL}${package}.js`;
         } else {
-          script.src = `${package_uri}`;
+          scriptSrc = `${package_uri}`;
         }
-        script.onerror = (e) => {
+        loadScript(scriptSrc, () => {}, () => {
           // If the package_uri fails to load, call monitorRunDependencies twice
           // (so packageCounter will still hit 0 and finish loading), and remove
           // the package from toLoad so we don't mark it as loaded.
-          console.error(`Couldn't load package from URL ${script.src}`)
+          console.error(`Couldn't load package from URL ${scriptSrc}`)
           let index = toLoad.indexOf(package);
           if (index !== -1) {
             toLoad.splice(index, 1);
           }
           for (let i = 0; i < 2; i++) {
-            window.pyodide._module.monitorRunDependencies();
+            self.pyodide._module.monitorRunDependencies();
           }
-        };
-        document.body.appendChild(script);
+        });
       }
 
       // We have to invalidate Python's import caches, or it won't
       // see the new files. This is done here so it happens in parallel
       // with the fetching over the network.
-      window.pyodide.runPython('import importlib as _importlib\n' +
-                               '_importlib.invalidate_caches()\n');
+      self.pyodide.runPython('import importlib as _importlib\n' +
+                             '_importlib.invalidate_caches()\n');
     });
 
     return promise;
@@ -275,7 +291,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   // Loading Pyodide
   let wasmURL = `${baseURL}pyodide.asm.wasm`;
   let Module = {};
-  window.Module = Module;
+  self.Module = Module;
 
   Module.noImageDecoding = true;
   Module.noAudioDecoding = true;
@@ -303,15 +319,15 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   Module.locateFile = (path) => baseURL + path;
   var postRunPromise = new Promise((resolve, reject) => {
     Module.postRun = () => {
-      delete window.Module;
+      delete self.Module;
       fetch(`${baseURL}packages.json`)
           .then((response) => response.json())
           .then((json) => {
-            fixRecursionLimit(window.pyodide);
-            window.pyodide.globals =
-                window.pyodide.runPython('import sys\nsys.modules["__main__"]');
-            window.pyodide = makePublicAPI(window.pyodide, PUBLIC_API);
-            window.pyodide._module.packages = json;
+            fixRecursionLimit(self.pyodide);
+            self.pyodide.globals =
+                self.pyodide.runPython('import sys\nsys.modules["__main__"]');
+            self.pyodide = makePublicAPI(self.pyodide, PUBLIC_API);
+            self.pyodide._module.packages = json;
             resolve();
           });
     };
@@ -329,28 +345,23 @@ var languagePluginLoader = new Promise((resolve, reject) => {
 
   Promise.all([ postRunPromise, dataLoadPromise ]).then(() => resolve());
 
-  let data_script = document.createElement('script');
-  data_script.src = `${baseURL}pyodide.asm.data.js`;
-  data_script.onload = (event) => {
-    let script = document.createElement('script');
-    script.src = `${baseURL}pyodide.asm.js`;
-    script.onload = () => {
+  const data_script_src = `${baseURL}pyodide.asm.data.js`;
+  loadScript(data_script_src, () => {
+    const scriptSrc = `${baseURL}pyodide.asm.js`;
+    loadScript(scriptSrc, () => {
       // The emscripten module needs to be at this location for the core
       // filesystem to install itself. Once that's complete, it will be replaced
       // by the call to `makePublicAPI` with a more limited public API.
-      window.pyodide = pyodide(Module);
-      window.pyodide.loadedPackages = new Array();
-      window.pyodide.loadPackage = loadPackage;
-    };
-    document.head.appendChild(script);
-  };
-
-  document.head.appendChild(data_script);
+      self.pyodide = pyodide(Module);
+      self.pyodide.loadedPackages = new Array();
+      self.pyodide.loadPackage = loadPackage;
+    }, () => {});
+  }, () => {});
 
   ////////////////////////////////////////////////////////////
   // Iodide-specific functionality, that doesn't make sense
   // if not using with Iodide.
-  if (window.iodide !== undefined) {
+  if (self.iodide !== undefined) {
     // Load the custom CSS for Pyodide
     let link = document.createElement('link');
     link.rel = 'stylesheet';
@@ -359,7 +370,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     document.getElementsByTagName('head')[0].appendChild(link);
 
     // Add a custom output handler for Python objects
-    window.iodide.addOutputHandler({
+    self.iodide.addOutputHandler({
       shouldHandle : (val) => {
         return (typeof val === 'function' &&
                 pyodide._module.PyProxy.isPyProxy(val));

src/pyproxy.c

@@ -159,10 +159,10 @@ EM_JS(int, pyproxy_init, (), {
     get: function (jsobj, jskey) {
       if (jskey === 'toString') {
         return function() {
-          if (window.pyodide.repr === undefined) {
-            window.pyodide.repr = window.pyodide.pyimport('repr');
+          if (self.pyodide.repr === undefined) {
+            self.pyodide.repr = self.pyodide.pyimport('repr');
           }
-          return window.pyodide.repr(jsobj);
+          return self.pyodide.repr(jsobj);
         }
       } else if (jskey === '$$') {
         return jsobj['$$'];

src/runpython.c

@@ -88,7 +88,7 @@ EM_JS(int, runpython_init_js, (), {
 
     if (jsimports.length) {
       var packageNames =
-        window.pyodide._module.packages.import_name_to_package_name;
+        self.pyodide._module.packages.import_name_to_package_name;
       var packages = {};
       for (var i = 0; i < jsimports.length; ++i) {
         var name = jsimports[i];

src/webworker.js

+self.languagePluginUrl = '{{DEPLOY}}'
+importScripts('./pyodide.js')
+
+var onmessage = function(e) { // eslint-disable-line no-unused-vars
+  languagePluginLoader.then(() => {
+    const data = e.data;
+    const keys = Object.keys(data);
+    for (let key of keys) {
+      if (key !== 'python') {
+        // Keys other than python must be arguments for the python script.
+        // Set them on self, so that `from js import key` works.
+        self[key] = data[key];
+      }
+    }
+
+    self.pyodide.runPythonAsync(data.python, () => {})
+        .then((results) => { self.postMessage({results}); })
+        .catch((err) => {
+          // if you prefer messages with the error
+          self.postMessage({error : err.message});
+          // if you prefer onerror events
+          // setTimeout(() => { throw err; });
+        });
+  });
+}

test/conftest.py

@@ -143,6 +143,68 @@ class SeleniumWrapper:
             catch (error) {{ console.log(error.stack); throw error; }}"""
         return self.driver.execute_script(catch)
 
+    def setup_webworker(self):
+        hostname = self.server_hostname
+        port = self.server_port
+        url = f'http://{hostname}:{port}/webworker_dev.js'
+        self.run_js(
+            f"""
+            window.done = false;
+            window.pyodideWorker = new Worker( '{url}' );
+
+            window.pyodideWorker.onerror = function(e) {{
+                window.output = e;
+                window.error = true;
+                window.done = true;
+            }};
+
+            window.pyodideWorker.onmessage = function(e) {{
+              if (e.data.results) {{
+                window.output = e.data.results;
+                window.error = false;
+              }} else {{
+                window.output = e.data.error;
+                window.error = true;
+              }}
+              window.done = true;
+            }};
+            """
+        )
+
+    def run_webworker(self, code):
+        from selenium.common.exceptions import TimeoutException
+        self.setup_webworker()
+        if isinstance(code, str) and code.startswith('\n'):
+            # we have a multiline string, fix indentation
+            code = textwrap.dedent(code)
+        self.run_js(
+            """
+            var data = {{
+              python: {!r}
+            }};
+
+            window.pyodideWorker.postMessage(data);
+            """.format(code)
+        )
+        try:
+            self.wait.until(PackageLoaded())
+        except TimeoutException:
+            _display_driver_logs(self.browser, self.driver)
+            print(self.logs)
+            raise TimeoutException('run_webworker timed out')
+        return self.run_js(
+            """
+            if (window.error) {
+                if (window.output instanceof Error) {
+                    throw window.output;
+                } else {
+                    throw new Error(String(window.output))
+                }
+            }
+            return window.output;
+            """
+        )
+
     def load_package(self, packages):
         self.run_js(
             'window.done = false\n' +

test/test_webworker.py

+def test_runwebworker(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        import numpy as np
+        x = np.zeros(5)
+        str(x)
+        """
+    )
+    assert output == '[0. 0. 0. 0. 0.]'
+
+
+def test_runwebworker_different_package_name(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        import dateutil
+        dateutil.__version__
+        """
+    )
+    assert isinstance(output, str)
+
+
+def test_runwebworker_no_imports(selenium_standalone):
+    output = selenium_standalone.run_webworker(
+        """
+        42
+        """
+    )
+    assert output == 42
+
+
+def test_runwebworker_missing_import(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            import foo
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ModuleNotFoundError" in str(e)
+    else:
+        assert False
+
+
+def test_runwebworker_exception(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False
+
+
+def test_runwebworker_exception_after_import(selenium_standalone):
+    try:
+        selenium_standalone.run_webworker(
+            """
+            import numpy as np
+            x = np.empty(5)
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False