Merge pull request #217 from mdboom/runpython-async

Add a package-loading version of runPython

docs/api_reference.md

@@ -55,13 +55,13 @@ The package needs to be imported from Python before it can be used.
 *Parameters*
 
 | name              | type            | description                           |
-|---------|-----------------|---------------------------------------|
+|-------------------|-----------------|---------------------------------------|
 | *names*           | {String, Array} | package name, or URL. Can be either a single element, or an array.          |
-
+| *messageCallback* | function        | A callback, called with progress messages. (optional) |
 
 *Returns*
 
-Loading is asynchronous, therefore, this returns a promise.
+Loading is asynchronous, therefore, this returns a `Promise`.
 
 
 ### pyodide.loadedPackage
@@ -137,6 +137,43 @@ Runs a string of code. The last part of the string may be an expression, in whic
 | *jsresult* | *any*   | Result, converted to Javascript |
 
 
+### pyodide.runPythonAsync(code, messageCallback)
+
+Runs Python code, possibly asynchronously loading any known packages that the code
+chunk imports.
+
+For example, given the following code chunk
+
+```python
+import numpy as np
+x = np.array([1, 2, 3])
+```
+
+pyodide will first call `pyodide.loadPackage(['numpy'])`, and then run the code
+chunk, returning the result. Since package fetching must happen asyncronously,
+this function returns a `Promise` which resolves to the output. For example, to
+use:
+
+```javascript
+pyodide.runPythonAsync(code, messageCallback)
+  .then((output) => handleOutput(output))
+```
+
+*Parameters*
+
+| name              | type     | description                    |
+|-------------------|----------|--------------------------------|
+| *code*            | String   | Python code to evaluate        |
+| *messageCallback* | function | Callback given status messages |
+|                   |          | (optional)                     |
+
+*Returns*
+
+| name       | type    | description                              |
+|------------|---------|------------------------------------------|
+| *result*   | Promise | Resolves to the result of the code chunk |
+
+
 ### pyodide.version()
 
 Returns the pyodide version.

packages/matplotlib/meta.yaml

@@ -41,3 +41,4 @@ requirements:
 test:
   imports:
     - matplotlib
+    - mpl_toolkits

pyodide_build/buildall.py

@@ -34,6 +34,7 @@ def build_packages(packagesdir, outputdir, args):
     # We have to build the packages in the correct order (dependencies first),
     # so first load in all of the package metadata and build a dependency map.
     dependencies = {}
+    import_name_to_package_name = {}
     for pkgdir in packagesdir.iterdir():
         pkgpath = pkgdir / 'meta.yaml'
         if pkgdir.is_dir() and pkgpath.is_file():
@@ -41,6 +42,9 @@ def build_packages(packagesdir, outputdir, args):
             name = pkg['package']['name']
             reqs = pkg.get('requirements', {}).get('run', [])
             dependencies[name] = reqs
+            imports = pkg.get('test', {}).get('imports', [name])
+            for imp in imports:
+                import_name_to_package_name[imp] = name
 
     for pkgname in dependencies.keys():
         build_package(pkgname, dependencies, packagesdir, outputdir, args)
@@ -51,7 +55,10 @@ def build_packages(packagesdir, outputdir, args):
 
     # This is done last so the Makefile can use it as a completion token.
     with open(outputdir / 'packages.json', 'w') as fd:
-        json.dump({'dependencies': dependencies}, fd)
+        json.dump({
+            'dependencies': dependencies,
+            'import_name_to_package_name': import_name_to_package_name,
+        }, fd)
 
 
 def make_parser(parser):

src/pyodide.js

@@ -75,7 +75,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
   }
   // clang-format on
 
-  let _loadPackage = (names) => {
+  let _loadPackage = (names, messageCallback) => {
     // DFS to find all dependencies of the requested packages
     let packages = window.pyodide._module.packages.dependencies;
     let loadedPackages = window.pyodide.loadedPackages;
@@ -140,13 +140,17 @@ var languagePluginLoader = new Promise((resolve, reject) => {
         resolve('No new packages to load');
       }
 
+      const packageList = Array.from(Object.keys(toLoad)).join(', ');
+      if (messageCallback !== undefined) {
+        messageCallback(`Loading ${packageList}`);
+      }
+
       window.pyodide._module.monitorRunDependencies = (n) => {
         if (n === 0) {
           for (let package in toLoad) {
             window.pyodide.loadedPackages[package] = toLoad[package];
           }
           delete window.pyodide._module.monitorRunDependencies;
-          const packageList = Array.from(Object.keys(toLoad)).join(', ');
           if (!isFirefox) {
             preloadWasm().then(() => {resolve(`Loaded ${packageList}`)});
           } else {
@@ -181,10 +185,11 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     return promise;
   };
 
-  let loadPackage = (names) => {
+  let loadPackage = (names, messageCallback) => {
     /* We want to make sure that only one loadPackage invocation runs at any
      * given time, so this creates a "chain" of promises. */
-    loadPackagePromise = loadPackagePromise.then(() => _loadPackage(names));
+    loadPackagePromise =
+        loadPackagePromise.then(() => _loadPackage(names, messageCallback));
     return loadPackagePromise;
   };
 
@@ -224,6 +229,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     'pyimport',
     'repr',
     'runPython',
+    'runPythonAsync',
     'version',
   ];
 

src/pyodide.py

@@ -2,8 +2,6 @@
 A library of helper utilities for connecting Python to the browser environment.
 """
 
-from js import XMLHttpRequest
-
 import ast
 import io
 
@@ -14,6 +12,8 @@ def open_url(url):
     """
     Fetches a given *url* and returns a io.StringIO to access its contents.
     """
+    from js import XMLHttpRequest
+
     req = XMLHttpRequest.new()
     req.open('GET', url, False)
     req.send(None)
@@ -39,4 +39,22 @@ def eval_code(code, ns):
         return None
 
 
-__all__ = ['open_url', 'eval_code']
+def find_imports(code):
+    """
+    Finds the imports in a string of code and returns a list of their package
+    names.
+    """
+    mod = ast.parse(code)
+    imports = set()
+    for node in ast.walk(mod):
+        if isinstance(node, ast.Import):
+            for name in node.names:
+                name = name.name
+                imports.add(name.split('.')[0])
+        elif isinstance(node, ast.ImportFrom):
+            name = node.module
+            imports.add(name.split('.')[0])
+    return list(imports)
+
+
+__all__ = ['open_url', 'eval_code', 'find_imports']

src/runpython.c

@@ -10,6 +10,7 @@
 extern PyObject* globals;
 
 PyObject* eval_code;
+PyObject* find_imports;
 
 int
 _runPython(char* code)
@@ -32,18 +33,79 @@ _runPython(char* code)
   return id;
 }
 
+int
+_findImports(char* code)
+{
+  PyObject* py_code;
+  py_code = PyUnicode_FromString(code);
+  if (py_code == NULL) {
+    return pythonexc2js();
+  }
+
+  PyObject* ret = PyObject_CallFunctionObjArgs(find_imports, py_code, NULL);
+
+  if (ret == NULL) {
+    return pythonexc2js();
+  }
+
+  int id = python2js(ret);
+  Py_DECREF(ret);
+  return id;
+}
+
 EM_JS(int, runpython_init_js, (), {
-  Module.runPython = function(code)
+  Module._runPythonInternal = function(pycode)
   {
-    var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL);
     var idresult = Module.__runPython(pycode);
-    jsresult = Module.hiwire_get_value(idresult);
+    var jsresult = Module.hiwire_get_value(idresult);
     Module.hiwire_decref(idresult);
     _free(pycode);
     return jsresult;
   };
 
-  return 0;
+  Module.runPython = function(code)
+  {
+    var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL);
+    return Module._runPythonInternal(pycode);
+  };
+
+  Module.runPythonAsync = function(code, messageCallback)
+  {
+    var pycode = allocate(intArrayFromString(code), 'i8', ALLOC_NORMAL);
+
+    var idimports = Module.__findImports(pycode);
+    var jsimports = Module.hiwire_get_value(idimports);
+    Module.hiwire_decref(idimports);
+
+    var internal = function(resolve, reject)
+    {
+      try {
+        resolve(Module._runPythonInternal(pycode));
+      } catch (e) {
+        reject(e);
+      }
+    };
+
+    if (jsimports.length) {
+      var packageNames =
+        window.pyodide._module.packages.import_name_to_package_name;
+      var packages = {};
+      for (var i = 0; i < jsimports.length; ++i) {
+        var name = jsimports[i];
+        // clang-format off
+        if (packageNames[name] !== undefined) {
+          // clang-format on
+          packages[packageNames[name]] = undefined;
+        }
+      }
+      if (Object.keys(packages).length) {
+        var runInternal = function() { return new Promise(internal); };
+        return Module.loadPackage(Object.keys(packages), messageCallback)
+          .then(runInternal);
+      }
+    }
+    return new Promise(internal);
+  };
 });
 
 int
@@ -64,6 +126,11 @@ runpython_init_py()
     return 1;
   }
 
+  find_imports = PyDict_GetItemString(d, "find_imports");
+  if (find_imports == NULL) {
+    return 1;
+  }
+
   Py_DECREF(m);
   Py_DECREF(d);
   return 0;

test/conftest.py

@@ -105,6 +105,37 @@ class SeleniumWrapper:
         return self.run_js(
             'return pyodide.runPython({!r})'.format(code))
 
+    def run_async(self, code):
+        from selenium.common.exceptions import TimeoutException
+        if isinstance(code, str) and code.startswith('\n'):
+            # we have a multiline string, fix indentation
+            code = textwrap.dedent(code)
+        self.run_js(
+            """
+            window.done = false;
+            pyodide.runPythonAsync({!r})
+              .then(function(output)
+                      {{ window.output = output; window.error = false; }},
+                    function(output)
+                      {{ window.output = output; window.error = true; }})
+              .finally(() => window.done = true);
+            """.format(code)
+        )
+        try:
+            self.wait.until(PackageLoaded())
+        except TimeoutException as exc:
+            _display_driver_logs(self.browser, self.driver)
+            print(self.logs)
+            raise TimeoutException('runPythonAsync timed out')
+        return self.run_js(
+            """
+            if (window.error) {
+              throw window.output;
+            }
+            return window.output;
+            """
+        )
+
     def run_js(self, code):
         if isinstance(code, str) and code.startswith('\n'):
             # we have a multiline string, fix indentation

test/test_python.py

@@ -448,3 +448,73 @@ def test_recursive_dict(selenium_standalone):
         """
     )
     selenium_standalone.run_js("x = pyodide.pyimport('x')")
+
+
+def test_runpythonasync(selenium_standalone):
+    output = selenium_standalone.run_async(
+        """
+        import numpy as np
+        np.zeros(5)
+        """
+    )
+    assert list(output) == [0, 0, 0, 0, 0]
+
+
+def test_runpythonasync_different_package_name(selenium_standalone):
+    output = selenium_standalone.run_async(
+        """
+        import dateutil
+        dateutil.__version__
+        """
+    )
+    assert isinstance(output, str)
+
+
+def test_runpythonasync_no_imports(selenium_standalone):
+    output = selenium_standalone.run_async(
+        """
+        42
+        """
+    )
+    assert output == 42
+
+
+def test_runpythonasync_missing_import(selenium_standalone):
+    try:
+        selenium_standalone.run_async(
+            """
+            import foo
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ModuleNotFoundError" in str(e)
+    else:
+        assert False
+
+
+def test_runpythonasync_exception(selenium_standalone):
+    try:
+        selenium_standalone.run_async(
+            """
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False
+
+
+def test_runpythonasync_exception_after_import(selenium_standalone):
+    try:
+        selenium_standalone.run_async(
+            """
+            import numpy as np
+            x = np.empty(5)
+            42 / 0
+            """
+        )
+    except selenium_standalone.JavascriptException as e:
+        assert "ZeroDivisionError" in str(e)
+    else:
+        assert False