Fix #20: Basic modularization of file system.

Makefile

 PYODIDE_ROOT=$(abspath .)
 include Makefile.envs
 
+FILEPACKAGER=emsdk/emsdk/emscripten/incoming/tools/file_packager.py
+
 PYVERSION=3.6.4
 PYMINOR=$(basename $(PYVERSION))
 CPYTHONROOT=cpython
@@ -55,7 +57,11 @@ all: build/pyodide.asm.html \
 	build/pyodide.js \
 	build/pyodide_dev.js \
 	build/python.html \
-	build/renderedhtml.css
+	build/renderedhtml.css \
+	build/numpy.data \
+	build/dateutil.data \
+	build/pytz.data \
+	build/pandas.data
 
 
 build/pyodide.asm.html: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
@@ -64,6 +70,9 @@ build/pyodide.asm.html: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python
 	[ -d build ] || mkdir build
 	$(CC) -s EXPORT_NAME="'pyodide'" --bind -o $@ $(filter %.bc,$^) $(LDFLAGS) \
 		$(foreach d,$(wildcard root/*),--preload-file $d@/$(notdir $d))
+	rm build/pyodide.asm.asm.js
+	rm build/pyodide.asm.wasm.pre
+	rm build/pyodide.asm.html
 
 
 build/pyodide_dev.js: src/pyodide.js
@@ -112,12 +121,25 @@ clean:
 	$(CC) -o $@ $< $(CFLAGS)
 
 
+# TODO: It would be nice to generalize this
+build/numpy.data: $(NUMPY_LIBS)
+	python2 $(FILEPACKAGER) build/numpy.data --preload $(NUMPY_ROOT)@/lib/python3.6/site-packages/numpy --js-output=build/numpy.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__
+
+
+build/dateutil.data: $(DATEUTIL_LIBS)
+	python2 $(FILEPACKAGER) build/dateutil.data --preload $(DATEUTIL_ROOT)@/lib/python3.6/site-packages/dateutil --js-output=build/dateutil.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__
+
+
+build/pytz.data: $(PYTZ_LIBS)
+	python2 $(FILEPACKAGER) build/pytz.data --preload $(PYTZ_ROOT)@/lib/python3.6/site-packages/pytz --js-output=build/pytz.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__
+
+
+build/pandas.data: $(PANDAS_LIBS)
+	python2 $(FILEPACKAGER) build/pandas.data --preload $(PANDAS_ROOT)@/lib/python3.6/site-packages/pandas --js-output=build/pandas.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__
+
+
 root/.built: \
 		$(CPYTHONLIB) \
-		$(NUMPY_LIBS) \
-		$(PANDAS_LIBS) \
-		$(DATEUTIL_LIBS) \
-		$(PYTZ_LIBS) \
 		$(SIX_LIBS) \
 		src/lazy_import.py \
 		src/sitecustomize.py \
@@ -126,12 +148,7 @@ root/.built: \
 	rm -rf root
 	mkdir -p root/lib
 	cp -a $(CPYTHONLIB)/ root/lib
-	cp -a numpy/build/numpy $(SITEPACKAGES)
-	cp -a pandas/build/pandas $(SITEPACKAGES)
-	cp -a $(DATEUTIL_ROOT) $(SITEPACKAGES)
-	cp -a $(PYTZ_ROOT) $(SITEPACKAGES)
 	cp $(SIX_LIBS) $(SITEPACKAGES)
-	rm -fr $(SITEPACKAGES)/numpy/distutils
 	cp src/lazy_import.py $(SITEPACKAGES)
 	cp src/sitecustomize.py $(SITEPACKAGES)
 	cp src/webbrowser.py root/lib/python$(PYMINOR)

src/pyodide.js

 var languagePluginLoader = new Promise((resolve, reject) => {
-    let baseURL = "{{DEPLOY}}";
+    const baseURL = '{{DEPLOY}}';
+
+    const packages = {
+        'dateutil': [],
+        'numpy': [],
+        'pandas': ['numpy', 'dateutil', 'pytz'],
+        'pytz': [],
+    };
+    let loadedPackages = new Set();
+    let loadPackage = (names) => {
+        if (Array.isArray(names)) {
+            names = [names];
+        }
+
+        // DFS to find all dependencies of the requested packages
+        let queue = new Array(names);
+        let toLoad = new Set();
+        while (queue.length) {
+            const package = queue.pop();
+            if (!packages.hasOwnProperty(package)) {
+                throw `Unknown package '${package}'`;
+            }
+            if (!loadedPackages.has(package)) {
+                toLoad.add(package);
+                packages[package].forEach((subpackage) => {
+                    if (!loadedPackages.has(subpackage) &&
+                        !toLoad.has(subpackage)) {
+                        queue.push(subpackage);
+                    }
+                });
+            }
+        }
+
+        let promise = new Promise((resolve, reject) => {
+            var n = toLoad.size;
+
+            if (n === 0) {
+                resolve('No new packages to load');
+            }
+
+            toLoad.forEach((package) => {
+                let script = document.createElement('script');
+                script.src = `${baseURL}${package}.js`;
+                console.log(script.src);
+                script.onload = (e) => {
+                    n--;
+                    loadedPackages.add(package);
+                    if (n <= 0) {
+                        // All of the requested packages are now loaded.
+                        // We have to invalidate Python's import caches, or it won't
+                        // see the new files.
+                        window.pyodide.runPython(
+                            'import importlib as _importlib\n' +
+                            '_importlib.invalidate_caches()\n');
+                        const packageList = Array.from(toLoad.keys()).join(', ');
+                        resolve(`Loaded ${packageList}`);
+                    }
+                };
+                script.onerror = (e) => {
+                    reject(e);
+                };
+                document.body.appendChild(script);
+            });
+        });
+
+        return promise;
+    };
+
     let wasmURL = `${baseURL}pyodide.asm.wasm`;
     let Module = {};
 
@@ -19,6 +86,9 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     script.src = `${baseURL}pyodide.asm.js`;
     script.onload = () => {
         window.pyodide = pyodide(Module);
+        if (window.iodide !== undefined) {
+            window.pyodide.loadPackage = loadPackage;
+        }
     };
     document.body.appendChild(script);
 

src/python.html

@@ -44,6 +44,8 @@ Pyodide adds support for Python in an Iodide notebook, running inside your brows
 
 (A major shortcoming is that `print` from Python currently prints to the Javascript debugger console, rather than to the notebook cell, so some of these examples are more contrived than they need to be.)
 
+**Also to note: If you have any issues, try disabling any ad or tracking blockers for this site.**
+
 First, let's use a plugin cell to load the Python interpreter and tell Iodide about the new cell type.
 
 %% plugin
@@ -215,7 +217,14 @@ document.getElementById("targetDiv").setAttribute("style", "background-color: re
 
 You bet, [Numpy](http://numpy.org) works.
 
-(Except the `numpy.fft` module -- we're working on that, but you can get a lot done without it in a lot of use cases...)
+To save on download times, isn't loaded by default. We need to manually use
+the `pyodide.loadPackage` function from a Javascript cell.
+
+%% js
+pyodide.loadPackage('numpy')
+
+%% md
+Now that the Numpy package has been loaded (i.e. transferred to your local browser), we can import it:
 
 %% code {"language":"py"}
 import numpy as np

src/sitecustomize.py

-import lazy_import
-
-print("Setting up lazy importing...")
-
-lazy_import.lazy_module("numpy.linalg")
-lazy_import.lazy_module("numpy.fft")
-lazy_import.lazy_module("numpy.polynomial")
-lazy_import.lazy_module("numpy.random")
-lazy_import.lazy_module("numpy.ctypeslib")
-
 import sys
 sys.argv = ['pyodide']