Webworker experiment

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/gadget_jsmd_eval.js.js

@@ -34,6 +34,7 @@
     cell_type_regexp = /^\%\% (\w+)\b/,
     language_type_regexp = /\{[\S\s]+\}/,
     is_pyodide_loaded = false,
+    is_matplotlib_used = true,
     Module = {},
     packages,
     loadedPackages = [],
@@ -41,17 +42,46 @@
     py_div_id_count = 0,
     py_div_id_count_2 = 0,
     props = {},
+    worker = new Worker('worker_load_pyodide.js'),
     // Regexp for validating package name and URI
     package_name_regexp = '[a-z0-9_][a-z0-9_\-]*',
     package_uri_regexp = new RegExp('^https?://.*?(' + package_name_regexp + ').js$', 'i');
 
   package_name_regexp = new RegExp('^' + package_name_regexp + '$', 'i');
   window.iodide = new IODide();
-
-  IODide.prototype.addOutputHandler = function () {
+  props.py_cell_list = [];
+  /*
+    function executePython() {
+      var queue = new RSVP.Queue(), i, len = props.py_cell_list.length;
+      queue.push(function () {
+        return initPyodide();
+      })
+        .push(function () {
+          for (i = 0; i < len; i += 1) {
+            executePyCell(props.py_cell_list[i]._line_list);
+          }
+        })
+        .push(undefined, function (error) {
+          console.log(error);
+        });
+      return queue;
+    }
+  */
+  IODide.prototype.addOutputHandler = function() {
     return;
   };
 
+  function sideEffectDiv(sideEffectClass, reportSideEffect) {
+    // appends a side effect div to the side effect area
+    var div = document.createElement("div");
+    div.setAttribute("class", sideEffectClass);
+    if (reportSideEffect === undefined) {
+      div.setAttribute("style", "display:");
+    }
+    document.body.appendChild(div);
+    return div;
+  }
+  /*
   function sideEffectDiv(sideEffectClass, reportSideEffect) {
     var div = document.getElementById(py_div_id_prefix + py_div_id_count_2),
         pre = div.getElementsByTagName('pre')[0],
@@ -65,6 +95,7 @@
     }
     return div;
   }
+  */
 
   // Copied from jio
   function ajax(param) {
@@ -125,6 +156,50 @@
     return null;
   }
 
+  function preloadWasm() {
+    // On Chrome, we have to instantiate wasm asynchronously. Since that
+    // can't be done synchronously within the call to dlopen, we instantiate
+    // every .so that comes our way up front, caching it in the
+    // `preloadedWasm` dictionary.
+
+    let promise = new Promise((resolve) => resolve());
+    let FS = pyodide._module.FS;
+
+    function recurseDir(rootpath) {
+      let dirs;
+      var entry;
+      try {
+        dirs = FS.readdir(rootpath);
+      } catch {
+        return;
+      }
+      for (entry of dirs) {
+        if (entry.startsWith('.')) {
+          continue;
+        }
+        const path = rootpath + entry;
+        if (entry.endsWith('.so')) {
+          if (Module['preloadedWasm'][path] === undefined) {
+            promise = promise
+              .then(() => Module['loadWebAssemblyModule'](
+                FS.readFile(path), {
+                  loadAsync: true
+                }))
+              .then((module) => {
+                Module['preloadedWasm'][path] = module;
+              });
+          }
+        } else if (FS.isDir(FS.lookupPath(path).node.mode)) {
+          recurseDir(path + '/');
+        }
+      }
+    }
+
+    recurseDir('/');
+
+    return promise;
+  }
+
   function pyodideLoadPackage(names) {
     // DFS to find all dependencies of the requested packages
     var queue, toLoad, package_uri, package_name, k,
@@ -179,7 +254,9 @@
           }
           delete pyodide.monitorRunDependencies;
           packageList = Array.from(Object.keys(toLoad)).join(', ');
-          resolve("Loaded " + packageList);
+          preloadWasm().then(() => {
+            resolve(`Loaded ${packageList}`)
+          });
         }
       };
 
@@ -241,6 +318,9 @@
         current_type = next_type;
         current_text_list = [];
       } else if (current_text_list !== undefined) {
+        if (current_line.indexOf("matplotlib") !== -1) {
+          is_matplotlib_used = true;
+        }
         current_text_list.push(current_line);
       }
     }
@@ -414,6 +494,7 @@
 
   function loadPyodide(info, receiveInstance) {
     var queue = new RSVP.Queue();
+
     queue.push(function () {
         return ajax({
           url: "pyodide.asm.wasm",
@@ -432,7 +513,36 @@
     return queue;
   }
 
+  function renderPyCodeblock(result_text, index) {
+    if (result_text !== undefined) {
+      var index_pos, index, code_text, div, pre, result;
+      index_pos = result_text.indexOf('_');
+      index = parseInt(result_text.slice(0, index_pos));
+      result_text = result_text.slice(index_pos + 1, -1);
+      div = document.getElementById(py_div_id_prefix + index);
+      pre = div.getElementsByTagName('pre')[0],
+        result = pre.getElementsByTagName('code')[0];
+
+      // py_div_id_count_2 += 1;
+      result.innerHTML = result_text;
+    }
+  }
+
   function renderCodeblock(result_text) {
+    var div = document.createElement('div'),
+      pre = document.createElement('pre'),
+      result = document.createElement('code');
+    div.style.border = '1px solid #C3CCD0';
+    div.style.margin = '40px 10px';
+    div.style.paddingLeft = '10px';
+
+    if (result_text !== undefined) {
+      result.innerHTML = result_text;
+      pre.appendChild(result);
+      div.appendChild(pre);
+      document.body.appendChild(div);
+    }
+    /*
     if (result_text !== undefined) {
       var div = document.getElementById(py_div_id_prefix + py_div_id_count_2),
           pre = div.getElementsByTagName('pre')[0],
@@ -441,6 +551,7 @@
       py_div_id_count_2 += 1;
       result.innerHTML = result_text;
     }
+    */
   }
 
   function addPyCellStub() {
@@ -551,21 +662,31 @@
         // empty block, do nothing.
         return;
       }
-      addPyCellStub();
-      if (!is_pyodide_loaded) {
-        props.queue = new RSVP.Queue()
-          .push(function () {
+      var queue = new RSVP.Queue();
+      if (is_pyodide_loaded === false) {
+        // worker.postMessage("pyodide.asm.wasm");
+        if (is_matplotlib_used === false) {
+          worker.postMessage("load pyodide");
+        } else {
+          queue.push(function () {
               return initPyodide();
             })
             .push(function () {
               return pyodideLoadPackage('matplotlib');
             });
+        }
         is_pyodide_loaded = true;
       }
 
-      props.queue.push(function () {
+      if (is_matplotlib_used === true) {
+        queue.push(function () {
           return executePyCell(cell._line_list);
         });
+        return queue
+      } else {
+        addPyCellStub();
+      }
+
       return;
     }
     return executeUnknownCellType(cell);
@@ -577,8 +698,51 @@
     };
   }
 
+  function collectPyCell(cell) {
+    if (cell._type === 'code_py') {
+      if (cell._line_list.length === 0) {
+        // empty block, do nothing.
+        return;
+      }
+      var i = 0,
+        len = cell._line_list.length;
+      for (i = 0; i < len; i += 1) {
+        if (cell._line_list[i].indexOf("matplotlib") !== -1) {
+          is_matplotlib_used = true;
+        }
+      }
+      var code_text = cell._line_list.join('\n');
+      props.py_cell_list.push(code_text);
+      return;
+    }
+    return;
+  }
 
-  document.addEventListener('DOMContentLoaded', function () {
+  worker.onmessage = function(event) {
+    console.log('Received message ' + event.data);
+    if (event.data == "pyodide loaded") {
+      var i = 0,
+        len = props.py_cell_list.length;
+      console.log(props.py_cell_list);
+      for (i = 0; i < len; i += 1) {
+        // the code text is like "1_import sys\n..."
+        // We want to pass the index to make sure the code blocks were executed in order
+        worker.postMessage(i + "_" + props.py_cell_list[i]);
+      }
+    } else if (event.data.indexOf('wasm_') == 0) {
+      props.response = event.data;
+      executePython();
+    } else {
+      console.log("Result");
+      console.log(event.data);
+      renderPyCodeblock(event.data);
+    }
+    // props.is_pyodide_loaded = true;
+    // props.response = event.data;
+    // executePython();
+  };
+
+  document.addEventListener('DOMContentLoaded', function() {
 
     var jsmd = document.querySelector('[type="text/x-jsmd"]').textContent,
       cell_list = parseJSMDCellList(jsmd),
@@ -588,14 +752,9 @@
 
     for (i = 0; i < len; i += 1) {
       queue.push(deferCellExecution(cell_list[i]));
+      collectPyCell(cell_list[i]);
     }
 
-    // Python packages loading and execution
-    queue
-      .push(function () {
-        return props.queue;
-    });
-
     return queue
       .push(function () {
         console.info('JSMD executed.');

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/worker_load_pyodide.js.js

+/*jslint nomen: true, indent: 2, maxerr: 3 */
+//   "use strict";
+// Copied from jio
+// importScripts("rsvp.js");
+var window = self,
+  Module = {},
+  packages,
+  loadedPackages = [],
+  props = {},
+  is_pyodide_loaded = false,
+  // Regexp for validating package name and URI
+  package_name_regexp = '[a-z0-9_][a-z0-9_\-]*',
+  package_uri_regexp = new RegExp('^https?://.*?(' + package_name_regexp + ').js$', 'i');
+
+var document = self.document = { parentNode: null, nodeType: 9, toString: function () { return "FakeDocument"; } };
+var window = self.window = self;
+var fakeElement = Object.create(document);
+fakeElement.nodeType = 1;
+fakeElement.toString = function () { return "FakeElement"; };
+fakeElement.parentNode = fakeElement.firstChild = fakeElement.lastChild = fakeElement;
+fakeElement.ownerDocument = document;
+
+document.head = document.body = fakeElement;
+document.ownerDocument = document.documentElement = document;
+document.getElementById = document.createElement = function () { return fakeElement; };
+document.createTextNode = function () {
+};
+document.createDocumentFragment = function () { return this; };
+document.getElementsByTagName = document.getElementsByClassName = function () { return [fakeElement]; };
+document.getAttribute = document.setAttribute = document.removeChild =
+    document.addEventListener = document.removeEventListener =
+    function () { return null; };
+document.cloneNode = document.appendChild = function () { return this; };
+document.appendChild = function (child) { return child; };
+document.childNodes = [];
+document.implementation = {
+  createHTMLDocument: function () { return document; }
+};
+
+importScripts("rsvp.js");
+package_name_regexp = new RegExp('^' + package_name_regexp + '$', 'i');
+
+function createIODide() {
+  var iodide = {
+    output: {
+      text: function (s, reportSideEffect) {
+        var i, div, line_list;
+        console.log(s);
+        line_list = s.toString().split("\n");
+        for (i = 0; i < line_list.length; i += 1) {
+          div = sideEffectDiv("side-effect-print", reportSideEffect);
+          div.textContent = line_list[i];
+        }
+      },
+      element: function (nodeType, reportSideEffect) {
+        var div, node;
+        div = sideEffectDiv("side-effect-element", reportSideEffect);
+        node = document.createElement(nodeType);
+        div.append(node);
+        props.figure = node.outerHTML;
+        return node;
+      }
+    }
+  };
+  return iodide;
+}
+
+window.iodide = createIODide();
+
+function ajax(param) {
+  var xhr = new XMLHttpRequest();
+  return new Promise(function (resolve, reject) {
+    var k;
+    xhr.open(param.type || "GET", param.url, true);
+    xhr.responseType = param.dataType || "";
+    if (typeof param.headers === 'object' && param.headers !== null) {
+      for (k in param.headers) {
+        if (param.headers.hasOwnProperty(k)) {
+          xhr.setRequestHeader(k, param.headers[k]);
+        }
+      }
+    }
+    xhr.addEventListener("load", function(e) {
+      if (e.target.status >= 400) {
+        return reject(e);
+      }
+      resolve(e);
+    });
+    xhr.addEventListener("error", reject);
+    if (typeof param.xhrFields === 'object' && param.xhrFields !== null) {
+      for (k in param.xhrFields) {
+        if (param.xhrFields.hasOwnProperty(k)) {
+          xhr[k] = param.xhrFields[k];
+        }
+      }
+    }
+    if (param.timeout !== undefined && param.timeout !== 0) {
+      xhr.timeout = param.timeout;
+      xhr.ontimeout = function() {
+        return reject(new Error("Gateway Timeout"));
+      };
+    }
+    if (typeof param.beforeSend === 'function') {
+      param.beforeSend(xhr);
+    }
+    xhr.send(param.data);
+  }, function() {
+    xhr.abort();
+  });
+}
+
+// clang-format off
+function preloadWasm() {
+  // On Chrome, we have to instantiate wasm asynchronously. Since that
+  // can't be done synchronously within the call to dlopen, we instantiate
+  // every .so that comes our way up front, caching it in the
+  // `preloadedWasm` dictionary.
+
+  let promise = new RSVP.Promise((resolve) => resolve());
+  let FS = pyodide._module.FS;
+
+  function recurseDir(rootpath) {
+    let dirs;
+    var entry;
+    try {
+      dirs = FS.readdir(rootpath);
+    } catch {
+      return;
+    }
+    for (entry of dirs) {
+      if (entry.startsWith('.')) {
+        continue;
+      }
+      const path = rootpath + entry;
+      if (entry.endsWith('.so')) {
+        if (Module['preloadedWasm'][path] === undefined) {
+          promise = promise
+            .then(() => Module['loadWebAssemblyModule'](
+              FS.readFile(path), {
+                loadAsync: true
+              }))
+            .then((module) => {
+              Module['preloadedWasm'][path] = module;
+            });
+        }
+      } else if (FS.isDir(FS.lookupPath(path).node.mode)) {
+        recurseDir(path + '/');
+      }
+    }
+  }
+
+  recurseDir('/');
+
+  return promise;
+}
+
+// Pyodide package loading
+function _uri_to_package_name(package_uri) {
+  // Generate a unique package name from URI
+  if (package_name_regexp.test(package_uri)) {
+    return package_uri;
+  }
+
+  if (package_uri_regexp.test(package_uri)) {
+    var match = package_uri_regexp.exec(package_uri);
+    // Get the regexp group corresponding to the package name
+    return match[1];
+  }
+
+  return null;
+}
+
+  // clang-format on
+  function pyodideLoadPackage(names) {
+    // DFS to find all dependencies of the requested packages
+    var queue, toLoad, package_uri, package_name, k,
+      subpackage, promise, packageList, script;
+    packages = window.pyodide.packages.dependencies;
+    queue = new Array(names);
+    toLoad = [];
+    while (queue.length) {
+      package_uri = queue.pop();
+      package_name = _uri_to_package_name(package_uri);
+
+      if (package_name === null) {
+        throw new Error("Invalid package name or URI " + package_uri);
+      }
+
+      if (package_name === package_uri) {
+        package_uri = 'default channel';
+      }
+
+      if (package_name in loadedPackages) {
+        if (package_uri !== loadedPackages[package_name]) {
+          throw new Error(
+            "URI mismatch, attempting to load package " +
+              package_name + " from " + package_uri + " while it is already " +
+              "loaded from " + loadedPackages[package_name] + " ! "
+          );
+        }
+      } else {
+        toLoad[package_name] = package_uri;
+        if (packages.hasOwnProperty(package_name)) {
+          for (k in packages[package_name]) {
+            subpackage = packages[package_name][k];
+            if (!(subpackage in loadedPackages) && !(subpackage in toLoad)) {
+              queue.push(subpackage);
+            }
+          }
+        } else {
+          throw new Error("Unknown package " + package_name);
+        }
+      }
+    }
+
+    promise = new RSVP.Promise(function (resolve, reject) {
+      if (Object.keys(toLoad).length === 0) {
+        resolve('No new packages to load');
+      }
+
+      pyodide.monitorRunDependencies = function (n) {
+        if (n === 0) {
+          for (package_name in toLoad) {
+            loadedPackages[package_name] = toLoad[package_name];
+          }
+          delete pyodide.monitorRunDependencies;
+          packageList = Array.from(Object.keys(toLoad)).join(', ');
+          preloadWasm().then(() => {resolve(`Loaded ${packageList}`)});
+          // resolve("Loaded " + packageList);
+        }
+      };
+
+      function script_reject(e) {
+        reject(e);
+      }
+
+      for (package_name in toLoad) {
+        // script = document.createElement('script');
+        package_uri = toLoad[package_name];
+        if (package_uri === 'default channel') {
+          script = package_name + ".js";
+          // script.src = package_name + ".js";
+        } else {
+          // script.src = package_uri;
+          script = package_uri;
+        }
+        console.log("loading dependencies");
+        console.log(script);
+        importScripts(script);
+        // script.onerror = script_reject;
+        // document.body.appendChild(script);
+      }
+
+      // We have to invalidate Python's import caches, or it won't
+      // see the new files. This is done here so it happens in parallel
+      // with the fetching over the network.
+      window.pyodide.runPython('import importlib as _importlib\n' +
+                               '_importlib.invalidate_caches()\n');
+    });
+    return promise;
+  }
+
+  function pyodideSetting() {
+    window.pyodide = pyodide(Module);
+    window.pyodide.loadPackage = pyodideLoadPackage;
+
+    var defer = RSVP.defer(), promise = defer.promise;
+
+    Module.postRun = defer.resolve;
+    promise.then(function () {
+      console.log("postRun get called");
+      delete window.Module;
+    });
+
+    return defer.promise;
+  }
+
+Module.checkABI = function(ABI_number) {
+  if (ABI_number !== parseInt('1')) {
+    var ABI_mismatch_exception = `ABI numbers differ. Expected 1, got ${ABI_number}`;
+    console.error(ABI_mismatch_exception);
+    throw ABI_mismatch_exception;
+  }
+  return true;
+};
+
+  function loadPyodide(info, receiveInstance) {
+    var queue = new RSVP.Queue();
+    queue.push(function () {
+      console.log("loading pyodide.asm.wasm");
+      return ajax({url: "pyodide.asm.wasm", dataType: "arraybuffer"})
+    })
+      .push(function (evt) {
+      console.log("WebAssembly.instantiate");
+        return WebAssembly.instantiate(evt.target.response, info);
+      })
+      .push(function (results) {
+      console.log("receiveInstance");
+        return receiveInstance(results.instance);
+      })
+      .push(undefined, function(error) {
+        console.log(error);
+      });
+    return queue;
+  }
+
+  function initPyodide() {
+    var queue = new RSVP.Queue();
+    queue.push(function () {
+      Module.instantiateWasm = loadPyodide;
+      window.Module = Module;
+      console.log("setting instantiateWasm");
+    })
+      .push(function () {
+      console.log("importing pyodide.asm.data.js");
+        return importScripts('pyodide.asm.data.js');
+      })
+      .push(function () {
+      console.log("pyodide.asm.js");
+        return importScripts('pyodide.asm.js');
+      })
+      .push(function () {
+      console.log("calling pyodideSetting");
+        return pyodideSetting();
+      })
+      .push(function () {
+      console.log("loading packages.json");
+        return ajax({url: 'packages.json'});
+      })
+      .push(function (evt) {
+      console.log("parse packages.json");
+        return JSON.parse(evt.target.response);
+      })
+      .push(function (json) {
+        window.pyodide._module = Module;
+        window.pyodide.loadedPackages = [];
+        window.pyodide._module.packages = json;
+      var b = document.createElement('div');
+      var a = self.document.createElement('div');
+      console.log(b.innerHTML);
+      console.log(a.innerHTML);
+      console.log("setting packages");
+        return;
+      })
+      .push(undefined, function (error) {
+        console.log(error);
+      });
+    return queue;
+  }
+
+/*
+  function executePyCell(line_list) {
+    var result, code_text = line_list.join('\n');
+    result = pyodide.runPython(code_text);
+    return result;
+  }
+
+  function pyodideSetting() {
+    window.pyodide = pyodide(Module);
+    window.pyodide.loadPackage = pyodideLoadPackage;
+
+    var defer = RSVP.defer(), promise = defer.promise;
+
+    Module.postRun = defer.resolve;
+    promise.then(function () {
+      console.log("postRun get called");
+      delete window.Module;
+    });
+
+    return defer.promise;
+  }
+*/
+addEventListener('message', function(e) {
+  console.log("Got it");
+  console.log(e.data);
+  var text = e.data;
+  if (text == "load pyodide") {
+    console.log("loading pyodide");
+    var promiseB = initPyodide();
+
+    return promiseB
+    .then(function () {
+      console.log("Loading matplotlib");
+      return pyodideLoadPackage("matplotlib");
+    })
+      .then(function () {
+      console.log("posting messasge");
+      // window.pyodide.runPython("import matplotlib");
+      // window.pyodide.runPython("from matplotlib import pyplot");
+      // window.pyodide.runPython("import matplotlib.pyplot as plt\n\nplt.plot([1,23,2,4])\nplt.ylabel('some numbers')\n\nplt.show()\n");
+      postMessage("pyodide loaded");
+    });
+  } else {
+    var index_pos, index, code_text, result;
+      index_pos = text.indexOf('_');
+      index = parseInt(text.slice(0, index_pos));
+      code_text = text.slice(index_pos + 1, -1);
+    console.log("Calculating");
+    result = window.pyodide.runPython(code_text);
+    postMessage(index + '_' + result);
+    console.log("result is");
+    console.log(result);
+  }
+}, false);
\ No newline at end of file