Use Proxy to make calling Python objects from JS more natural

src/js2python.cpp

@@ -27,8 +27,13 @@ PyObject *jsToPython(val x) {
     Py_INCREF(Py_False);
     return Py_False;
   } else if (!x["$$"].isUndefined() &&
-             x["$$"]["ptrType"]["name"].equals(val("Py*"))) {
-    Py py_x = x.as<Py>();
+             x["$$"]["ptrType"]["name"].equals(val("PyObject*"))) {
+    PyObject *py_x = x.as<PyObject *>(emscripten::allow_raw_pointers());
+    Py_INCREF(py_x);
+    return py_x;
+  } else if (!x["$$"].isUndefined() &&
+             x["$$"]["ptrType"]["name"].equals(val("PyCallable*"))) {
+    PyCallable py_x = x.as<PyCallable>();
     PyObject *pypy_x = py_x.x;
     Py_INCREF(pypy_x);
     return pypy_x;

src/main.cpp

@@ -38,14 +38,17 @@ EMSCRIPTEN_BINDINGS(python) {
   emscripten::function("runPython", &runPython);
   emscripten::function("pyimport", &pyimport);
   emscripten::function("repr", &repr);
+  emscripten::class_<PyObject>("PyObject");
   emscripten::class_<Py>("Py")
-    .function<val>("call", &Py::call)
-    .function<val>("getattr", &Py::getattr)
-    .function<void>("setattr", &Py::setattr)
-    .function<val>("hasattr", &Py::hasattr)
-    .function<val>("getitem", &Py::getitem)
-    .function<void>("setitem", &Py::setitem)
-    .function<val>("hasitem", &Py::hasitem);
+    .class_function<bool>("isExtensible", &Py::isExtensible)
+    .class_function<bool>("has", &Py::has)
+    .class_function<val>("get", &Py::get)
+    .class_function<val>("set", &Py::set)
+    .class_function<val>("deleteProperty", &Py::deleteProperty)
+    .class_function<val>("ownKeys", &Py::ownKeys)
+    .class_function<val>("enumerate", &Py::enumerate);
+  emscripten::class_<PyCallable>("PyCallable")
+    .function<val>("call", &PyCallable::call);
 }
 
 extern "C" {

src/pyodide.js

@@ -67,6 +67,19 @@ var languagePluginLoader = new Promise((resolve, reject) => {
         return promise;
     };
 
+    let makeCallableProxy = (obj) => {
+        var clone = obj.clone();
+        return (args, kwargs) => {
+            if (args === undefined) {
+                args = [];
+            }
+            if (kwargs === undefined) {
+                kwargs = {};
+            }
+            return clone.call(args, kwargs);
+        };
+    };
+
     let wasmURL = `${baseURL}pyodide.asm.wasm`;
     let Module = {};
 
@@ -86,11 +99,10 @@ var languagePluginLoader = new Promise((resolve, reject) => {
     script.src = `${baseURL}pyodide.asm.js`;
     script.onload = () => {
         window.pyodide = pyodide(Module);
-        if (window.iodide !== undefined) {
-            window.pyodide.loadPackage = loadPackage;
-        }
+        window.pyodide.loadPackage = loadPackage;
+        window.pyodide.makeCallableProxy = makeCallableProxy;
     };
-    document.body.appendChild(script);
+    document.head.appendChild(script);
 
     if (window.iodide !== undefined) {
         // Load the custom CSS for Pyodide
@@ -105,15 +117,15 @@ var languagePluginLoader = new Promise((resolve, reject) => {
             shouldHandle: (val) => {
                 return (typeof val === 'object' &&
                         val['$$'] !== undefined &&
-                        val['$$']['ptrType']['name'] === 'Py*');
+                        val['$$']['ptrType']['name'] === 'PyObject*');
             },
 
             render: (val) => {
                 let div = document.createElement('div');
                 div.className = 'rendered_html';
-                if (val.hasattr('_repr_html_')) {
+                if ('_repr_html_' in val) {
                     div.appendChild(new DOMParser().parseFromString(
-                        val.getattr('_repr_html_').call([], {}), 'text/html').body.firstChild);
+                        val._repr_html_(), 'text/html').body.firstChild);
                 } else {
                     let pre = document.createElement('pre');
                     pre.textContent = window.pyodide.repr(val);

src/pyproxy.cpp

@@ -2,43 +2,29 @@
 
 using emscripten::val;
 
-Py::Py(PyObject *obj) : x(obj) {
-  Py_INCREF(x);
+val Py::makeProxy(PyObject *obj) {
+  Py_INCREF(obj);
+  return val::global("Proxy").new_(val(obj), val::global("pyodide")["Py"]);
 }
 
-Py::Py(const Py& o) : x(o.x) {
-  Py_INCREF(x);
+bool Py::isExtensible(val obj, val proxy) {
+  return true;
 }
 
-Py::~Py() {
-  Py_DECREF(x);
+bool Py::has(val obj, val idx) {
+  PyObject *x = obj.as<PyObject *>(emscripten::allow_raw_pointers());
+  PyObject *pyidx = jsToPython(idx);
+  bool result = PyObject_HasAttr(x, pyidx) ? true: false;
+  Py_DECREF(pyidx);
+  return result;
 }
 
-val Py::call(val args, val kwargs) {
-  PyObject *pyargs = jsToPythonArgs(args);
-  if (pyargs == NULL) {
-    return pythonExcToJs();
-  }
-
-  PyObject *pykwargs = jsToPythonKwargs(kwargs);
-  if (pykwargs == NULL) {
-    Py_DECREF(pyargs);
-    return pythonExcToJs();
-  }
-
-  PyObject *pyret = PyObject_Call(x, pyargs, pykwargs);
-  Py_DECREF(pyargs);
-  Py_DECREF(pykwargs);
-  if (pyret == NULL) {
-    return pythonExcToJs();
+val Py::get(val obj, val idx, val proxy) {
+  if (idx.equals(val("$$"))) {
+    return obj["$$"];
   }
 
-  val ret = pythonToJs(pyret);
-  Py_DECREF(pyret);
-  return ret;
-}
-
-val Py::getattr(val idx) {
+  PyObject *x = obj.as<PyObject *>(emscripten::allow_raw_pointers());
   PyObject *pyidx = jsToPython(idx);
   PyObject *attr = PyObject_GetAttr(x, pyidx);
   Py_DECREF(pyidx);
@@ -49,55 +35,85 @@ val Py::getattr(val idx) {
   val ret = pythonToJs(attr);
   Py_DECREF(attr);
   return ret;
-}
+};
 
-void Py::setattr(val idx, val v) {
+val Py::set(val obj, val idx, val value, val proxy) {
+  PyObject *x = obj.as<PyObject *>(emscripten::allow_raw_pointers());
   PyObject *pyidx = jsToPython(idx);
-  PyObject *pyv = jsToPython(v);
-
-  int ret = PyObject_SetAttr(x, pyidx, pyv);
+  PyObject *pyvalue = jsToPython(value);
+  int ret = PyObject_SetAttr(x, pyidx, pyvalue);
   Py_DECREF(pyidx);
-  Py_DECREF(pyv);
+  Py_DECREF(pyvalue);
+
   if (ret) {
-    pythonExcToJs();
+    return pythonExcToJs();
   }
+  return value;
 }
 
-val Py::hasattr(val idx) {
+val Py::deleteProperty(val obj, val idx) {
+  PyObject *x = obj.as<PyObject *>(emscripten::allow_raw_pointers());
   PyObject *pyidx = jsToPython(idx);
-  val result(PyObject_HasAttr(x, pyidx) ? true : false);
-  Py_DECREF(pyidx);
-  return result;
-}
 
-val Py::getitem(val idx) {
-  PyObject *pyidx = jsToPython(idx);
-  PyObject *item = PyObject_GetItem(x, pyidx);
+  int ret = PyObject_DelAttr(x, pyidx);
   Py_DECREF(pyidx);
-  if (item == NULL) {
+
+  printf("return %d\n", ret);
+
+  if (ret) {
     return pythonExcToJs();
   }
 
-  val ret = pythonToJs(item);
-  Py_DECREF(item);
-  return ret;
+  return val::global("undefined");
 }
 
-void Py::setitem(val idx, val v) {
-  PyObject *pyidx = jsToPython(idx);
-  PyObject *pyv = jsToPython(v);
+val Py::ownKeys(val obj) {
+  PyObject *x = obj.as<PyObject *>(emscripten::allow_raw_pointers());
+  PyObject *dir = PyObject_Dir(x);
+  if (dir == NULL) {
+    return pythonExcToJs();
+  }
 
-  int ret = PyObject_SetItem(x, pyidx, pyv);
-  Py_DECREF(pyidx);
-  Py_DECREF(pyv);
-  if (ret) {
-    pythonExcToJs();
+  val result = val::global("Array").new_();
+  result.call<int>("push", val("$$"));
+  Py_ssize_t n = PyList_Size(dir);
+  for (Py_ssize_t i = 0; i < n; ++i) {
+    PyObject *entry = PyList_GetItem(dir, i);
+    result.call<int>("push", pythonToJs(entry));
   }
-}
+  Py_DECREF(dir);
 
-val Py::hasitem(val idx) {
-  PyObject *pyidx = jsToPython(idx);
-  val result(PySequence_Contains(x, pyidx) ? true : false);
-  Py_DECREF(pyidx);
   return result;
 }
+
+val Py::enumerate(val obj) {
+  return Py::ownKeys(obj);
+}
+
+val PyCallable::makeCallableProxy(PyObject *obj) {
+  return val::global("pyodide").call<val>("makeCallableProxy", PyCallable(obj));
+}
+
+val PyCallable::call(val args, val kwargs) {
+  PyObject *pyargs = jsToPythonArgs(args);
+  if (pyargs == NULL) {
+    return pythonExcToJs();
+  }
+
+  PyObject *pykwargs = jsToPythonKwargs(kwargs);
+  if (pykwargs == NULL) {
+    Py_DECREF(pyargs);
+    return pythonExcToJs();
+  }
+
+  PyObject *pyret = PyObject_Call(x, pyargs, pykwargs);
+  Py_DECREF(pyargs);
+  Py_DECREF(pykwargs);
+  if (pyret == NULL) {
+    return pythonExcToJs();
+  }
+
+  val ret = pythonToJs(pyret);
+  Py_DECREF(pyret);
+  return ret;
+}

src/pyproxy.hpp

@@ -9,21 +9,50 @@
 #include "js2python.hpp"
 #include "python2js.hpp"
 
+// This implements the Javascript Proxy handler interface as defined here:
+//     https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
+
 class Py {
+public:
+  static emscripten::val makeProxy(PyObject *obj);
+
+  static bool isExtensible(
+      emscripten::val obj, emscripten::val proxy);
+  static bool has(
+      emscripten::val obj, emscripten::val idx);
+  static emscripten::val get(
+      emscripten::val obj, emscripten::val idx, emscripten::val proxy);
+  static emscripten::val set(
+      emscripten::val obj, emscripten::val idx, emscripten::val value,
+      emscripten::val proxy);
+  static emscripten::val deleteProperty(
+      emscripten::val obj, emscripten::val idx);
+  static emscripten::val ownKeys(
+      emscripten::val obj);
+  static emscripten::val enumerate(
+      emscripten::val obj);
+};
+
+class PyCallable {
 public:
   PyObject *x;
 
-  Py(PyObject *obj);
-  Py(const Py& o);
-  ~Py();
-
-  emscripten::val call(emscripten::val args, emscripten::val kwargs);
-  emscripten::val getattr(emscripten::val idx);
-  void setattr(emscripten::val idx, emscripten::val v);
-  emscripten::val hasattr(emscripten::val idx);
-  emscripten::val getitem(emscripten::val idx);
-  void setitem(emscripten::val idx, emscripten::val v);
-  emscripten::val hasitem(emscripten::val idx);
+  PyCallable(PyObject *x_) : x(x_) {
+    Py_INCREF(x);
+  }
+
+  PyCallable(const PyCallable& o) : x(o.x) {
+    Py_INCREF(x);
+  }
+
+  ~PyCallable() {
+    Py_DECREF(x);
+  }
+
+  emscripten::val call(
+      emscripten::val args, emscripten::val kwargs);
+
+  static emscripten::val makeCallableProxy(PyObject *obj);
 };
 
 #endif /* PYPROXY_H */

src/python.html

@@ -32,7 +32,7 @@
       "pluginType": "language"
     }
   },
-  "lastExport": "2018-04-06T14:10:51.879Z"
+  "lastExport": "2018-05-04T17:13:00.489Z"
 }
 
 %% md
@@ -124,12 +124,12 @@ def square(x, integer=False):
   return x * x
 
 %% md
-To call a Python callable from Javascript, we use its `call` method, which takes two arguments: the positional arguments as an array, and the keyword arguments as an object.
+Since calling conventions are a bit different in Python than in Javascript, all Python callables take two arguments when called from Javascript: the positional arguments as an array, and the keyword arguments as an object.
 
 %% js
 // javascript
 var square = pyodide.pyimport("square")
-square.call([2.5], {integer: true})
+square([2.5], {integer: true})
 
 %% md
 This is equivalent to the following Python syntax:
@@ -155,12 +155,12 @@ We can get the value of its `val` property as so:
 %% js
 // javascript
 var foo = pyodide.pyimport("foo")
-foo.getattr("val")
+foo.val
 
 %% md
 ### Using Javascript objects from Python
 
-Likewise, you can use Javascript objects from Python, and since the overloading features of Python are a little more dynamic, the syntax for doing so is also a little bit easier.
+Likewise, you can use Javascript objects from Python.
 
 %% js
 // javascript
@@ -266,4 +266,4 @@ A couple things that already work that will be coming to this example notebook s
 <div id='page'></div>
 <script src='https://iodide-project.github.io/dist/iodide.pyodide-20180420.js'></script>
 </body>
-</html>
+</html>
\ No newline at end of file

src/python2js.cpp

@@ -17,49 +17,62 @@ val pythonExcToJs() {
 
   val excval("");
 
-  PyObject *tbmod = PyImport_ImportModule("traceback");
-  if (tbmod == NULL) {
-    excval = pythonToJs(PyObject_Repr(value));
-  } else {
-    PyObject *format_exception;
-    if (traceback == NULL || traceback == Py_None) {
-      no_traceback = true;
-      format_exception = PyObject_GetAttrString(tbmod, "format_exception_only");
-    } else {
-      format_exception = PyObject_GetAttrString(tbmod, "format_exception");
-    }
-    if (format_exception == NULL) {
-      excval = val("Couldn't get format_exception function");
+  if (type == NULL || type == Py_None ||
+      value == NULL || value == Py_None) {
+    excval = val("No exception type or value");
+    PyErr_Print();
+    PyErr_Clear();
+    goto exit;
+  }
+
+  {
+    PyObject *tbmod = PyImport_ImportModule("traceback");
+    if (tbmod == NULL) {
+      excval = pythonToJs(PyObject_Repr(value));
     } else {
-      PyObject *pylines;
-      if (no_traceback) {
-        pylines = PyObject_CallFunctionObjArgs
-          (format_exception, type, value, NULL);
+      PyObject *format_exception;
+      if (traceback == NULL || traceback == Py_None) {
+        no_traceback = true;
+        format_exception = PyObject_GetAttrString(tbmod, "format_exception_only");
       } else {
-        pylines = PyObject_CallFunctionObjArgs
-          (format_exception, type, value, traceback, NULL);
+        format_exception = PyObject_GetAttrString(tbmod, "format_exception");
       }
-      if (pylines == NULL) {
-        excval = val("Error calling traceback.format_exception");
-        PyErr_Print();
+      if (format_exception == NULL) {
+        excval = val("Couldn't get format_exception function");
       } else {
-        PyObject *newline = PyUnicode_FromString("\n");
-        PyObject *pystr = PyUnicode_Join(newline, pylines);
-        PyObject_Print(pystr, stderr, 0);
-        excval = pythonToJs(pystr);
-        Py_DECREF(pystr);
-        Py_DECREF(newline);
-        Py_DECREF(pylines);
+        PyObject *pylines;
+        if (no_traceback) {
+          pylines = PyObject_CallFunctionObjArgs
+            (format_exception, type, value, NULL);
+        } else {
+          pylines = PyObject_CallFunctionObjArgs
+            (format_exception, type, value, traceback, NULL);
+        }
+        if (pylines == NULL) {
+          excval = val("Error calling traceback.format_exception");
+          PyErr_Print();
+          PyErr_Clear();
+          goto exit;
+        } else {
+          PyObject *newline = PyUnicode_FromString("\n");
+          PyObject *pystr = PyUnicode_Join(newline, pylines);
+          PyObject_Print(pystr, stderr, 0);
+          excval = pythonToJs(pystr);
+          Py_DECREF(pystr);
+          Py_DECREF(newline);
+          Py_DECREF(pylines);
+        }
+        Py_DECREF(format_exception);
       }
-      Py_DECREF(format_exception);
+      Py_DECREF(tbmod);
     }
-    Py_DECREF(tbmod);
   }
 
+ exit:
   val exc = val::global("Error").new_(excval);
 
-  Py_DECREF(type);
-  Py_DECREF(value);
+  Py_XDECREF(type);
+  Py_XDECREF(value);
   Py_XDECREF(traceback);
 
   PyErr_Clear();
@@ -114,9 +127,10 @@ val pythonToJs(PyObject *x) {
     for (size_t i = 0; i < length; ++i) {
       PyObject *item = PySequence_GetItem(x, i);
       if (item == NULL) {
-        // If something goes wrong converting the sequence, fallback to the Python proxy
+        // If something goes wrong converting the sequence (as is the case with
+        // Pandas data frames), fallback to the Python object proxy
         PyErr_Clear();
-        return val(Py(x));
+        return Py::makeProxy(x);
       }
       x_array.call<int>("push", pythonToJs(item));
       Py_DECREF(item);
@@ -131,8 +145,10 @@ val pythonToJs(PyObject *x) {
       x_object.set(pythonToJs(k), pythonToJs(v));
     }
     return x_object;
+  } else if (PyCallable_Check(x)) {
+    return PyCallable::makeCallableProxy(x);
   } else {
-    return val(Py(x));
+    return Py::makeProxy(x);
   }
 }
 

src/test.html

@@ -15,4 +15,5 @@
         </script>
         <script src="pyodide_dev.js"></script>
     </head>
+    <body></body>
 </html>

test/test_python.py

@@ -26,6 +26,28 @@ def test_import_js(selenium):
     assert result == 'Foo'
 
 
+def test_py_proxy(selenium):
+    selenium.run(
+        "class Foo:\n  bar = 42\n  def get_value(self):\n    return 64\nf = Foo()\n")
+    assert selenium.run_js("return pyodide.pyimport('f').get_value()") == 64
+    assert selenium.run_js("return pyodide.pyimport('f').bar") == 42
+    assert selenium.run_js("return ('bar' in pyodide.pyimport('f'))") == True
+    selenium.run_js("f = pyodide.pyimport('f'); f.baz = 32")
+    assert selenium.run("f.baz") == 32
+    assert set(selenium.run_js(
+        "return Object.getOwnPropertyNames(pyodide.pyimport('f'))")) == set(
+            ['$$', '__class__', '__delattr__', '__dict__', '__dir__',
+             '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__',
+             '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__',
+             '__lt__', '__module__', '__ne__', '__new__', '__reduce__',
+             '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__',
+             '__str__', '__subclasshook__', '__weakref__', 'bar', 'baz',
+             'get_value'])
+    assert selenium.run("hasattr(f, 'baz')") == True
+    selenium.run_js("delete pyodide.pyimport('f').baz")
+    assert selenium.run("hasattr(f, 'baz')") == False
+
+
 def test_run_core_python_test(python_test, selenium):
     selenium.run(
         "import sys\n"