Merge pull request #96 from mdboom/fix93-string-decoding

Fix #93 by avoiding use of TextDecoder

Makefile

@@ -32,7 +32,8 @@ LDFLAGS=\
 	-s USE_LIBPNG=1 \
 	-std=c++14 \
   -lstdc++ \
-  --memory-init-file 0
+  --memory-init-file 0 \
+  -s TEXTDECODER=0
 
 SIX_ROOT=six/six-1.11.0/build/lib
 SIX_LIBS=$(SIX_ROOT)/six.py

src/hiwire.c

@@ -36,10 +36,31 @@ EM_JS(int, hiwire_double, (double val), {
   return Module.hiwire_new_value(val);
 });
 
-EM_JS(int, hiwire_string_utf8_length, (int ptr, int len), {
-  var bytes = new Uint8Array(Module.HEAPU8.buffer, ptr, len);
-  var jsval = new TextDecoder('utf-8').decode(bytes);
-  return Module.hiwire_new_value(jsval);
+EM_JS(int, hiwire_string_ucs4, (int ptr, int len), {
+  var jsstr = "";
+  var idx = ptr / 4;
+  for (var i = 0; i < len; ++i) {
+    jsstr += String.fromCharCode(Module.HEAPU32[idx + i]);
+  }
+  return Module.hiwire_new_value(jsstr);
+});
+
+EM_JS(int, hiwire_string_ucs2, (int ptr, int len), {
+  var jsstr = "";
+  var idx = ptr / 2;
+  for (var i = 0; i < len; ++i) {
+    jsstr += String.fromCharCode(Module.HEAPU16[idx + i]);
+  }
+  return Module.hiwire_new_value(jsstr);
+});
+
+EM_JS(int, hiwire_string_ucs1, (int ptr, int len), {
+  var jsstr = "";
+  var idx = ptr;
+  for (var i = 0; i < len; ++i) {
+    jsstr += String.fromCharCode(Module.HEAPU8[idx + i]);
+  }
+  return Module.hiwire_new_value(jsstr);
 });
 
 EM_JS(int, hiwire_string_utf8, (int ptr), {

src/hiwire.h

@@ -51,13 +51,31 @@ int
 hiwire_double(double val);
 
 /**
- * Create a new Javascript string, given a pointer to a buffer containing UTF8
- * and a length, in bytes.  The string data itself is copied.
+ * Create a new Javascript string, given a pointer to a buffer
+ * containing UCS4 and a length. The string data itself is copied.
  *
  * Returns: New reference
  */
 int
-hiwire_string_utf8_length(int ptr, int len);
+hiwire_string_ucs4(int ptr, int len);
+
+/**
+ * Create a new Javascript string, given a pointer to a buffer
+ * containing UCS2 and a length. The string data itself is copied.
+ *
+ * Returns: New reference
+ */
+int
+hiwire_string_ucs2(int ptr, int len);
+
+/**
+ * Create a new Javascript string, given a pointer to a buffer
+ * containing UCS1 and a length. The string data itself is copied.
+ *
+ * Returns: New reference
+ */
+int
+hiwire_string_ucs1(int ptr, int len);
 
 /**
  * Create a new Javascript string, given a pointer to a null-terminated buffer

src/python2js.c

@@ -130,12 +130,20 @@ python2js_int(PyObject* x)
     }
     return hiwire_double(x_double);
   } else if (PyUnicode_Check(x)) {
-    Py_ssize_t length;
-    char* chars = PyUnicode_AsUTF8AndSize(x, &length);
-    if (chars == NULL) {
+    int kind = PyUnicode_KIND(x);
+    int data = (int)PyUnicode_DATA(x);
+    int length = (int)PyUnicode_GET_LENGTH(x);
+    switch (kind) {
+    case PyUnicode_1BYTE_KIND:
+      return hiwire_string_ucs1(data, length);
+    case PyUnicode_2BYTE_KIND:
+      return hiwire_string_ucs2(data, length);
+    case PyUnicode_4BYTE_KIND:
+      return hiwire_string_ucs4(data, length);
+    default:
+      PyErr_SetString(PyExc_ValueError, "Unknown Unicode KIND");
       return -1;
     }
-    return hiwire_string_utf8_length((int)(void*)chars, length);
   } else if (PyBytes_Check(x)) {
     char* x_buff;
     Py_ssize_t length;

test/test_python.py

@@ -25,6 +25,12 @@ def test_python2js(selenium):
     assert selenium.run_js('return pyodide.runPython("False") === false')
     assert selenium.run_js('return pyodide.runPython("42") === 42')
     assert selenium.run_js('return pyodide.runPython("3.14") === 3.14')
+    # Need to test all three internal string representations in Python: UCS1,
+    # UCS2 and UCS4
+    assert selenium.run_js(
+        'return pyodide.runPython("\'ascii\'") === "ascii"')
+    assert selenium.run_js(
+        'return pyodide.runPython("\'ιωδιούχο\'") === "ιωδιούχο"')
     assert selenium.run_js(
         'return pyodide.runPython("\'碘化物\'") === "碘化物"')
     assert selenium.run_js(
@@ -311,3 +317,18 @@ def test_recursive_repr(selenium):
         "except RecursionError:\n"
         "   result = False\n"
         "result")
+
+
+def test_load_package_after_convert_string(selenium):
+    """
+    See #93.
+    """
+    selenium.run(
+        "import sys\n"
+        "x = sys.version")
+    selenium.run_js(
+        "var x = pyodide.pyimport('x')\n"
+        "console.log(x)")
+    selenium.load_package('kiwisolver')
+    selenium.run(
+        "import kiwisolver")