Merge pull request #77 from mdboom/python37

Fix #71: Upgrade to Python 3.7

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 defaults: &defaults
   working_directory: ~/repo
   docker:
-    - image: circleci/python:3.6.5-stretch-browsers
+    - image: circleci/python:3.7.0-stretch-browsers
 
 jobs:
   build:
@@ -27,11 +27,11 @@ jobs:
             sudo ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format
 
             sudo pip install virtualenv
-            
+
             virtualenv pyodide-env
 
             source pyodide-env/bin/activate
-            
+
             pip install pytest pytest-xdist pytest-instafail selenium PyYAML
 
             # Get recent version of Firefox and geckodriver

.editorconfig

@@ -16,3 +16,6 @@ indent_size = 4
 
 [Makefile]
 indent_style = tab
+
+[**.patch]
+trim_trailing_whitespace = false

Makefile

@@ -56,7 +56,7 @@ all: build/pyodide.asm.js \
 
 build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
 											src/pyimport.bc src/pyproxy.bc src/python2js.bc \
-											src/runpython.bc src/dummy_thread.bc src/hiwire.bc
+											src/runpython.bc src/hiwire.bc
 	[ -d build ] || mkdir build
 	$(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \
 	  $(LDFLAGS) -s FORCE_FILESYSTEM=1
@@ -143,7 +143,7 @@ build/test.data: $(CPYTHONLIB)
 	)
 	( \
 		cd build; \
-		python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.6/test --js-output=test.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__ \
+		python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__ \
   )
 	uglifyjs build/test.js -o build/test.js
 

Makefile.envs

@@ -6,7 +6,7 @@ export EM_CACHE = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten_cache
 export EMSCRIPTEN = $(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.10
 export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.10_64bit_binaryen
 
-export PYVERSION=3.6.4
+export PYVERSION=3.7.0
 export PYMINOR=$(basename $(PYVERSION))
 export HOSTPYTHONROOT=$(PYODIDE_ROOT)/cpython/build/$(PYVERSION)/host
 export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3

README.md

@@ -24,7 +24,7 @@ need to build it yourself prior.
 Additional build prerequisites are:
 
 - A working native compiler toolchain, enough to build CPython.
-- A native Python 3.6 or later to run the build scripts.
+- A native Python 3.7 to run the build scripts.
 - PyYAML
 - [lessc](https://lesscss.org/) to compile less to css.
 - [uglifyjs](https://github.com/mishoo/UglifyJS) to minify Javascript builds.

cpython/Makefile

@@ -15,6 +15,12 @@ URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz
 LIB=libpython$(PYMINOR).a
 
 
+ZLIBVERSION = 1.2.11
+ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz
+ZLIBBUILD=$(ROOT)/build/zlib-$(ZLIBVERSION)
+ZLIBURL=https://zlib.net/zlib-1.2.11.tar.gz
+
+
 all: $(INSTALL)/lib/$(LIB)
 
 
@@ -42,6 +48,11 @@ $(TARBALL):
 	md5sum --quiet --check checksums || (rm $@; false)
 
 
+$(ZLIBTARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(ZLIBURL)
+
+
 $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
 	mkdir -p $(HOSTINSTALL)
 	[ -d $(HOSTBUILD) ] || tar -C $(HOSTINSTALL) -xf $(TARBALL)
@@ -61,7 +72,14 @@ $(BUILD)/.patched: $(TARBALL)
 	touch $@
 
 
-$(BUILD)/Makefile: $(BUILD)/.patched
+$(ZLIBBUILD)/.patched: $(ZLIBTARBALL)
+	[ -d $(ROOT)/build ] || (mkdir $(ROOT)/build)
+	tar -C $(ROOT)/build/ -xf $(ROOT)/downloads/zlib-1.2.11.tar.gz
+	cat patches/zlib/*.patch | (cd $(ZLIBBUILD) ; patch -p1)
+	touch $@
+
+
+$(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched
 	cp config.site $(BUILD)/
 	( \
 		cd $(BUILD); \
@@ -71,10 +89,6 @@ $(BUILD)/Makefile: $(BUILD)/.patched
 
 $(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local
 	cp Setup.local $(BUILD)/Modules/
-	( \
-		cd $(BUILD)/Modules/zlib; \
-		emconfigure ./configure --static \
-	)
 	( \
 		cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \
 		cd $(BUILD); \

cpython/Setup.local

@@ -7,6 +7,7 @@
 array arraymodule.c	# array objects
 math mathmodule.c _math.c # -lm # math library functions, e.g. sin()
 cmath cmathmodule.c # complex math functions
+_contextvars _contextvarsmodule.c
 _struct _struct.c	# binary structure packing/unpacking
 _random _randommodule.c	# Random number generator
 _bisect _bisectmodule.c	# Bisection algorithms
@@ -21,10 +22,9 @@ parser parsermodule.c
 _socket socketmodule.c
 select selectmodule.c
 _posixsubprocess _posixsubprocess.c
-
 binascii binascii.c
 
-zlib zlibmodule.c -IModules/zlib
+zlib zlibmodule.c -I../../zlib-1.2.11
 
 pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY
 
@@ -33,7 +33,9 @@ _sha256 sha256module.c
 _sha512 sha512module.c
 _sha3 _sha3/sha3module.c
 _md5 md5module.c
-_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.6.4/Modules/_blake2/blake2s_impl.c
+_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.7.0/Modules/_blake2/blake2s_impl.c
+
+_queue _queuemodule.c
 
 #future_builtins future_builtins.c
 

cpython/checksums

-9de6494314ea199e3633211696735f65 ./downloads/Python-3.6.4.tgz
+41b6595deb4147a1ed517a7d9a580271  downloads/Python-3.7.0.tgz

cpython/config.site

@@ -2,4 +2,5 @@ ac_cv_file__dev_ptmx=no
 ac_cv_file__dev_ptc=no
 ac_cv_func_dlopen=yes
 ac_cv_lib_socket_socket=no
-ac_cv_little_endian_double=yes
+ac_cv_header_linux_vm_sockets_h=yes
\ No newline at end of file
+ac_cv_little_endian_double=yes

cpython/patches/add-emscripten-host.patch

+diff --git a/config.sub b/config.sub
+index ba37cf99e2..d772f33065 100755
+--- a/config.sub
++++ b/config.sub
+@@ -118,7 +118,8 @@ case $maybe_os in
+   linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
+   knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
+   kopensolaris*-gnu* | cloudabi*-eabi* | \
+-  storm-chaos* | os2-emx* | rtmk-nova*)
++  storm-chaos* | os2-emx* | rtmk-nova* | \
++  emscripten)
+     os=-$maybe_os
+     basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+     ;;
+@@ -377,6 +378,7 @@ case $basic_machine in
+ 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
+ 	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
+ 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
++  | asmjs \
+ 	| avr-* | avr32-* \
+ 	| ba-* \
+ 	| be32-* | be64-* \
+@@ -1522,6 +1524,8 @@ case $os in
+ 			;;
+ 		esac
+ 		;;
++	-emscripten)
++		;;
+ 	-nacl*)
+ 		;;
+ 	-ios)
 diff --git a/configure b/configure
-index 8cf777e..263719f 100755
+index e0389649ae..e6ab1613cf 100755
 --- a/configure
 +++ b/configure
-@@ -3202,6 +3202,9 @@ then
+@@ -3257,6 +3257,9 @@ then
  	*-*-cygwin*)
  		ac_sys_system=Cygwin
  		;;
@@ -12,7 +43,7 @@ index 8cf777e..263719f 100755
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-@@ -3248,6 +3251,9 @@ if test "$cross_compiling" = yes; then
+@@ -3301,6 +3304,9 @@ if test "$cross_compiling" = yes; then
  	*-*-cygwin*)
  		_host_cpu=
  		;;
@@ -23,10 +54,10 @@ index 8cf777e..263719f 100755
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
 diff --git a/configure.ac b/configure.ac
-index 78fe3c7..d7665b4 100644
+index b5beb08574..8051e5a8ed 100644
 --- a/configure.ac
 +++ b/configure.ac
-@@ -322,6 +322,9 @@ then
+@@ -379,6 +379,9 @@ then
  	*-*-cygwin*)
  		ac_sys_system=Cygwin
  		;;
@@ -36,7 +67,7 @@ index 78fe3c7..d7665b4 100644
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-@@ -368,6 +371,9 @@ if test "$cross_compiling" = yes; then
+@@ -423,6 +426,9 @@ if test "$cross_compiling" = yes; then
  	*-*-cygwin*)
  		_host_cpu=
  		;;
@@ -46,34 +77,3 @@ index 78fe3c7..d7665b4 100644
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-diff --git a/config.sub b/config.sub
-index d654d03..0d8236f 100755
---- a/config.sub
-+++ b/config.sub
-@@ -119,7 +119,8 @@ case $maybe_os in
-   linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
-   knetbsd*-gnu* | netbsd*-gnu* | \
-   kopensolaris*-gnu* | cloudabi*-eabi* | \
--  storm-chaos* | os2-emx* | rtmk-nova*)
-+  storm-chaos* | os2-emx* | rtmk-nova* | \
-+  emscripten)
-     os=-$maybe_os
-     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
-     ;;
-@@ -254,6 +255,7 @@ case $basic_machine in
- 	| am33_2.0 \
- 	| arc | arceb \
- 	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
-+	| asmjs \
- 	| avr | avr32 \
- 	| be32 | be64 \
- 	| bfin \
-@@ -1510,6 +1512,8 @@ case $os in
- 	-dicos*)
- 		os=-dicos
- 		;;
-+	-emscripten)
-+		;;
- 	-nacl*)
- 		;;
- 	-none)

cpython/patches/disable-set_inheritable.patch

-*** a/Python/fileutils.c	2016-10-25 02:01:16.270148214 +0100
---- b/Python/fileutils.c	2016-10-25 02:01:27.086148626 +0100
-***************
-*** 785,790 ****
---- 785,793 ----
-  static int
-  set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
-  {
-+ #ifdef EMSCRIPTEN
-+     return 0;
-+ #else
-  #ifdef MS_WINDOWS
-      HANDLE handle;
-      DWORD flags;
-***************
-*** 897,902 ****
---- 900,906 ----
-      }
-      return 0;
-  #endif
-+ #endif
-  }
-  
-  /* Make the file descriptor non-inheritable.
+diff --git a/Python/fileutils.c b/Python/fileutils.c
+index 35869c81ac..1af4ead7f5 100644
+--- a/Python/fileutils.c
++++ b/Python/fileutils.c
+@@ -964,6 +964,9 @@ _Py_get_inheritable(int fd)
+ static int
+ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
+ {
++#ifdef EMSCRIPTEN
++    return 0;
++#else
+ #ifdef MS_WINDOWS
+     HANDLE handle;
+     DWORD flags;
+@@ -1080,6 +1083,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
+     }
+     return 0;
+ #endif
++#endif
+ }
+ 
+ /* Make the file descriptor non-inheritable.

cpython/patches/no_af_vsock.patch

+diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h
+index fdb4e871ce..c990003ff3 100644
+--- a/Modules/socketmodule.h
++++ b/Modules/socketmodule.h
+@@ -144,6 +144,8 @@ typedef int socklen_t;
+ extern "C" {
+ #endif
+ 
++#undef AF_VSOCK
++
+ /* Python module and C API name */
+ #define PySocket_MODULE_NAME    "_socket"
+ #define PySocket_CAPI_NAME      "CAPI"

cpython/patches/testing.patch

 diff --git a/Lib/platform.py b/Lib/platform.py
-index cc2db9870d..ac4e3c538f 100755
+index 20f9817f4f..bcf96874c0 100755
 --- a/Lib/platform.py
 +++ b/Lib/platform.py
 @@ -748,7 +748,7 @@ def _syscmd_uname(option, default=''):
@@ -21,18 +21,18 @@ index cc2db9870d..ac4e3c538f 100755
          return default
      target = _follow_symlinks(target)
 diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
-index ca5f9c20dd..97934039ee 100644
+index 64b25aab80..7cf67fd149 100644
 --- a/Lib/test/support/script_helper.py
 +++ b/Lib/test/support/script_helper.py
-@@ -11,6 +11,7 @@ import subprocess
+@@ -9,6 +9,7 @@ import os.path
+ import subprocess
  import py_compile
- import contextlib
- import shutil
-+import unittest
  import zipfile
++import unittest
  
  from importlib.util import source_from_cache
-@@ -37,6 +38,8 @@ def interpreter_requires_environment():
+ from test.support import make_legacy_pyc, strip_python_stderr
+@@ -34,6 +35,8 @@ def interpreter_requires_environment():
      situation.  PYTHONPATH or PYTHONUSERSITE are other common environment
      variables that might impact whether or not the interpreter can start.
      """
@@ -40,16 +40,16 @@ index ca5f9c20dd..97934039ee 100644
 +
      global __cached_interp_requires_environment
      if __cached_interp_requires_environment is None:
-         # Try running an interpreter with -E to see if it works or not.
-@@ -165,6 +168,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
+         # If PYTHONHOME is set, assume that we need it
+@@ -172,6 +175,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
      kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
      object.
      """
 +    raise unittest.SkipTest("no subprocess")
 +
-     cmd_line = [sys.executable, '-E']
-     cmd_line.extend(args)
-     # Under Fedora (?), GNU readline can output junk on stderr when initialized,
+     cmd_line = [sys.executable]
+     if not interpreter_requires_environment():
+         cmd_line.append('-E')
 diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
 index 55faf4c427..b2201c09e7 100644
 --- a/Lib/test/test_code.py
@@ -67,7 +67,7 @@ index 55faf4c427..b2201c09e7 100644
  import weakref
  try:
 diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
-index b73a96f757..0d1c411f9a 100644
+index 1fc4de11e1..e6c91707ae 100644
 --- a/Lib/test/test_import/__init__.py
 +++ b/Lib/test/test_import/__init__.py
 @@ -10,7 +10,10 @@ import py_compile

cpython/patches/zlib/include_unistd.patch

+diff -u zlib-1.2.11-orig/gzguts.h zlib-1.2.11/gzguts.h
+--- a/gzguts.h
++++ b/gzguts.h
+@@ -3,6 +3,8 @@
+  * For conditions of distribution and use, see copyright notice in zlib.h
+  */
+ 
++#include <unistd.h>
++
+ #ifdef _LARGEFILE64_SOURCE
+ #  ifndef _LARGEFILE_SOURCE
+ #    define _LARGEFILE_SOURCE 1

docs/new_packages.md

@@ -107,8 +107,7 @@ Extra arguments to pass to the linker when building for WebAssembly.
 Shell commands to run after building the library. These are run inside of
 `bash`, and there are two special environment variables defined:
 
-- `$BUILD`: The root of the built package. (`build/lib.XXX/` inside of the
-  source directory). This is what will be installed into Python site-packages.
+- `$SITEPACKAGES`: The `site-packages` directory into which the package has been installed.
 - `$PKGDIR`: The directory in which the `meta.yaml` file resides.
 
 (This key is not in the Conda spec).

packages/matplotlib/meta.yaml

@@ -26,10 +26,10 @@ build:
   cflags: -s USE_FREETYPE=1 -s USE_LIBPNG=1 -s USE_ZLIB=1
   ldflags: ../../../../emsdk/emsdk/.emscripten_cache/asmjs/libpng.bc
   post: |
-    rm -rf $BUILD/matplotlib/backends/qt_editor
-    rm -rf $BUILD/matplotlib/backends/web_backend
-    rm -rf $BUILD/sphinxext
-    cp $PKGDIR/src/fontList.json $BUILD/matplotlib
+    rm -rf $SITEPACKAGES/matplotlib/backends/qt_editor
+    rm -rf $SITEPACKAGES/matplotlib/backends/web_backend
+    rm -rf $SITEPACKAGES/sphinxext
+    cp $PKGDIR/src/fontList.json $SITEPACKAGES/matplotlib
 
 requirements:
   run:

packages/numpy/meta.yaml

 package:
   name: numpy
-  version: 1.14.1
+  version: 1.14.5
 
 source:
-  url: https://pypi.python.org/packages/a3/99/74aa456fc740a7e8f733af4e8302d8e61e123367ec660cd89c53a3cd4d70/numpy-1.14.1.zip
-  md5: b8324ef90ac9064cd0eac46b8b388674
+  url: https://files.pythonhosted.org/packages/d5/6e/f00492653d0fdf6497a181a1c1d46bbea5a2383e7faf4c8ca6d6f3d2581d/numpy-1.14.5.zip
+  sha256: a4a433b3a264dbc9aa9c7c241e87c0358a503ea6394f8737df1683c7c9a102ac
 
   patches:
     - patches/add-emscripten-cpu.patch

packages/pandas/meta.yaml

 package:
   name: pandas
-  version: 0.22.0
+  version: 0.23.3
 
 source:
-  url: https://pypi.python.org/packages/08/01/803834bc8a4e708aedebb133095a88a4dad9f45bbaf5ad777d2bea543c7e/pandas-0.22.0.tar.gz
-  md5: c7a2757b607748255f3270221ac61d59
+  url: https://files.pythonhosted.org/packages/3e/56/82c4d4c049294f87ebd05b65fdcbc9ed68bd23fb0a7e4469caf9a75d199f/pandas-0.23.3.tar.gz
+  sha256: 9cd3614b4e31a0889388ff1bd19ae857ad52658b33f776065793c293a29cf612
 
   patches:
     - patches/avoid-mmap.patch
 
 build:
-  cflags: -include ../../src/state.h
+  cflags: -Wno-implicit-function-declaration
 
 requirements:
   run:

packages/pandas/patches/avoid-mmap.patch

-diff -ru pandas-0.22.0.orig/pandas/io/common.py pandas-0.22.0/pandas/io/common.py
---- pandas-0.22.0.orig/pandas/io/common.py	2018-04-24 17:23:06.833240372 -0400
-+++ pandas-0.22.0/pandas/io/common.py	2018-04-24 17:42:33.148934940 -0400
+diff --git a/pandas/io/common.py b/pandas/io/common.py
+index ac9077f2d..9303fe721 100644
+--- a/pandas/io/common.py
++++ b/pandas/io/common.py
 @@ -3,7 +3,10 @@
  import os
  import csv
@@ -11,23 +12,23 @@ diff -ru pandas-0.22.0.orig/pandas/io/common.py pandas-0.22.0/pandas/io/common.p
 +except ImportError:
 +    mmap = type(None)
  from contextlib import contextmanager, closing
-
- from pandas.compat import StringIO, BytesIO, string_types, text_type
-@@ -203,7 +206,7 @@
-
+ import zipfile
+ 
+@@ -210,7 +213,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
+ 
      if isinstance(filepath_or_buffer, (compat.string_types,
                                         compat.binary_type,
 -                                       mmap.mmap)):
 +                                       mmap)):
-         return _expand_user(filepath_or_buffer), None, compression
-
+         return _expand_user(filepath_or_buffer), None, compression, False
+ 
      if not is_file_like(filepath_or_buffer):
-@@ -432,7 +435,7 @@
+@@ -464,7 +467,7 @@ class MMapWrapper(BaseIterator):
      """
-
+ 
      def __init__(self, f):
 -        self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
 +        self.mmap = mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
-
+ 
      def __getattr__(self, name):
          return getattr(self.mmap, name)

packages/pandas/src/state.h

-#include "Python.h"
-
-inline PyGILState_STATE PyGILState_Ensure(void) { return PyGILState_UNLOCKED; }
-
-inline void PyGILState_Release(PyGILState_STATE state) { }

src/dummy_thread.c

-#include <Python.h>
-#include <pythread.h>
-
-/* A set of mock-ups for thread lock functionality that is required for
-   Cython-generated modules to work.
-
-   It is compiled into the main module, as if it came from Python itself.
-*/
-
-PyThread_type_lock
-PyThread_allocate_lock(void)
-{
-  return (PyThread_type_lock)0x1;
-}
-
-void
-PyThread_free_lock(PyThread_type_lock _)
-{}
-
-int
-PyThread_acquire_lock(PyThread_type_lock _, int __)
-{
-  return PY_LOCK_ACQUIRED;
-}

src/jsimport.c

@@ -43,7 +43,7 @@ JsImport_Call(PyObject* self, PyObject* args, PyObject* kwargs)
         if (key == NULL) {
           return NULL;
         }
-        char* c = PyUnicode_AsUTF8(key);
+        const char* c = PyUnicode_AsUTF8(key);
         if (c == NULL) {
           Py_DECREF(key);
           return NULL;

src/jsproxy.c

@@ -46,7 +46,7 @@ JsProxy_GetAttr(PyObject* o, PyObject* attr_name)
     return NULL;
   }
 
-  char* key = PyUnicode_AsUTF8(str);
+  const char* key = PyUnicode_AsUTF8(str);
 
   if (strncmp(key, "new", 4) == 0 || strncmp(key, "_has_bytes", 11) == 0) {
     Py_DECREF(str);
@@ -81,7 +81,7 @@ JsProxy_SetAttr(PyObject* o, PyObject* attr_name, PyObject* pyvalue)
   if (attr_name_py_str == NULL) {
     return -1;
   }
-  char* key = PyUnicode_AsUTF8(attr_name_py_str);
+  const char* key = PyUnicode_AsUTF8(attr_name_py_str);
 
   if (pyvalue == NULL) {
     hiwire_delete_member_string(self->js, (int)key);

test/make_test_list.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 
 TEST_DIR = (Path(__file__).parent
-            / "cpython/build/3.6.4/host/lib/python3.6/test")
+            / "cpython/build/3.6.4/host/lib/python3.7/test")
 
 
 def collect_tests(base_dir):

test/python_tests.txt

@@ -4,8 +4,10 @@
 # Following reason codes are skipped:
 # - platform-specific: This is testing something about a particular platform
 #   that isn't relevant here
+# - async: relies on async
 # - audioop: Requires the audioop module
 # - floating point: Failures caused by floating-point differences
+# - threading: Failures due to lack of a threading implementation
 # - subprocess: Failures caused by no subprocess module. Some of these are
 #   because the underlying functionality depends on subprocess, and others are
 #   just a side-effect of the way the test is written. The latter should
@@ -17,7 +19,8 @@
 # - strftime: Failures due to differences / shortcomings in WebAssembly's
 #   implementation of date/time formatting in strftime and strptime
 # - permissions: Issues with the test writing to the virtual filesystem
-# - locale: Fails due to include locale implementation.
+# - locale: Fails due to limitations in the included locale implementation.
+# - multiprocessing: Fails due to no multiprocessing implementation.
 # - nonsense: This functionality doesn't make sense in this context. Includes
 #   things like `pip`, `distutils`
 #
@@ -28,7 +31,7 @@
 # - crash-chrome: Same as crash but only affecting Chrome
 # - crash-firefox: Same as crash but only affecting Firefox
 
-test___all__
+test___all__       multiprocessing
 test___future__
 test__locale  locale
 test__opcode
@@ -40,30 +43,31 @@ test_argparse
 test_array
 test_asdl_parser
 test_ast
-test_asyncgen
-test_asynchat
-test_asyncio.test_base_events
-test_asyncio.test_events
-test_asyncio.test_futures
-test_asyncio.test_locks
-test_asyncio.test_pep492
-test_asyncio.test_proactor_events
-test_asyncio.test_queues
-test_asyncio.test_selector_events
-test_asyncio.test_sslproto
-test_asyncio.test_streams
-test_asyncio.test_subprocess
-test_asyncio.test_tasks
-test_asyncio.test_transports
-test_asyncio.test_unix_events
-test_asyncio.test_windows_events
-test_asyncio.test_windows_utils
-test_asyncore bad ioctl syscall
+test_asyncgen async
+test_asynchat async
+test_asyncio.test_base_events async
+test_asyncio.test_events      async
+test_asyncio.test_futures     async
+test_asyncio.test_locks       async
+test_asyncio.test_pep492      async
+test_asyncio.test_proactor_events async
+test_asyncio.test_queues          async
+test_asyncio.test_selector_events async
+test_asyncio.test_sslproto        async
+test_asyncio.test_streams         async
+test_asyncio.test_subprocess      async
+test_asyncio.test_tasks           async
+test_asyncio.test_transports      async
+test_asyncio.test_unix_events     async
+test_asyncio.test_windows_events  async
+test_asyncio.test_windows_utils   async
+test_asyncore bad ioctl syscall   async
 test_atexit
 test_audioop  audioop
 test_augassign
 test_base64
 test_baseexception
+test_bdb
 test_bigaddrspace
 test_bigmem
 test_binascii
@@ -76,6 +80,7 @@ test_bufio
 test_builtin  floating point
 test_bytes
 test_bz2
+test_c_locale_coercion
 test_calendar
 test_call
 test_capi
@@ -112,16 +117,19 @@ test_complex
 test_concurrent_futures
 test_configparser
 test_contains
+test_context
 test_contextlib
+test_contextlib_async async
 test_copy
 test_copyreg  dbm
-test_coroutines
+test_coroutines async
 test_cprofile _lsprof
 test_crashers
 test_crypt
 test_csv
 test_ctypes
 test_curses
+test_dataclasses
 test_datetime strftime
 test_dbm      permissions
 test_dbm_dumb permissions
@@ -143,7 +151,7 @@ test_dis
 test_distutils crash
 test_doctest   subprocess
 test_doctest2
-test_docxmlrpc
+test_docxmlrpc  socket
 test_dtrace platform
 test_dummy_thread
 test_dummy_threading
@@ -155,7 +163,7 @@ test_email.test__header_value_parser
 test_email.test_asian_codecs
 test_email.test_contentmanager
 test_email.test_defect_handling
-test_email.test_email
+test_email.test_email threading
 test_email.test_generator
 test_email.test_headerregistry
 test_email.test_inversion
@@ -164,8 +172,9 @@ test_email.test_parser
 test_email.test_pickleable
 test_email.test_policy
 test_email.test_utils
+test_embed
 test_ensurepip  nonsense
-test_enum
+test_enum       threading
 test_enumerate
 test_eof
 test_epoll  crash
@@ -185,14 +194,15 @@ test_finalization
 test_float  floating point
 test_flufl
 test_fnmatch
-test_fork1
+test_fork1  threading
 test_format
 test_fractions
 test_frame
+test_frozen
 test_fstring
-test_ftplib
+test_ftplib syscall 21537
 test_funcattrs
-test_functools
+test_functools  threading
 test_future
 test_future3
 test_future4
@@ -201,6 +211,7 @@ test_gc
 test_gdb
 test_generator_stop
 test_generators
+test_genericclass
 test_genericpath  permissions
 test_genexps
 test_getargs2
@@ -213,7 +224,7 @@ test_grammar
 test_grp
 test_gzip
 test_hash
-test_hashlib
+test_hashlib  threading
 test_heapq
 test_hmac
 test_html
@@ -221,9 +232,9 @@ test_htmlparser
 test_http_cookiejar
 test_http_cookies
 test_httplib  socket
-test_httpservers
+test_httpservers  threading
 test_idle
-test_imaplib
+test_imaplib  socket
 test_imghdr
 test_imp  crash
 test_importlib.builtin.test_finder
@@ -244,15 +255,19 @@ test_importlib.import_.test_packages
 test_importlib.import_.test_path
 test_importlib.import_.test_relative_imports
 test_importlib.source.test_case_sensitivity
-test_importlib.source.test_file_loader
+test_importlib.source.test_file_loader  unittest has no attribute mock
 test_importlib.source.test_finder
 test_importlib.source.test_path_hook
 test_importlib.source.test_source_encoding
 test_importlib.test_abc
 test_importlib.test_api
 test_importlib.test_lazy
-test_importlib.test_locks
+test_importlib.test_locks threading
 test_importlib.test_namespace_pkgs
+test_importlib.test_open
+test_importlib.test_path
+test_importlib.test_read
+test_importlib.test_resource
 test_importlib.test_spec
 test_importlib.test_util
 test_importlib.test_windows platform-specific
@@ -298,7 +313,6 @@ test_long
 test_longexp
 test_lzma
 test_macpath  platform-specific
-test_macurl2path
 test_mailbox  crash
 test_mailcap  nonsense
 test_marshal
@@ -338,14 +352,14 @@ test_peepholer
 test_pickle dbm
 test_pickletools  dbm
 test_pipes        platform-specific
-test_pkg
+test_pkg          unknown
 test_pkgimport
 test_pkgutil
 test_platform   subprocess
 test_plistlib
 test_poll subprocess
 test_popen  subprocess
-test_poplib
+test_poplib bad ioctl syscall 21537
 test_posix  crash
 test_posixpath  crash
 test_pow
@@ -361,10 +375,10 @@ test_py_compile
 test_pyclbr
 test_pydoc  crash
 test_pyexpat
-test_queue
+test_queue  threading
 test_quopri subprocess
 test_raise
-test_random
+test_random subprocess
 test_range
 test_re locale
 test_readline
@@ -374,10 +388,10 @@ test_reprlib
 test_resource
 test_richcmp
 test_rlcompleter  crash
-test_robotparser
+test_robotparser  socket
 test_runpy
 test_sax
-test_sched
+test_sched  threading
 test_scope
 test_script_helper
 test_secrets
@@ -392,7 +406,7 @@ test_signal
 test_site subprocess
 test_slice
 test_smtpd
-test_smtplib
+test_smtplib  baad ioctl syscall 21537
 test_smtpnet
 test_sndhdr audioop
 test_socket
@@ -425,20 +439,20 @@ test_symtable
 test_syntax
 test_sys  subprocess
 test_sys_setprofile
-test_sys_settrace
+test_sys_settrace async
 test_sysconfig  nonsense
 test_syslog
 test_tarfile  crash
 test_tcl
-test_telnetlib
+test_telnetlib  bad ioctl syscall 21537
 test_tempfile crash
 test_textwrap
-test_thread
-test_threaded_import
-test_threadedtempfile
-test_threading
-test_threading_local
-test_threadsignals
+test_thread threading
+test_threaded_import  floating point
+test_threadedtempfile threading
+test_threading        threading
+test_threading_local  threading
+test_threadsignals    threading
 test_time
 test_timeit
 test_timeout
@@ -477,7 +491,7 @@ test_unpack
 test_unpack_ex
 test_urllib crash
 test_urllib2  subprocess
-test_urllib2_localnet
+test_urllib2_localnet socket
 test_urllib2net
 test_urllib_response
 test_urllibnet
@@ -485,14 +499,15 @@ test_urlparse
 test_userdict
 test_userlist
 test_userstring
+test_utf8_mode
 test_utf8source
 test_uu
 test_uuid subprocess
 test_venv crash
-test_wait3
-test_wait4
+test_wait3  threading
+test_wait4  threading
 test_wave crash
-test_weakref
+test_weakref  threading
 test_weakset
 test_webbrowser replaced
 test_winconsoleio
@@ -506,6 +521,7 @@ test_xml_etree
 test_xml_etree_c
 test_xmlrpc networking
 test_xmlrpc_net
+test_xxtestfuzz
 test_yield_from
 test_zipapp
 test_zipfile

test/test_common.py

@@ -43,4 +43,7 @@ def test_import(name, selenium_standalone):
 
     for import_name in meta.get('test', {}).get('imports', []):
         selenium_standalone.load_package(name)
-        selenium_standalone.run('import %s' % import_name)
+        try:
+            selenium_standalone.run('import %s' % import_name)
+        except Exception as e:
+            print(selenium_standalone.logs)

test/test_pandas.py

@@ -6,7 +6,7 @@ def test_pandas(selenium, request):
         request.applymarker(pytest.mark.xfail(
             run=False, reason='chrome not supported'))
     selenium.load_package("pandas")
-    assert len(selenium.run("import pandas\ndir(pandas)")) == 179
+    assert len(selenium.run("import pandas\ndir(pandas)")) == 140
 
 
 def test_extra_import(selenium, request):

test/test_python.py

@@ -297,9 +297,15 @@ def test_run_core_python_test(python_test, selenium, request):
     try:
         selenium.run(
             "from test.libregrtest import main\n"
-            "main(['{}'], verbose=True, verbose3=True)".format(name))
+            "try:\n"
+            "    main(['{}'], verbose=True, verbose3=True)\n"
+            "except SystemExit as e:\n"
+            "    if e.code != 0:\n"
+            "        raise RuntimeError(f'Failed with code: {{e.code}}')\n"
+            .format(name))
     except selenium.JavascriptException as e:
-        assert 'SystemExit: 0' in str(e)
+        print(selenium.logs)
+        raise
 
 
 def pytest_generate_tests(metafunc):
@@ -326,18 +332,6 @@ def pytest_generate_tests(metafunc):
                              ids=test_modules_ids)
 
 
-def test_recursive_repr(selenium):
-    assert not selenium.run(
-        "d = {}\n"
-        "d[42] = d.values()\n"
-        "result = True\n"
-        "try:\n"
-        "   repr(d)\n"
-        "except RecursionError:\n"
-        "   result = False\n"
-        "result")
-
-
 def test_load_package_after_convert_string(selenium):
     """
     See #93.

tools/buildall

@@ -45,7 +45,11 @@ def build_packages(packagesdir, outputdir, args):
     for pkgname in dependencies.keys():
         build_package(pkgname, dependencies, packagesdir, outputdir, args)
 
-    # This is done last so the main Makefile can use it as a completion token
+    # The "test" package is built in a different way, so we hardcode its
+    # existence here.
+    dependencies['test'] = []
+
+    # This is done last so the Makefile can use it as a completion token.
     with open(outputdir / 'packages.json', 'w') as fd:
         json.dump({'dependencies': dependencies}, fd)
 

tools/buildpkg.py

@@ -79,25 +79,6 @@ def patch(path, srcpath, pkg, args):
         fd.write(b'\n')
 
 
-def get_libdir(srcpath, args):
-    # Get the name of the build/lib.XXX directory that distutils wrote its
-    # output to
-    slug = subprocess.check_output([
-        str(Path(args.host) / 'bin' / 'python3'),
-        '-c',
-        'import sysconfig, sys; '
-        'print("{}-{}.{}".format('
-        'sysconfig.get_platform(), '
-        'sys.version_info[0], '
-        'sys.version_info[1]))']).decode('ascii').strip()
-    purelib = srcpath / 'build' / 'lib'
-    if purelib.is_dir():
-        libdir = purelib
-    else:
-        libdir = srcpath / 'build' / ('lib.' + slug)
-    return libdir
-
-
 def compile(path, srcpath, pkg, args):
     if (srcpath / '.built').is_file():
         return
@@ -121,10 +102,11 @@ def compile(path, srcpath, pkg, args):
 
     post = pkg.get('build', {}).get('post')
     if post is not None:
-        libdir = get_libdir(srcpath, args)
+        site_packages_dir = (
+            srcpath / 'install' / 'lib' / 'python3.7' / 'site-packages')
         pkgdir = path.parent.resolve()
         env = {
-            'BUILD': libdir,
+            'SITEPACKAGES': site_packages_dir,
             'PKGDIR': pkgdir
         }
         subprocess.run([

tools/common.py

@@ -2,8 +2,8 @@ from pathlib import Path
 
 
 ROOTDIR = Path(__file__).parent.resolve()
-HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.6.4' / 'host'
-TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.6.4'
+HOSTPYTHON = ROOTDIR / '..' / 'cpython' / 'build' / '3.7.0' / 'host'
+TARGETPYTHON = ROOTDIR / '..' / 'cpython' / 'installs' / 'python-3.7.0'
 DEFAULTCFLAGS = ''
 DEFAULTLDFLAGS = ' '.join([
     '-O3',