Lots of pandas fixes:

- Install dependencies: dateutil, pytz and six
- Patch things for threading and mmap
- Disable deployment (for related reasons)

.gitignore

 *.bc
 *.a
+.patched
 .pytest_cache/
 __pycache__
 geckodriver.log
@@ -16,9 +17,16 @@ geckodriver.log
 /numpy/build
 /numpy/host
 /numpy/downloads
-/numpy/.patched
 
 /pandas/build
 /pandas/host
 /pandas/downloads
-/pandas/.patched
+
\ No newline at end of file
+/dateutil/downloads
+/dateutil/python-dateutil-2.7.2
+
+/pytz/downloads
+/pytz/pytz-2018.4
+
+/six/downloads
+/six/six-1.11.0

.travis.yml

@@ -30,17 +30,17 @@ install:
 script:
   - make test
 
-deploy:
-  provider: pages
-  github-token: "$GITHUB_TOKEN"
-  skip-cleanup: true
-  keep-history: false
-  repo: iodide-project/pyodide-demo
-  verbose: false
-  local-dir: build
-  target-branch: master
-  on:
-    branch: master
+# deploy:
+#   provider: pages
+#   github-token: "$GITHUB_TOKEN"
+#   skip-cleanup: true
+#   keep-history: false
+#   repo: iodide-project/pyodide-demo
+#   verbose: false
+#   local-dir: build
+#   target-branch: master
+#   on:
+#     branch: master
 
 env:
   global:

Makefile

@@ -11,13 +11,14 @@ HOSTPYTHON=$(CPYTHONROOT)/build/$(PYVERSION)/host/bin/python3
 CC=emcc
 CXX=em++
 OPTFLAGS=-O3
-CXXFLAGS=-std=c++14 $(OPTFLAGS) -g -I$(CPYTHONINC) -Wno-warn-absolute-paths
+CFLAGS=$(OPTFLAGS) -g -I$(CPYTHONINC) -Wno-warn-absolute-paths
+CXXFLAGS=$(CFLAGS) -std=c++14
 LDFLAGS=\
 	-O3 \
 	-s MODULARIZE=1 \
 	$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
   -s "BINARYEN_METHOD='native-wasm'" \
-  -s TOTAL_MEMORY=268435456 \
+  -s TOTAL_MEMORY=536870912 \
 	-s MAIN_MODULE=1 \
 	-s EMULATED_FUNCTION_POINTERS=1 \
   -s EMULATE_FUNCTION_POINTER_CASTS=1 \
@@ -38,6 +39,15 @@ PANDAS_ROOT=pandas/build/pandas
 PANDAS_LIBS=\
 	$(PANDAS_ROOT)/_libs/lib.so
 
+DATEUTIL_ROOT=dateutil/python-dateutil-2.7.2/build/lib/dateutil
+DATEUTIL_LIBS=$(DATEUTIL_ROOT)/__init__.py
+
+PYTZ_ROOT=pytz/pytz-2018.4/build/lib/pytz
+PYTZ_LIBS=$(PYTZ_ROOT)/__init__.py
+
+SIX_ROOT=six/six-1.11.0/build/lib
+SIX_LIBS=$(SIX_ROOT)/six.py
+
 SITEPACKAGES=root/lib/python$(PYMINOR)/site-packages
 
 all: build/pyodide.asm.html build/pyodide.js build/pyodide_dev.js build/python.html
@@ -45,7 +55,7 @@ all: build/pyodide.asm.html build/pyodide.js build/pyodide_dev.js build/python.h
 
 build/pyodide.asm.html: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
                         src/pyimport.bc src/pyproxy.bc src/python2js.bc \
-												src/runpython.bc root/.built
+												src/runpython.bc src/dummy_thread.bc root/.built
 	[ -d build ] || mkdir build
 	$(CC) -s EXPORT_NAME="'pyodide'" --bind -o $@ $(filter %.bc,$^) $(LDFLAGS) \
 		$(foreach d,$(wildcard root/*),--preload-file $d@/$(notdir $d))
@@ -90,10 +100,17 @@ clean:
 	$(CXX) --bind -o $@ $< $(CXXFLAGS)
 
 
+%.bc: %.c $(CPYTHONLIB)
+	$(CC) -o $@ $< $(CFLAGS)
+
+
 root/.built: \
 		$(CPYTHONLIB) \
 		$(NUMPY_LIBS) \
 		$(PANDAS_LIBS) \
+		$(DATEUTIL_LIBS) \
+		$(PYTZ_LIBS) \
+		$(SIX_LIBS) \
 		src/lazy_import.py \
 		src/sitecustomize.py \
 		src/webbrowser.py \
@@ -103,6 +120,9 @@ root/.built: \
 	cp -a $(CPYTHONLIB)/ root/lib
 	cp -a numpy/build/numpy $(SITEPACKAGES)
 	cp -a pandas/build/pandas $(SITEPACKAGES)
+	cp -a $(DATEUTIL_ROOT) $(SITEPACKAGES)
+	cp -a $(PYTZ_ROOT) $(SITEPACKAGES)
+	cp $(SIX_LIBS) $(SITEPACKAGES)
 	rm -fr $(SITEPACKAGES)/numpy/distutils
 	cp src/lazy_import.py $(SITEPACKAGES)
 	cp src/sitecustomize.py $(SITEPACKAGES)
@@ -127,8 +147,22 @@ $(CPYTHONLIB): emsdk/emsdk/emsdk
 $(NUMPY_LIBS): $(CPYTHONLIB)
 	make -C numpy
 
+
 $(PANDAS_LIBS): $(NUMPY_LIBS)
 	make -C pandas
 
+
+$(DATEUTIL_LIBS): $(CPYTHONLIB)
+	make -C dateutil
+
+
+$(PYTZ_LIBS): $(CPYTHONLIB)
+	make -C pytz
+
+
+$(SIX_LIBS): $(CPYTHONLIB)
+	make -C six
+
+
 emsdk/emsdk/emsdk:
 	make -C emsdk

dateutil/Makefile

+PYODIDE_ROOT=$(abspath ..)
+include ../Makefile.envs
+
+PYVERSION=3.6.4
+PYMINOR=$(basename $(PYVERSION))
+
+DATEUTILVERSION=2.7.2
+
+ROOT=$(abspath .)
+
+HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
+HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
+
+SRC=$(ROOT)/python-dateutil-$(DATEUTILVERSION)
+BUILD=$(SRC)/build/lib/dateutil
+TARBALL=$(ROOT)/downloads/python-dateutil-$(DATEUTILVERSION).tgz
+URL=https://files.pythonhosted.org/packages/c5/39/4da7c2dbc4f023fba5fb2325febcadf0d0ce0efdc8bd12083a0f65d20653/python-dateutil-2.7.2.tar.gz
+
+
+all: $(BUILD)/__init__.py
+
+
+clean:
+	-rm -fr downloads
+	-rm -fr $(SRC)
+
+
+$(TARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(URL)
+	md5sum --quiet --check checksums || (rm $@; false)
+
+
+$(SRC)/setup.py: $(TARBALL)
+	tar -C . -xf $(TARBALL)
+	touch $(SRC)/setup.py
+
+
+$(BUILD)/__init__.py: $(ROOT)/.patched
+	( \
+		cd $(SRC) ; \
+		$(HOSTPYTHON) setup.py build ; \
+		touch build/lib/dateutil/__init__.py \
+	)
+
+
+$(ROOT)/.patched: $(SRC)/setup.py
+	cat patches/*.patch | (cd $(SRC) ; patch -p1)
+	touch $@

dateutil/checksums

+03a08c8bcf0a2b29f1cd21b9de4d12fb  ./downloads/python-dateutil-2.7.2.tgz

dateutil/patches/dummy-thread-lock.patch

+diff -ru python-dateutil-2.7.2.orig/dateutil/tz/tz.py python-dateutil-2.7.2/dateutil/tz/tz.py
+--- python-dateutil-2.7.2.orig/dateutil/tz/tz.py	2018-04-24 18:33:38.436301176 -0400
++++ python-dateutil-2.7.2/dateutil/tz/tz.py	2018-04-24 18:34:39.653365591 -0400
+@@ -16,7 +16,6 @@
+
+ import six
+ from six import string_types
+-from six.moves import _thread
+ from ._common import tzname_in_python2, _tzinfo
+ from ._common import tzrangebase, enfold
+ from ._common import _validate_fromutc_inputs
+@@ -33,6 +32,14 @@
+ EPOCHORDINAL = EPOCH.toordinal()
+
+
++class _dummy_lock:
++    def __enter__(self, *args, **kwargs):
++        pass
++
++    def __exit__(self, *args, **kwargs):
++        pass
++
++
+ @six.add_metaclass(_TzSingleton)
+ class tzutc(datetime.tzinfo):
+     """
+@@ -1104,7 +1111,7 @@
+         self._comps = comps
+         self._cachedate = []
+         self._cachecomp = []
+-        self._cache_lock = _thread.allocate_lock()
++        self._cache_lock = _dummy_lock()
+
+     def _find_comp(self, dt):
+         if len(self._comps) == 1:
+@@ -1407,7 +1414,7 @@
+         def __init__(self):
+
+             self.__instances = {}
+-            self._cache_lock = _thread.allocate_lock()
++            self._cache_lock = _dummy_lock()
+
+         def __call__(self, name=None):
+             with self._cache_lock:

pandas/Makefile

@@ -46,12 +46,15 @@ all: \
 	$(BUILD)/_libs/index.so \
 	$(BUILD)/_libs/interval.so \
 	$(BUILD)/_libs/join.so \
+	$(BUILD)/_libs/reshape.so \
 	$(BUILD)/_libs/json.so \
 	$(BUILD)/_libs/lib.so \
 	$(BUILD)/_libs/parsers.so \
 	$(BUILD)/_libs/period.so \
+	$(BUILD)/_libs/tslibs/parsing.so \
 	$(BUILD)/_libs/properties.so \
 	$(BUILD)/_libs/sparse.so \
+	$(BUILD)/_libs/hashing.so \
 	$(BUILD)/_libs/tslib.so \
 	$(BUILD)/_libs/tslibs/fields.so \
 	$(BUILD)/_libs/tslibs/frequencies.so \
@@ -61,6 +64,7 @@ all: \
 	$(BUILD)/_libs/window.so \
 	$(BUILD)/io/msgpack/_packer.so \
 	$(BUILD)/io/msgpack/_unpacker.so \
+	$(BUILD)/_libs/testing.so \
 	$(BUILD)/io/sas/_sas.so \
 	$(BUILD)/util/_move.so
 
@@ -160,6 +164,12 @@ $(BUILD)/_libs/period.so: \
 	mv tmp.wasm $@
 
 
+$(BUILD)/_libs/tslibs/parsing.so: \
+		$(HOSTDIR)/pandas/_libs/tslibs/parsing.bc
+	$(CC) $(LDFLAGS) $^ -o tmp.wasm
+	mv tmp.wasm $@
+
+
 $(BUILD)/_libs/tslibs/frequencies.so: \
 		$(HOSTDIR)/pandas/_libs/tslibs/frequencies.bc
 	$(CC) $(LDFLAGS) $^ -o tmp.wasm
@@ -193,6 +203,12 @@ $(BUILD)/_libs/join.so: \
 	mv tmp.wasm $@
 
 
+$(BUILD)/_libs/reshape.so: \
+		$(HOSTDIR)/pandas/_libs/reshape.bc
+	$(CC) $(LDFLAGS) $^ -o tmp.wasm
+	mv tmp.wasm $@
+
+
 $(BUILD)/_libs/interval.so: \
 		$(HOSTDIR)/pandas/_libs/interval.bc
 	$(CC) $(LDFLAGS) $^ -o tmp.wasm
@@ -231,6 +247,12 @@ $(BUILD)/io/sas/_sas.so: \
 	mv tmp.wasm $@
 
 
+$(BUILD)/_libs/testing.so: \
+		$(HOSTDIR)/pandas/_libs/testing.bc
+	$(CC) $(LDFLAGS) $^ -o tmp.wasm
+	mv tmp.wasm $@
+
+
 $(BUILD)/io/msgpack/_packer.so: \
 		$(HOSTDIR)/pandas/io/msgpack/_packer.bc
 	$(CC) $(LDFLAGS) $^ -o tmp.wasm

pandas/patches/avoid-mmap.patch

+diff -ru pandas-0.22.0.orig/pandas/io/common.py pandas-0.22.0/pandas/io/common.py
+--- pandas-0.22.0.orig/pandas/io/common.py	2018-04-24 17:23:06.833240372 -0400
++++ pandas-0.22.0/pandas/io/common.py	2018-04-24 17:42:33.148934940 -0400
+@@ -3,7 +3,10 @@
+ import os
+ import csv
+ import codecs
+-import mmap
++try:
++    from mmap import mmap
++except ImportError:
++    mmap = None
+ from contextlib import contextmanager, closing
+
+ from pandas.compat import StringIO, BytesIO, string_types, text_type
+@@ -203,7 +206,7 @@
+
+     if isinstance(filepath_or_buffer, (compat.string_types,
+                                        compat.binary_type,
+-                                       mmap.mmap)):
++                                       mmap)):
+         return _expand_user(filepath_or_buffer), None, compression
+
+     if not is_file_like(filepath_or_buffer):
+@@ -432,7 +435,7 @@
+     """
+
+     def __init__(self, f):
+-        self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
++        self.mmap = mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+
+     def __getattr__(self, name):
+         return getattr(self.mmap, name)

pytz/Makefile

+PYODIDE_ROOT=$(abspath ..)
+include ../Makefile.envs
+
+PYVERSION=3.6.4
+PYMINOR=$(basename $(PYVERSION))
+
+PYTZVERSION=2018.4
+
+ROOT=$(abspath .)
+
+HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
+HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
+
+SRC=$(ROOT)/pytz-$(PYTZVERSION)
+BUILD=$(SRC)/build/lib/pytz
+TARBALL=$(ROOT)/downloads/pytz-$(PYTZVERSION).tgz
+URL=https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-$(PYTZVERSION).tar.gz
+
+
+all: $(BUILD)/__init__.py
+
+
+
+clean:
+	-rm -fr downloads
+	-rm -fr $(SRC)
+
+
+$(TARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(URL)
+	md5sum --quiet --check checksums || (rm $@; false)
+
+
+$(SRC)/setup.py: $(TARBALL)
+	tar -C . -xf $(TARBALL)
+	touch $(SRC)/setup.py
+
+
+$(BUILD)/__init__.py: $(ROOT)/.patched
+	( \
+		cd $(SRC) ; \
+		$(HOSTPYTHON) setup.py build ; \
+		touch build/lib/pytz/__init__.py \
+	)
+
+
+$(ROOT)/.patched: $(SRC)/setup.py
+	cat patches/*.patch | (cd $(SRC) ; patch -p1)
+	touch $@

pytz/checksums

+f054437920c895dd14a4509fabafe029  ./downloads/pytz-2018.4.tgz

pytz/patches/dummy-threading.patch

+diff -ru pytz-2018.4.orig/pytz/lazy.py pytz-2018.4/pytz/lazy.py
+--- pytz-2018.4.orig/pytz/lazy.py	2018-04-24 15:47:46.293980100 -0400
++++ pytz-2018.4/pytz/lazy.py	2018-04-24 15:56:06.281648286 -0400
+@@ -1,4 +1,7 @@
+-from threading import RLock
++try:
++    from threading import RLock
++except (ImportError, ModuleNotFoundError):
++    from dummy_threading import RLock
+ try:
+     from collections.abc import Mapping as DictMixin
+ except ImportError:  # Python < 3.3
+Binary files pytz-2018.4.orig/pytz/__pycache__/lazy.cpython-36.pyc and pytz-2018.4/pytz/__pycache__/lazy.cpython-36.pyc differ

remove_modules.txt

@@ -2,7 +2,6 @@ _osx_support.py
 ctypes
 curses
 dbm
-distutils
 ensurepip
 idlelib
 lib2to3

six/Makefile

+PYODIDE_ROOT=$(abspath ..)
+include ../Makefile.envs
+
+PYVERSION=3.6.4
+PYMINOR=$(basename $(PYVERSION))
+
+SIXVERSION=1.11.0
+
+ROOT=$(abspath .)
+
+HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
+HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
+
+SRC=$(ROOT)/six-$(SIXVERSION)
+BUILD=$(SRC)/build/lib/six
+TARBALL=$(ROOT)/downloads/six-$(SIXVERSION).tgz
+URL=https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz
+
+
+all: $(BUILD)/__init__.py
+
+
+
+clean:
+	-rm -fr downloads
+	-rm -fr $(SRC)
+
+
+$(TARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(URL)
+	md5sum --quiet --check checksums || (rm $@; false)
+
+
+$(SRC)/setup.py: $(TARBALL)
+	tar -C . -xf $(TARBALL)
+	touch $(SRC)/setup.py
+
+
+$(BUILD)/__init__.py: $(SRC)/setup.py
+	( \
+		cd $(SRC) ; \
+		$(HOSTPYTHON) setup.py build ; \
+		touch build/lib/six.py \
+	)

six/checksums

+d12789f9baf7e9fb2524c0c64f1773f8  downloads/six-1.11.0.tgz

src/dummy_thread.c

+#include <Python.h>
+#include <pythread.h>
+
+/* A set of mock-ups for thread lock functionality that is required for
+   Cython-generated modules to work. */
+
+PyThread_type_lock PyThread_allocate_lock(void) {
+  return (PyThread_type_lock)0x1;
+}
+
+void PyThread_free_lock(PyThread_type_lock _) {
+
+}
+
+int PyThread_acquire_lock(PyThread_type_lock _, int __) {
+  return PY_LOCK_ACQUIRED;
+}

test/test_pandas.py

 def test_pandas(selenium):
     selenium.run("import pandas")
+
+
+def test_extra_import(selenium):
+    selenium.run("from pandas import Series, DataFrame, Panel")