Skip to content

P
pyodide
Commit 5f42b618 authored by Michael Droettboom's avatar Michael Droettboom
Browse files
Options

Lots of pandas fixes: 

- Install dependencies: dateutil, pytz and six
- Patch things for threading and mmap
- Disable deployment (for related reasons)
parent 4d854354
Hide whitespace changes
Inline Side-by-side
Showing with 338 additions and 17 deletions
.gitignore
View file @ 5f42b618
*.bc
*.a
.patched
.pytest_cache/
__pycache__
geckodriver.log
......@@ -16,9 +17,16 @@ geckodriver.log
/numpy/build
/numpy/host
/numpy/downloads
/numpy/.patched
/pandas/build
/pandas/host
/pandas/downloads
/pandas/.patched
\ No newline at end of file
/dateutil/downloads
/dateutil/python-dateutil-2.7.2
/pytz/downloads
/pytz/pytz-2018.4
/six/downloads
/six/six-1.11.0
.travis.yml
View file @ 5f42b618
......@@ -30,17 +30,17 @@ install:
script:
- make test
deploy:
provider: pages
github-token: "$GITHUB_TOKEN"
skip-cleanup: true
keep-history: false
repo: iodide-project/pyodide-demo
verbose: false
local-dir: build
target-branch: master
on:
branch: master
# deploy:
# provider: pages
# github-token: "$GITHUB_TOKEN"
# skip-cleanup: true
# keep-history: false
# repo: iodide-project/pyodide-demo
# verbose: false
# local-dir: build
# target-branch: master
# on:
# branch: master
env:
global:
......
Makefile
View file @ 5f42b618
......@@ -11,13 +11,14 @@ HOSTPYTHON=$(CPYTHONROOT)/build/$(PYVERSION)/host/bin/python3
CC=emcc
CXX=em++
OPTFLAGS=-O3
CXXFLAGS=-std=c++14 $(OPTFLAGS) -g -I$(CPYTHONINC) -Wno-warn-absolute-paths
CFLAGS=$(OPTFLAGS) -g -I$(CPYTHONINC) -Wno-warn-absolute-paths
CXXFLAGS=$(CFLAGS) -std=c++14
LDFLAGS=\
-O3 \
-s MODULARIZE=1 \
$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
-s "BINARYEN_METHOD='native-wasm'" \
-s TOTAL_MEMORY=268435456 \
-s TOTAL_MEMORY=536870912 \
-s MAIN_MODULE=1 \
-s EMULATED_FUNCTION_POINTERS=1 \
-s EMULATE_FUNCTION_POINTER_CASTS=1 \
......@@ -38,6 +39,15 @@ PANDAS_ROOT=pandas/build/pandas
PANDAS_LIBS=\
$(PANDAS_ROOT)/_libs/lib.so
DATEUTIL_ROOT=dateutil/python-dateutil-2.7.2/build/lib/dateutil
DATEUTIL_LIBS=$(DATEUTIL_ROOT)/__init__.py
PYTZ_ROOT=pytz/pytz-2018.4/build/lib/pytz
PYTZ_LIBS=$(PYTZ_ROOT)/__init__.py
SIX_ROOT=six/six-1.11.0/build/lib
SIX_LIBS=$(SIX_ROOT)/six.py
SITEPACKAGES=root/lib/python$(PYMINOR)/site-packages
all: build/pyodide.asm.html build/pyodide.js build/pyodide_dev.js build/python.html
......@@ -45,7 +55,7 @@ all: build/pyodide.asm.html build/pyodide.js build/pyodide_dev.js build/python.h
build/pyodide.asm.html: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
src/pyimport.bc src/pyproxy.bc src/python2js.bc \
src/runpython.bc root/.built
src/runpython.bc src/dummy_thread.bc root/.built
[ -d build ] || mkdir build
$(CC) -s EXPORT_NAME="'pyodide'" --bind -o $@ $(filter %.bc,$^) $(LDFLAGS) \
$(foreach d,$(wildcard root/*),--preload-file $d@/$(notdir $d))
......@@ -90,10 +100,17 @@ clean:
$(CXX) --bind -o $@ $< $(CXXFLAGS)
%.bc: %.c $(CPYTHONLIB)
$(CC) -o $@ $< $(CFLAGS)
root/.built: \
$(CPYTHONLIB) \
$(NUMPY_LIBS) \
$(PANDAS_LIBS) \
$(DATEUTIL_LIBS) \
$(PYTZ_LIBS) \
$(SIX_LIBS) \
src/lazy_import.py \
src/sitecustomize.py \
src/webbrowser.py \
......@@ -103,6 +120,9 @@ root/.built: \
cp -a $(CPYTHONLIB)/ root/lib
cp -a numpy/build/numpy $(SITEPACKAGES)
cp -a pandas/build/pandas $(SITEPACKAGES)
cp -a $(DATEUTIL_ROOT) $(SITEPACKAGES)
cp -a $(PYTZ_ROOT) $(SITEPACKAGES)
cp $(SIX_LIBS) $(SITEPACKAGES)
rm -fr $(SITEPACKAGES)/numpy/distutils
cp src/lazy_import.py $(SITEPACKAGES)
cp src/sitecustomize.py $(SITEPACKAGES)
......@@ -127,8 +147,22 @@ $(CPYTHONLIB): emsdk/emsdk/emsdk
$(NUMPY_LIBS): $(CPYTHONLIB)
make -C numpy
$(PANDAS_LIBS): $(NUMPY_LIBS)
make -C pandas
$(DATEUTIL_LIBS): $(CPYTHONLIB)
make -C dateutil
$(PYTZ_LIBS): $(CPYTHONLIB)
make -C pytz
$(SIX_LIBS): $(CPYTHONLIB)
make -C six
emsdk/emsdk/emsdk:
make -C emsdk
dateutil/Makefile 0 → 100644
View file @ 5f42b618
PYODIDE_ROOT=$(abspath ..)
include ../Makefile.envs
PYVERSION=3.6.4
PYMINOR=$(basename $(PYVERSION))
DATEUTILVERSION=2.7.2
ROOT=$(abspath .)
HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
SRC=$(ROOT)/python-dateutil-$(DATEUTILVERSION)
BUILD=$(SRC)/build/lib/dateutil
TARBALL=$(ROOT)/downloads/python-dateutil-$(DATEUTILVERSION).tgz
URL=https://files.pythonhosted.org/packages/c5/39/4da7c2dbc4f023fba5fb2325febcadf0d0ce0efdc8bd12083a0f65d20653/python-dateutil-2.7.2.tar.gz
all: $(BUILD)/__init__.py
clean:
-rm -fr downloads
-rm -fr $(SRC)
$(TARBALL):
[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
wget -q -O $@ $(URL)
md5sum --quiet --check checksums || (rm $@; false)
$(SRC)/setup.py: $(TARBALL)
tar -C . -xf $(TARBALL)
touch $(SRC)/setup.py
$(BUILD)/__init__.py: $(ROOT)/.patched
( \
cd $(SRC) ; \
$(HOSTPYTHON) setup.py build ; \
touch build/lib/dateutil/__init__.py \
)
$(ROOT)/.patched: $(SRC)/setup.py
cat patches/*.patch | (cd $(SRC) ; patch -p1)
touch $@
dateutil/checksums 0 → 100644
View file @ 5f42b618
03a08c8bcf0a2b29f1cd21b9de4d12fb ./downloads/python-dateutil-2.7.2.tgz
dateutil/patches/dummy-thread-lock.patch 0 → 100644
View file @ 5f42b618
diff -ru python-dateutil-2.7.2.orig/dateutil/tz/tz.py python-dateutil-2.7.2/dateutil/tz/tz.py
--- python-dateutil-2.7.2.orig/dateutil/tz/tz.py 2018-04-24 18:33:38.436301176 -0400
+++ python-dateutil-2.7.2/dateutil/tz/tz.py 2018-04-24 18:34:39.653365591 -0400
@@ -16,7 +16,6 @@
import six
from six import string_types
-from six.moves import _thread
from ._common import tzname_in_python2, _tzinfo
from ._common import tzrangebase, enfold
from ._common import _validate_fromutc_inputs
@@ -33,6 +32,14 @@
EPOCHORDINAL = EPOCH.toordinal()
+class _dummy_lock:
+ def __enter__(self, *args, **kwargs):
+ pass
+
+ def __exit__(self, *args, **kwargs):
+ pass
+
+
@six.add_metaclass(_TzSingleton)
class tzutc(datetime.tzinfo):
"""
@@ -1104,7 +1111,7 @@
self._comps = comps
self._cachedate = []
self._cachecomp = []
- self._cache_lock = _thread.allocate_lock()
+ self._cache_lock = _dummy_lock()
def _find_comp(self, dt):
if len(self._comps) == 1:
@@ -1407,7 +1414,7 @@
def __init__(self):
self.__instances = {}
- self._cache_lock = _thread.allocate_lock()
+ self._cache_lock = _dummy_lock()
def __call__(self, name=None):
with self._cache_lock:
pandas/Makefile
View file @ 5f42b618
......@@ -46,12 +46,15 @@ all: \
$(BUILD)/_libs/index.so \
$(BUILD)/_libs/interval.so \
$(BUILD)/_libs/join.so \
$(BUILD)/_libs/reshape.so \
$(BUILD)/_libs/json.so \
$(BUILD)/_libs/lib.so \
$(BUILD)/_libs/parsers.so \
$(BUILD)/_libs/period.so \
$(BUILD)/_libs/tslibs/parsing.so \
$(BUILD)/_libs/properties.so \
$(BUILD)/_libs/sparse.so \
$(BUILD)/_libs/hashing.so \
$(BUILD)/_libs/tslib.so \
$(BUILD)/_libs/tslibs/fields.so \
$(BUILD)/_libs/tslibs/frequencies.so \
......@@ -61,6 +64,7 @@ all: \
$(BUILD)/_libs/window.so \
$(BUILD)/io/msgpack/_packer.so \
$(BUILD)/io/msgpack/_unpacker.so \
$(BUILD)/_libs/testing.so \
$(BUILD)/io/sas/_sas.so \
$(BUILD)/util/_move.so
......@@ -160,6 +164,12 @@ $(BUILD)/_libs/period.so: \
mv tmp.wasm $@
$(BUILD)/_libs/tslibs/parsing.so: \
$(HOSTDIR)/pandas/_libs/tslibs/parsing.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
mv tmp.wasm $@
$(BUILD)/_libs/tslibs/frequencies.so: \
$(HOSTDIR)/pandas/_libs/tslibs/frequencies.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
......@@ -193,6 +203,12 @@ $(BUILD)/_libs/join.so: \
mv tmp.wasm $@
$(BUILD)/_libs/reshape.so: \
$(HOSTDIR)/pandas/_libs/reshape.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
mv tmp.wasm $@
$(BUILD)/_libs/interval.so: \
$(HOSTDIR)/pandas/_libs/interval.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
......@@ -231,6 +247,12 @@ $(BUILD)/io/sas/_sas.so: \
mv tmp.wasm $@
$(BUILD)/_libs/testing.so: \
$(HOSTDIR)/pandas/_libs/testing.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
mv tmp.wasm $@
$(BUILD)/io/msgpack/_packer.so: \
$(HOSTDIR)/pandas/io/msgpack/_packer.bc
$(CC) $(LDFLAGS) $^ -o tmp.wasm
......
pandas/patches/avoid-mmap.patch 0 → 100644
View file @ 5f42b618
diff -ru pandas-0.22.0.orig/pandas/io/common.py pandas-0.22.0/pandas/io/common.py
--- pandas-0.22.0.orig/pandas/io/common.py 2018-04-24 17:23:06.833240372 -0400
+++ pandas-0.22.0/pandas/io/common.py 2018-04-24 17:42:33.148934940 -0400
@@ -3,7 +3,10 @@
import os
import csv
import codecs
-import mmap
+try:
+ from mmap import mmap
+except ImportError:
+ mmap = None
from contextlib import contextmanager, closing
from pandas.compat import StringIO, BytesIO, string_types, text_type
@@ -203,7 +206,7 @@
if isinstance(filepath_or_buffer, (compat.string_types,
compat.binary_type,
- mmap.mmap)):
+ mmap)):
return _expand_user(filepath_or_buffer), None, compression
if not is_file_like(filepath_or_buffer):
@@ -432,7 +435,7 @@
"""
def __init__(self, f):
- self.mmap = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+ self.mmap = mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
def __getattr__(self, name):
return getattr(self.mmap, name)
pytz/Makefile 0 → 100644
View file @ 5f42b618
PYODIDE_ROOT=$(abspath ..)
include ../Makefile.envs
PYVERSION=3.6.4
PYMINOR=$(basename $(PYVERSION))
PYTZVERSION=2018.4
ROOT=$(abspath .)
HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
SRC=$(ROOT)/pytz-$(PYTZVERSION)
BUILD=$(SRC)/build/lib/pytz
TARBALL=$(ROOT)/downloads/pytz-$(PYTZVERSION).tgz
URL=https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-$(PYTZVERSION).tar.gz
all: $(BUILD)/__init__.py
clean:
-rm -fr downloads
-rm -fr $(SRC)
$(TARBALL):
[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
wget -q -O $@ $(URL)
md5sum --quiet --check checksums || (rm $@; false)
$(SRC)/setup.py: $(TARBALL)
tar -C . -xf $(TARBALL)
touch $(SRC)/setup.py
$(BUILD)/__init__.py: $(ROOT)/.patched
( \
cd $(SRC) ; \
$(HOSTPYTHON) setup.py build ; \
touch build/lib/pytz/__init__.py \
)
$(ROOT)/.patched: $(SRC)/setup.py
cat patches/*.patch | (cd $(SRC) ; patch -p1)
touch $@
pytz/checksums 0 → 100644
View file @ 5f42b618
f054437920c895dd14a4509fabafe029 ./downloads/pytz-2018.4.tgz
pytz/patches/dummy-threading.patch 0 → 100644
View file @ 5f42b618
diff -ru pytz-2018.4.orig/pytz/lazy.py pytz-2018.4/pytz/lazy.py
--- pytz-2018.4.orig/pytz/lazy.py 2018-04-24 15:47:46.293980100 -0400
+++ pytz-2018.4/pytz/lazy.py 2018-04-24 15:56:06.281648286 -0400
@@ -1,4 +1,7 @@
-from threading import RLock
+try:
+ from threading import RLock
+except (ImportError, ModuleNotFoundError):
+ from dummy_threading import RLock
try:
from collections.abc import Mapping as DictMixin
except ImportError: # Python < 3.3
Binary files pytz-2018.4.orig/pytz/__pycache__/lazy.cpython-36.pyc and pytz-2018.4/pytz/__pycache__/lazy.cpython-36.pyc differ
remove_modules.txt
View file @ 5f42b618
......@@ -2,7 +2,6 @@ _osx_support.py
ctypes
curses
dbm
distutils
ensurepip
idlelib
lib2to3
......
six/Makefile 0 → 100644
View file @ 5f42b618
PYODIDE_ROOT=$(abspath ..)
include ../Makefile.envs
PYVERSION=3.6.4
PYMINOR=$(basename $(PYVERSION))
SIXVERSION=1.11.0
ROOT=$(abspath .)
HOSTPYTHONINSTALL=$(ROOT)/../cpython/build/$(PYVERSION)/host
HOSTPYTHON=$(HOSTPYTHONINSTALL)/bin/python3$(EXE)
SRC=$(ROOT)/six-$(SIXVERSION)
BUILD=$(SRC)/build/lib/six
TARBALL=$(ROOT)/downloads/six-$(SIXVERSION).tgz
URL=https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz
all: $(BUILD)/__init__.py
clean:
-rm -fr downloads
-rm -fr $(SRC)
$(TARBALL):
[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
wget -q -O $@ $(URL)
md5sum --quiet --check checksums || (rm $@; false)
$(SRC)/setup.py: $(TARBALL)
tar -C . -xf $(TARBALL)
touch $(SRC)/setup.py
$(BUILD)/__init__.py: $(SRC)/setup.py
( \
cd $(SRC) ; \
$(HOSTPYTHON) setup.py build ; \
touch build/lib/six.py \
)
six/checksums 0 → 100644
View file @ 5f42b618
d12789f9baf7e9fb2524c0c64f1773f8 downloads/six-1.11.0.tgz
src/dummy_thread.c 0 → 100644
View file @ 5f42b618
#include <Python.h>
#include <pythread.h>
/* A set of mock-ups for thread lock functionality that is required for
Cython-generated modules to work. */
PyThread_type_lock PyThread_allocate_lock(void) {
return (PyThread_type_lock)0x1;
}
void PyThread_free_lock(PyThread_type_lock _) {
}
int PyThread_acquire_lock(PyThread_type_lock _, int __) {
return PY_LOCK_ACQUIRED;
}
test/test_pandas.py
View file @ 5f42b618
def test_pandas(selenium):
selenium.run("import pandas")
def test_extra_import(selenium):
selenium.run("from pandas import Series, DataFrame, Panel")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7