Merge pull request #155 from mdboom/allow-memory-growth

Set ALLOW_MEMORY_GROWTH=1

Makefile

@@ -21,7 +21,8 @@ LDFLAGS=\
 	-s MODULARIZE=1 \
 	$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
   -s "BINARYEN_METHOD='native-wasm'" \
-  -s TOTAL_MEMORY=536870912 \
+  -s TOTAL_MEMORY=1073741824 \
+  -s ALLOW_MEMORY_GROWTH=1 \
 	-s MAIN_MODULE=1 \
 	-s EMULATED_FUNCTION_POINTERS=1 \
   -s EMULATE_FUNCTION_POINTER_CASTS=1 \
@@ -50,7 +51,6 @@ all: build/pyodide.asm.js \
 	build/renderedhtml.css \
   build/test.data \
   build/packages.json \
-  build/test_data.txt \
   build/test.html
 
 
@@ -107,10 +107,6 @@ test: all
 	pytest test/ -v
 
 
-build/test_data.txt: test/data.txt
-	cp test/data.txt build/test_data.txt
-
-
 lint:
 	flake8 src
 	flake8 test

test/conftest.py

@@ -214,16 +214,29 @@ def run_web_server(q):
     print("Running webserver...")
 
     os.chdir(BUILD_PATH)
-    Handler = http.server.SimpleHTTPRequestHandler
-    Handler.extensions_map['.wasm'] = 'application/wasm'
 
-    def dummy_log(*args, **kwargs):
-        pass
-    Handler.log_message = dummy_log
+    class Handler(http.server.CGIHTTPRequestHandler):
+        def translate_path(self, path):
+            if path.startswith('/test/'):
+                return TEST_PATH / path[6:]
+            return super(Handler, self).translate_path(path)
+
+        def is_cgi(self):
+            if self.path.startswith('/test/') and self.path.endswith('.cgi'):
+                self.cgi_info = '/test', self.path[6:]
+                return True
+            return False
+
+        def log_message(self, *args, **kwargs):
+            pass
+
+    Handler.extensions_map['.wasm'] = 'application/wasm'
 
     with socketserver.TCPServer(("", 0), Handler) as httpd:
         host, port = httpd.server_address
         print("serving at port", port)
+        httpd.server_name = 'test-server'
+        httpd.server_port = port
         q.put(port)
 
         def service_actions():

test/data.cgi

+#!/usr/bin/env python
+print("Content-Type: text/text")
+print()
+print("HELLO")

test/largish.json.cgi

+#!/usr/bin/env python
+
+import json
+import random
+import sys
+
+random.seed(0)
+
+columns = [
+    ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'),
+    ('column1', lambda: random.choice([
+        'notification-interval-longer', 'notification-interval-short', 'control'])),
+    ('column2', lambda: random.choice([True, False])),
+    ('column3', lambda: random.randint(0, 4)),
+    ('column4', lambda: random.randint(0, 4)),
+    ('column5', lambda: random.randint(0, 4)),
+    ('column6', lambda: random.randint(0, 4)),
+    ('column7', lambda: random.randint(0, 4))
+]
+
+N_ROWS = 91746  # the output JSON size will be ~15 MB/10k rows
+
+
+class StreamDict(dict):
+    """
+    To serialize to JSON, we create an iterable object that inherits from a
+    known supported object type: dict.
+    """
+    def __init__(self, generator):
+        self.generator = generator
+
+    def items(self):
+        for i in range(N_ROWS):
+            yield i, self.generator()
+
+    def __len__(self):
+        return 1
+
+
+data = {}
+for name, generator in columns:
+    data[name] = StreamDict(generator)
+
+
+print("Content-Type: application/json")
+print()
+json.dump(data, sys.stdout)

test/test_pandas.py

@@ -16,3 +16,22 @@ def test_extra_import(selenium, request):
 
     selenium.load_package("pandas")
     selenium.run("from pandas import Series, DataFrame, Panel")
+
+
+def test_load_largish_file(selenium_standalone, request):
+    selenium = selenium_standalone
+
+    if selenium.browser == 'chrome':
+        request.applymarker(pytest.mark.xfail(
+            run=False, reason='chrome not supported'))
+
+    selenium.load_package("pandas")
+    selenium.load_package("matplotlib")
+
+    selenium.run("""
+        import pyodide
+        import matplotlib.pyplot as plt
+        import pandas as pd
+
+        pd.read_json(pyodide.open_url('test/largish.json.cgi'))
+    """)

test/test_python.py

@@ -317,7 +317,15 @@ def test_open_url(selenium):
     assert selenium.run(
         """
         import pyodide
-        pyodide.open_url('test_data.txt').read()
+        pyodide.open_url('test/data.txt').read()
+        """) == 'HELLO\n'
+
+
+def test_open_url_cgi(selenium):
+    assert selenium.run(
+        """
+        import pyodide
+        pyodide.open_url('test/data.cgi').read()
         """) == 'HELLO\n'