Commit 65ebb994 authored by Michael Droettboom's avatar Michael Droettboom Committed by GitHub

Merge pull request #293 from mdboom/upgrade-emscripten-1.38.22

Upgrade emscripten to 1.38.22
parents 00a2bf9b 5c3bf202
...@@ -18,7 +18,7 @@ jobs: ...@@ -18,7 +18,7 @@ jobs:
- restore_cache: - restore_cache:
keys: keys:
- v1-emsdk-{{ checksum "emsdk/Makefile" }}-v12- - v1-emsdk-{{ checksum "emsdk/Makefile" }}-v13-
- run: - run:
name: build name: build
...@@ -36,7 +36,7 @@ jobs: ...@@ -36,7 +36,7 @@ jobs:
paths: paths:
- ./emsdk/emsdk - ./emsdk/emsdk
- ~/.ccache - ~/.ccache
key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v12-{{ .BuildNum }} key: v1-emsdk-{{ checksum "emsdk/Makefile" }}-v13-{{ .BuildNum }}
- persist_to_workspace: - persist_to_workspace:
root: . root: .
......
...@@ -16,22 +16,20 @@ CFLAGS=$(OPTFLAGS) -g -I$(PYTHONINCLUDE) -Wno-warn-absolute-paths ...@@ -16,22 +16,20 @@ CFLAGS=$(OPTFLAGS) -g -I$(PYTHONINCLUDE) -Wno-warn-absolute-paths
CXXFLAGS=$(CFLAGS) -std=c++14 CXXFLAGS=$(CFLAGS) -std=c++14
# __ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv is in
# EXPORTED_FUNCTIONS to keep the C++ standard library in the core, even though
# there isn't any C++ there, for the sake of loading dynamic modules written in
# C++, such as those in matplotlib.
LDFLAGS=\ LDFLAGS=\
-O3 \ -O3 \
-s MODULARIZE=1 \ -s MODULARIZE=1 \
$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \ $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
lz4/lz4-1.8.3/lib/liblz4.a \ $(LZ4LIB) \
-s "BINARYEN_METHOD='native-wasm'" \ -s "BINARYEN_METHOD='native-wasm'" \
-s TOTAL_MEMORY=1073741824 \ -s TOTAL_MEMORY=1073741824 \
-s ALLOW_MEMORY_GROWTH=1 \ -s ALLOW_MEMORY_GROWTH=1 \
-s MAIN_MODULE=1 \ -s MAIN_MODULE=1 \
-s EMULATED_FUNCTION_POINTERS=1 \ -s EMULATED_FUNCTION_POINTERS=1 \
-s EMULATE_FUNCTION_POINTER_CASTS=1 \ -s EMULATE_FUNCTION_POINTER_CASTS=1 \
-s EXPORTED_FUNCTIONS='["_main", "__ZNKSt3__220__vector_base_commonILb1EE20__throw_length_errorEv", "__ZNSt11logic_errorC2EPKc"]' \ -s LINKABLE=1 \
-s EXPORT_ALL=1 \
-s EXPORTED_FUNCTIONS='["___cxa_guard_acquire"]' \
-s WASM=1 \ -s WASM=1 \
-s SWAPPABLE_ASM_MODULE=1 \ -s SWAPPABLE_ASM_MODULE=1 \
-s USE_FREETYPE=1 \ -s USE_FREETYPE=1 \
...@@ -68,8 +66,6 @@ build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.b ...@@ -68,8 +66,6 @@ build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.b
[ -d build ] || mkdir build [ -d build ] || mkdir build
$(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \ $(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \
$(LDFLAGS) -s FORCE_FILESYSTEM=1 $(LDFLAGS) -s FORCE_FILESYSTEM=1
rm build/pyodide.asm.asm.js
rm build/pyodide.asm.wasm.pre
rm build/pyodide.asm.html rm build/pyodide.asm.html
...@@ -149,7 +145,7 @@ build/test.data: $(CPYTHONLIB) ...@@ -149,7 +145,7 @@ build/test.data: $(CPYTHONLIB)
) )
( \ ( \
cd build; \ cd build; \
python $(FILEPACKAGER) test.data --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude \*.wasm.pre --exclude __pycache__ \ python $(FILEPACKAGER) test.data --lz4 --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide._module --exclude __pycache__ \
) )
uglifyjs build/test.js -o build/test.js uglifyjs build/test.js -o build/test.js
...@@ -174,7 +170,6 @@ root/.built: \ ...@@ -174,7 +170,6 @@ root/.built: \
cd root/lib/python$(PYMINOR); \ cd root/lib/python$(PYMINOR); \
rm -fr `cat ../../../remove_modules.txt`; \ rm -fr `cat ../../../remove_modules.txt`; \
rm -fr test; \ rm -fr test; \
find . -name "*.wasm.pre" -type f -delete ; \
find -type d -name __pycache__ -prune -exec rm -rf {} \; \ find -type d -name __pycache__ -prune -exec rm -rf {} \; \
) )
touch root/.built touch root/.built
...@@ -185,7 +180,7 @@ ccache/emcc: ...@@ -185,7 +180,7 @@ ccache/emcc:
if hash ccache &>/dev/null; then \ if hash ccache &>/dev/null; then \
ln -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \ ln -s `which ccache` $(PYODIDE_ROOT)/ccache/emcc ; \
else \ else \
ln -s emsdk/emsdk/emscripten/tag-1.38.12/emcc $(PYODIDE_ROOT)/ccache/emcc; \ ln -s emsdk/emsdk/emscripten/tag-$(EMSCRIPTEN_VERSION)/emcc $(PYODIDE_ROOT)/ccache/emcc; \
fi fi
...@@ -194,7 +189,7 @@ ccache/em++: ...@@ -194,7 +189,7 @@ ccache/em++:
if hash ccache &>/dev/null; then \ if hash ccache &>/dev/null; then \
ln -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \ ln -s `which ccache` $(PYODIDE_ROOT)/ccache/em++ ; \
else \ else \
ln -s emsdk/emsdk/emscripten/tag-1.38.12/em++ $(PYODIDE_ROOT)/ccache/em++; \ ln -s emsdk/emsdk/emscripten/tag-$(EMSCRIPTEN_VERSION)/em++ $(PYODIDE_ROOT)/ccache/em++; \
fi fi
......
export PATH := $(PYODIDE_ROOT)/ccache:$(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e1.38.12/build_tag-e1.38.12_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.12:$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.12_64bit_binaryen/bin:$(PATH) export EMSCRIPTEN_VERSION = 1.38.22
export PATH := $(PYODIDE_ROOT)/ccache:$(PYODIDE_ROOT)/emsdk/emsdk:$(PYODIDE_ROOT)/emsdk/emsdk/clang/tag-e$(EMSCRIPTEN_VERSION)/build_tag-e$(EMSCRIPTEN_VERSION)_64/bin:$(PYODIDE_ROOT)/emsdk/emsdk/node/8.9.1_64bit/bin:$(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-$(EMSCRIPTEN_VERSION):$(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-$(EMSCRIPTEN_VERSION)_64bit_binaryen/bin:$(PATH)
export EMSDK = $(PYODIDE_ROOT)/emsdk/emsdk export EMSDK = $(PYODIDE_ROOT)/emsdk/emsdk
export EM_CONFIG = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten export EM_CONFIG = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten
export EM_CACHE = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten_cache export EM_CACHE = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten_cache
export EMSCRIPTEN = $(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.12 export EMSCRIPTEN = $(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-$(EMSCRIPTEN_VERSION)
export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.12_64bit_binaryen export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-$(EMSCRIPTEN_VERSION)_64bit_binaryen
export PYVERSION=3.7.0 export PYVERSION=3.7.0
export PYMINOR=$(basename $(PYVERSION)) export PYMINOR=$(basename $(PYVERSION))
...@@ -22,4 +24,6 @@ export SIDE_LDFLAGS=\ ...@@ -22,4 +24,6 @@ export SIDE_LDFLAGS=\
-s SIDE_MODULE=1 \ -s SIDE_MODULE=1 \
-s WASM=1 \ -s WASM=1 \
-s "BINARYEN_TRAP_MODE='clamp'" \ -s "BINARYEN_TRAP_MODE='clamp'" \
--memory-init-file 0 --memory-init-file 0 \
-s LINKABLE=1 \
-s EXPORT_ALL=1
PYODIDE_ROOT=$(abspath ..)
include ../Makefile.envs
all: emsdk/.complete all: emsdk/.complete
# We hack the CPU_CORES, because if you use all of the cores on Circle-CI, you # We hack the CPU_CORES, because if you use all of the cores on Circle-CI, you
...@@ -8,14 +11,15 @@ emsdk/.complete: ...@@ -8,14 +11,15 @@ emsdk/.complete:
git clone https://github.com/juj/emsdk.git git clone https://github.com/juj/emsdk.git
sed -i -e "s#CPU_CORES = max(multiprocessing.cpu_count()-1, 1)#CPU_CORES = 3#g" emsdk/emsdk sed -i -e "s#CPU_CORES = max(multiprocessing.cpu_count()-1, 1)#CPU_CORES = 3#g" emsdk/emsdk
( \ ( \
cd emsdk ; \ cd emsdk && \
./emsdk install --build=Release sdk-tag-1.38.12-64bit binaryen-tag-1.38.12-64bit ; \ ./emsdk install --build=Release sdk-tag-$(EMSCRIPTEN_VERSION)-64bit binaryen-tag-$(EMSCRIPTEN_VERSION)-64bit && \
cd .. ; \ cd .. && \
(cat patches/*.patch | patch -p1) ; \ (cat patches/*.patch | patch -p1) ; \
cd emsdk/binaryen/tag-1.38.12_64bit_binaryen/ ; \ cd emsdk/binaryen/tag-$(EMSCRIPTEN_VERSION)_64bit_binaryen/ && \
make ; \ make && \
cd ../.. ; \ cd ../.. && \
./emsdk activate --embedded --build=Release sdk-tag-1.38.12-64bit binaryen-tag-1.38.12-64bit ; \ cp binaryen/tag-$(EMSCRIPTEN_VERSION)/bin/wasm.js binaryen/tag-$(EMSCRIPTEN_VERSION)_64bit_binaryen/bin && \
./emsdk activate --embedded --build=Release sdk-tag-$(EMSCRIPTEN_VERSION)-64bit binaryen-tag-$(EMSCRIPTEN_VERSION)-64bit && \
touch .complete \ touch .complete \
) )
......
diff --git a/emsdk/emscripten/tag-1.38.12/src/library.js b/emsdk/emscripten/tag-1.38.12/src/library.js diff --git a/emsdk/emscripten/tag-1.38.22/src/support.js b//emsdk/emscripten/tag-1.38.22/src/support.js
index 82537bb3e..8e2e43128 100644 index 8e1df8e82..4d07d6bef 100644
--- a/emsdk/emscripten/tag-1.38.12/src/library.js --- a/emsdk/emscripten/tag-1.38.22/src/support.js
+++ b/emsdk/emscripten/tag-1.38.12/src/library.js +++ b/emsdk/emscripten/tag-1.38.22/src/support.js
@@ -1781,6 +1781,12 @@ LibraryManager.library = { @@ -437,7 +437,18 @@ function loadWebAssemblyModule(binary, flags) {
// present in the dynamic library but not in the main JS,
// and the dynamic library cannot provide JS for it. Use
// the generic "X" invoke for it.
- return env[prop] = invoke_X;
+ var dynCallName = 'dynCall_' + prop.slice(7);
+ return env[prop] = function() {
+ var sp = stackSave();
+ try {
+ var args = Array.prototype.slice.call(arguments);
+ return Module[dynCallName].apply(null, args);
+ } catch(e) {
+ stackRestore(sp);
+ if (typeof e !== 'number' && e !== 'longjmp') throw e;
+ Module["setThrew"](1, 0);
+ }
+ }
} }
} // if not a global, then a function - call it indirectly
return env[prop] = function() {
+ for (var sym in lib_module) { @@ -502,6 +513,11 @@ function loadWebAssemblyModule(binary, flags) {
+ if (sym.startsWith('dynCall_') && !Module.hasOwnProperty(sym)) { }
+ Module[sym] = lib_module[sym]; #endif
+ } }
+ } + if (e.startsWith("dynCall_")) {
+ + if (!Module.hasOwnProperty(e)) {
// Not all browsers support Object.keys(). + Module[e] = value;
var handle = 1;
for (var key in DLFCN.loadedLibs) {
diff --git a/emsdk/emscripten/tag-1.38.12/src/support.js b/emsdk/emscripten/tag-1.38.12/src/support.js
index cff68fbe2..3a4e51dca 100644
--- a/emsdk/emscripten/tag-1.38.12/src/support.js
+++ b/emsdk/emscripten/tag-1.38.12/src/support.js
@@ -211,9 +211,21 @@ function loadWebAssemblyModule(binary, loadAsync) {
if (prop.startsWith('invoke_')) {
// A missing invoke, i.e., an invoke for a function type
// present in the dynamic library but not in the main JS,
- // and the dynamic library cannot provide JS for it. Use
- // the generic "X" invoke for it.
- return env[prop] = invoke_X;
+ // and the dynamic library cannot provide JS for it. Generate
+ // a closure for it.
+ var dynCallName = 'dynCall_' + prop.slice(7);
+ env[prop] = function() {
+ var sp = stackSave();
+ try {
+ var args = Array.prototype.slice.call(arguments);
+ return Module[dynCallName].apply(null, args);
+ } catch(e) {
+ stackRestore(sp);
+ if (typeof e !== 'number' && e !== 'longjmp') throw e;
+ Module["setThrew"](1, 0);
+ } + }
+ } + }
+ return env[prop]; exports[e] = value;
} }
// if not a global, then a function - call it indirectly // initialize the module
return env[prop] = function() {
diff --git a/emsdk/emscripten/tag-1.38.12/src/library_lz4.js b/emsdk/emscripten/tag-1.38.12/src/library_lz4.js diff --git a/emsdk/emscripten/tag-1.38.22/src/library_lz4.js b/emsdk/emscripten/tag-1.38.22/src/library_lz4.js
index 4c3f583b7..5291002a4 100644 index 4c3f583b7..5291002a4 100644
--- a/emsdk/emscripten/tag-1.38.12/src/library_lz4.js --- a/emsdk/emscripten/tag-1.38.22/src/library_lz4.js
+++ b/emsdk/emscripten/tag-1.38.12/src/library_lz4.js +++ b/emsdk/emscripten/tag-1.38.22/src/library_lz4.js
@@ -5,26 +5,14 @@ mergeInto(LibraryManager.library, { @@ -5,26 +5,14 @@ mergeInto(LibraryManager.library, {
DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */, DIR_MODE: {{{ cDefine('S_IFDIR') }}} | 511 /* 0777 */,
FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */, FILE_MODE: {{{ cDefine('S_IFREG') }}} | 511 /* 0777 */,
......
diff --git a/emsdk/binaryen/master/src/passes/FuncCastEmulation.cpp b/emsdk/binaryen/master/src/passes/FuncCastEmulation.cpp diff --git a/emsdk/binaryen/master/src/passes/FuncCastEmulation.cpp b/emsdk/binaryen/master/src/passes/FuncCastEmulation.cpp
index 013e9403..d95fc282 100644 index 013e9403..d95fc282 100644
--- a/emsdk/binaryen/tag-1.38.12/src/passes/FuncCastEmulation.cpp --- a/emsdk/binaryen/tag-1.38.22/src/passes/FuncCastEmulation.cpp
+++ b/emsdk/binaryen/tag-1.38.12/src/passes/FuncCastEmulation.cpp +++ b/emsdk/binaryen/tag-1.38.22/src/passes/FuncCastEmulation.cpp
@@ -39,7 +39,7 @@ namespace wasm { @@ -39,7 +39,7 @@ namespace wasm {
// This should be enough for everybody. (As described above, we need this // This should be enough for everybody. (As described above, we need this
// to match when dynamically linking, and also dynamic linking is why we // to match when dynamically linking, and also dynamic linking is why we
......
# flake8: noqa # flake8: noqa
# This is forked from emscripten 1.38.10 # This is forked from emscripten 1.38.22, with the original copyright notice
# below.
# Copyright 2012 The Emscripten Authors. All rights reserved.
# Emscripten is available under two separate licenses, the MIT license and the
# University of Illinois/NCSA Open Source License. Both these licenses can be
# found in the LICENSE file.
''' '''
A tool that generates FS API calls to generate a filesystem, and packages the files A tool that generates FS API calls to generate a filesystem, and packages the files
...@@ -59,19 +65,19 @@ Notes: ...@@ -59,19 +65,19 @@ Notes:
''' '''
from __future__ import print_function from __future__ import print_function
import os, sys, shutil, random, uuid, ctypes import os
import sys
sys.path.insert( import shutil
1, import random
os.path.join( import uuid
os.path.dirname( import ctypes
os.path.dirname(
os.path.abspath(__file__) emscripten_dir = os.path.join(
) os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
), 'emsdk', 'emsdk', 'emscripten'
'emsdk', 'emsdk', 'emscripten', 'tag-1.38.12'
)
) )
tag_dir = sorted(os.listdir(emscripten_dir), key=lambda x: len(x))[0]
sys.path.insert(1, os.path.join(emscripten_dir, tag_dir))
from tools.toolchain_profiler import ToolchainProfiler from tools.toolchain_profiler import ToolchainProfiler
if __name__ == '__main__': if __name__ == '__main__':
...@@ -79,9 +85,8 @@ if __name__ == '__main__': ...@@ -79,9 +85,8 @@ if __name__ == '__main__':
import posixpath import posixpath
from tools import shared from tools import shared
from tools.shared import suffix, unsuffixed
from tools.jsrun import run_js from tools.jsrun import run_js
from subprocess import Popen, PIPE, STDOUT from subprocess import PIPE
import fnmatch import fnmatch
import json import json
...@@ -96,11 +101,13 @@ data_target = sys.argv[1] ...@@ -96,11 +101,13 @@ data_target = sys.argv[1]
IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp') IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3') AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' } AUDIO_MIMETYPES = {'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg'}
DDS_HEADER_SIZE = 128 DDS_HEADER_SIZE = 128
AV_WORKAROUND = 0 # Set to 1 to randomize file order and add some padding, to work around silly av false positives # Set to 1 to randomize file order and add some padding,
# to work around silly av false positives
AV_WORKAROUND = 0
data_files = [] data_files = []
excluded_patterns = [] excluded_patterns = []
...@@ -112,16 +119,22 @@ plugins = [] ...@@ -112,16 +119,22 @@ plugins = []
jsoutput = None jsoutput = None
from_emcc = False from_emcc = False
force = True force = True
# If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally cache VFS XHR so that subsequent # If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally
# page loads can read the data from the offline cache instead. # cache VFS XHR so that subsequent page loads can read the data from the
# offline cache instead.
use_preload_cache = False use_preload_cache = False
indexeddb_name = 'EM_PRELOAD_CACHE' indexeddb_name = 'EM_PRELOAD_CACHE'
# If set to True, the blob received from XHR is moved to the Emscripten HEAP, optimizing for mmap() performance. # If set to True, the blob received from XHR is moved to the Emscripten HEAP,
# If set to False, the XHR blob is kept intact, and fread()s etc. are performed directly to that data. This optimizes for minimal memory usage and fread() performance. # optimizing for mmap() performance.
# If set to False, the XHR blob is kept intact, and fread()s etc. are performed
# directly to that data. This optimizes for minimal memory usage and fread()
# performance.
no_heap_copy = True no_heap_copy = True
# If set to True, the package metadata is stored separately from js-output file which makes js-output file immutable to the package content changes. # If set to True, the package metadata is stored separately from js-output
# If set to False, the package metadata is stored inside the js-output file which makes js-output file to mutate on each invocation of this packager tool. # file which makes js-output file immutable to the package content changes.
separate_metadata = False # If set to False, the package metadata is stored inside the js-output file
# which makes js-output file to mutate on each invocation of this packager tool.
separate_metadata = False
lz4 = False lz4 = False
use_preload_plugins = False use_preload_plugins = False
...@@ -170,16 +183,22 @@ for arg in sys.argv[2:]: ...@@ -170,16 +183,22 @@ for arg in sys.argv[2:]:
leading = '' leading = ''
elif leading == 'preload' or leading == 'embed': elif leading == 'preload' or leading == 'embed':
mode = leading mode = leading
at_position = arg.replace('@@', '__').find('@') # position of @ if we're doing 'src@dst'. '__' is used to keep the index same with the original if they escaped with '@@'. # position of @ if we're doing 'src@dst'. '__' is used to keep the index
uses_at_notation = (at_position != -1) # '@@' in input string means there is an actual @ character, a single '@' means the 'src@dst' notation. # same with the original if they escaped with '@@'.
at_position = arg.replace('@@', '__').find('@')
# '@@' in input string means there is an actual @ character, a single '@'
# means the 'src@dst' notation.
uses_at_notation = (at_position != -1)
if uses_at_notation: if uses_at_notation:
srcpath = arg[0:at_position].replace('@@', '@') # split around the @ srcpath = arg[0:at_position].replace('@@', '@') # split around the @
dstpath = arg[at_position+1:].replace('@@', '@') dstpath = arg[at_position + 1:].replace('@@', '@')
else: else:
srcpath = dstpath = arg.replace('@@', '@') # Use source path as destination path. # Use source path as destination path.
srcpath = dstpath = arg.replace('@@', '@')
if os.path.isfile(srcpath) or os.path.isdir(srcpath): if os.path.isfile(srcpath) or os.path.isdir(srcpath):
data_files.append({ 'srcpath': srcpath, 'dstpath': dstpath, 'mode': mode, 'explicit_dst_path': uses_at_notation }) data_files.append({'srcpath': srcpath, 'dstpath': dstpath, 'mode': mode,
'explicit_dst_path': uses_at_notation})
else: else:
print('Warning: ' + arg + ' does not exist, ignoring.', file=sys.stderr) print('Warning: ' + arg + ' does not exist, ignoring.', file=sys.stderr)
elif leading == 'exclude': elif leading == 'exclude':
...@@ -190,14 +209,19 @@ for arg in sys.argv[2:]: ...@@ -190,14 +209,19 @@ for arg in sys.argv[2:]:
if (not force) and len(data_files) == 0: if (not force) and len(data_files) == 0:
has_preloaded = False has_preloaded = False
if not has_preloaded or jsoutput == None: if not has_preloaded or jsoutput is None:
assert not separate_metadata, 'cannot separate-metadata without both --preloaded files and a specified --js-output' assert not separate_metadata, (
'cannot separate-metadata without both --preloaded files '
'and a specified --js-output')
if not from_emcc: if not from_emcc:
print('Remember to build the main file with -s FORCE_FILESYSTEM=1 so that it includes support for loading this file package', file=sys.stderr) print('Remember to build the main file with -s FORCE_FILESYSTEM=1 '
'so that it includes support for loading this file package',
file=sys.stderr)
ret = '' ret = ''
# emcc.py will add this to the output itself, so it is only needed for standalone calls # emcc.py will add this to the output itself, so it is only needed for
# standalone calls
if not from_emcc: if not from_emcc:
ret = ''' ret = '''
var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {}; var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {};
...@@ -219,22 +243,26 @@ code = ''' ...@@ -219,22 +243,26 @@ code = '''
} }
''' '''
# Win32 code to test whether the given file has the hidden property set.
def has_hidden_attribute(filepath): def has_hidden_attribute(filepath):
"""Win32 code to test whether the given file has the hidden property set."""
if sys.platform != 'win32': if sys.platform != 'win32':
return False return False
try: try:
attrs = ctypes.windll.kernel32.GetFileAttributesW(unicode(filepath)) attrs = ctypes.windll.kernel32.GetFileAttributesW(
u'%s' % filepath)
assert attrs != -1 assert attrs != -1
result = bool(attrs & 2) result = bool(attrs & 2)
except: except Exception:
result = False result = False
return result return result
# The packager should never preload/embed files if the file is hidden (Win32).
# or it matches any pattern specified in --exclude
def should_ignore(fullname): def should_ignore(fullname):
"""The packager should never preload/embed files if the file
is hidden (Win32) or it matches any pattern specified in --exclude"""
if has_hidden_attribute(fullname): if has_hidden_attribute(fullname):
return True return True
...@@ -243,10 +271,15 @@ def should_ignore(fullname): ...@@ -243,10 +271,15 @@ def should_ignore(fullname):
return True return True
return False return False
# Expand directories into individual files
def add(mode, rootpathsrc, rootpathdst): def add(mode, rootpathsrc, rootpathdst):
# rootpathsrc: The path name of the root directory on the local FS we are adding to emscripten virtual FS. """Expand directories into individual files
# rootpathdst: The name we want to make the source path available on the emscripten virtual FS.
rootpathsrc: The path name of the root directory on the local FS we are
adding to emscripten virtual FS.
rootpathdst: The name we want to make the source path available on the
emscripten virtual FS.
"""
for dirpath, dirnames, filenames in os.walk(rootpathsrc): for dirpath, dirnames, filenames in os.walk(rootpathsrc):
new_dirnames = [] new_dirnames = []
for name in dirnames: for name in dirnames:
...@@ -254,17 +287,23 @@ def add(mode, rootpathsrc, rootpathdst): ...@@ -254,17 +287,23 @@ def add(mode, rootpathsrc, rootpathdst):
if not should_ignore(fullname): if not should_ignore(fullname):
new_dirnames.append(name) new_dirnames.append(name)
elif DEBUG: elif DEBUG:
print('Skipping directory "' + fullname + '" from inclusion in the emscripten virtual file system.', file=sys.stderr) print('Skipping directory "%s" from inclusion in the emscripten '
'virtual file system.' % fullname, file=sys.stderr)
for name in filenames: for name in filenames:
fullname = os.path.join(dirpath, name) fullname = os.path.join(dirpath, name)
if not should_ignore(fullname): if not should_ignore(fullname):
dstpath = os.path.join(rootpathdst, os.path.relpath(fullname, rootpathsrc)) # Convert source filename relative to root directory of target FS. # Convert source filename relative to root directory of target FS.
new_data_files.append({ 'srcpath': fullname, 'dstpath': dstpath, 'mode': mode, 'explicit_dst_path': True }) dstpath = os.path.join(rootpathdst,
os.path.relpath(fullname, rootpathsrc))
new_data_files.append({'srcpath': fullname, 'dstpath': dstpath,
'mode': mode, 'explicit_dst_path': True})
elif DEBUG: elif DEBUG:
print('Skipping file "' + fullname + '" from inclusion in the emscripten virtual file system.', file=sys.stderr) print('Skipping file "%s" from inclusion in the emscripten '
'virtual file system.' % fullname, file=sys.stderr)
del dirnames[:] del dirnames[:]
dirnames.extend(new_dirnames) dirnames.extend(new_dirnames)
new_data_files = [] new_data_files = []
for file_ in data_files: for file_ in data_files:
if not should_ignore(file_['srcpath']): if not should_ignore(file_['srcpath']):
...@@ -272,43 +311,67 @@ for file_ in data_files: ...@@ -272,43 +311,67 @@ for file_ in data_files:
add(file_['mode'], file_['srcpath'], file_['dstpath']) add(file_['mode'], file_['srcpath'], file_['dstpath'])
else: else:
new_data_files.append(file_) new_data_files.append(file_)
data_files = [file_ for file_ in new_data_files if not os.path.isdir(file_['srcpath'])] data_files = [file_ for file_ in new_data_files
if not os.path.isdir(file_['srcpath'])]
if len(data_files) == 0: if len(data_files) == 0:
print('Nothing to do!', file=sys.stderr) print('Nothing to do!', file=sys.stderr)
sys.exit(1) sys.exit(1)
# Absolutize paths, and check that they make sense # Absolutize paths, and check that they make sense
curr_abspath = os.path.abspath(os.getcwd()) # os.getcwd() always returns the hard path with any symbolic links resolved, even if we cd'd into a symbolic link. # os.getcwd() always returns the hard path with any symbolic links resolved,
# even if we cd'd into a symbolic link.
curr_abspath = os.path.abspath(os.getcwd())
for file_ in data_files: for file_ in data_files:
if not file_['explicit_dst_path']: if not file_['explicit_dst_path']:
# This file was not defined with src@dst, so we inferred the destination from the source. In that case, # This file was not defined with src@dst, so we inferred the destination
# we require that the destination not be under the current location # from the source. In that case, we require that the destination not be
# under the current location
path = file_['dstpath'] path = file_['dstpath']
abspath = os.path.realpath(os.path.abspath(path)) # Use os.path.realpath to resolve any symbolic links to hard paths, to match the structure in curr_abspath. # Use os.path.realpath to resolve any symbolic links to hard paths,
if DEBUG: print(path, abspath, curr_abspath, file=sys.stderr) # to match the structure in curr_abspath.
abspath = os.path.realpath(os.path.abspath(path))
if DEBUG:
print(path, abspath, curr_abspath, file=sys.stderr)
if not abspath.startswith(curr_abspath): if not abspath.startswith(curr_abspath):
print('Error: Embedding "%s" which is below the current directory "%s". This is invalid since the current directory becomes the root that the generated code will see' % (path, curr_abspath), file=sys.stderr) print('Error: Embedding "%s" which is below the current directory '
'"%s". This is invalid since the current directory becomes the '
'root that the generated code will see' % (path, curr_abspath),
file=sys.stderr)
sys.exit(1) sys.exit(1)
file_['dstpath'] = abspath[len(curr_abspath)+1:] file_['dstpath'] = abspath[len(curr_abspath) + 1:]
if os.path.isabs(path): if os.path.isabs(path):
print('Warning: Embedding an absolute file/directory name "' + path + '" to the virtual filesystem. The file will be made available in the relative path "' + file_['dstpath'] + '". You can use the explicit syntax --preload-file srcpath@dstpath to explicitly specify the target location the absolute source path should be directed to.', file=sys.stderr) print('Warning: Embedding an absolute file/directory name "%s" to the '
'virtual filesystem. The file will be made available in the '
'relative path "%s". You can use the explicit syntax '
'--preload-file srcpath@dstpath to explicitly specify the target '
'location the absolute source path should be directed to.'
% (path, file_['dstpath']), file=sys.stderr)
for file_ in data_files: for file_ in data_files:
file_['dstpath'] = file_['dstpath'].replace(os.path.sep, '/') # name in the filesystem, native and emulated # name in the filesystem, native and emulated
if file_['dstpath'].endswith('/'): # If user has submitted a directory name as the destination but omitted the destination filename, use the filename from source file file_['dstpath'] = file_['dstpath'].replace(os.path.sep, '/')
# If user has submitted a directory name as the destination but omitted
# the destination filename, use the filename from source file
if file_['dstpath'].endswith('/'):
file_['dstpath'] = file_['dstpath'] + os.path.basename(file_['srcpath']) file_['dstpath'] = file_['dstpath'] + os.path.basename(file_['srcpath'])
# make destination path always relative to the root # make destination path always relative to the root
file_['dstpath'] = posixpath.normpath(os.path.join('/', file_['dstpath'])) file_['dstpath'] = posixpath.normpath(os.path.join('/', file_['dstpath']))
if DEBUG: if DEBUG:
print('Packaging file "' + file_['srcpath'] + '" to VFS in path "' + file_['dstpath'] + '".', file=sys.stderr) print('Packaging file "%s" to VFS in path "%s".'
% (file_['srcpath'], file_['dstpath']), file=sys.stderr)
# Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/) # Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/)
seen = {} seen = {}
def was_seen(name): def was_seen(name):
if seen.get(name): return True if seen.get(name):
return True
seen[name] = 1 seen[name] = 1
return False return False
data_files = [file_ for file_ in data_files if not was_seen(file_['dstpath'])] data_files = [file_ for file_ in data_files if not was_seen(file_['dstpath'])]
if AV_WORKAROUND: if AV_WORKAROUND:
...@@ -329,27 +392,32 @@ for file_ in data_files: ...@@ -329,27 +392,32 @@ for file_ in data_files:
if dirname != '': if dirname != '':
parts = dirname.split('/') parts = dirname.split('/')
for i in range(len(parts)): for i in range(len(parts)):
partial = '/'.join(parts[:i+1]) partial = '/'.join(parts[:i + 1])
if partial not in partial_dirs: if partial not in partial_dirs:
code += '''Module['FS_createPath']('/%s', '%s', true, true);\n''' % ('/'.join(parts[:i]), parts[i]) code += ('''Module['FS_createPath']('/%s', '%s', true, true);\n'''
% ('/'.join(parts[:i]), parts[i]))
partial_dirs.append(partial) partial_dirs.append(partial)
if has_preloaded: if has_preloaded:
# Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead. # Bundle all datafiles into one archive. Avoids doing lots of simultaneous
# XHRs which has overhead.
data = open(data_target, 'wb') data = open(data_target, 'wb')
start = 0 start = 0
for file_ in data_files: for file_ in data_files:
file_['data_start'] = start file_['data_start'] = start
curr = open(file_['srcpath'], 'rb').read() curr = open(file_['srcpath'], 'rb').read()
file_['data_end'] = start + len(curr) file_['data_end'] = start + len(curr)
if AV_WORKAROUND: curr += '\x00' if AV_WORKAROUND:
#print >> sys.stderr, 'bundling', file_['srcpath'], file_['dstpath'], file_['data_start'], file_['data_end'] curr += '\x00'
start += len(curr) start += len(curr)
data.write(curr) data.write(curr)
data.close() data.close()
# TODO: sha256sum on data_target # TODO: sha256sum on data_target
if start > 256*1024*1024: if start > 256 * 1024 * 1024:
print('warning: file packager is creating an asset bundle of %d MB. this is very large, and browsers might have trouble loading it. see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/' % (start/(1024*1024)), file=sys.stderr) print('warning: file packager is creating an asset bundle of %d MB. '
'this is very large, and browsers might have trouble loading it. '
'see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/'
% (start / (1024 * 1024)), file=sys.stderr)
create_preloaded = ''' create_preloaded = '''
Module['FS_createPreloadedFile'](this.name, null, byteArray, true, true, function() { Module['FS_createPreloadedFile'](this.name, null, byteArray, true, true, function() {
...@@ -367,7 +435,8 @@ if has_preloaded: ...@@ -367,7 +435,8 @@ if has_preloaded:
Module['removeRunDependency']('fp ' + that.name); Module['removeRunDependency']('fp ' + that.name);
''' '''
# Data requests - for getting a block of data out of the big archive - have a similar API to XHRs # Data requests - for getting a block of data out of the big archive - have
# a similar API to XHRs
code += ''' code += '''
function DataRequest(start, end, audio) { function DataRequest(start, end, audio) {
this.start = start; this.start = start;
...@@ -414,10 +483,12 @@ for file_ in data_files: ...@@ -414,10 +483,12 @@ for file_ in data_files:
chunk_size = 10240 chunk_size = 10240
start = 0 start = 0
while start < len(data): while start < len(data):
parts.append('''fileData%d.push.apply(fileData%d, %s);\n''' % (counter, counter, str(data[start:start+chunk_size]))) parts.append('''fileData%d.push.apply(fileData%d, %s);\n'''
% (counter, counter, str(data[start:start + chunk_size])))
start += chunk_size start += chunk_size
code += ''.join(parts) code += ''.join(parts)
code += '''Module['FS_createDataFile']('%s', '%s', fileData%d, true, true, false);\n''' % (dirname, basename, counter) code += ('''Module['FS_createDataFile']('%s', '%s', fileData%d, true, true, false);\n'''
% (dirname, basename, counter))
counter += 1 counter += 1
elif file_['mode'] == 'preload': elif file_['mode'] == 'preload':
# Preload # Preload
...@@ -439,7 +510,6 @@ if has_preloaded: ...@@ -439,7 +510,6 @@ if has_preloaded:
use_data = ''' use_data = '''
// copy the entire loaded file into a spot in the heap. Files will refer to slices in that. They cannot be freed though // copy the entire loaded file into a spot in the heap. Files will refer to slices in that. They cannot be freed though
// (we may be allocating before malloc is ready, during startup). // (we may be allocating before malloc is ready, during startup).
if (Module['SPLIT_MEMORY']) err('warning: you should run the file packager with --no-heap-copy when SPLIT_MEMORY is used, otherwise copying into the heap may fail due to the splitting');
var ptr = Module['getMemory'](byteArray.length); var ptr = Module['getMemory'](byteArray.length);
Module['HEAPU8'].set(byteArray, ptr); Module['HEAPU8'].set(byteArray, ptr);
DataRequest.prototype.byteArray = Module['HEAPU8'].subarray(ptr, ptr+byteArray.length); DataRequest.prototype.byteArray = Module['HEAPU8'].subarray(ptr, ptr+byteArray.length);
...@@ -455,13 +525,17 @@ if has_preloaded: ...@@ -455,13 +525,17 @@ if has_preloaded:
DataRequest.prototype.requests[files[i].filename].onload(); DataRequest.prototype.requests[files[i].filename].onload();
} }
''' '''
use_data += " Module['removeRunDependency']('datafile_%s');\n" % shared.JS.escape_for_js_string(data_target) use_data += (" Module['removeRunDependency']('datafile_%s');\n"
% shared.JS.escape_for_js_string(data_target))
else: else:
# LZ4FS usage # LZ4FS usage
temp = data_target + '.orig' temp = data_target + '.orig'
shutil.move(data_target, temp) shutil.move(data_target, temp)
meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'), shared.NODE_JS, [shared.path_from_root('src', 'mini-lz4.js'), temp, data_target], stdout=PIPE) meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'),
shared.NODE_JS,
[shared.path_from_root('src', 'mini-lz4.js'),
temp, data_target], stdout=PIPE)
os.unlink(temp) os.unlink(temp)
use_data = ''' use_data = '''
var compressedData = %s; var compressedData = %s;
...@@ -473,8 +547,7 @@ if has_preloaded: ...@@ -473,8 +547,7 @@ if has_preloaded:
package_uuid = uuid.uuid4() package_uuid = uuid.uuid4()
package_name = data_target package_name = data_target
statinfo = os.stat(package_name) remote_package_size = os.path.getsize(package_name)
remote_package_size = statinfo.st_size
remote_package_name = os.path.basename(package_name) remote_package_name = os.path.basename(package_name)
ret += r''' ret += r'''
var PACKAGE_PATH; var PACKAGE_PATH;
...@@ -493,7 +566,8 @@ if has_preloaded: ...@@ -493,7 +566,8 @@ if has_preloaded:
err('warning: you defined Module.locateFilePackage, that has been renamed to Module.locateFile (using your locateFilePackage for now)'); err('warning: you defined Module.locateFilePackage, that has been renamed to Module.locateFile (using your locateFilePackage for now)');
} }
var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE; var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;
''' % (shared.JS.escape_for_js_string(data_target), shared.JS.escape_for_js_string(remote_package_name)) ''' % (shared.JS.escape_for_js_string(data_target),
shared.JS.escape_for_js_string(remote_package_name))
metadata['remote_package_size'] = remote_package_size metadata['remote_package_size'] = remote_package_size
metadata['package_uuid'] = str(package_uuid) metadata['package_uuid'] = str(package_uuid)
ret += ''' ret += '''
...@@ -538,59 +612,122 @@ if has_preloaded: ...@@ -538,59 +612,122 @@ if has_preloaded:
}; };
}; };
// This is needed as chromium has a limit on per-entry files in IndexedDB
// https://cs.chromium.org/chromium/src/content/renderer/indexed_db/webidbdatabase_impl.cc?type=cs&sq=package:chromium&g=0&l=177
// https://cs.chromium.org/chromium/src/out/Debug/gen/third_party/blink/public/mojom/indexeddb/indexeddb.mojom.h?type=cs&sq=package:chromium&g=0&l=60
// We set the chunk size to 64MB to stay well-below the limit
var CHUNK_SIZE = 64 * 1024 * 1024;
function cacheRemotePackage(
db,
packageName,
packageData,
packageMeta,
callback,
errback
) {
var transactionPackages = db.transaction([PACKAGE_STORE_NAME], IDB_RW);
var packages = transactionPackages.objectStore(PACKAGE_STORE_NAME);
var chunkSliceStart = 0;
var nextChunkSliceStart = 0;
var chunkCount = Math.ceil(packageData.byteLength / CHUNK_SIZE);
var finishedChunks = 0;
for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
nextChunkSliceStart += CHUNK_SIZE;
var putPackageRequest = packages.put(
packageData.slice(chunkSliceStart, nextChunkSliceStart),
'package/' + packageName + '/' + chunkId
);
chunkSliceStart = nextChunkSliceStart;
putPackageRequest.onsuccess = function(event) {
finishedChunks++;
if (finishedChunks == chunkCount) {
var transaction_metadata = db.transaction(
[METADATA_STORE_NAME],
IDB_RW
);
var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME);
var putMetadataRequest = metadata.put(
{
uuid: packageMeta.uuid,
chunkCount: chunkCount
},
'metadata/' + packageName
);
putMetadataRequest.onsuccess = function(event) {
callback(packageData);
};
putMetadataRequest.onerror = function(error) {
errback(error);
};
}
};
putPackageRequest.onerror = function(error) {
errback(error);
};
}
}
/* Check if there's a cached package, and if so whether it's the latest available */ /* Check if there's a cached package, and if so whether it's the latest available */
function checkCachedPackage(db, packageName, callback, errback) { function checkCachedPackage(db, packageName, callback, errback) {
var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO); var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO);
var metadata = transaction.objectStore(METADATA_STORE_NAME); var metadata = transaction.objectStore(METADATA_STORE_NAME);
var getRequest = metadata.get('metadata/' + packageName);
var getRequest = metadata.get("metadata/" + packageName);
getRequest.onsuccess = function(event) { getRequest.onsuccess = function(event) {
var result = event.target.result; var result = event.target.result;
if (!result) { if (!result) {
return callback(false); return callback(false, null);
} else { } else {
return callback(PACKAGE_UUID === result.uuid); return callback(PACKAGE_UUID === result.uuid, result);
} }
}; };
getRequest.onerror = function(error) { getRequest.onerror = function(error) {
errback(error); errback(error);
}; };
}; }
function fetchCachedPackage(db, packageName, callback, errback) { function fetchCachedPackage(db, packageName, metadata, callback, errback) {
var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO); var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO);
var packages = transaction.objectStore(PACKAGE_STORE_NAME); var packages = transaction.objectStore(PACKAGE_STORE_NAME);
var getRequest = packages.get("package/" + packageName); var chunksDone = 0;
getRequest.onsuccess = function(event) { var totalSize = 0;
var result = event.target.result; var chunks = new Array(metadata.chunkCount);
callback(result);
}; for (var chunkId = 0; chunkId < metadata.chunkCount; chunkId++) {
getRequest.onerror = function(error) { var getRequest = packages.get('package/' + packageName + '/' + chunkId);
errback(error); getRequest.onsuccess = function(event) {
}; // If there's only 1 chunk, there's nothing to concatenate it with so we can just return it now
}; if (metadata.chunkCount == 1) {
callback(event.target.result);
function cacheRemotePackage(db, packageName, packageData, packageMeta, callback, errback) { } else {
var transaction_packages = db.transaction([PACKAGE_STORE_NAME], IDB_RW); chunksDone++;
var packages = transaction_packages.objectStore(PACKAGE_STORE_NAME); totalSize += event.target.result.byteLength;
chunks.push(event.target.result);
var putPackageRequest = packages.put(packageData, "package/" + packageName); if (chunksDone == metadata.chunkCount) {
putPackageRequest.onsuccess = function(event) { if (chunksDone == 1) {
var transaction_metadata = db.transaction([METADATA_STORE_NAME], IDB_RW); callback(event.target.result);
var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME); } else {
var putMetadataRequest = metadata.put(packageMeta, "metadata/" + packageName); var tempTyped = new Uint8Array(totalSize);
putMetadataRequest.onsuccess = function(event) { var byteOffset = 0;
callback(packageData); for (var chunkId in chunks) {
var buffer = chunks[chunkId];
tempTyped.set(new Uint8Array(buffer), byteOffset);
byteOffset += buffer.byteLength;
buffer = undefined;
}
chunks = undefined;
callback(tempTyped.buffer);
tempTyped = undefined;
}
}
}
}; };
putMetadataRequest.onerror = function(error) { getRequest.onerror = function(error) {
errback(error); errback(error);
}; };
}; }
putPackageRequest.onerror = function(error) { }
errback(error);
};
};
''' '''
ret += r''' ret += r'''
...@@ -657,7 +794,9 @@ if has_preloaded: ...@@ -657,7 +794,9 @@ if has_preloaded:
%s %s
}; };
Module['addRunDependency']('datafile_%s'); Module['addRunDependency']('datafile_%s');
''' % (use_data, shared.JS.escape_for_js_string(data_target)) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file ''' % (use_data, shared.JS.escape_for_js_string(data_target))
# use basename because from the browser's point of view,
# we need to find the datafile in the same dir as the html file
code += r''' code += r'''
if (!Module.preloadResults) Module.preloadResults = {}; if (!Module.preloadResults) Module.preloadResults = {};
...@@ -674,11 +813,11 @@ if has_preloaded: ...@@ -674,11 +813,11 @@ if has_preloaded:
openDatabase( openDatabase(
function(db) { function(db) {
checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME,
function(useCached) { function(useCached, metadata) {
Module.preloadResults[PACKAGE_NAME] = {fromCache: useCached}; Module.preloadResults[PACKAGE_NAME] = {fromCache: useCached};
if (useCached) { if (useCached) {
console.info('loading ' + PACKAGE_NAME + ' from cache'); console.info('loading ' + PACKAGE_NAME + ' from cache');
fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, processPackageData, preloadFallback); fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, metadata, processPackageData, preloadFallback);
} else { } else {
console.info('loading ' + PACKAGE_NAME + ' from remote'); console.info('loading ' + PACKAGE_NAME + ' from remote');
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE,
...@@ -699,8 +838,10 @@ if has_preloaded: ...@@ -699,8 +838,10 @@ if has_preloaded:
if (Module['setStatus']) Module['setStatus']('Downloading...'); if (Module['setStatus']) Module['setStatus']('Downloading...');
''' '''
else: else:
# Not using preload cache, so we might as well start the xhr ASAP, potentially before JS parsing of the main codebase if it's after us. # Not using preload cache, so we might as well start the xhr ASAP,
# Only tricky bit is the fetch is async, but also when runWithFS is called is async, so we handle both orderings. # potentially before JS parsing of the main codebase if it's after us.
# Only tricky bit is the fetch is async, but also when runWithFS is called
# is async, so we handle both orderings.
ret += r''' ret += r'''
var fetchedCallback = null; var fetchedCallback = null;
var fetched = Module['getPreloadedPackage'] ? Module['getPreloadedPackage'](REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE) : null; var fetched = Module['getPreloadedPackage'] ? Module['getPreloadedPackage'](REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE) : null;
...@@ -739,9 +880,8 @@ ret += ''' ...@@ -739,9 +880,8 @@ ret += '''
} }
''' '''
ret += '''%s if separate_metadata:
})(); _metadata_template = '''
''' % ('''
Module['removeRunDependency']('%(metadata_file)s'); Module['removeRunDependency']('%(metadata_file)s');
} }
...@@ -765,16 +905,26 @@ ret += '''%s ...@@ -765,16 +905,26 @@ ret += '''%s
if (!Module['preRun']) Module['preRun'] = []; if (!Module['preRun']) Module['preRun'] = [];
Module["preRun"].push(runMetaWithFS); Module["preRun"].push(runMetaWithFS);
} }
''' % {'metadata_file': os.path.basename(jsoutput + '.metadata')} if separate_metadata else ''' ''' % {'metadata_file': os.path.basename(jsoutput + '.metadata')}
else:
_metadata_template = '''
} }
loadPackage(%s); loadPackage(%s);
''' % json.dumps(metadata)) ''' % json.dumps(metadata)
ret += '''%s
})();
''' % _metadata_template
if force or len(data_files): if force or len(data_files):
if jsoutput == None: if jsoutput is None:
print(ret) print(ret)
else: else:
# Overwrite the old jsoutput file (if exists) only when its content differs from the current generated one, otherwise leave the file untouched preserving its old timestamp # Overwrite the old jsoutput file (if exists) only when its content
# differs from the current generated one, otherwise leave the file
# untouched preserving its old timestamp
if os.path.isfile(jsoutput): if os.path.isfile(jsoutput):
f = open(jsoutput, 'r+') f = open(jsoutput, 'r+')
old = f.read() old = f.read()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment