aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/file_packager.py232
-rw-r--r--tools/shared.py46
2 files changed, 223 insertions, 55 deletions
diff --git a/tools/file_packager.py b/tools/file_packager.py
index e434e813..eef268cb 100644
--- a/tools/file_packager.py
+++ b/tools/file_packager.py
@@ -28,6 +28,8 @@ Usage:
--no-force Don't create output if no valid input file is specified.
+ --use-preload-cache Stores package in IndexedDB so that subsequent loads don't need to do XHR. Checks package version.
+
Notes:
* The file packager generates unix-style file paths. So if you are on windows and a file is accessed at
@@ -37,14 +39,14 @@ TODO: You can also provide .crn files yourself, pre-crunched. With this o
to dds files in the browser, exactly the same as if this tool compressed them.
'''
-import os, sys, shutil, random
+import os, sys, shutil, random, uuid
import shared
from shared import Compression, execute, suffix, unsuffixed
from subprocess import Popen, PIPE, STDOUT
if len(sys.argv) == 1:
- print '''Usage: file_packager.py TARGET [--preload A...] [--embed B...] [--compress COMPRESSION_DATA] [--pre-run] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force]
+ print '''Usage: file_packager.py TARGET [--preload A...] [--embed B...] [--compress COMPRESSION_DATA] [--pre-run] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache]
See the source for more details.'''
sys.exit(0)
@@ -70,6 +72,7 @@ crunch = 0
plugins = []
jsoutput = None
force = True
+use_preload_cache = False
for arg in sys.argv[1:]:
if arg == '--preload':
@@ -93,6 +96,8 @@ for arg in sys.argv[1:]:
in_compress = 0
elif arg == '--no-force':
force = False
+ elif arg == '--use-preload-cache':
+ use_preload_cache = True
elif arg.startswith('--js-output'):
jsoutput = arg.split('=')[1] if '=' in arg else None
elif arg.startswith('--crunch'):
@@ -256,6 +261,7 @@ if has_preloaded:
start += len(curr)
data.write(curr)
data.close()
+ # TODO: sha256sum on data_target
if Compression.on:
Compression.compress(data_target)
@@ -356,8 +362,9 @@ if has_preloaded:
byteArray = new Uint8Array(decompressed);
%s
});
-''' % use_data
+ ''' % use_data
+ package_uuid = uuid.uuid4();
code += r'''
if (!Module.expectedDataFileDownloads) {
Module.expectedDataFileDownloads = 0;
@@ -365,49 +372,202 @@ if has_preloaded:
}
Module.expectedDataFileDownloads++;
- var dataFile = new XMLHttpRequest();
- dataFile.onprogress = function(event) {
- var url = '%s';
- if (event.loaded && event.total) {
- if (!dataFile.addedTotal) {
- dataFile.addedTotal = true;
- if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
- Module.dataFileDownloads[url] = {
- loaded: event.loaded,
- total: event.total
- };
- } else {
- Module.dataFileDownloads[url].loaded = event.loaded;
+ var PACKAGE_PATH = window.encodeURIComponent(window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/');
+ var PACKAGE_NAME = '%s';
+ var REMOTE_PACKAGE_NAME = '%s';
+ var PACKAGE_UUID = '%s';
+ ''' % (data_target, os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target), package_uuid)
+
+ if use_preload_cache:
+ code += r'''
+ var indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
+ var IDB_RO = "readonly";
+ var IDB_RW = "readwrite";
+ var DB_NAME = 'EM_PRELOAD_CACHE';
+ var DB_VERSION = 1;
+ var METADATA_STORE_NAME = 'METADATA';
+ var PACKAGE_STORE_NAME = 'PACKAGES';
+ function openDatabase(callback, errback) {
+ try {
+ var openRequest = indexedDB.open(DB_NAME, DB_VERSION);
+ } catch (e) {
+ return errback(e);
}
- var total = 0;
- var loaded = 0;
- var num = 0;
- for (var download in Module.dataFileDownloads) {
+ openRequest.onupgradeneeded = function(event) {
+ var db = event.target.result;
+
+ if(db.objectStoreNames.contains(PACKAGE_STORE_NAME)) {
+ db.deleteObjectStore(PACKAGE_STORE_NAME);
+ }
+ var packages = db.createObjectStore(PACKAGE_STORE_NAME);
+
+ if(db.objectStoreNames.contains(METADATA_STORE_NAME)) {
+ db.deleteObjectStore(METADATA_STORE_NAME);
+ }
+ var metadata = db.createObjectStore(METADATA_STORE_NAME);
+ };
+ openRequest.onsuccess = function(event) {
+ var db = event.target.result;
+ callback(db);
+ };
+ openRequest.onerror = function(error) {
+ errback(error);
+ };
+ };
+
+ /* Check if there's a cached package, and if so whether it's the latest available */
+ function checkCachedPackage(db, packageName, callback, errback) {
+ var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO);
+ var metadata = transaction.objectStore(METADATA_STORE_NAME);
+
+ var getRequest = metadata.get(packageName);
+ getRequest.onsuccess = function(event) {
+ var result = event.target.result;
+ if (!result) {
+ return callback(false);
+ } else {
+ return callback(PACKAGE_UUID === result.uuid);
+ }
+ };
+ getRequest.onerror = function(error) {
+ errback(error);
+ };
+ };
+
+ function fetchCachedPackage(db, packageName, callback, errback) {
+ var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO);
+ var packages = transaction.objectStore(PACKAGE_STORE_NAME);
+
+ var getRequest = packages.get(packageName);
+ getRequest.onsuccess = function(event) {
+ var result = event.target.result;
+ callback(result);
+ };
+ getRequest.onerror = function(error) {
+ errback(error);
+ };
+ };
+
+ function cacheRemotePackage(db, packageName, packageData, packageMeta, callback, errback) {
+ var transaction = db.transaction([PACKAGE_STORE_NAME, METADATA_STORE_NAME], IDB_RW);
+ var packages = transaction.objectStore(PACKAGE_STORE_NAME);
+ var metadata = transaction.objectStore(METADATA_STORE_NAME);
+
+ var putPackageRequest = packages.put(packageData, packageName);
+ putPackageRequest.onsuccess = function(event) {
+ var putMetadataRequest = metadata.put(packageMeta, packageName);
+ putMetadataRequest.onsuccess = function(event) {
+ callback(packageData);
+ };
+ putMetadataRequest.onerror = function(error) {
+ errback(error);
+ };
+ };
+ putPackageRequest.onerror = function(error) {
+ errback(error);
+ };
+ };
+ '''
+
+ code += r'''
+ function fetchRemotePackage(packageName, callback, errback) {
+ var xhr = new XMLHttpRequest();
+ xhr.open('GET', packageName, true);
+ xhr.responseType = 'arraybuffer';
+ xhr.onprogress = function(event) {
+ var url = packageName;
+ if (event.loaded && event.total) {
+ if (!xhr.addedTotal) {
+ xhr.addedTotal = true;
+ if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
+ Module.dataFileDownloads[url] = {
+ loaded: event.loaded,
+ total: event.total
+ };
+ } else {
+ Module.dataFileDownloads[url].loaded = event.loaded;
+ }
+ var total = 0;
+ var loaded = 0;
+ var num = 0;
+ for (var download in Module.dataFileDownloads) {
var data = Module.dataFileDownloads[download];
- total += data.total;
- loaded += data.loaded;
- num++;
+ total += data.total;
+ loaded += data.loaded;
+ num++;
+ }
+ total = Math.ceil(total * Module.expectedDataFileDownloads/num);
+ Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
+ } else if (!Module.dataFileDownloads) {
+ Module['setStatus']('Downloading data...');
}
- total = Math.ceil(total * Module.expectedDataFileDownloads/num);
- Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
- } else if (!Module.dataFileDownloads) {
- Module['setStatus']('Downloading data...');
- }
- }
- dataFile.open('GET', '%s', true);
- dataFile.responseType = 'arraybuffer';
- dataFile.onload = function() {
+ };
+ xhr.onload = function(event) {
+ var packageData = xhr.response;
+ callback(packageData);
+ };
+ xhr.send(null);
+ };
+
+ function processPackageData(arrayBuffer) {
Module.finishedDataFileDownloads++;
- var arrayBuffer = dataFile.response;
assert(arrayBuffer, 'Loading data file failed.');
var byteArray = new Uint8Array(arrayBuffer);
var curr;
%s
};
Module['addRunDependency']('datafile_%s');
- dataFile.send(null);
- if (Module['setStatus']) Module['setStatus']('Downloading...');
- ''' % (data_target, os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target), use_data, data_target) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file
+
+ function handleError(error) {
+ console.error('package error:', error);
+ };
+ ''' % (use_data, data_target) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file
+
+ code += r'''
+ if (!Module.preloadResults)
+ Module.preloadResults = {};
+ '''
+
+ if use_preload_cache:
+ code += r'''
+ function preloadFallback(error) {
+ console.error(error);
+ console.error('falling back to default preload behavior');
+ fetchRemotePackage(REMOTE_PACKAGE_NAME, processPackageData, handleError);
+ };
+
+ openDatabase(
+ function(db) {
+ checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME,
+ function(useCached) {
+ Module.preloadResults[PACKAGE_NAME] = {fromCache: useCached};
+ if (useCached) {
+ console.info('loading ' + PACKAGE_NAME + ' from cache');
+ fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, processPackageData, preloadFallback);
+ } else {
+ console.info('loading ' + PACKAGE_NAME + ' from remote');
+ fetchRemotePackage(REMOTE_PACKAGE_NAME,
+ function(packageData) {
+ cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}, processPackageData,
+ function(error) {
+ console.error(error);
+ processPackageData(packageData);
+ });
+ }
+ , preloadFallback);
+ }
+ }
+ , preloadFallback);
+ }
+ , preloadFallback);
+
+ if (Module['setStatus']) Module['setStatus']('Downloading...');
+ '''
+ else:
+ code += r'''
+ Module.preloadResults[PACKAGE_NAME] = {fromCache: false};
+ fetchRemotePackage(REMOTE_PACKAGE_NAME, processPackageData, handleError);
+ '''
if pre_run:
ret += '''
diff --git a/tools/shared.py b/tools/shared.py
index bc1d4352..09277d06 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -199,29 +199,33 @@ def check_node_version():
# we re-check sanity when the settings are changed)
# We also re-check sanity and clear the cache when the version changes
-EMSCRIPTEN_VERSION = '1.3.7'
+EMSCRIPTEN_VERSION = '1.4.1'
+
+def generate_sanity():
+ return EMSCRIPTEN_VERSION + '|' + LLVM_TARGET
def check_sanity(force=False):
try:
- if not force:
- if not CONFIG_FILE:
- return # config stored directly in EM_CONFIG => skip sanity checks
+ reason = None
+ if not CONFIG_FILE:
+ if not force: return # config stored directly in EM_CONFIG => skip sanity checks
+ else:
settings_mtime = os.stat(CONFIG_FILE).st_mtime
sanity_file = CONFIG_FILE + '_sanity'
- reason = 'unknown'
- try:
- sanity_mtime = os.stat(sanity_file).st_mtime
- if sanity_mtime <= settings_mtime:
- reason = 'settings file has changed'
- else:
- sanity_version = open(sanity_file).read()
- if sanity_version != EMSCRIPTEN_VERSION:
- reason = 'version bump'
+ if os.path.exists(sanity_file):
+ try:
+ sanity_mtime = os.stat(sanity_file).st_mtime
+ if sanity_mtime <= settings_mtime:
+ reason = 'settings file has changed'
else:
- return # all is well
- except:
- pass
-
+ sanity_data = open(sanity_file).read()
+ if sanity_data != generate_sanity():
+ reason = 'system change: %s vs %s' % (generate_sanity(), sanity_data)
+ else:
+ if not force: return # all is well
+ except Exception, e:
+ reason = 'unknown: ' + str(e)
+ if reason:
print >> sys.stderr, '(Emscripten: %s, clearing cache)' % reason
Cache.erase()
@@ -262,7 +266,7 @@ def check_sanity(force=False):
if not force:
# Only create/update this file if the sanity check succeeded, i.e., we got here
f = open(sanity_file, 'w')
- f.write(EMSCRIPTEN_VERSION)
+ f.write(generate_sanity())
f.close()
except Exception, e:
@@ -394,6 +398,9 @@ except:
# Additional compiler options
+# Target choice. Must be synced with src/settings.js (TARGET_*)
+LLVM_TARGET = os.environ.get('EMCC_LLVM_TARGET') or 'i386-pc-linux-gnu' # 'le32-unknown-nacl'
+
try:
COMPILER_OPTS # Can be set in EM_CONFIG, optionally
except:
@@ -402,7 +409,8 @@ except:
COMPILER_OPTS = COMPILER_OPTS + ['-m32', '-U__i386__', '-U__i386', '-Ui386',
'-U__SSE__', '-U__SSE_MATH__', '-U__SSE2__', '-U__SSE2_MATH__', '-U__MMX__',
'-DEMSCRIPTEN', '-D__EMSCRIPTEN__', '-U__STRICT_ANSI__',
- '-target', 'i386-pc-linux-gnu', '-D__IEEE_LITTLE_ENDIAN', '-fno-math-errno']
+ '-D__IEEE_LITTLE_ENDIAN', '-fno-math-errno',
+ '-target', LLVM_TARGET]
USE_EMSDK = not os.environ.get('EMMAKEN_NO_SDK')
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-07-24 18:09:20 +0000