diff options
| -rwxr-xr-x | emcc | 256 | ||||
| -rw-r--r-- | tools/file_packager.py | 264 | ||||
| -rw-r--r-- | tools/shared.py | 26 |
3 files changed, 306 insertions, 240 deletions
@@ -77,15 +77,7 @@ emcc can be influenced by a few environment variables: import os, sys, shutil, tempfile, subprocess, shlex from subprocess import PIPE, STDOUT from tools import shared - -def execute(cmd, *args, **kw): - try: - return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that) - except: - if not isinstance(cmd, str): - cmd = ' '.join(cmd) - print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>' - raise +from tools.shared import Compression, execute # Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt # levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get @@ -343,10 +335,6 @@ STATICLIB_SUFFIXES = ('.a',) ASSEMBLY_SUFFIXES = ('.ll',) LIB_PREFIXES = ('', 'lib') -IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp') -AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3') -AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' } - def suffix(name): return name.split('.')[-1] @@ -418,21 +406,6 @@ else: def in_temp(name): return os.path.join(temp_dir, name) -class Compression: - on = False - - @staticmethod - def compressed_name(filename): - return filename + '.compress' - - @staticmethod - def compress(filename): - execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb')) - - @staticmethod - def worth_it(original, compressed): - return compressed < original - 1500 # save at least one TCP packet or so - try: call = CXX if use_cxx else CC @@ -448,8 +421,8 @@ try: pre_js = None post_js = None minify_whitespace = None - data_files = [] - has_preloaded_files = False + preload_files = [] + embed_files = [] compression = None ignore_dynamic_linking = False shell_path = shared.path_from_root('src', 'shell.html') @@ -503,13 +476,12 @@ try: newargs[i+1] = '' elif newargs[i].startswith('--embed-file'): check_bad_eq(newargs[i]) - data_files.append({ 'name': newargs[i+1], 'mode': 'embed' }) + embed_files.append(newargs[i+1]) newargs[i] = '' newargs[i+1] = '' elif newargs[i].startswith('--preload-file'): check_bad_eq(newargs[i]) - data_files.append({ 'name': newargs[i+1], 'mode': 'preload' }) - has_preloaded_files = True + preload_files.append(newargs[i+1]) newargs[i] = '' newargs[i+1] = '' elif newargs[i].startswith('--compression'): @@ -875,214 +847,18 @@ try: if DEBUG: save_intermediate('original') # Embed and preload files - if len(data_files) > 0: + if len(preload_files) + len(embed_files) > 0: if DEBUG: print >> sys.stderr, 'emcc: setting up files' - code = '' - - if has_preloaded_files: - code += ''' - var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs")); - var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs"); - var hasBlobConstructor; - try { - new Blob(); - hasBlobConstructor = true; - } catch(e) { - hasBlobConstructor = false; - console.log("warning: no blob constructor, cannot create blobs with mimetypes"); - } - ''' - - code += 'var preloadedImages = {}; // maps url to image data\n' - code += 'var preloadedAudios = {}; // maps url to audio data\n' - - # Expand directories into individual files - def add(mode, dirname, names): - for name in names: - fullname = os.path.join(dirname, name) - if not os.path.isdir(fullname): - data_files.append({ 'name': fullname, 'mode': mode }) - - for file_ in data_files: - if os.path.isdir(file_['name']): - os.path.walk(file_['name'], add, file_['mode']) - data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files) - - for file_ in data_files: - file_['name'] = file_['name'].replace(os.path.sep, '/') - file_['net_name'] = file_['name'] - - data_target = unsuffixed(target) + '.data' - - # Set up folders - partial_dirs = [] - for file_ in data_files: - dirname = os.path.dirname(file_['name']) - dirname = dirname.lstrip('/') # absolute paths start with '/', remove that - if dirname != '': - parts = dirname.split('/') - for i in range(len(parts)): - partial = '/'.join(parts[:i+1]) - if partial not in partial_dirs: - code += '''Module['FS_createFolder']('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i]) - partial_dirs.append(partial) - - if has_preloaded_files: - # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead. - data = open(data_target, 'wb') - start = 0 - for file_ in data_files: - file_['data_start'] = start - curr = open(file_['name'], 'rb').read() - file_['data_end'] = start + len(curr) - start += len(curr) - data.write(curr) - data.close() - if Compression.on: - Compression.compress(data_target) - - # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs - code += ''' - function DataRequest() {} - DataRequest.prototype = { - requests: {}, - open: function(mode, name) { - this.requests[name] = this; - }, - send: function() {} - }; - ''' - - counter = 0 - for file_ in data_files: - filename = file_['name'] - if file_['mode'] == 'embed': - # Embed - code += '''Module['FS_createDataFile']('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read()))) - elif file_['mode'] == 'preload': - # Preload - varname = 'filePreload%d' % counter - counter += 1 - image = filename.endswith(IMAGE_SUFFIXES) - audio = filename.endswith(AUDIO_SUFFIXES) - - if image: - finish = ''' - var bb = new BlobBuilder(); - bb.append(byteArray.buffer); - var b = bb.getBlob(); - var url = URLObject.createObjectURL(b); - var img = new Image(); - img.onload = function() { - assert(img.complete, 'Image %(filename)s could not be decoded'); - var canvas = document.createElement('canvas'); - canvas.width = img.width; - canvas.height = img.height; - var ctx = canvas.getContext('2d'); - ctx.drawImage(img, 0, 0); - preloadedImages['%(filename)s'] = canvas; - URLObject.revokeObjectURL(url); - removeRunDependency(); - }; - img.onerror = function(event) { - console.log('Image %(filename)s could not be decoded'); - }; - img.src = url; -''' % { 'filename': filename } - elif audio: - # Need actual blob constructor here, to set the mimetype or else audios fail to decode - finish = ''' - if (hasBlobConstructor) { - var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' }); - var url = URLObject.createObjectURL(b); // XXX we never revoke this! - var audio = new Audio(); - audio.removedDependency = false; - audio['oncanplaythrough'] = function() { // XXX string for closure - audio['oncanplaythrough'] = null; - preloadedAudios['%(filename)s'] = audio; - if (!audio.removedDependency) { - removeRunDependency(); - audio.removedDependency = true; - } - }; - audio.onerror = function(event) { - if (!audio.removedDependency) { - console.log('Audio %(filename)s could not be decoded or timed out trying to decode'); - removeRunDependency(); - audio.removedDependency = true; - } - }; - setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang) - audio.src = url; - } else { - preloadedAudios['%(filename)s'] = new Audio(); // empty shim - removeRunDependency(); - } -''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] } - else: - finish = 'removeRunDependency();\n' - - code += ''' - var %(varname)s = new %(request)s(); - %(varname)s.open('GET', '%(netname)s', true); - %(varname)s.responseType = 'arraybuffer'; - %(varname)s.onload = function() { - var arrayBuffer = %(varname)s.response; - assert(arrayBuffer, 'Loading file %(filename)s failed.'); - var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer; - Module['FS_createDataFile']('/%(dirname)s', '%(basename)s', byteArray, true, true); - %(finish)s - }; - addRunDependency(); - %(varname)s.send(null); -''' % { - 'request': 'DataRequest', # In the past we also supported XHRs here - 'varname': varname, - 'filename': filename, - 'netname': file_['net_name'], - 'dirname': os.path.dirname(filename), - 'basename': os.path.basename(filename), - 'finish': finish - } - else: - assert 0 - - if has_preloaded_files: - # Get the big archive and split it up - use_data = '' - for file_ in data_files: - if file_['mode'] == 'preload': - use_data += ''' - curr = DataRequest.prototype.requests['%s']; - curr.response = byteArray.subarray(%d,%d); - curr.onload(); - ''' % (file_['name'], file_['data_start'], file_['data_end']) - use_data += ' removeRunDependency();\n' - - if Compression.on: - use_data = ''' - Module["decompress"](byteArray, function(decompressed) { - byteArray = new Uint8Array(decompressed); - %s - }); - ''' % use_data - - code += ''' - var dataFile = new XMLHttpRequest(); - dataFile.open('GET', '%s', true); - dataFile.responseType = 'arraybuffer'; - dataFile.onload = function() { - var arrayBuffer = dataFile.response; - assert(arrayBuffer, 'Loading data file failed.'); - var byteArray = new Uint8Array(arrayBuffer); - var curr; - %s - }; - addRunDependency(); - dataFile.send(null); - if (Module['setStatus']) Module['setStatus']('Downloading...'); - ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data) - + file_args = [] + if len(preload_files) > 0: + file_args.append('--preload') + file_args += preload_files + if len(embed_files) > 0: + file_args.append('--embed') + file_args += embed_files + if Compression.on: + file_args += ['--compress', Compression.encoder, Compression.decoder, Compression.js_name] + code = execute(shared.ENV_PREFIX + ['python', shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0] src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code) final += '.files.js' open(final, 'w').write(src) diff --git a/tools/file_packager.py b/tools/file_packager.py new file mode 100644 index 00000000..ebd3d37a --- /dev/null +++ b/tools/file_packager.py @@ -0,0 +1,264 @@ +''' +A tool that generates FS API calls to generate a filesystem, and packages the files +to work with that. + +This is called by emcc. You can also call it yourself. + +Usage: + + file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--compress COMPRESSION_DATA] + +''' + +import os, sys + +from shared import Compression, execute + +def suffix(name): + return name.split('.')[-1] + +IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp') +AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3') +AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' } + +data_files = [] +in_preload = False +in_embed = False +has_preloaded = False +in_compress = 0 +for arg in sys.argv[1:]: + if arg == '--preload': + in_preload = True + in_embed = False + has_preloaded = True + elif arg == '--embed': + in_embed = True + in_preload = False + elif arg == '--compress': + Compression.on = True + in_compress = 1 + in_preload = False + in_embed = False + elif in_preload: + data_files.append({ 'name': arg, 'mode': 'preload' }) + elif in_embed: + data_files.append({ 'name': arg, 'mode': 'embed' }) + elif in_compress: + if in_compress == 1: + Compression.encoder = arg + in_compress = 2 + elif in_compress == 2: + Compression.decoder = arg + in_compress = 3 + elif in_compress == 3: + Compression.js_name = arg + in_compress = 0 + +code = '' + +if has_preloaded: + code += ''' + var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs")); + var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs"); + var hasBlobConstructor; + try { + new Blob(); + hasBlobConstructor = true; + } catch(e) { + hasBlobConstructor = false; + console.log("warning: no blob constructor, cannot create blobs with mimetypes"); + } +''' + + code += 'var preloadedImages = {}; // maps url to image data\n' + code += 'var preloadedAudios = {}; // maps url to audio data\n' + +# Expand directories into individual files +def add(mode, dirname, names): + for name in names: + fullname = os.path.join(dirname, name) + if not os.path.isdir(fullname): + data_files.append({ 'name': fullname, 'mode': mode }) + +for file_ in data_files: + if os.path.isdir(file_['name']): + os.path.walk(file_['name'], add, file_['mode']) +data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files) + +for file_ in data_files: + file_['name'] = file_['name'].replace(os.path.sep, '/') + file_['net_name'] = file_['name'] + +data_target = sys.argv[1] + +# Set up folders +partial_dirs = [] +for file_ in data_files: + dirname = os.path.dirname(file_['name']) + dirname = dirname.lstrip('/') # absolute paths start with '/', remove that + if dirname != '': + parts = dirname.split('/') + for i in range(len(parts)): + partial = '/'.join(parts[:i+1]) + if partial not in partial_dirs: + code += '''Module['FS_createFolder']('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i]) + partial_dirs.append(partial) + +if has_preloaded: + # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead. + data = open(data_target, 'wb') + start = 0 + for file_ in data_files: + file_['data_start'] = start + curr = open(file_['name'], 'rb').read() + file_['data_end'] = start + len(curr) + start += len(curr) + data.write(curr) + data.close() + if Compression.on: + Compression.compress(data_target) + + # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs + code += ''' + function DataRequest() {} + DataRequest.prototype = { + requests: {}, + open: function(mode, name) { + this.requests[name] = this; + }, + send: function() {} + }; + ''' + +counter = 0 +for file_ in data_files: + filename = file_['name'] + if file_['mode'] == 'embed': + # Embed + code += '''Module['FS_createDataFile']('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read()))) + elif file_['mode'] == 'preload': + # Preload + varname = 'filePreload%d' % counter + counter += 1 + image = filename.endswith(IMAGE_SUFFIXES) + audio = filename.endswith(AUDIO_SUFFIXES) + + if image: + finish = ''' + var bb = new BlobBuilder(); + bb.append(byteArray.buffer); + var b = bb.getBlob(); + var url = URLObject.createObjectURL(b); + var img = new Image(); + img.onload = function() { + assert(img.complete, 'Image %(filename)s could not be decoded'); + var canvas = document.createElement('canvas'); + canvas.width = img.width; + canvas.height = img.height; + var ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0); + preloadedImages['%(filename)s'] = canvas; + URLObject.revokeObjectURL(url); + removeRunDependency(); + }; + img.onerror = function(event) { + console.log('Image %(filename)s could not be decoded'); + }; + img.src = url; +''' % { 'filename': filename } + elif audio: + # Need actual blob constructor here, to set the mimetype or else audios fail to decode + finish = ''' + if (hasBlobConstructor) { + var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' }); + var url = URLObject.createObjectURL(b); // XXX we never revoke this! + var audio = new Audio(); + audio.removedDependency = false; + audio['oncanplaythrough'] = function() { // XXX string for closure + audio['oncanplaythrough'] = null; + preloadedAudios['%(filename)s'] = audio; + if (!audio.removedDependency) { + removeRunDependency(); + audio.removedDependency = true; + } + }; + audio.onerror = function(event) { + if (!audio.removedDependency) { + console.log('Audio %(filename)s could not be decoded or timed out trying to decode'); + removeRunDependency(); + audio.removedDependency = true; + } + }; + setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang) + audio.src = url; + } else { + preloadedAudios['%(filename)s'] = new Audio(); // empty shim + removeRunDependency(); + } +''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] } + else: + finish = 'removeRunDependency();\n' + + code += ''' + var %(varname)s = new %(request)s(); + %(varname)s.open('GET', '%(netname)s', true); + %(varname)s.responseType = 'arraybuffer'; + %(varname)s.onload = function() { + var arrayBuffer = %(varname)s.response; + assert(arrayBuffer, 'Loading file %(filename)s failed.'); + var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer; + Module['FS_createDataFile']('/%(dirname)s', '%(basename)s', byteArray, true, true); + %(finish)s + }; + addRunDependency(); + %(varname)s.send(null); +''' % { + 'request': 'DataRequest', # In the past we also supported XHRs here + 'varname': varname, + 'filename': filename, + 'netname': file_['net_name'], + 'dirname': os.path.dirname(filename), + 'basename': os.path.basename(filename), + 'finish': finish + } + else: + assert 0 + +if has_preloaded: + # Get the big archive and split it up + use_data = '' + for file_ in data_files: + if file_['mode'] == 'preload': + use_data += ''' + curr = DataRequest.prototype.requests['%s']; + curr.response = byteArray.subarray(%d,%d); + curr.onload(); + ''' % (file_['name'], file_['data_start'], file_['data_end']) + use_data += ' removeRunDependency();\n' + + if Compression.on: + use_data = ''' + Module["decompress"](byteArray, function(decompressed) { + byteArray = new Uint8Array(decompressed); + %s + }); +''' % use_data + + code += ''' + var dataFile = new XMLHttpRequest(); + dataFile.open('GET', '%s', true); + dataFile.responseType = 'arraybuffer'; + dataFile.onload = function() { + var arrayBuffer = dataFile.response; + assert(arrayBuffer, 'Loading data file failed.'); + var byteArray = new Uint8Array(arrayBuffer); + var curr; + %s + }; + addRunDependency(); + dataFile.send(null); + if (Module['setStatus']) Module['setStatus']('Downloading...'); + ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data) + +print code + diff --git a/tools/shared.py b/tools/shared.py index f71bec72..3e2ce04e 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -131,6 +131,7 @@ BINDINGS_GENERATOR = path_from_root('tools', 'bindings_generator.py') EXEC_LLVM = path_from_root('tools', 'exec_llvm.py') VARIABLE_ELIMINATOR = path_from_root('tools', 'eliminator', 'eliminator.coffee') JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') +FILE_PACKAGER = path_from_root('tools', 'file_packager.py') # Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use TEMP_DIR/emscripten_temp @@ -932,3 +933,28 @@ class Cache: shutil.copyfile(creator(), cachename) return cachename +# Compression of code and data for smaller downloads +class Compression: + on = False + + @staticmethod + def compressed_name(filename): + return filename + '.compress' + + @staticmethod + def compress(filename): + execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb')) + + @staticmethod + def worth_it(original, compressed): + return compressed < original - 1500 # save at least one TCP packet or so + +def execute(cmd, *args, **kw): + try: + return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that) + except: + if not isinstance(cmd, str): + cmd = ' '.join(cmd) + print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>' + raise + |