diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/file_packager.py | 63 | ||||
| -rw-r--r-- | tools/scons/site_scons/site_tools/emscripten/emscripten.py | 2 | ||||
| -rw-r--r-- | tools/settings_template_readonly.py | 47 | ||||
| -rw-r--r-- | tools/shared.py | 27 |
4 files changed, 125 insertions, 14 deletions
diff --git a/tools/file_packager.py b/tools/file_packager.py index 7797e58c..e5a9025d 100644 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -4,6 +4,11 @@ to work with that. This is called by emcc. You can also call it yourself. +You can split your files into "asset bundles", and create each bundle separately +with this tool. Then just include the generated js for each and they will load +the data and prepare it accordingly. This allows you to share assets and reduce +data downloads. + Usage: file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--compress COMPRESSION_DATA] [--pre-run] [--crunch[=X]] @@ -25,7 +30,7 @@ TODO: You can also provide .crn files yourself, pre-crunched. With this o to dds files in the browser, exactly the same as if this tool compressed them. ''' -import os, sys, shutil +import os, sys, shutil, random from shared import Compression, execute, suffix, unsuffixed import shared @@ -41,6 +46,8 @@ CRUNCH_OUTPUT_SUFFIX = '.crn' DDS_HEADER_SIZE = 128 +AV_WORKAROUND = 0 # Set to 1 to randomize file order and add some padding, to work around silly av false positives + data_files = [] in_preload = False in_embed = False @@ -97,6 +104,10 @@ for arg in sys.argv[1:]: Compression.js_name = arg in_compress = 0 +print ''' +(function() { +''' + code = ''' function assert(check, msg) { if (!check) throw msg + new Error().stack; @@ -127,6 +138,9 @@ def was_seen(name): return False data_files = filter(lambda file_: not was_seen(file_['name']), data_files) +if AV_WORKAROUND: + random.shuffle(data_files) + # Apply plugins for file_ in data_files: for plugin in plugins: @@ -199,7 +213,7 @@ for file_ in data_files: for i in range(len(parts)): partial = '/'.join(parts[:i+1]) if partial not in partial_dirs: - code += '''Module['FS_createFolder']('/%s', '%s', true, true);\n''' % ('/'.join(parts[:i]), parts[i]) + code += '''Module['FS_createPath']('/%s', '%s', true, true);\n''' % ('/'.join(parts[:i]), parts[i]) partial_dirs.append(partial) if has_preloaded: @@ -210,6 +224,7 @@ if has_preloaded: file_['data_start'] = start curr = open(file_['localname'], 'rb').read() file_['data_end'] = start + len(curr) + if AV_WORKAROUND: curr += '\x00' print >> sys.stderr, 'bundling', file_['name'], file_['localname'], file_['data_start'], file_['data_end'] start += len(curr) data.write(curr) @@ -296,7 +311,7 @@ if has_preloaded: curr.response = byteArray.subarray(%d,%d); curr.onload(); ''' % (file_['name'], file_['data_start'], file_['data_end']) - use_data += " Module['removeRunDependency']('datafile');\n" + use_data += " Module['removeRunDependency']('datafile_%s');\n" % data_target if Compression.on: use_data = ''' @@ -306,28 +321,56 @@ if has_preloaded: }); ''' % use_data - code += ''' + code += r''' + if (!Module.expectedDataFileDownloads) { + Module.expectedDataFileDownloads = 0; + Module.finishedDataFileDownloads = 0; + } + Module.expectedDataFileDownloads++; + var dataFile = new XMLHttpRequest(); dataFile.onprogress = function(event) { + var url = '%s'; if (event.loaded && event.total) { - Module.setStatus('Downloading data... (' + event.loaded + '/' + event.total + ')'); - } else { + if (!dataFile.addedTotal) { + dataFile.addedTotal = true; + if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; + Module.dataFileDownloads[url] = { + loaded: event.loaded, + total: event.total + }; + } else { + Module.dataFileDownloads[url].loaded = event.loaded; + } + var total = 0; + var loaded = 0; + var num = 0; + for (var download in Module.dataFileDownloads) { + var data = Module.dataFileDownloads[download]; + total += data.total; + loaded += data.loaded; + num++; + } + total = Math.ceil(total * Module.expectedDataFileDownloads/num); + Module.setStatus('Downloading data... (' + loaded + '/' + total + ')'); + } else if (!Module.dataFileDownloads) { Module.setStatus('Downloading data...'); } } dataFile.open('GET', '%s', true); dataFile.responseType = 'arraybuffer'; dataFile.onload = function() { + Module.finishedDataFileDownloads++; var arrayBuffer = dataFile.response; assert(arrayBuffer, 'Loading data file failed.'); var byteArray = new Uint8Array(arrayBuffer); var curr; %s }; - Module['addRunDependency']('datafile'); + Module['addRunDependency']('datafile_%s'); dataFile.send(null); if (Module['setStatus']) Module['setStatus']('Downloading...'); - ''' % (os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target), use_data) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file + ''' % (data_target, os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target), use_data, data_target) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file if pre_run: print ''' @@ -349,3 +392,7 @@ if crunch: }); ''' +print ''' +})(); +''' + diff --git a/tools/scons/site_scons/site_tools/emscripten/emscripten.py b/tools/scons/site_scons/site_tools/emscripten/emscripten.py index 7ccf2c5f..ce704bf8 100644 --- a/tools/scons/site_scons/site_tools/emscripten/emscripten.py +++ b/tools/scons/site_scons/site_tools/emscripten/emscripten.py @@ -21,6 +21,8 @@ def generate(env, emscripten_path=None, **kw): emscripten_path = EMSCRIPTEN_ROOT + env['EMSCRIPTEN_ROOT'] = emscripten_path + try: emscPath = emscripten_path.abspath except: diff --git a/tools/settings_template_readonly.py b/tools/settings_template_readonly.py new file mode 100644 index 00000000..45a30569 --- /dev/null +++ b/tools/settings_template_readonly.py @@ -0,0 +1,47 @@ +# This file will be copied to ~/.emscripten if that file doesn't exist. Or, this is that copy. +# IMPORTANT: Edit the *copy* with the right paths! +# Note: If you put paths relative to the home directory, do not forget os.path.expanduser + +import os + +# this helps projects using emscripten find it +EMSCRIPTEN_ROOT = os.path.expanduser(os.getenv('EMSCRIPTEN') or '/opt/emscripten') +LLVM_ROOT = os.path.expanduser(os.getenv('LLVM') or '/usr/bin') + +# See below for notes on which JS engine(s) you need +NODE_JS = 'node' +SPIDERMONKEY_ENGINE = [ + os.path.expanduser(os.getenv('SPIDERMONKEY') or 'js'), '-m', '-n'] +V8_ENGINE = os.path.expanduser(os.getenv('V8') or 'd8') + +JAVA = 'java' + +TEMP_DIR = '/tmp' # You will need to modify this on Windows + +#CLOSURE_COMPILER = '..' # define this to not use the bundled version + +######################################################################################################## + + +# Pick the JS engine to use for running the compiler. This engine must exist, or +# nothing can be compiled. +# +# Recommendation: If you already have node installed, use that. Otherwise, build v8 or +# spidermonkey from source. Any of these three is fine, as long as it's +# a recent version (especially for v8 and spidermonkey). + +COMPILER_ENGINE = NODE_JS +#COMPILER_ENGINE = V8_ENGINE +#COMPILER_ENGINE = SPIDERMONKEY_ENGINE + + +# All JS engines to use when running the automatic tests. Not all the engines in this list +# must exist (if they don't, they will be skipped in the test runner). +# +# Recommendation: If you already have node installed, use that. If you can, also build +# spidermonkey from source as well to get more test coverage (node can't +# run all the tests due to node issue 1669). v8 is currently not recommended +# here because of v8 issue 1822. + +JS_ENGINES = [NODE_JS, SPIDERMONKEY_ENGINE] + diff --git a/tools/shared.py b/tools/shared.py index ed019999..700849e8 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -20,7 +20,7 @@ if '\n' in EM_CONFIG: else: CONFIG_FILE = os.path.expanduser(EM_CONFIG) if not os.path.exists(CONFIG_FILE): - shutil.copy(path_from_root('settings.py'), CONFIG_FILE) + shutil.copy(path_from_root('tools', 'settings_template_readonly.py'), CONFIG_FILE) print >> sys.stderr, ''' ============================================================================== Welcome to Emscripten! @@ -103,7 +103,7 @@ def check_sanity(force=False): try: subprocess.call([JAVA, '-version'], stdout=PIPE, stderr=PIPE) except: - print >> sys.stderr, 'WARNING: java does not seem to exist, required for closure compiler. -O2 and above will fail. You need to define JAVA in ~/.emscripten (see settings.py)' + print >> sys.stderr, 'WARNING: java does not seem to exist, required for closure compiler. -O2 and above will fail. You need to define JAVA in ~/.emscripten' if not os.path.exists(CLOSURE_COMPILER): print >> sys.stderr, 'WARNING: Closure compiler (%s) does not exist, check the paths in %s. -O2 and above will fail' % (CLOSURE_COMPILER, EM_CONFIG) @@ -235,6 +235,7 @@ if USE_EMSDK: '-Xclang', '-isystem' + path_from_root('system', 'include', 'bsd'), # posix stuff '-Xclang', '-isystem' + path_from_root('system', 'include', 'libc'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'libcxx'), + '-Xclang', '-isystem' + path_from_root('system', 'lib', 'libcxxabi', 'include'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'gfx'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'net'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'SDL'), @@ -512,6 +513,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e def make(args, stdout=None, stderr=None, env=None): if env is None: env = Building.get_building_env() + #args += ['VERBOSE=1'] Popen(args, stdout=stdout, stderr=stderr, env=env).communicate() @staticmethod @@ -549,9 +551,17 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |link| call) Building.configure(configure + configure_args, stdout=open(os.path.join(project_dir, 'configure_'), 'w'), stderr=open(os.path.join(project_dir, 'configure_err'), 'w'), env=env) - for i in range(2): # workaround for some build systems that need to be run twice to succeed (e.g. poppler) - Building.make(make + make_args, stdout=open(os.path.join(project_dir, 'make_' + str(i)), 'w'), - stderr=open(os.path.join(project_dir, 'make_err' + str(i)), 'w'), env=env) + def open_make_out(i, mode='r'): + return open(os.path.join(project_dir, 'make_' + str(i)), mode) + + def open_make_err(i, mode='r'): + return open(os.path.join(project_dir, 'make_err' + str(i)), mode) + + for i in range(2): # FIXME: Sad workaround for some build systems that need to be run twice to succeed (e.g. poppler) + with open_make_out(i, 'w') as make_out: + with open_make_err(i, 'w') as make_err: + Building.make(make + make_args, stdout=make_out, + stderr=make_err, env=env) try: if cache is not None: cache[cache_name] = [] @@ -560,7 +570,12 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e cache[cache_name].append((basename, open(f, 'rb').read())) break except: - if i > 0: raise Exception('could not build library ' + name) + if i > 0: + # Due to the ugly hack above our best guess is to output the first run + with open_make_err(0) as ferr: + for line in ferr: + sys.stderr.write(line) + raise Exception('could not build library ' + name) if old_dir: os.chdir(old_dir) return generated_libs |