diff options
50 files changed, 11049 insertions, 5080 deletions
@@ -10,3 +10,4 @@ src/relooper.js.raw.js src/relooper/*.o src/relooper/*.out +tests/fake/
\ No newline at end of file @@ -421,6 +421,22 @@ Options that are modified or new in %s include: 'jsfuncs' will be cached. So avoid modifying globals to let caching work fully. + To work around the problem mentioned in the + previous paragraph, you can use + + emscripten_jcache_printf + + when adding debug printfs to your code. That + function is specially preprocessed so that it + does not create a constant string global for + its first argument. See emscripten.h for more + details. Note in particular that you need to + already have a call to that function in your + code *before* you add one and do an incremental + build, so that adding an external reference + (also a global property) does not invalidate + everything. + --clear-cache Manually clears the cache of compiled emscripten system libraries (libc++, libc++abi, libc). This is normally @@ -978,6 +994,7 @@ try: for input_file in input_files: if input_file.endswith(SOURCE_SUFFIXES): if DEBUG: print >> sys.stderr, 'emcc: compiling source file: ', input_file + input_file = shared.Building.preprocess(input_file, in_temp(uniquename(input_file))) output_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') temp_files.append(output_file) args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file] @@ -1268,9 +1285,6 @@ try: execute(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)]) if DEBUG: save_intermediate('transformed') - if shared.Settings.ASM_JS: # XXX temporary wrapping for testing purposes - print >> sys.stderr, 'emcc: ASM_JS mode is highly experimental, and will not work on most codebases yet. It is NOT recommended that you try this yet.' - # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing js_optimizer_queue = [] def flush_js_optimizer_queue(): diff --git a/emscripten.py b/emscripten.py index 8bd1b58a..0b9244c2 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,21 +9,9 @@ header files (so that the JS compiler can see the constants in those headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, multiprocessing +import os, sys, json, optparse, subprocess, re, time, multiprocessing, functools -if not os.environ.get('EMSCRIPTEN_SUPPRESS_USAGE_WARNING'): - print >> sys.stderr, ''' -============================================================== -WARNING: You should normally never use this! Use emcc instead. -============================================================== - ''' - -from tools import shared - -DEBUG = os.environ.get('EMCC_DEBUG') -if DEBUG == "0": - DEBUG = None -DEBUG_CACHE = DEBUG and "cache" in DEBUG +from tools import jsrun, cache as cache_module, tempfiles __rootpath__ = os.path.abspath(os.path.dirname(__file__)) def path_from_root(*pathelems): @@ -32,11 +20,6 @@ def path_from_root(*pathelems): """ return os.path.join(__rootpath__, *pathelems) -temp_files = shared.TempFiles() - -compiler_engine = None -jcache = False - def scan(ll, settings): # blockaddress(@main, %23) blockaddrs = [] @@ -50,16 +33,20 @@ NUM_CHUNKS_PER_CORE = 1.25 MIN_CHUNK_SIZE = 1024*1024 MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes -def process_funcs(args): - i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args +def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files)): ll = ''.join(funcs) + '\n' + meta funcs_file = temp_files.get('.func_%d.ll' % i).name open(funcs_file, 'w').write(ll) - out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) - shared.try_delete(funcs_file) + out = jsrun.run_js( + compiler, + engine=compiler_engine, + args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, + stdout=subprocess.PIPE) + tempfiles.try_delete(funcs_file) return out -def emscript(infile, settings, outfile, libraries=[]): +def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, + jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: @@ -78,7 +65,7 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: print >> sys.stderr, 'emscript: ll=>js' - if jcache: shared.JCache.ensure() + if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() @@ -147,13 +134,13 @@ def emscript(infile, settings, outfile, libraries=[]): out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] - shortkey = shared.JCache.get_shortkey(keys) + shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey - out = shared.JCache.get(shortkey, keys) + out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: - dfpath = os.path.join(shared.TEMP_DIR, "ems_" + shortkey) + dfpath = os.path.join(configuration.TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input); dfp.write("\n\n========================== settings_text\n\n"); @@ -166,10 +153,10 @@ def emscript(infile, settings, outfile, libraries=[]): if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) - out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) + out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE) if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' - shared.JCache.set(shortkey, keys, out) + jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) @@ -194,15 +181,17 @@ def emscript(infile, settings, outfile, libraries=[]): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() - chunks = shared.JCache.chunkify(funcs, chunk_size, 'emscript_files' if jcache else None) + chunks = cache_module.chunkify( + funcs, chunk_size, + jcache.get_cachename('emscript_files') if jcache else None) if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] - shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code - out = shared.JCache.get(shortkey, keys) # this is relatively expensive (pickling?) + shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code + out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False @@ -215,12 +204,16 @@ def emscript(infile, settings, outfile, libraries=[]): # TODO: minimize size of forwarded data from funcs to what we actually need - if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' + if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: + assert len(chunks) == 1, 'no point in splitting up without multiple cores' if len(chunks) > 0: if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) - commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))] + commands = [ + (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files) + for i, chunk in enumerate(chunks) + ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) @@ -235,15 +228,15 @@ def emscript(infile, settings, outfile, libraries=[]): for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] - shortkey = shared.JCache.get_shortkey(keys) - shared.JCache.set(shortkey, keys, outputs[i]) + shortkey = jcache.get_shortkey(keys) + jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: - assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[1] + assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() @@ -311,7 +304,7 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data - out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) + out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) @@ -495,8 +488,7 @@ Runtime.stackRestore = function(top) { asm.stackRestore(top) }; outfile.close() - -def main(args): +def main(args, compiler_engine, cache, jcache, relooper, temp_files, DEBUG, DEBUG_CACHE): # Prepare settings for serialization to JSON. settings = {} for setting in args.settings: @@ -570,16 +562,23 @@ def main(args): libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else [] # Compile the assembly to Javascript. - if settings.get('RELOOP'): shared.Building.ensure_relooper() - - emscript(args.infile, settings, args.outfile, libraries) - -if __name__ == '__main__': + if settings.get('RELOOP'): + if not relooper: + relooper = cache.get_path('relooper.js') + settings.setdefault('RELOOPER', relooper) + if not os.path.exists(relooper): + from tools import shared + shared.Building.ensure_relooper(relooper) + + emscript(args.infile, settings, args.outfile, libraries, compiler_engine=compiler_engine, + jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE) + +def _main(environ): parser = optparse.OptionParser( - usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile', - description=('You should normally never use this! Use emcc instead. ' - 'This is a wrapper around the JS compiler, converting .ll to .js.'), - epilog='') + usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile', + description=('You should normally never use this! Use emcc instead. ' + 'This is a wrapper around the JS compiler, converting .ll to .js.'), + epilog='') parser.add_option('-H', '--headers', default=[], action='append', @@ -592,8 +591,11 @@ if __name__ == '__main__': default=sys.stdout, help='Where to write the output; defaults to stdout.') parser.add_option('-c', '--compiler', - default=shared.COMPILER_ENGINE, + default=None, help='Which JS engine to use to run the compiler; defaults to the one in ~/.emscripten.') + parser.add_option('--relooper', + default=None, + help='Which relooper file to use if RELOOP is enabled.') parser.add_option('-s', '--setting', dest='settings', default=[], @@ -605,16 +607,82 @@ if __name__ == '__main__': action='store_true', default=False, help=('Enable jcache (ccache-like caching of compilation results, for faster incremental builds).')) + parser.add_option('-T', '--temp-dir', + default=None, + help=('Where to create temporary files.')) + parser.add_option('-v', '--verbose', + action='store_true', + dest='verbose', + help='Displays debug output') + parser.add_option('-q', '--quiet', + action='store_false', + dest='verbose', + help='Hides debug output') + parser.add_option('--suppressUsageWarning', + action='store_true', + default=environ.get('EMSCRIPTEN_SUPPRESS_USAGE_WARNING'), + help=('Suppress usage warning')) # Convert to the same format that argparse would have produced. keywords, positional = parser.parse_args() + + if not keywords.suppressUsageWarning: + print >> sys.stderr, ''' +============================================================== +WARNING: You should normally never use this! Use emcc instead. +============================================================== + ''' + if len(positional) != 1: raise RuntimeError('Must provide exactly one positional argument.') keywords.infile = os.path.abspath(positional[0]) if isinstance(keywords.outfile, basestring): keywords.outfile = open(keywords.outfile, 'w') - compiler_engine = keywords.compiler - jcache = keywords.jcache - temp_files.run_and_clean(lambda: main(keywords)) + if keywords.relooper: + relooper = os.path.abspath(keywords.relooper) + else: + relooper = None # use the cache + + def get_configuration(): + if hasattr(get_configuration, 'configuration'): + return get_configuration.configuration + + from tools import shared + configuration = shared.Configuration(environ=os.environ) + get_configuration.configuration = configuration + return configuration + + if keywords.temp_dir is None: + temp_files = get_configuration().get_temp_files() + else: + temp_dir = os.path.abspath(keywords.temp_dir) + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + temp_files = tempfiles.TempFiles(temp_dir) + + if keywords.compiler is None: + from tools import shared + keywords.compiler = shared.COMPILER_ENGINE + + if keywords.verbose is None: + DEBUG = get_configuration().DEBUG + DEBUG_CACHE = get_configuration().DEBUG_CACHE + else: + DEBUG = keywords.verbose + DEBUG_CACHE = keywords.verbose + + cache = cache_module.Cache() + temp_files.run_and_clean(lambda: main( + keywords, + compiler_engine=keywords.compiler, + cache=cache, + jcache=cache_module.JCache(cache) if keywords.jcache else None, + relooper=relooper, + temp_files=temp_files, + DEBUG=DEBUG, + DEBUG_CACHE=DEBUG_CACHE, + )) +if __name__ == '__main__': + _main(environ=os.environ) diff --git a/src/analyzer.js b/src/analyzer.js index f9b0c5af..ecb5ea6b 100644 --- a/src/analyzer.js +++ b/src/analyzer.js @@ -122,7 +122,8 @@ function analyzer(data, sidePass) { // Legalization if (USE_TYPED_ARRAYS == 2) { function getLegalVars(base, bits, allowLegal) { - if (allowLegal && bits <= 32) return [{ ident: base, bits: bits }]; + bits = bits || 32; // things like pointers are all i32, but show up as 0 bits from getBits + if (allowLegal && bits <= 32) return [{ ident: base + ('i' + bits in Runtime.INT_TYPES ? '' : '$0'), bits: bits }]; if (isNumber(base)) return getLegalLiterals(base, bits); var ret = new Array(Math.ceil(bits/32)); var i = 0; @@ -647,13 +648,7 @@ function analyzer(data, sidePass) { default: throw 'Invalid mathop for legalization: ' + [value.op, item.lineNum, dump(item)]; } // Do the legalization - var sourceElements; - if (sourceBits <= 32) { - // The input is a legal type - sourceElements = [{ ident: value.params[0].ident, bits: sourceBits }]; - } else { - sourceElements = getLegalVars(value.params[0].ident, sourceBits); - } + var sourceElements = getLegalVars(value.params[0].ident, sourceBits, true); if (!isNumber(shifts)) { // We can't statically legalize this, do the operation at runtime TODO: optimize assert(sourceBits == 64, 'TODO: handle nonconstant shifts on != 64 bits'); diff --git a/src/compiler.js b/src/compiler.js index 14816f1e..3047daf1 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -172,13 +172,14 @@ assert(!(USE_TYPED_ARRAYS === 2 && QUANTUM_SIZE !== 4), 'For USE_TYPED_ARRAYS == if (ASM_JS) { assert(!ALLOW_MEMORY_GROWTH, 'Cannot grow asm.js heap'); assert((TOTAL_MEMORY&(TOTAL_MEMORY-1)) == 0, 'asm.js heap must be power of 2'); + assert(DISABLE_EXCEPTION_CATCHING == 1, 'asm.js does not support C++ exceptions yet'); } assert(!(!NAMED_GLOBALS && BUILD_AS_SHARED_LIB)); // shared libraries must have named globals // Output some info and warnings based on settings if (phase == 'pre') { - if (!MICRO_OPTS || !RELOOP || ASSERTIONS || CHECK_SIGNS || CHECK_OVERFLOWS || INIT_STACK || INIT_HEAP || + if (!MICRO_OPTS || !RELOOP || ASSERTIONS || CHECK_SIGNS || CHECK_OVERFLOWS || INIT_HEAP || !SKIP_STACK_IN_SMALL || SAFE_HEAP || !DISABLE_EXCEPTION_CATCHING) { print('// Note: Some Emscripten settings will significantly limit the speed of the generated code.'); } else { @@ -198,7 +199,7 @@ load('parseTools.js'); load('intertyper.js'); load('analyzer.js'); load('jsifier.js'); -if (RELOOP) load('relooper.js') +if (RELOOP) load(RELOOPER) globalEval(processMacros(preprocess(read('runtime.js')))); Runtime.QUANTUM_SIZE = QUANTUM_SIZE; diff --git a/src/intertyper.js b/src/intertyper.js index c1a98354..6c88e765 100644 --- a/src/intertyper.js +++ b/src/intertyper.js @@ -741,10 +741,12 @@ function intertyper(data, sidePass, baseLineNums) { processItem: function(item) { item.intertype = 'atomic'; if (item.tokens[0].text == 'atomicrmw') { + if (item.tokens[1].text == 'volatile') item.tokens.splice(1, 1); item.op = item.tokens[1].text; item.tokens.splice(1, 1); } else { assert(item.tokens[0].text == 'cmpxchg') + if (item.tokens[1].text == 'volatile') item.tokens.splice(1, 1); item.op = 'cmpxchg'; } var last = getTokenIndexByText(item.tokens, ';'); diff --git a/src/jsifier.js b/src/jsifier.js index 7066f8c5..ff58ece2 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -1204,10 +1204,13 @@ function JSify(data, functionsOnly, givenFunctions) { switch (item.op) { case 'add': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue+' + param2, type, null, null, null, null, ',') + ',tempValue)'; case 'sub': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue-' + param2, type, null, null, null, null, ',') + ',tempValue)'; + case 'or': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue|' + param2, type, null, null, null, null, ',') + ',tempValue)'; + case 'and': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue&' + param2, type, null, null, null, null, ',') + ',tempValue)'; + case 'xor': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue^' + param2, type, null, null, null, null, ',') + ',tempValue)'; case 'xchg': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, param2, type, null, null, null, null, ',') + ',tempValue)'; case 'cmpxchg': { var param3 = finalizeLLVMParameter(item.params[2]); - return '(tempValue=' + makeGetValue(param1, 0, type) + ',(' + makeGetValue(param1, 0, type) + '==' + param2 + ' ? ' + makeSetValue(param1, 0, param3, type, null, null, null, null, ',') + ' : 0),tempValue)'; + return '(tempValue=' + makeGetValue(param1, 0, type) + ',(' + makeGetValue(param1, 0, type) + '==(' + param2 + '|0) ? ' + makeSetValue(param1, 0, param3, type, null, null, null, null, ',') + ' : 0),tempValue)'; } default: throw 'unhandled atomic op: ' + item.op; } diff --git a/src/library.js b/src/library.js index 1676a82c..a288b739 100644 --- a/src/library.js +++ b/src/library.js @@ -2403,6 +2403,7 @@ LibraryManager.library = { case {{{ cDefine('_SC_STREAM_MAX') }}}: return 16; case {{{ cDefine('_SC_TZNAME_MAX') }}}: return 6; case {{{ cDefine('_SC_THREAD_DESTRUCTOR_ITERATIONS') }}}: return 4; + case {{{ cDefine('_SC_NPROCESSORS_ONLN') }}}: return 1; } ___setErrNo(ERRNO_CODES.EINVAL); return -1; @@ -6064,6 +6065,15 @@ LibraryManager.library = { __timespec_struct_layout: Runtime.generateStructInfo([ ['i32', 'tv_sec'], ['i32', 'tv_nsec']]), + nanosleep__deps: ['usleep', '__timespec_struct_layout'], + nanosleep: function(rqtp, rmtp) { + // int nanosleep(const struct timespec *rqtp, struct timespec *rmtp); + var seconds = {{{ makeGetValue('rqtp', '___timespec_struct_layout.tv_sec', 'i32') }}}; + var nanoseconds = {{{ makeGetValue('rqtp', '___timespec_struct_layout.tv_nsec', 'i32') }}}; + {{{ makeSetValue('rmtp', '___timespec_struct_layout.tv_sec', '0', 'i32') }}} + {{{ makeSetValue('rmtp', '___timespec_struct_layout.tv_nsec', '0', 'i32') }}} + return _usleep((seconds * 1e6) + (nanoseconds / 1000)); + }, // TODO: Implement these for real. clock_gettime__deps: ['__timespec_struct_layout'], clock_gettime: function(clk_id, tp) { @@ -7368,6 +7378,23 @@ LibraryManager.library = { emscripten_random: function() { return Math.random(); }, + + emscripten_jcache_printf___deps: ['_formatString'], + emscripten_jcache_printf_: function(varargs) { + var MAX = 10240; + if (!_emscripten_jcache_printf_.buffer) { + _emscripten_jcache_printf_.buffer = _malloc(MAX); + } + var i = 0; + do { + var curr = {{{ makeGetValue('varargs', 'i*4', 'i8') }}}; + {{{ makeSetValue('_emscripten_jcache_printf_.buffer', 'i', 'curr', 'i8') }}}; + i++; + assert(i*4 < MAX); + } while (curr != 0); + Module.print(intArrayToString(__formatString(_emscripten_jcache_printf_.buffer, varargs + i*4)).replace('\\n', '')); + Runtime.stackAlloc(-4*i); // free up the stack space we know is ok to free + }, }; |