diff options
Diffstat (limited to 'emscripten.py')
| -rwxr-xr-x | emscripten.py | 497 |
1 files changed, 365 insertions, 132 deletions
diff --git a/emscripten.py b/emscripten.py index 8fa1c3e5..248d0ce4 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,18 +9,10 @@ header files (so that the JS compiler can see the constants in those headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, multiprocessing +import os, sys, json, optparse, subprocess, re, time, multiprocessing, functools -if not os.environ.get('EMSCRIPTEN_SUPPRESS_USAGE_WARNING'): - print >> sys.stderr, ''' -============================================================== -WARNING: You should normally never use this! Use emcc instead. -============================================================== - ''' - -from tools import shared - -DEBUG = os.environ.get('EMCC_DEBUG') +from tools import jsrun, cache as cache_module, tempfiles +from tools.response_file import read_response_file __rootpath__ = os.path.abspath(os.path.dirname(__file__)) def path_from_root(*pathelems): @@ -29,10 +21,14 @@ def path_from_root(*pathelems): """ return os.path.join(__rootpath__, *pathelems) -temp_files = shared.TempFiles() +def get_configuration(): + if hasattr(get_configuration, 'configuration'): + return get_configuration.configuration -compiler_engine = None -jcache = False + from tools import shared + configuration = shared.Configuration(environ=os.environ) + get_configuration.configuration = configuration + return configuration def scan(ll, settings): # blockaddress(@main, %23) @@ -43,20 +39,30 @@ def scan(ll, settings): if len(blockaddrs) > 0: settings['NECESSARY_BLOCKADDRS'] = blockaddrs -NUM_CHUNKS_PER_CORE = 5 +NUM_CHUNKS_PER_CORE = 1.25 MIN_CHUNK_SIZE = 1024*1024 MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes -def process_funcs(args): - i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args - ll = ''.join(funcs) + '\n' + meta +def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): funcs_file = temp_files.get('.func_%d.ll' % i).name - open(funcs_file, 'w').write(ll) - out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) - shared.try_delete(funcs_file) + f = open(funcs_file, 'w') + f.write(funcs) + funcs = None + f.write('\n') + f.write(meta) + f.close() + out = jsrun.run_js( + compiler, + engine=compiler_engine, + args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, + stdout=subprocess.PIPE, + cwd=path_from_root('src')) + tempfiles.try_delete(funcs_file) + if DEBUG: print >> sys.stderr, '.' return out -def emscript(infile, settings, outfile, libraries=[]): +def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, + jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: @@ -75,7 +81,7 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: print >> sys.stderr, 'emscript: ll=>js' - if jcache: shared.JCache.ensure() + if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() @@ -88,25 +94,27 @@ def emscript(infile, settings, outfile, libraries=[]): # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later - func_idents = [] meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() + curr_func = None for line in ll_lines: if in_func: - funcs[-1][1].append(line) + curr_func.append(line) if line.startswith('}'): in_func = False - funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1])) - pre.append(line) # pre needs it to, so we know about all implemented functions + funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier + # pre needs to know about all implemented functions, even for non-pre func + pre.append(curr_func[0]) + pre.append(line) + curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True - funcs.append((line, [line])) # use the entire line as the identifier - pre.append(line) # pre needs it to, so we know about all implemented functions + curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): @@ -131,7 +139,7 @@ def emscript(infile, settings, outfile, libraries=[]): settings_file = temp_files.get('.txt').name def save_settings(): global settings_text - settings_text = json.dumps(settings) + settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() @@ -144,15 +152,31 @@ def emscript(infile, settings, outfile, libraries=[]): out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] - shortkey = shared.JCache.get_shortkey(keys) - out = shared.JCache.get(shortkey, keys) + shortkey = jcache.get_shortkey(keys) + if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey + + out = jcache.get(shortkey, keys) + + if DEBUG_CACHE and not out: + dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) + dfp = open(dfpath, 'w') + dfp.write(pre_input); + dfp.write("\n\n========================== settings_text\n\n"); + dfp.write(settings_text); + dfp.write("\n\n========================== libraries\n\n"); + dfp.write("\n".join(libraries)) + dfp.close() + print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath + if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) - out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) + out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, + cwd=path_from_root('src')) + assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' - shared.JCache.set(shortkey, keys, out) + jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) @@ -160,12 +184,12 @@ def emscript(infile, settings, outfile, libraries=[]): # Phase 2 - func - cores = multiprocessing.cpu_count() + cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: - intended_num_chunks = cores * NUM_CHUNKS_PER_CORE + intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) - chunk_size += 3*len(meta) # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) + chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size @@ -177,15 +201,19 @@ def emscript(infile, settings, outfile, libraries=[]): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() - chunks = shared.JCache.chunkify(funcs, chunk_size, 'emscript_files' if jcache else None) + chunks = cache_module.chunkify( + funcs, chunk_size, + jcache.get_cachename('emscript_files') if jcache else None) + + funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] - shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code - out = shared.JCache.get(shortkey, keys) # this is relatively expensive (pickling?) + shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code + out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False @@ -198,18 +226,25 @@ def emscript(infile, settings, outfile, libraries=[]): # TODO: minimize size of forwarded data from funcs to what we actually need - if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' - if len(chunks) > 0: + if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: + assert len(chunks) == 1, 'no point in splitting up without multiple cores' + if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) - commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))] + commands = [ + (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) + for i, chunk in enumerate(chunks) + ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] + + commands = None + else: outputs = [] @@ -218,13 +253,17 @@ def emscript(infile, settings, outfile, libraries=[]): for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] - shortkey = shared.JCache.get_shortkey(keys) - shared.JCache.set(shortkey, keys, outputs[i]) + shortkey = jcache.get_shortkey(keys) + jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) + chunks = None + if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] + for output in outputs: + assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() @@ -232,8 +271,8 @@ def emscript(infile, settings, outfile, libraries=[]): # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise - for additional_export in ['_malloc', '_free']: # additional functions to export from asm, if they are implemented - all_exported_functions.add(additional_export) + for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented + all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) @@ -243,12 +282,21 @@ def emscript(infile, settings, outfile, libraries=[]): for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): + export_bindings = settings['EXPORT_BINDINGS'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): - if key in all_exported_functions: exported_implemented_functions.add(key) + if key in all_exported_functions or (export_bindings and key.startswith('_emscripten_bind')): + exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value + for key, value in curr_forwarded_json['Functions']['neededTables'].iteritems(): + forwarded_json['Functions']['neededTables'][key] = value - funcs_js = ''.join([output[0] for output in outputs]) + if settings.get('ASM_JS'): + parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') + if len(parts) > 1: + pre = parts[0] + outputs.append([parts[1]]) + funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) @@ -256,20 +304,39 @@ def emscript(infile, settings, outfile, libraries=[]): # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} - i = 2 + i = 2 # universal counter + if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS'] + table_counters = {} # table-specific counters + alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] + sig = None for indexed in indexed_functions: - #print >> sys.stderr, 'indaxx', indexed, i - forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize - i += 2 - forwarded_json['Functions']['nextIndex'] = i + if alias: + sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed) + assert sig, indexed + if sig not in table_counters: + table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS'] + curr = table_counters[sig] + table_counters[sig] += 2 + else: + curr = i + i += 2 + #print >> sys.stderr, 'function indexing', indexed, curr, sig + forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize + + def split_32(x): + x = int(x) + return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): - return re.sub(r'{{{ FI_([\w\d_$]+) }}}', lambda m: str(indexing[m.groups(0)[0]]), js) + # In the global initial allocation, we need to split up into Uint8 format + ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) + return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): - return re.sub(r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) + ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) + return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) #if DEBUG: outfile.write('// pre\n') outfile.write(blockaddrsize(indexize(pre))) @@ -287,54 +354,117 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data - out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) - post, last_forwarded_data = out.split('//FORWARDED_DATA:') + out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, + cwd=path_from_root('src')) + post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): + post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') + post = post_rest + + # Move preAsms to their right place + def move_preasm(m): + contents = m.groups(0)[0] + outfile.write(contents + '\n') + return '' + post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) + + funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] + simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 + pre_tables = last_forwarded_json['Functions']['tables']['pre'] + del last_forwarded_json['Functions']['tables']['pre'] + def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) - coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] == 'd' else '', p, '' if sig[p+1] == 'd' else '|0') for p in range(len(sig)-1)]) + ';' - ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] == 'd' else '')) - return 'function %s(%s) { %s abort(%d); %s };\n' % (bad, params, coercions, i, ret) + raw.replace('[0,', '[' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']') - function_tables_defs = '\n'.join([make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()]) - - maths = ['Runtime.bitshift64', 'Math.floor', 'Math.min'] - if settings['USE_MATH_IMUL']: - maths += ['Math.imul'] - asm_setup = '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in maths]) - fundamentals = ['buffer', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] - basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in maths] + coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' + ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) + start = raw.index('[') + end = raw.rindex(']') + body = raw[start+1:end].split(',') + for j in range(settings['RESERVED_FUNCTION_POINTERS']): + body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j) + def fix_item(item): + newline = '\n' in item + return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') + body = ','.join(map(fix_item, body)) + return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:]) + infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] + function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) + + asm_setup = '' + maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] + fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] + math_envs = ['Math.min'] # TODO: move min to maths + asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) + + basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat'] + [m.replace('.', '_') for m in math_envs] + if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') + if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] + if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] + if settings['ASSERTIONS']: + basic_funcs += ['nullFunc'] + asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n' + basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] - if forwarded_json['Types']['preciseI64MathUsed']: - basic_funcs += ['i64Math_' + op for op in ['add', 'subtract', 'multiply', 'divide', 'modulo']] - asm_setup += ''' -var i64Math_add = function(a, b, c, d) { i64Math.add(a, b, c, d) }; -var i64Math_subtract = function(a, b, c, d) { i64Math.subtract(a, b, c, d) }; -var i64Math_multiply = function(a, b, c, d) { i64Math.multiply(a, b, c, d) }; -var i64Math_divide = function(a, b, c, d, e) { i64Math.divide(a, b, c, d, e) }; -var i64Math_modulo = function(a, b, c, d, e) { i64Math.modulo(a, b, c, d, e) }; -''' + basic_float_vars = ['NaN', 'Infinity'] + + if forwarded_json['Types']['preciseI64MathUsed'] or \ + forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ + forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): + basic_vars += ['cttz_i8', 'ctlz_i8'] + asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables + def asm_coerce(value, sig): + if sig == 'v': return value + return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') + function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) - arg_coercions = ' '.join(['a' + str(i) + '=' + ('+' if sig[i] == 'd' else '') + 'a' + str(i) + ('|0' if sig[i] == 'i' else '') + ';' for i in range(1, len(sig))]) + arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) + coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) + ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s - %sFUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s); + %s; } -''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, 'return ' if sig[0] != 'v' else '', sig, sig, args)) +''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) + + for i in range(settings['RESERVED_FUNCTION_POINTERS']): + jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) + function_tables_impls.append(''' + function jsCall_%s_%s(%s) { + %s + %s; + } + +''' % (sig, i, args, arg_coercions, jsret)) + args = ','.join(['a' + str(i) for i in range(1, len(sig))]) + args = 'index' + (',' if args else '') + args + # C++ exceptions are numbers, and longjmp is a string 'longjmp' + asm_setup += ''' +function invoke_%s(%s) { + try { + %sModule["dynCall_%s"](%s); + } catch(e) { + if (typeof e !== 'number' && e !== 'longjmp') throw e; + asm["setThrew"](1, 0); + } +} +''' % (sig, args, 'return ' if sig[0] != 'v' else '', sig, args) + basic_funcs.append('invoke_%s' % sig) + # calculate exports exported_implemented_functions = list(exported_implemented_functions) exports = [] @@ -351,41 +481,60 @@ var i64Math_modulo = function(a, b, c, d, e) { i64Math.modulo(a, b, c, d, e) }; pass # If no named globals, only need externals global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) - global_funcs = ['_' + x for x in forwarded_json['Functions']['libraryFunctions'].keys()] - asm_global_funcs = ''.join([' var ' + g + '=env.' + g + ';\n' for g in basic_funcs + global_funcs]) - asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] + def math_fix(g): + return g if not g.startswith('Math_') else g.split('_')[1]; + asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ + ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) + asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ + ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # sent data - sending = '{ ' + ', '.join([s + ': ' + s for s in fundamentals + basic_funcs + global_funcs + basic_vars + global_vars]) + ' }' + the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' + sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received if not simple: - receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables]) + receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) else: receiving = 'var _main = Module["_main"] = asm;' + # finalize - funcs_js = ''' + + if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) + + funcs_js = [''' %s -var asmPre = (function(env, buffer) { +function asmPrintInt(x, y) { + Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); +} +function asmPrintFloat(x, y) { + Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); +} +// EMSCRIPTEN_START_ASM +var asm = (function(global, env, buffer) { 'use asm'; - var HEAP8 = new env.Int8Array(buffer); - var HEAP16 = new env.Int16Array(buffer); - var HEAP32 = new env.Int32Array(buffer); - var HEAPU8 = new env.Uint8Array(buffer); - var HEAPU16 = new env.Uint16Array(buffer); - var HEAPU32 = new env.Uint32Array(buffer); - var HEAPF32 = new env.Float32Array(buffer); - var HEAPF64 = new env.Float64Array(buffer); + var HEAP8 = new global.Int8Array(buffer); + var HEAP16 = new global.Int16Array(buffer); + var HEAP32 = new global.Int32Array(buffer); + var HEAPU8 = new global.Uint8Array(buffer); + var HEAPU16 = new global.Uint16Array(buffer); + var HEAPU32 = new global.Uint32Array(buffer); + var HEAPF32 = new global.Float32Array(buffer); + var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup,) + '\n' + asm_global_vars + ''' var __THREW__ = 0; + var threwValue = 0; + var setjmpId = 0; var undef = 0; - var tempInt = 0; + var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' +// EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; - STACKTOP = ((STACKTOP + 3)>>2)<<2; +''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' return ret|0; } function stackSave() { @@ -395,32 +544,31 @@ var asmPre = (function(env, buffer) { top = top|0; STACKTOP = top; } - function setThrew(threw) { + function setThrew(threw, value) { threw = threw|0; - __THREW__ = threw; + value = value|0; + if ((__THREW__|0) == 0) { + __THREW__ = threw; + threwValue = value; + } } ''' + ''.join([''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } -''' % (i, i) for i in range(10)]) + funcs_js.replace('\n', '\n ') + ''' - +''' % (i, i) for i in range(10)])] + funcs_js + [''' %s return %s; -}); -if (asmPre.toSource) { // works in sm but not v8, so we get full coverage between those two - asmPre = asmPre.toSource(); - asmPre = asmPre.substr(25, asmPre.length-28); - asmPre = new Function('env', 'buffer', asmPre); -} -var asm = asmPre(%s, buffer); // pass through Function to prevent seeing outside scope +}) +// EMSCRIPTEN_END_ASM +(%s, %s, buffer); %s; -Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) }; -Runtime.stackSave = function() { return asm.stackSave() }; -Runtime.stackRestore = function(top) { asm.stackRestore(top) }; -''' % (function_tables_defs.replace('\n', '\n ') + '\n' + '\n'.join(function_tables_impls), exports, sending, receiving) +Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; +Runtime.stackSave = function() { return asm['stackSave']() }; +Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; +''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] # Set function table masks def function_table_maskize(js): @@ -431,16 +579,22 @@ Runtime.stackRestore = function(top) { asm.stackRestore(top) }; default = sig def fix(m): sig = m.groups(0)[0] - if not sig in masks: - print >> sys.stderr, 'warning: function table use without functions for it!', sig - return masks[default] # TODO: generate empty function tables for this case, even though it would fail at runtime if used return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] - funcs_js = function_table_maskize(funcs_js) + funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) - outfile.write(blockaddrsize(indexize(funcs_js))) + funcs_js = [''' +// EMSCRIPTEN_START_FUNCS +'''] + funcs_js + [''' +// EMSCRIPTEN_END_FUNCS +'''] + + for funcs_js_item in funcs_js: # do this loop carefully to save memory + funcs_js_item = indexize(funcs_js_item) + funcs_js_item = blockaddrsize(funcs_js_item) + outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) @@ -448,8 +602,7 @@ Runtime.stackRestore = function(top) { asm.stackRestore(top) }; outfile.close() - -def main(args): +def main(args, compiler_engine, cache, jcache, relooper, temp_files, DEBUG, DEBUG_CACHE): # Prepare settings for serialization to JSON. settings = {} for setting in args.settings: @@ -523,16 +676,35 @@ def main(args): libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else [] # Compile the assembly to Javascript. - if settings.get('RELOOP'): shared.Building.ensure_relooper() - - emscript(args.infile, settings, args.outfile, libraries) + if settings.get('RELOOP'): + if not relooper: + relooper = cache.get_path('relooper.js') + settings.setdefault('RELOOPER', relooper) + if not os.path.exists(relooper): + from tools import shared + shared.Building.ensure_relooper(relooper) + + emscript(args.infile, settings, args.outfile, libraries, compiler_engine=compiler_engine, + jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE) + +def _main(environ): + response_file = True + while response_file: + response_file = None + for index in range(1, len(sys.argv)): + if sys.argv[index][0] == '@': + # found one, loop again next time + response_file = True + response_file_args = read_response_file(sys.argv[index]) + # slice in extra_args in place of the response file arg + sys.argv[index:index+1] = response_file_args + break -if __name__ == '__main__': parser = optparse.OptionParser( - usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile', - description=('You should normally never use this! Use emcc instead. ' - 'This is a wrapper around the JS compiler, converting .ll to .js.'), - epilog='') + usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile', + description=('You should normally never use this! Use emcc instead. ' + 'This is a wrapper around the JS compiler, converting .ll to .js.'), + epilog='') parser.add_option('-H', '--headers', default=[], action='append', @@ -545,8 +717,11 @@ if __name__ == '__main__': default=sys.stdout, help='Where to write the output; defaults to stdout.') parser.add_option('-c', '--compiler', - default=shared.COMPILER_ENGINE, + default=None, help='Which JS engine to use to run the compiler; defaults to the one in ~/.emscripten.') + parser.add_option('--relooper', + default=None, + help='Which relooper file to use if RELOOP is enabled.') parser.add_option('-s', '--setting', dest='settings', default=[], @@ -558,16 +733,74 @@ if __name__ == '__main__': action='store_true', default=False, help=('Enable jcache (ccache-like caching of compilation results, for faster incremental builds).')) + parser.add_option('-T', '--temp-dir', + default=None, + help=('Where to create temporary files.')) + parser.add_option('-v', '--verbose', + action='store_true', + dest='verbose', + help='Displays debug output') + parser.add_option('-q', '--quiet', + action='store_false', + dest='verbose', + help='Hides debug output') + parser.add_option('--suppressUsageWarning', + action='store_true', + default=environ.get('EMSCRIPTEN_SUPPRESS_USAGE_WARNING'), + help=('Suppress usage warning')) # Convert to the same format that argparse would have produced. keywords, positional = parser.parse_args() + + if not keywords.suppressUsageWarning: + print >> sys.stderr, ''' +============================================================== +WARNING: You should normally never use this! Use emcc instead. +============================================================== + ''' + if len(positional) != 1: raise RuntimeError('Must provide exactly one positional argument.') keywords.infile = os.path.abspath(positional[0]) if isinstance(keywords.outfile, basestring): keywords.outfile = open(keywords.outfile, 'w') - compiler_engine = keywords.compiler - jcache = keywords.jcache - temp_files.run_and_clean(lambda: main(keywords)) + if keywords.relooper: + relooper = os.path.abspath(keywords.relooper) + else: + relooper = None # use the cache + if keywords.temp_dir is None: + temp_files = get_configuration().get_temp_files() + temp_dir = get_configuration().TEMP_DIR + else: + temp_dir = os.path.abspath(keywords.temp_dir) + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + temp_files = tempfiles.TempFiles(temp_dir) + + if keywords.compiler is None: + from tools import shared + keywords.compiler = shared.COMPILER_ENGINE + + if keywords.verbose is None: + DEBUG = get_configuration().DEBUG + DEBUG_CACHE = get_configuration().DEBUG_CACHE + else: + DEBUG = keywords.verbose + DEBUG_CACHE = keywords.verbose + + cache = cache_module.Cache() + temp_files.run_and_clean(lambda: main( + keywords, + compiler_engine=keywords.compiler, + cache=cache, + jcache=cache_module.JCache(cache) if keywords.jcache else None, + relooper=relooper, + temp_files=temp_files, + DEBUG=DEBUG, + DEBUG_CACHE=DEBUG_CACHE, + )) + +if __name__ == '__main__': + _main(environ=os.environ) |