diff options
Diffstat (limited to 'emscripten.py')
| -rwxr-xr-x | emscripten.py | 1029 |
1 files changed, 794 insertions, 235 deletions
diff --git a/emscripten.py b/emscripten.py index 248d0ce4..c96c56a0 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,8 +9,9 @@ header files (so that the JS compiler can see the constants in those headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, multiprocessing, functools +import os, sys, json, optparse, subprocess, re, time, multiprocessing, string, logging, shutil +from tools import shared from tools import jsrun, cache as cache_module, tempfiles from tools.response_file import read_response_file @@ -25,7 +26,6 @@ def get_configuration(): if hasattr(get_configuration, 'configuration'): return get_configuration.configuration - from tools import shared configuration = shared.Configuration(environ=os.environ) get_configuration.configuration = configuration return configuration @@ -39,26 +39,31 @@ def scan(ll, settings): if len(blockaddrs) > 0: settings['NECESSARY_BLOCKADDRS'] = blockaddrs -NUM_CHUNKS_PER_CORE = 1.25 +NUM_CHUNKS_PER_CORE = 1.0 MIN_CHUNK_SIZE = 1024*1024 MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes -def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): - funcs_file = temp_files.get('.func_%d.ll' % i).name - f = open(funcs_file, 'w') - f.write(funcs) - funcs = None - f.write('\n') - f.write(meta) - f.close() - out = jsrun.run_js( - compiler, - engine=compiler_engine, - args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, - stdout=subprocess.PIPE, - cwd=path_from_root('src')) - tempfiles.try_delete(funcs_file) - if DEBUG: print >> sys.stderr, '.' +STDERR_FILE = os.environ.get('EMCC_STDERR_FILE') +if STDERR_FILE: + STDERR_FILE = os.path.abspath(STDERR_FILE) + logging.info('logging stderr in js compiler phase into %s' % STDERR_FILE) + STDERR_FILE = open(STDERR_FILE, 'w') + +def process_funcs((i, funcs_file, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, DEBUG)): + try: + #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments, + # # just copy temp dir to under this one + out = jsrun.run_js( + compiler, + engine=compiler_engine, + args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, + stdout=subprocess.PIPE, + stderr=STDERR_FILE, + cwd=path_from_root('src')) + except KeyboardInterrupt: + # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt + raise Exception() + if DEBUG: logging.debug('.') return out def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, @@ -79,7 +84,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. - if DEBUG: print >> sys.stderr, 'emscript: ll=>js' + if DEBUG: logging.debug('emscript: ll=>js') if jcache: jcache.ensure() @@ -88,52 +93,55 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) - ll = None # allow collection - if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: scan took %s seconds' % (time.time() - t)) # Split input into the relevant parts for each phase + + if DEBUG: t = time.time() + pre = [] funcs = [] # split up functions here, for parallelism later - meta = [] # needed by each function XXX - if DEBUG: t = time.time() - in_func = False - ll_lines = open(infile).readlines() - curr_func = None - for line in ll_lines: - if in_func: - curr_func.append(line) - if line.startswith('}'): - in_func = False - funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier - # pre needs to know about all implemented functions, even for non-pre func - pre.append(curr_func[0]) - pre.append(line) - curr_func = None - else: - if line.startswith(';'): continue - if line.startswith('define '): - in_func = True - curr_func = [line] - elif line.find(' = type { ') > 0: - pre.append(line) # type - elif line.startswith('!'): - if line.startswith('!llvm.module'): continue # we can ignore that - meta.append(line) # metadata - else: - pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there - ll_lines = None - meta = ''.join(meta) - if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' - if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) + meta_start = ll.find('\n!') + if meta_start > 0: + meta = ll[meta_start:] + else: + meta = '' + meta_start = -1 + + start = ll.find('\n') if ll[0] == ';' else 0 # ignore first line, which contains ; ModuleID = '/dir name' + + func_start = start + last = func_start + while 1: + last = func_start + func_start = ll.find('\ndefine ', func_start) + if func_start > last: + pre.append(ll[last:min(func_start+1, meta_start) if meta_start > 0 else func_start+1] + '\n') + if func_start < 0: + pre.append(ll[last:meta_start] + '\n') + break + header = ll[func_start+1:ll.find('\n', func_start+1)+1] + end = ll.find('\n}', func_start) + last = end+3 + funcs.append((header, ll[func_start+1:last])) + pre.append(header + '}\n') + func_start = last + ll = None + + if DEBUG and len(meta) > 1024*1024: logging.debug('emscript warning: large amounts of metadata, will slow things down') + if DEBUG: logging.debug(' emscript: split took %s seconds' % (time.time() - t)) + + if len(funcs) == 0: + logging.error('No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)') #if DEBUG: - # print >> sys.stderr, '========= pre ================\n' - # print >> sys.stderr, ''.join(pre) - # print >> sys.stderr, '========== funcs ===============\n' + # logging.debug('========= pre ================\n') + # logging.debug(''.join(pre)) + # logging.debug('========== funcs ===============\n') # for func in funcs: - # print >> sys.stderr, '\n// ===\n\n', ''.join(func) - # print >> sys.stderr, '=========================\n' + # logging.debug('\n// ===\n\n', ''.join(func)) + # logging.debug('=========================\n') # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name @@ -153,34 +161,41 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) - if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey + if DEBUG_CACHE: logging.debug('shortkey', shortkey) out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') - dfp.write(pre_input); - dfp.write("\n\n========================== settings_text\n\n"); - dfp.write(settings_text); - dfp.write("\n\n========================== libraries\n\n"); + dfp.write(pre_input) + dfp.write("\n\n========================== settings_text\n\n") + dfp.write(settings_text) + dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() - print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath + logging.debug(' cache miss, key data dumped to %s' % dfpath) - if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' + if out and DEBUG: logging.debug(' loading pre from jcache') if not out: open(pre_file, 'w').write(pre_input) - out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, + #print >> sys.stderr, 'running', str([settings_file, pre_file, 'pre'] + libraries).replace("'/", "'") # see funcs + out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: - if DEBUG: print >> sys.stderr, ' saving pre to jcache' + if DEBUG: logging.debug(' saving pre to jcache') jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name + pre_input = None open(forwarded_file, 'w').write(forwarded_data) - if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 1 took %s seconds' % (time.time() - t)) + + indexed_functions = set() + forwarded_json = json.loads(forwarded_data) + for key in forwarded_json['Functions']['indexedFunctions'].iterkeys(): + indexed_functions.add(key) # Phase 2 - func @@ -189,14 +204,12 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) - chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) + chunk_size += 3*len(meta) # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() - forwarded_json = json.loads(forwarded_data) - indexed_functions = set() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() @@ -205,6 +218,9 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) + #sys.exit(1) + #chunks = [chunks[0]] # pick specific chunks for debugging/profiling + funcs = None if jcache: @@ -220,7 +236,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: - if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) + if out and DEBUG: logging.debug(' loading %d funcchunks from jcache' % len(cached_outputs)) else: cached_outputs = [] @@ -230,12 +246,22 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' - if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) + if DEBUG: logging.debug(' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))) - commands = [ - (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) - for i, chunk in enumerate(chunks) - ] + commands = [] + for i in range(len(chunks)): + funcs_file = temp_files.get('.func_%d.ll' % i).name + f = open(funcs_file, 'w') + f.write(chunks[i]) + if not jcache: + chunks[i] = None # leave chunks array alive (need its length later) + f.write('\n') + f.write(meta) + f.close() + commands.append( + (i, funcs_file, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine,# + ['--prof'], + DEBUG) + ) if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) @@ -255,7 +281,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) - if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) + if out and DEBUG and len(chunks) > 0: logging.debug(' saving %d funcchunks to jcache' % len(chunks)) chunks = None @@ -265,7 +291,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] - if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2 took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # merge forwarded data @@ -276,6 +302,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) + forwarded_json['Types']['hasInlineJS'] = forwarded_json['Types']['hasInlineJS'] or curr_forwarded_json['Types']['hasInlineJS'] + forwarded_json['Types']['usesSIMD'] = forwarded_json['Types']['usesSIMD'] or curr_forwarded_json['Types']['usesSIMD'] forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value @@ -283,8 +311,9 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] + export_all = settings['EXPORT_ALL'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): - if key in all_exported_functions or (export_bindings and key.startswith('_emscripten_bind')): + if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value @@ -299,13 +328,14 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs_js = [output[0] for output in outputs] outputs = None - if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2b took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} - i = 2 # universal counter - if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS'] + i = settings['FUNCTION_POINTER_ALIGNMENT'] # universal counter + if settings['ASM_JS']: i += settings['RESERVED_FUNCTION_POINTERS']*settings['FUNCTION_POINTER_ALIGNMENT'] + base_fp = i table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None @@ -314,13 +344,13 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: - table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS'] + table_counters[sig] = base_fp curr = table_counters[sig] - table_counters[sig] += 2 + table_counters[sig] += settings['FUNCTION_POINTER_ALIGNMENT'] else: curr = i - i += 2 - #print >> sys.stderr, 'function indexing', indexed, curr, sig + i += settings['FUNCTION_POINTER_ALIGNMENT'] + #logging.debug('function indexing ' + str([indexed, curr, sig])) forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): @@ -328,18 +358,29 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] + def indexize_mem(js): + return re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) def indexize(js): - # In the global initial allocation, we need to split up into Uint8 format - ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) - return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) + return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), js) blockaddrs = forwarded_json['Functions']['blockAddresses'] + def blockaddrsize_mem(js): + return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) def blockaddrsize(js): - ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) - return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) + return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) + + pre = blockaddrsize(blockaddrsize_mem(indexize(indexize_mem(pre)))) + + if settings.get('ASM_JS'): + # move postsets into the asm module + class PostSets: js = '' + def handle_post_sets(m): + PostSets.js = m.group(0) + return '\n' + pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre) #if DEBUG: outfile.write('// pre\n') - outfile.write(blockaddrsize(indexize(pre))) + outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') @@ -348,13 +389,13 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) - if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2c took %s seconds' % (time.time() - t)) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data - out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, + out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) @@ -375,6 +416,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 + j = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] @@ -383,19 +425,24 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) - coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' - ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) + coercions = ';'.join(['p%d = %s' % (p, shared.JS.make_coercion('p%d' % p, sig[p+1], settings)) for p in range(len(sig)-1)]) + ';' + ret = '' if sig[0] == 'v' else ('return %s' % shared.JS.make_initializer(sig[0], settings)) start = raw.index('[') end = raw.rindex(']') body = raw[start+1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): - body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j) + body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = 'jsCall_%s_%s' % (sig, j) + Counter.j = 0 def fix_item(item): - newline = '\n' in item - return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') + Counter.j += 1 + newline = Counter.j % 30 == 29 + if item == '0': return bad if not newline else (bad + '\n') + return item if not newline else (item + '\n') body = ','.join(map(fix_item, body)) - return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:]) + return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), ''.join([raw[:start+1], body, raw[end:]])) + infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] + function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' @@ -404,9 +451,11 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) - basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat'] + [m.replace('.', '_') for m in math_envs] + if settings['PRECISE_F32']: maths += ['Math.fround'] + + basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat'] + [m.replace('.', '_') for m in math_envs] if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') - if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] + if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE'] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] if settings['ASSERTIONS']: basic_funcs += ['nullFunc'] @@ -420,19 +469,24 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] + if settings.get('DLOPEN_SUPPORT'): + for sig in last_forwarded_json['Functions']['tables'].iterkeys(): + basic_vars.append('F_BASE_%s' % sig) + asm_setup += ' var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n' + + if '_rand' in exported_implemented_functions or '_srand' in exported_implemented_functions: + basic_vars += ['___rand_seed'] + asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables - def asm_coerce(value, sig): - if sig == 'v': return value - return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') - function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] + for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) - arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) - coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) - ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) + arg_coercions = ' '.join(['a' + str(i) + '=' + shared.JS.make_coercion('a' + str(i), sig[i], settings) + ';' for i in range(1, len(sig))]) + coerced_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings) for i in range(1, len(sig))]) + ret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0], settings) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; @@ -442,7 +496,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) for i in range(settings['RESERVED_FUNCTION_POINTERS']): - jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) + jsret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0], settings) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s @@ -450,23 +504,16 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, } ''' % (sig, i, args, arg_coercions, jsret)) - args = ','.join(['a' + str(i) for i in range(1, len(sig))]) - args = 'index' + (',' if args else '') + args - # C++ exceptions are numbers, and longjmp is a string 'longjmp' - asm_setup += ''' -function invoke_%s(%s) { - try { - %sModule["dynCall_%s"](%s); - } catch(e) { - if (typeof e !== 'number' && e !== 'longjmp') throw e; - asm["setThrew"](1, 0); - } -} -''' % (sig, args, 'return ' if sig[0] != 'v' else '', sig, args) + shared.Settings.copy(settings) + asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) + if settings.get('DLOPEN_SUPPORT'): + asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n' + basic_funcs.append('extCall_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) + exported_implemented_functions.append('runPostSets') exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: @@ -474,6 +521,7 @@ function invoke_%s(%s) { exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' + # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable @@ -483,11 +531,20 @@ function invoke_%s(%s) { global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] def math_fix(g): - return g if not g.startswith('Math_') else g.split('_')[1]; + return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) + # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules + if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): + assert settings.get('TARGET_LE32'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' + for key, value in forwarded_json['Variables']['globals'].iteritems(): + if value.get('linkable'): + init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE + if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' + asm_global_vars += ' var %s=%s;\n' % (key, str(init)) + # sent data the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' @@ -499,7 +556,7 @@ function invoke_%s(%s) { # finalize - if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) + if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) funcs_js = [''' %s @@ -511,7 +568,7 @@ function asmPrintFloat(x, y) { } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { - 'use asm'; + %s var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); @@ -520,44 +577,62 @@ var asm = (function(global, env, buffer) { var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); -''' % (asm_setup,) + '\n' + asm_global_vars + ''' +''' % (asm_setup, "'use asm';" if not forwarded_json['Types']['hasInlineJS'] and not settings['SIDE_MODULE'] and settings['ASM_JS'] == 1 else "'almost asm';") + '\n' + asm_global_vars + ''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' - var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' + var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + [' var tempFloat = %s;\n' % ('Math_fround(0)' if settings.get('PRECISE_F32') else '0.0')] + [''' // EMSCRIPTEN_START_FUNCS - function stackAlloc(size) { - size = size|0; - var ret = 0; - ret = STACKTOP; - STACKTOP = (STACKTOP + size)|0; -''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' - return ret|0; - } - function stackSave() { - return STACKTOP|0; - } - function stackRestore(top) { - top = top|0; - STACKTOP = top; - } - function setThrew(threw, value) { - threw = threw|0; - value = value|0; - if ((__THREW__|0) == 0) { - __THREW__ = threw; - threwValue = value; - } +function stackAlloc(size) { + size = size|0; + var ret = 0; + ret = STACKTOP; + STACKTOP = (STACKTOP + size)|0; +''' + ('STACKTOP = (STACKTOP + 3)&-4;' if settings['TARGET_X86'] else 'STACKTOP = (STACKTOP + 7)&-8;') + ''' + return ret|0; +} +function stackSave() { + return STACKTOP|0; +} +function stackRestore(top) { + top = top|0; + STACKTOP = top; +} +function setThrew(threw, value) { + threw = threw|0; + value = value|0; + if ((__THREW__|0) == 0) { + __THREW__ = threw; + threwValue = value; } +} +function copyTempFloat(ptr) { + ptr = ptr|0; + HEAP8[tempDoublePtr] = HEAP8[ptr]; + HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; + HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; + HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; +} +function copyTempDouble(ptr) { + ptr = ptr|0; + HEAP8[tempDoublePtr] = HEAP8[ptr]; + HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; + HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; + HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; + HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0]; + HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0]; + HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0]; + HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0]; +} ''' + ''.join([''' - function setTempRet%d(value) { - value = value|0; - tempRet%d = value; - } -''' % (i, i) for i in range(10)])] + funcs_js + [''' +function setTempRet%d(value) { + value = value|0; + tempRet%d = value; +} +''' % (i, i) for i in range(10)])] + [PostSets.js + '\n'] + funcs_js + [''' %s return %s; @@ -565,23 +640,36 @@ var asm = (function(global, env, buffer) { // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; +''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] + + if not settings.get('SIDE_MODULE'): + funcs_js.append(''' Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; Runtime.stackSave = function() { return asm['stackSave']() }; Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; -''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] +''') # Set function table masks - def function_table_maskize(js): - masks = {} - default = None - for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): - masks[sig] = str(table.count(',')) - default = sig + masks = {} + max_mask = 0 + for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): + mask = table.count(',') + masks[sig] = str(mask) + max_mask = max(mask, max_mask) + def function_table_maskize(js, masks): def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] - funcs_js = map(function_table_maskize, funcs_js) + funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) + + if settings.get('DLOPEN_SUPPORT'): + funcs_js.append(''' + asm.maxFunctionIndex = %(max_mask)d; + DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module); + Module.SYMBOL_TABLE = SYMBOL_TABLE; +''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) + else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) @@ -591,17 +679,547 @@ Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; // EMSCRIPTEN_END_FUNCS '''] - for funcs_js_item in funcs_js: # do this loop carefully to save memory + # Create symbol table for self-dlopen + if settings.get('DLOPEN_SUPPORT'): + symbol_table = {} + for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems(): + if forwarded_json['Variables']['globals'][k]['named']: + symbol_table[k] = str(v + forwarded_json['Runtime']['GLOBAL_BASE']) + for raw in last_forwarded_json['Functions']['tables'].itervalues(): + if raw == '': continue + table = map(string.strip, raw[raw.find('[')+1:raw.find(']')].split(",")) + for i in range(len(table)): + value = table[i] + if value != '0': + if settings.get('SIDE_MODULE'): + symbol_table[value] = 'FUNCTION_TABLE_OFFSET+' + str(i) + else: + symbol_table[value] = str(i) + outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', '')) + + for i in range(len(funcs_js)): # do this loop carefully to save memory + funcs_js_item = funcs_js[i] + funcs_js[i] = None funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) - if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 3 took %s seconds' % (time.time() - t)) outfile.close() +# emscript_fast: emscript'en code using the 'fast' compilation path, using +# an LLVM backend +# FIXME: this is just a copy-paste of normal emscript(), and we trample it +# if the proper env var is set (see below). we should refactor to +# share code between the two, once emscript_fast stabilizes (or, +# leaving it separate like it is will make it trivial to rip out +# if the experiment fails) + +def emscript_fast(infile, settings, outfile, libraries=[], compiler_engine=None, + jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): + """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible + + Args: + infile: The path to the input LLVM assembly file. + settings: JSON-formatted settings that override the values + defined in src/settings.js. + outfile: The file where the output is written. + """ + + assert(settings['ASM_JS']) + + # Overview: + # * Run LLVM backend to emit JS. JS includes function bodies, memory initializer, + # and various metadata + # * Run compiler.js on the metadata to emit the shell js code, pre/post-ambles, + # JS library dependencies, etc. + + temp_js = temp_files.get('.4.js').name + backend_compiler = os.path.join(shared.LLVM_ROOT, 'llc') + backend_args = [backend_compiler, infile, '-march=js', '-filetype=asm', '-o', temp_js] + if settings['PRECISE_F32']: + backend_args += ['-emscripten-precise-f32'] + if settings['WARN_UNALIGNED']: + backend_args += ['-emscripten-warn-unaligned'] + if DEBUG: + logging.debug('emscript: llvm backend: ' + ' '.join(backend_args)) + t = time.time() + shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE)) + if DEBUG: + logging.debug(' emscript: llvm backend took %s seconds' % (time.time() - t)) + t = time.time() + + # Split up output + backend_output = open(temp_js).read() + #if DEBUG: print >> sys.stderr, backend_output + + start_funcs_marker = '// EMSCRIPTEN_START_FUNCTIONS' + end_funcs_marker = '// EMSCRIPTEN_END_FUNCTIONS' + metadata_split_marker = '// EMSCRIPTEN_METADATA' + + start_funcs = backend_output.index(start_funcs_marker) + end_funcs = backend_output.rindex(end_funcs_marker) + metadata_split = backend_output.rindex(metadata_split_marker) + + funcs = backend_output[start_funcs+len(start_funcs_marker):end_funcs] + metadata_raw = backend_output[metadata_split+len(metadata_split_marker):] + #if DEBUG: print >> sys.stderr, "METAraw", metadata_raw + metadata = json.loads(metadata_raw) + mem_init = backend_output[end_funcs+len(end_funcs_marker):metadata_split] + #if DEBUG: print >> sys.stderr, "FUNCS", funcs + #if DEBUG: print >> sys.stderr, "META", metadata + #if DEBUG: print >> sys.stderr, "meminit", mem_init + + # function table masks + + table_sizes = {} + for k, v in metadata['tables'].iteritems(): + table_sizes[k] = str(v.count(',')) # undercounts by one, but that is what we want + funcs = re.sub(r"#FM_(\w+)#", lambda m: table_sizes[m.groups(0)[0]], funcs) + + # fix +float into float.0, if not running js opts + if not settings['RUNNING_JS_OPTS']: + def fix_dot_zero(m): + num = m.group(3) + # TODO: handle 0x floats? + if num.find('.') < 0: + |