diff options
Diffstat (limited to 'emscripten.py')
| -rwxr-xr-x | emscripten.py | 123 |
1 files changed, 73 insertions, 50 deletions
diff --git a/emscripten.py b/emscripten.py index 19e2160d..2e90fa48 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,7 +9,7 @@ header files (so that the JS compiler can see the constants in those headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, multiprocessing, string +import os, sys, json, optparse, subprocess, re, time, multiprocessing, string, logging from tools import jsrun, cache as cache_module, tempfiles from tools.response_file import read_response_file @@ -46,7 +46,7 @@ MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # con STDERR_FILE = os.environ.get('EMCC_STDERR_FILE') if STDERR_FILE: STDERR_FILE = os.path.abspath(STDERR_FILE) - print >> sys.stderr, 'logging stderr in js compiler phase into %s' % STDERR_FILE + logging.info('logging stderr in js compiler phase into %s' % STDERR_FILE) STDERR_FILE = open(STDERR_FILE, 'w') def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): @@ -58,6 +58,8 @@ def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libr f.write('\n') f.write(meta) f.close() + #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments, + # # just copy temp dir to under this one out = jsrun.run_js( compiler, engine=compiler_engine, @@ -68,9 +70,7 @@ def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libr except KeyboardInterrupt: # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt raise Exception() - finally: - tempfiles.try_delete(funcs_file) - if DEBUG: print >> sys.stderr, '.' + if DEBUG: logging.debug('.') return out def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, @@ -91,7 +91,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. - if DEBUG: print >> sys.stderr, 'emscript: ll=>js' + if DEBUG: logging.debug('emscript: ll=>js') if jcache: jcache.ensure() @@ -101,7 +101,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection - if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: scan took %s seconds' % (time.time() - t)) # Split input into the relevant parts for each phase pre = [] @@ -136,19 +136,19 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) - if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' - if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) + if DEBUG and len(meta) > 1024*1024: logging.debug('emscript warning: large amounts of metadata, will slow things down') + if DEBUG: logging.debug(' emscript: split took %s seconds' % (time.time() - t)) if len(funcs) == 0: - print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' + logging.error('No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)') #if DEBUG: - # print >> sys.stderr, '========= pre ================\n' - # print >> sys.stderr, ''.join(pre) - # print >> sys.stderr, '========== funcs ===============\n' + # logging.debug('========= pre ================\n') + # logging.debug(''.join(pre)) + # logging.debug('========== funcs ===============\n') # for func in funcs: - # print >> sys.stderr, '\n// ===\n\n', ''.join(func) - # print >> sys.stderr, '=========================\n' + # logging.debug('\n// ===\n\n', ''.join(func)) + # logging.debug('=========================\n') # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name @@ -168,7 +168,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) - if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey + if DEBUG_CACHE: logging.debug('shortkey', shortkey) out = jcache.get(shortkey, keys) @@ -181,21 +181,22 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() - print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath + logging.debug(' cache miss, key data dumped to %s' % dfpath) - if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' + if out and DEBUG: logging.debug(' loading pre from jcache') if not out: open(pre_file, 'w').write(pre_input) + #print >> sys.stderr, 'running', str([settings_file, pre_file, 'pre'] + libraries).replace("'/", "'") # see funcs out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: - if DEBUG: print >> sys.stderr, ' saving pre to jcache' + if DEBUG: logging.debug(' saving pre to jcache') jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) - if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 1 took %s seconds' % (time.time() - t)) indexed_functions = set() forwarded_json = json.loads(forwarded_data) @@ -223,6 +224,9 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) + #sys.exit(1) + #chunks = [chunks[0]] # pick specific chunks for debugging/profiling + funcs = None if jcache: @@ -238,7 +242,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: - if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) + if out and DEBUG: logging.debug(' loading %d funcchunks from jcache' % len(cached_outputs)) else: cached_outputs = [] @@ -248,10 +252,11 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' - if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) + if DEBUG: logging.debug(' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))) commands = [ - (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) + (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine,# + ['--prof'], + temp_files, DEBUG) for i, chunk in enumerate(chunks) ] @@ -273,7 +278,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) - if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) + if out and DEBUG and len(chunks) > 0: logging.debug(' saving %d funcchunks to jcache' % len(chunks)) chunks = None @@ -283,7 +288,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] - if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2 took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # merge forwarded data @@ -319,13 +324,14 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs_js = [output[0] for output in outputs] outputs = None - if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2b took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} - i = 2 # universal counter - if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS'] + i = settings['FUNCTION_POINTER_ALIGNMENT'] # universal counter + if settings['ASM_JS']: i += settings['RESERVED_FUNCTION_POINTERS']*settings['FUNCTION_POINTER_ALIGNMENT'] + base_fp = i table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None @@ -334,13 +340,13 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: - table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS'] + table_counters[sig] = base_fp curr = table_counters[sig] - table_counters[sig] += 2 + table_counters[sig] += settings['FUNCTION_POINTER_ALIGNMENT'] else: curr = i - i += 2 - #print >> sys.stderr, 'function indexing', indexed, curr, sig + i += settings['FUNCTION_POINTER_ALIGNMENT'] + #logging.debug('function indexing', indexed, curr, sig) forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): @@ -348,17 +354,18 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] + def indexize_mem(js): + return re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) def indexize(js): - # In the global initial allocation, we need to split up into Uint8 format - ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) - return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) + return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), js) blockaddrs = forwarded_json['Functions']['blockAddresses'] + def blockaddrsize_mem(js): + return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) def blockaddrsize(js): - ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) - return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) + return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) - pre = blockaddrsize(indexize(pre)) + pre = blockaddrsize(blockaddrsize_mem(indexize(indexize_mem(pre)))) if settings.get('ASM_JS'): # move postsets into the asm module @@ -378,7 +385,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) - if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2c took %s seconds' % (time.time() - t)) # Phase 3 - post if DEBUG: t = time.time() @@ -405,6 +412,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 + j = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] @@ -419,13 +427,18 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, end = raw.rindex(']') body = raw[start+1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): - body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j) + body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = 'jsCall_%s_%s' % (sig, j) + Counter.j = 0 def fix_item(item): - newline = '\n' in item - return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') + Counter.j += 1 + newline = Counter.j % 30 == 29 + if item == '0': return bad if not newline else (bad + '\n') + return item if not newline else (item + '\n') body = ','.join(map(fix_item, body)) - return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:]) + return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), ''.join([raw[:start+1], body, raw[end:]])) + infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] + function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' @@ -465,6 +478,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] + for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) @@ -539,7 +553,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, # finalize - if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) + if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) funcs_js = [''' %s @@ -574,7 +588,7 @@ function stackAlloc(size) { var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; -''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' +''' + ('STACKTOP = (STACKTOP + 3)&-4;' if settings['TARGET_X86'] else 'STACKTOP = (STACKTOP + 7)&-8;') + ''' return ret|0; } function stackSave() { @@ -680,14 +694,16 @@ Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; symbol_table[value] = str(i) outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', '')) - for funcs_js_item in funcs_js: # do this loop carefully to save memory + for i in range(len(funcs_js)): # do this loop carefully to save memory + funcs_js_item = funcs_js[i] + funcs_js[i] = None funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) - if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 3 took %s seconds' % (time.time() - t)) outfile.close() @@ -709,7 +725,14 @@ def main(args, compiler_engine, cache, jcache, relooper, temp_files, DEBUG, DEBU if not os.path.exists(relooper): from tools import shared shared.Building.ensure_relooper(relooper) - + + settings.setdefault('STRUCT_INFO', cache.get_path('struct_info.compiled.json')) + struct_info = settings.get('STRUCT_INFO') + + if not os.path.exists(struct_info): + from tools import shared + shared.Building.ensure_struct_info(struct_info) + emscript(args.infile, settings, args.outfile, libraries, compiler_engine=compiler_engine, jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE) @@ -779,11 +802,11 @@ def _main(environ): keywords, positional = parser.parse_args() if not keywords.suppressUsageWarning: - print >> sys.stderr, ''' + logging.warning(''' ============================================================== WARNING: You should normally never use this! Use emcc instead. ============================================================== - ''' + ''') if len(positional) != 1: raise RuntimeError('Must provide exactly one positional argument. Got ' + str(len(positional)) + ': "' + '", "'.join(positional) + '"') |