diff options
45 files changed, 1296 insertions, 247 deletions
@@ -40,4 +40,5 @@ a license to everyone to use it as detailed in LICENSE.) * Lars Schneider <lars.schneider@autodesk.com> (copyright owned by Autodesk, Inc.) * Joel Martin <github@martintribe.org> * Manuel Wellmann <manuel.wellmann@autodesk.com> (copyright owned by Autodesk, Inc.) +* Xuejie Xiao <xxuejie@gmail.com> @@ -116,7 +116,7 @@ shared.check_sanity(force=DEBUG) if len(sys.argv) == 1: print 'emcc: no input files' - exit(0) + exit(1) if sys.argv[1] == '--version': print '''emcc (Emscripten GCC-like replacement) 2.0 @@ -327,6 +327,20 @@ Options that are modified or new in %s include: llvm-link's behavior is not as permissive as ld is. + --jcache Use a JavaScript cache. This is disabled by + default. When enabled, emcc will store the + results of compilation in a cache and check + the cache when compiling later, something + like what ccache does. This allows incremental + builds - where you are compiling a large + program but only modified a small part of it - + to be much faster (at the cost of more disk + IO for cache accesses). Note that you need + to enable --jcache for both loading and saving + of data, so you must enable it on a full build + for a later incremental build (where you also + enable it) to be sped up. + --clear-cache Manually clears the cache of compiled emscripten system libraries (libc++, libc++abi, dlmalloc). This is normally @@ -336,7 +350,10 @@ Options that are modified or new in %s include: mechanism can get confused. Clearing the cache can fix weird problems related to cache incompatibilities, like clang failing - to link with library files. + to link with library files. This also clears + other cached data like the jcache and + the bootstrapped relooper. After the cache + is cleared, this process will exit. The target file, if specified (-o <target>), defines what will be generated: @@ -571,6 +588,7 @@ try: remove_duplicates = False keep_debug = False bind = False + jcache = False def check_bad_eq(arg): assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)' @@ -678,10 +696,14 @@ try: elif newargs[i] == '--remove-duplicates': remove_duplicates = True newargs[i] = '' + elif newargs[i] == '--jcache': + jcache = True + newargs[i] = '' elif newargs[i] == '--clear-cache': newargs[i] = '' print >> sys.stderr, 'emcc: clearing cache' shared.Cache.erase() + sys.exit(0) elif newargs[i].startswith(('-I/', '-L/')): if not absolute_warning_shown: print >> sys.stderr, 'emcc: warning: -I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript)' # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not @@ -695,6 +717,8 @@ try: minify_whitespace = closure # if closure is run, minify whitespace if opt_level <= 0: keep_debug = True # always keep debug in -O0 + if DEBUG: start_time = time.time() # done after parsing arguments, which might affect debug state + if closure: assert os.path.exists(shared.CLOSURE_COMPILER), 'emcc: fatal: Closure compiler (%s) does not exist' % shared.CLOSURE_COMPILER @@ -1044,6 +1068,7 @@ try: # Emscripten if DEBUG: print >> sys.stderr, 'emcc: LLVM => JS' extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))] + if jcache: extra_args.append('--jcache') final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args) if DEBUG: save_intermediate('original') @@ -1100,12 +1125,12 @@ try: if len(js_optimizer_queue) > 0: if DEBUG < 2: if DEBUG: print >> sys.stderr, 'emcc: applying js optimization passes:', js_optimizer_queue - final = shared.Building.js_optimizer(final, js_optimizer_queue) + final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache) if DEBUG: save_intermediate('js_opts') else: for name in js_optimizer_queue: print >> sys.stderr, 'emcc: applying js optimization pass:', name - final = shared.Building.js_optimizer(final, [name]) + final = shared.Building.js_optimizer(final, [name], jcache) save_intermediate(name) js_optimizer_queue = [] @@ -1114,7 +1139,7 @@ try: if DEBUG >= 2: # Clean up the syntax a bit - final = shared.Building.js_optimizer(final, []) + final = shared.Building.js_optimizer(final, [], jcache) if DEBUG: save_intermediate('pretty') def get_eliminate(): @@ -1221,6 +1246,8 @@ try: # copy final JS to output shutil.move(final, target) + if DEBUG: print >> sys.stderr, 'emcc: total time: %.2f seconds' % (time.time() - start_time) + finally: if not TEMP_DIR: try: diff --git a/emscripten.py b/emscripten.py index 91b1de5a..15beb4ee 100755 --- a/emscripten.py +++ b/emscripten.py @@ -32,6 +32,7 @@ def path_from_root(*pathelems): temp_files = shared.TempFiles() compiler_engine = None +jcache = False def scan(ll, settings): # blockaddress(@main, %23) @@ -47,12 +48,13 @@ MIN_CHUNK_SIZE = 1024*1024 MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes def process_funcs(args): - i, ll, settings_file, compiler, forwarded_file, libraries = args + i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args + ll = ''.join(funcs) + '\n' + meta funcs_file = temp_files.get('.func_%d.ll' % i).name open(funcs_file, 'w').write(ll) out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) shared.try_delete(funcs_file) - return out.split('//FORWARDED_DATA:') + return out def emscript(infile, settings, outfile, libraries=[]): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible @@ -73,6 +75,8 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: print >> sys.stderr, 'emscript: ll=>js' + if jcache: shared.JCache.ensure() + # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() @@ -84,31 +88,31 @@ def emscript(infile, settings, outfile, libraries=[]): # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later + func_idents = [] meta = [] # needed by each function XXX - post = [] if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() for line in ll_lines: if in_func: - funcs[-1].append(line) + funcs[-1][1].append(line) if line.startswith('}'): in_func = False - funcs[-1] = ''.join(funcs[-1]) + funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1])) pre.append(line) # pre needs it to, so we know about all implemented functions else: + if line.startswith(';'): continue if line.startswith('define '): in_func = True - funcs.append([line]) + funcs.append((line, [line])) # use the entire line as the identifier pre.append(line) # pre needs it to, so we know about all implemented functions elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): meta.append(line) # metadata else: - post.append(line) # global - pre.append(line) # pre needs it to, so we know about globals in pre and funcs + pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' @@ -120,24 +124,32 @@ def emscript(infile, settings, outfile, libraries=[]): # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) - # print >> sys.stderr, '========== post ==============\n' - # print >> sys.stderr, ''.join(post) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name + settings_text = json.dumps(settings) s = open(settings_file, 'w') - s.write(json.dumps(settings)) + s.write(settings_text) s.close() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name - open(pre_file, 'w').write(''.join(pre) + '\n' + meta) - out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) - js, forwarded_data = out.split('//FORWARDED_DATA:') - outfile.write(js) - js = None + pre_input = ''.join(pre) + '\n' + meta + out = None + if jcache: + keys = [pre_input, settings_text, ','.join(libraries)] + shortkey = shared.JCache.get_shortkey(keys) + out = shared.JCache.get(shortkey, keys) + if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' + if not out: + open(pre_file, 'w').write(pre_input) + out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) + if jcache: + if DEBUG: print >> sys.stderr, ' saving pre to jcache' + shared.JCache.set(shortkey, keys, out) + pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) @@ -157,28 +169,58 @@ def emscript(infile, settings, outfile, libraries=[]): if DEBUG: t = time.time() forwarded_json = json.loads(forwarded_data) indexed_functions = set() - chunks = [] # bundles of functions - curr = '' - for i in range(len(funcs)): - func = funcs[i] - if len(curr) + len(func) < chunk_size: - curr += func + + chunks = shared.JCache.chunkify(funcs, chunk_size, 'emscript_files' if jcache else None) + + if jcache: + # load chunks from cache where we can # TODO: ignore small chunks + cached_outputs = [] + def load_from_cache(chunk): + keys = [settings_text, forwarded_data, chunk] + shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code + out = shared.JCache.get(shortkey, keys) # this is relatively expensive (pickling?) + if out: + cached_outputs.append(out) + return False + return True + chunks = filter(load_from_cache, chunks) + if len(cached_outputs) > 0: + if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) else: - chunks.append(curr) - curr = func - if curr: - chunks.append(curr) - curr = '' + cached_outputs = [] + + # TODO: minimize size of forwarded data from funcs to what we actually need + if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' - if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) - commands = [(i, chunks[i] + '\n' + meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))] + if len(chunks) > 0: + if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) + + commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))] - if len(chunks) > 1: - pool = multiprocessing.Pool(processes=cores) - outputs = pool.map(process_funcs, commands, chunksize=1) + if len(chunks) > 1: + pool = multiprocessing.Pool(processes=cores) + outputs = pool.map(process_funcs, commands, chunksize=1) + elif len(chunks) == 1: + outputs = [process_funcs(commands[0])] else: - outputs = [process_funcs(commands[0])] + outputs = [] + + if jcache: + # save chunks to cache + for i in range(len(chunks)): + chunk = chunks[i] + keys = [settings_text, forwarded_data, chunk] + shortkey = shared.JCache.get_shortkey(keys) + shared.JCache.set(shortkey, keys, outputs[i]) + if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) + + if jcache: outputs += cached_outputs # TODO: preserve order + + outputs = [output.split('//FORWARDED_DATA:') for output in outputs] + + if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) + if DEBUG: t = time.time() funcs_js = ''.join([output[0] for output in outputs]) @@ -191,7 +233,7 @@ def emscript(infile, settings, outfile, libraries=[]): for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) outputs = None - if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) + if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data @@ -201,23 +243,35 @@ def emscript(infile, settings, outfile, libraries=[]): forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize i += 2 forwarded_json['Functions']['nextIndex'] = i + indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): return re.sub(r'{{{ FI_([\w\d_$]+) }}}', lambda m: str(indexing[m.groups(0)[0]]), js) - outfile.write(indexize(funcs_js)) + + blockaddrs = forwarded_json['Functions']['blockAddresses'] + def blockaddrsize(js): + return re.sub(r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) + + #if DEBUG: outfile.write('// pre\n') + outfile.write(blockaddrsize(indexize(pre))) + pre = None + + #if DEBUG: outfile.write('// funcs\n') + outfile.write(blockaddrsize(indexize(funcs_js))) funcs_js = None # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name - open(forwarded_file, 'w').write(forwarded_data) - if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) + open(forwarded_file, 'w').write(indexize(forwarded_data)) + if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name - open(post_file, 'w').write(''.join(post) + '\n' + meta) + open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) + #if DEBUG: outfile.write('// post\n') outfile.write(indexize(out)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) @@ -329,6 +383,10 @@ if __name__ == '__main__': metavar='FOO=BAR', help=('Overrides for settings defined in settings.js. ' 'May occur multiple times.')) + parser.add_option('-j', '--jcache', + action='store_true', + default=False, + help=('Enable jcache (ccache-like caching of compilation results, for faster incremental builds).')) # Convert to the same format that argparse would have produced. keywords, positional = parser.parse_args() @@ -338,6 +396,7 @@ if __name__ == '__main__': if isinstance(keywords.outfile, basestring): keywords.outfile = open(keywords.outfile, 'w') compiler_engine = keywords.compiler + jcache = keywords.jcache temp_files.run_and_clean(lambda: main(keywords)) diff --git a/src/analyzer.js b/src/analyzer.js index c09739e9..014579f4 100644 --- a/src/analyzer.js +++ b/src/analyzer.js @@ -690,6 +690,8 @@ function analyzer(data, sidePass) { var subType = check[2]; addTypeInternal(subType, data); // needed for anonymous structure definitions (see below) + // Huge structural types are represented very inefficiently, both here and in generated JS. Best to avoid them - for example static char x[10*1024*1024]; is bad, while static char *x = malloc(10*1024*1024) is fine. + if (num >= 10*1024*1024) warnOnce('warning: very large fixed-size structural type: ' + type + ' - can you reduce it? (compilation may be slow)'); Types.types[nonPointing] = { name_: nonPointing, fields: range(num).map(function() { return subType }), diff --git a/src/intertyper.js b/src/intertyper.js index fbad353a..8e7bb418 100644 --- a/src/intertyper.js +++ b/src/intertyper.js @@ -165,7 +165,7 @@ function intertyper(data, sidePass, baseLineNums) { function makeToken(text) { if (text.length == 0) return; // merge certain tokens - if (lastToken && ( (lastToken.text == '%' && text[0] == '"') || /^\**$/.exec(text) ) ) { + if (lastToken && ( (lastToken.text == '%' && text[0] == '"') || /^\**$/.test(text) ) ) { lastToken.text += text; return; } @@ -182,7 +182,7 @@ function intertyper(data, sidePass, baseLineNums) { // merge certain tokens if (lastToken && isType(lastToken.text) && isFunctionDef(token)) { lastToken.text += ' ' + text; - } else if (lastToken && /^}\**$/.exec(text)) { // }, }*, etc. + } else if (lastToken && text[0] == '}') { // }, }*, etc. var openBrace = tokens.length-1; while (tokens[openBrace].text.substr(-1) != '{') openBrace --; token = combineTokens(tokens.slice(openBrace+1)); @@ -674,7 +674,7 @@ function intertyper(data, sidePass, baseLineNums) { // Inline assembly is just JavaScript that we paste into the code item.intertype = 'value'; if (tokensLeft[0].text == 'sideeffect') tokensLeft.splice(0, 1); - item.ident = tokensLeft[0].text.substr(1, tokensLeft[0].text.length-2); + item.ident = tokensLeft[0].text.substr(1, tokensLeft[0].text.length-2) || ';'; // use ; for empty inline assembly return { forward: null, ret: [item], item: item }; } if (item.ident.substr(-2) == '()') { diff --git a/src/jsifier.js b/src/jsifier.js index fae92f70..595e057c 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -65,24 +65,27 @@ function JSify(data, functionsOnly, givenFunctions) { // Add additional necessary items for the main pass. We can now do this since types are parsed (types can be used through // generateStructInfo in library.js) LibraryManager.load(); - var libFuncsToInclude; - if (INCLUDE_FULL_LIBRARY) { - assert(!BUILD_AS_SHARED_LIB, 'Cannot have both INCLUDE_FULL_LIBRARY and BUILD_AS_SHARED_LIB set.') - libFuncsToInclude = []; - for (var key in LibraryManager.library) { - if (!key.match(/__(deps|postset|inline)$/)) { - libFuncsToInclude.push(key); + + if (phase == 'pre') { + var libFuncsToInclude; + if (INCLUDE_FULL_LIBRARY) { + assert(!BUILD_AS_SHARED_LIB, 'Cannot have both INCLUDE_FULL_LIBRARY and BUILD_AS_SHARED_LIB set.') + libFuncsToInclude = []; + for (var key in LibraryManager.library) { + if (!key.match(/__(deps|postset|inline)$/)) { + libFuncsToInclude.push(key); + } } + } else { + libFuncsToInclude = DEFAULT_LIBRARY_FUNCS_TO_INCLUDE; } - } else { - libFuncsToInclude = DEFAULT_LIBRARY_FUNCS_TO_INCLUDE; - } - libFuncsToInclude.forEach(function(ident) { - data.functionStubs.push({ - intertype: 'functionStub', - ident: '_' + ident + libFuncsToInclude.forEach(function(ident) { + data.functionStubs.push({ + intertype: 'functionStub', + ident: '_' + ident + }); }); - }); + } } // Functions @@ -329,7 +332,7 @@ function JSify(data, functionsOnly, givenFunctions) { if (item.ident.substr(0, 5) == '__ZTV') { js += '\n' + makePointer('[0]', null, BUILD_AS_SHARED_LIB ? 'ALLOC_NORMAL' : 'ALLOC_STATIC', ['void*']) + ';'; } - if (item.ident in EXPORTED_GLOBALS) { + if (EXPORT_ALL || (item.ident in EXPORTED_GLOBALS)) { js += '\nModule["' + item.ident + '"] = ' + item.ident + ';'; } if (BUILD_AS_SHARED_LIB == 2 && !item.private_) { @@ -439,7 +442,7 @@ function JSify(data, functionsOnly, givenFunctions) { } var text = (deps ? '\n' + deps.map(addFromLibrary).filter(function(x) { return x != '' }).join('\n') : ''); text += isFunction ? snippet : 'var ' + ident + '=' + snippet + ';'; - if (ident in EXPORTED_FUNCTIONS) { + if (EXPORT_ALL || (ident in EXPORTED_FUNCTIONS)) { text += '\nModule["' + ident + '"] = ' + ident + ';'; } return text; @@ -616,6 +619,7 @@ function JSify(data, functionsOnly, givenFunctions) { } // otherwise, should have been set before! if (func.setjmpTable) { var setjmpTable = {}; + ret += indent + 'var setjmped = false;'; // set to true if we setjmp in this invocation ret += indent + 'var setjmpTable = {'; func.setjmpTable.forEach(function(triple) { // original label, label we created for right after the setjmp, variable setjmp result goes into ret += '"' + getLabelId(triple[0]) + '": ' + 'function(value) { label = ' + getLabelId(triple[1]) + '; ' + triple[2] + ' = value },'; @@ -634,7 +638,7 @@ function JSify(data, functionsOnly, givenFunctions) { }).join('\n'); ret += '\n' + indent + ' default: assert(0, "bad label: " + label);\n' + indent + '}'; if (func.setjmpTable) { - ret += ' } catch(e) { if (!e.longjmp) throw(e); setjmpTable[e.label](e.value) }'; + ret += ' } catch(e) { if (!setjmped) throw(e); if (!e.longjmp) throw(e); setjmpTable[e.label](e.value) }'; } } else { ret += (SHOW_LABELS ? indent + '/* ' + block.entries[0] + ' */' : '') + '\n' + getLabelLines(block.labels[0], indent); @@ -699,7 +703,7 @@ function JSify(data, functionsOnly, givenFunctions) { func.JS += '\n//FUNCTION_END_MARKER_OF_SOURCE_FILE_' + associatedSourceFile + '\n'; } - if (func.ident in EXPORTED_FUNCTIONS) { + if (EXPORT_ALL || (func.ident in EXPORTED_FUNCTIONS)) { func.JS += 'Module["' + func.ident + '"] = ' + func.ident + ';'; } @@ -1066,12 +1070,12 @@ function JSify(data, functionsOnly, givenFunctions) { var param1 = finalizeLLVMParameter(item.params[0]); var param2 = finalizeLLVMParameter(item.params[1]); switch (item.op) { - case 'add': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue+' + param2, type) + ',tempValue)'; - case 'sub': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue-' + param2, type) + ',tempValue)'; - case 'xchg': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, param2, type) + ',tempValue)'; + case 'add': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue+' + param2, type, null, null, null, null, ',') + ',tempValue)'; + case 'sub': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, 'tempValue-' + param2, type, null, null, null, null, ',') + ',tempValue)'; + case 'xchg': return '(tempValue=' + makeGetValue(param1, 0, type) + ',' + makeSetValue(param1, 0, param2, type, null, null, null, null, ',') + ',tempValue)'; case 'cmpxchg': { var param3 = finalizeLLVMParameter(item.params[2]); - return '(tempValue=' + makeGetValue(param1, 0, type) + ',(' + makeGetValue(param1, 0, type) + '==' + param2 + ' && (' + makeSetValue(param1, 0, param3, type) + ')),tempValue)'; + return '(tempValue=' + makeGetValue(param1, 0, type) + ',(' + makeGetValue(param1, 0, type) + '==' + param2 + ' && (' + makeSetValue(param1, 0, param3, type, null, null, null, null, ',') + ')),tempValue)'; } default: throw 'unhandled atomic op: ' + item.op; } @@ -1279,6 +1283,19 @@ function JSify(data, functionsOnly, givenFunctions) { return; } + // Print out global variables and postsets TODO: batching + if (phase == 'pre') { + legalizedI64s = false; + JSify(analyzer(intertyper(data.unparsedGlobalss[0].lines, true), true), true, Functions); + data.unparsedGlobalss = null; + + var generated = itemsDict.functionStub.concat(itemsDict.GlobalVariablePostSet); + generated.forEach(function(item) { print(indentify(item.JS || '', 2)); }); + } else { + assert(data.unparsedGlobalss[0].lines.length == 0, dump([phase, data.unparsedGlobalss])); + assert(itemsDict.functionStub.length == 0, dump([phase, itemsDict.functionStub])); + } + if (phase == 'pre' || phase == 'funcs') { // serialize out the data that later passes need PassManager.serialize(); // XXX for funcs pass, do not serialize it all. I think we just need which were indexized. @@ -1299,8 +1316,6 @@ function JSify(data, functionsOnly, givenFunctions) { print(read('headless.js').replace("'%s'", "'http://emscripten.org'").replace("'?%s'", "''").replace('%s,', 'null,').replace('%d', '0')); print('}'); } - var generated = itemsDict.functionStub.concat(itemsDict.GlobalVariablePostSet); - generated.forEach(function(item) { print(indentify(item.JS || '', 2)); }); if (RUNTIME_TYPE_INFO) { Types.cleanForRuntime(); print('Runtime.typeInfo = ' + JSON.stringify(Types.types)); @@ -1310,11 +1325,6 @@ function JSify(data, functionsOnly, givenFunctions) { var postParts = processMacros(preprocess(read(postFile))).split('{{GLOBAL_VARS}}'); print(postParts[0]); - // Print out global variables and postsets TODO: batching - legalizedI64s = false; - JSify(analyzer(intertyper(data.unparsedGlobalss[0].lines, true), true), true, Functions); - data.unparsedGlobalss = null; - print(Functions.generateIndexing()); // done last, as it may rely on aliases set in postsets // Load runtime-linked libraries diff --git a/src/library.js b/src/library.js index fd5e0fae..06661d59 100644 --- a/src/library.js +++ b/src/library.js @@ -2457,20 +2457,23 @@ LibraryManager.library = { var argIndex = 0; var next; - next = 1; mainLoop: - for (var formatIndex = 0; formatIndex < format.length; formatIndex++) { + for (var formatIndex = 0; formatIndex < format.length;) { + if (format[formatIndex] === '%' && format[formatIndex+1] == 'n') { + var argPtr = {{{ makeGetValue('varargs', 'argIndex', 'void*') }}}; + argIndex += Runtime.getNativeFieldSize('void*'); + {{{ makeSetValue('argPtr', 0, 'soFar', 'i32') }}}; + formatIndex += 2; + continue; + } + // remove whitespace while (1) { next = get(); if (next == 0) return fields; if (!(next in __scanString.whiteSpace)) break; - } - unget(next); - - if (next <= 0) return fields; - var next = get(); - if (next <= 0) return fields; // End of input. + } + unget(); if (format[formatIndex] === '%') { formatIndex++; @@ -2504,6 +2507,7 @@ LibraryManager.library = { // Read characters according to the format. floats are trickier, they may be in an unfloat state in the middle, then be a valid float later if (type == 'f') { var last = 0; + next = get(); while (next > 0) { buffer.push(String.fromCharCode(next)); if (__isFloat(buffer.join(''))) { @@ -2511,12 +2515,12 @@ LibraryManager.library = { } next = get(); } - unget(next); - while (buffer.length > last) { - unget(buffer.pop().charCodeAt(0)); + for (var i = 0; i < buffer.length - last + 1; i++) { + unget(); } + buffer.length = last; + } else { next = get(); - } else if (type != 'n') { var first = true; while ((curr < max_ || isNaN(max_)) && next > 0) { if (!(next in __scanString.whiteSpace) && // stop on whitespace @@ -2530,13 +2534,14 @@ LibraryManager.library = { buffer.push(String.fromCharCode(next)); next = get(); curr++; + first = false; } else { break; } - first = false; } + unget(); } - if (buffer.length === 0 && type != 'n') return 0; // Failure. + if (buffer.length === 0) return 0; // Failure. var text = buffer.join(''); var argPtr = {{{ makeGetValue('varargs', 'argIndex', 'void*') }}}; argIndex += Runtime.getNativeFieldSize('void*'); @@ -2566,31 +2571,26 @@ LibraryManager.library = { {{{ makeSetValue('argPtr', 'j', 'array[j]', 'i8') }}} } break; - case 'n': - {{{ makeSetValue('argPtr', 0, 'soFar-1', 'i32') }}} - break; } - if (type != 'n') fields++; - if (next <= 0) break mainLoop; // End of input. + fields++; } else if (format[formatIndex] in __scanString.whiteSpace) { + next = get(); while (next in __scanString.whiteSpace) { - next = get(); if (next <= 0) break mainLoop; // End of input. + next = get(); } unget(next); + formatIndex++; } else { // Not a specifier. + next = get(); if (format[formatIndex].charCodeAt(0) !== next) { unget(next); break mainLoop; } + formatIndex++; } } - // 'n' is special in that it needs no input. so it can be at the end, even with nothing left to read - if (format[formatIndex-1] == '%' && format[formatIndex] == 'n') { - var argPtr = {{{ makeGetValue('varargs', 'argIndex', 'void*') }}}; - {{{ makeSetValue('argPtr', 0, 'soFar-1', 'i32') }}} - } return fields; }, // Performs prtinf-style formatting. @@ -2741,7 +2741,7 @@ LibraryManager.library = { var signed = next == 'd'.charCodeAt(0) || next == 'i'.charCodeAt(0); argSize = argSize || 4; var currArg = getNextArg('i' + (argSize * 8)); -#if PRECISE_I64_MATH == 1 +#if PRECISE_I64_MATH var origArg = currArg; #endif var argText; @@ -2760,12 +2760,12 @@ LibraryManager.library = { var currAbsArg = Math.abs(currArg); var prefix = ''; if (next == 'd'.charCodeAt(0) || next == 'i'.charCodeAt(0)) { -#if PRECISE_I64_MATH == 1 - if (argSize == 8 && i64Math) argText = i64Math.stringify(origArg[0], origArg[1]); else +#if PRECISE_I64_MATH + if (argSize == 8 && i64Math) argText = i64Math.stringify(origArg[0], origArg[1], null); else #endif argText = reSign(currArg, 8 * argSize, 1).toString(10); } else if (next == 'u'.charCodeAt(0)) { -#if PRECISE_I64_MATH == 1 +#if PRECISE_I64_MATH if (argSize == 8 && i64Math) argText = i64Math.stringify(origArg[0], origArg[1], true); else #endif argText = unSign(currArg, 8 * argSize, 1).toString(10); @@ -2774,6 +2774,9 @@ LibraryManager.library = { argText = (flagAlternative ? '0' : '') + currAbsArg.toString(8); } else if (next == 'x'.charCodeAt(0) || next == 'X'.charCodeAt(0)) { prefix = flagAlternative ? '0x' : ''; +#if PRECISE_I64_MATH + if (argSize == 8 && i64Math) argText = (origArg[1]>>>0).toString(16) + (origArg[0]>>>0).toString(16); else +#endif if (currArg < 0) { // Represent negative numbers in hex as 2's complement. currArg = -currArg; @@ -3452,8 +3455,9 @@ LibraryManager.library = { // int fscanf(FILE *restrict stream, const char *restrict format, ... ); // http://pubs.opengroup.org/onlinepubs/000095399/functions/scanf.html if (FS.streams[stream]) { - var get = function() { return _fgetc(stream); }; - var unget = function(c) { return _ungetc(c, stream); }; + var stack = []; + var get = function() { var ret = _fgetc(stream); stack.push(ret); return ret }; + var unget = function(c) { return _ungetc(stack.pop(), stream) }; return __scanString(format, get, unget, varargs); } else { return -1; @@ -3735,6 +3739,7 @@ LibraryManager.library = { strtod_l: 'strtod', // no locale support yet strtold: 'strtod', // XXX add real support for long double strtold_l: 'strtold', // no locale support yet + strtof: 'strtod', // use stdtod to handle strtof _parseInt__deps: ['isspace', '__setErrNo', '$ERRNO_CODES'], _parseInt: function(str, endptr, base, min, max, bits, unsign) { @@ -4035,6 +4040,10 @@ LibraryManager.library = { return Math.floor(Math.random()*0x80000000); }, + drand48: function() { + return Math.random(); + }, + realpath__deps: ['$FS', '__setErrNo'], realpath: function(file_name, resolved_name) { // char *realpath(const char *restrict file_name, char *restrict resolved_name); @@ -4742,6 +4751,10 @@ LibraryManager.library = { return 32; }, + llvm_trap: function() { + throw 'trap! ' + new Error().stack; + }, + __assert_fail: function(condition, file, line) { ABORT = true; throw 'Assertion failed: ' + Pointer_stringify(condition);//JSON.stringify(arguments)//condition; @@ -5105,6 +5118,34 @@ LibraryManager.library = { llvm_objectsize_i32: function() { return -1 }, // TODO: support this // ========================================================================== + // llvm-mono integration + // ========================================================================== + + llvm_mono_load_i8_p0i8: function(ptr) { + return {{{ makeGetValue('ptr', 0, 'i8') }}}; + }, + + llvm_mono_store_i8_p0i8: function(value, ptr) { + {{{ makeSetValue('ptr', 0, 'value', 'i8') }}}; + }, + + llvm_mono_load_i16_p0i16: function(ptr) { + return {{{ makeGetValue('ptr', 0, 'i16') }}}; + }, + + llvm_mono_store_i16_p0i16: function(value, ptr) { + {{{ makeSetValue('ptr', 0, 'value', 'i16') }}}; + }, + + llvm_mono_load_i32_p0i32: function(ptr) { + return {{{ makeGetValue('ptr', 0, 'i32') }}}; + }, + + llvm_mono_store_i32_p0i32: function(value, ptr) { + {{{ makeSetValue('ptr', 0, 'value', 'i32') }}}; + }, + + // ========================================================================== // math.h // ========================================================================== @@ -5124,6 +5165,64 @@ LibraryManager.library = { atan2f: 'Math.atan2', exp: 'Math.exp', expf: 'Math.exp', + + // The erf and erfc functions are inspired from + // http://www.digitalmars.com/archives/cplusplus/3634.html + // and mruby source code at + // https://github.com/mruby/mruby/blob/master/src/math.c + erfc: function (x) { + var MATH_TOLERANCE = 1E-12; + var ONE_SQRTPI = 0.564189583547756287; + var a = 1; + var b = x; + var c = x; + var d = x * x + 0.5; + var n = 1.0; + var q2 = b / d; + var q1, t; + + if (Math.abs(x) < 2.2) { + return 1.0 - _erf(x); + } + if (x < 0) { + return 2.0 - _erfc(-x); + } + do { + t = a * n + b * x; + a = b; + b = t; + t = c * n + d * x; + c = d; + d = t; + n += 0.5; + q1 = q2; + q2 = b / d; + } while (Math.abs(q1 - q2) / q2 > MATH_TOLERANCE); + return (ONE_SQRTPI * Math.exp(- x * x) * q2); + }, + erfcf: 'erfcf', + erf: function (x) { + var MATH_TOLERANCE = 1E-12; + var TWO_SQRTPI = 1.128379167095512574; + var sum = x; + var term = x; + var xsqr = x*x; + var j = 1; + + if (Math.abs(x) > 2.2) { + return 1.0 - _erfc(x); + } + do { + term *= xsqr / j; + sum -= term / (2 * j + 1); + ++j; + term *= xsqr / j; + sum += term / (2 * j + 1); + ++j; + } while (Math.abs(term / sum) > MATH_TOLERANCE); + return (TWO_SQRTPI * sum); + }, + erff: 'erf', log: 'Math.log', logf: 'Math.log', sqrt: 'Math.sqrt', @@ -5266,7 +5365,7 @@ LibraryManager.library = { }, fmaxf: 'fmax', fmin: function(x, y) { - return isNaN(x) ? y : isNaN(y) ? x : Math.max(x, y); + return isNaN(x) ? y : isNaN(y) ? x : Math.min(x, y); }, fminf: 'fmin', fma: function(x, y, z) { @@ -5841,7 +5940,7 @@ LibraryManager.library = { setjmp__inline: function(env) { // Save the label - return '(' + makeSetValue(env, '0', 'label', 'i32') + ', 0)'; + return '(setjmped = true, ' + makeSetValue(env, '0', 'label', 'i32') + ', 0)'; }, longjmp: function(env, value) { @@ -6404,6 +6503,17 @@ LibraryManager.library = { _pthread_key_create.keys[key] = value; }, + pthread_cleanup_push: function(routine, arg) { + __ATEXIT__.push({ func: function() { FUNCTION_TABLE[routine](arg) } }) + _pthread_cleanup_push.level = __ATEXIT__.length; + }, + + pthread_cleanup_pop: function() { + assert(_pthread_cleanup_push.level == __ATEXIT__.length, 'cannot pop if something else added meanwhile!'); + __ATEXIT__.pop(); + _pthread_cleanup_push.level = __ATEXIT__.length; + }, + // ========================================================================== // malloc.h // ========================================================================== @@ -6833,6 +6943,18 @@ LibraryManager.library = { return eval(Pointer_stringify(ptr)); }, + emscripten_run_script_string: function(ptr) { + var s = eval(Pointer_stringify(ptr)); + var me = _emscripten_run_script_string; + if (!me.bufferSize || me.bufferSize < s.length+1) { + if (me.bufferSize) _free(me.buffer); + me.bufferSize = s.length+1; + me.buffer = _malloc(me.bufferSize); + } + writeStringToMemory(s, me.buffer); + return me.buffer; + }, + emscripten_random: function() { return Math.random(); }, diff --git a/src/library_gc.js b/src/library_gc.js index bf0a6aff..a06e2f01 100644 --- a/src/library_gc.js +++ b/src/library_gc.js @@ -16,6 +16,11 @@ if (GC_SUPPORT) { init: function() { assert(!GC.initted); GC.initted = true; + + _GC_finalize_on_demand = _malloc(4); setValue(_GC_finalize_on_demand, 0, 'i32') + _GC_java_finalization = _malloc(4); setValue(_GC_java_finalization, 0, 'i32'); + _GC_finalizer_notifier = _malloc(4); setValue(_GC_finalizer_notifier, 0, 'i32'); + if (ENVIRONMENT_IS_WEB) { setInterval(function() { GC.maybeCollect(); @@ -159,7 +164,13 @@ if (GC_SUPPORT) { GC_FORCE_COLLECT__deps: ['$GC'], GC_FORCE_COLLECT: function() { GC.collect(); - } + }, + + GC_finalize_on_demand: 0, + GC_java_finalization: 0, + GC_finalizer_notifier: 0, + + GC_enable_incremental: function(){}, }; mergeInto(LibraryManager.library, LibraryGC); diff --git a/src/library_gl.js b/src/library_gl.js index 4d83572e..b4098813 100644 --- a/src/library_gl.js +++ b/src/library_gl.js @@ -454,8 +454,7 @@ var LibraryGL = { }, glBufferSubData: function(target, offset, size, data) { - var floatArray = {{{ makeHEAPView('F32', 'data', 'data+size') }}}; - Module.ctx.bufferSubData(target, offset, floatArray); + Module.ctx.bufferSubData(target, offset, HEAPU8.subarray(data, data+size)); }, glIsBuffer: function(buffer) { diff --git a/src/modules.js b/src/modules.js index 60b4ff87..9ef87691 100644 --- a/src/modules.js +++ b/src/modules.js @@ -5,11 +5,11 @@ var LLVM = { LINKAGES: set('private', 'linker_private', 'linker_private_weak', 'linker_private_weak_def_auto', 'internal', 'available_externally', 'linkonce', 'common', 'weak', 'appending', 'extern_weak', 'linkonce_odr', - 'weak_odr', 'externally_visible', 'dllimport', 'dllexport', 'unnamed_addr'), + 'weak_odr', 'externally_visible', 'dllimport', 'dllexport', 'unnamed_addr', 'thread_local'), VISIBILITIES: set('default', 'hidden', 'protected'), PARAM_ATTR: set('noalias', 'signext', 'zeroext', 'inreg', 'sret', 'nocapture', 'nest'), FUNC_ATTR: set('hidden', 'nounwind', 'define', 'inlinehint', '{'), - CALLING_CONVENTIONS: set('ccc', 'fastcc', 'coldcc', 'cc10', 'x86_fastcallcc', 'x86_stdcallcc'), + CALLING_CONVENTIONS: set('ccc', 'fastcc', 'coldcc', 'cc10', 'x86_fastcallcc', 'x86_stdcallcc', 'cc11'), ACCESS_OPTIONS: set('volatile', 'atomic'), INVOKE_MODIFIERS: set('alignstack', 'alwaysinline', 'inlinehint', 'naked', 'noimplicitfloat', 'noinline', 'alwaysinline attribute.', 'noredzone', 'noreturn', 'nounwind', 'optsize', 'readnone', 'readonly', 'ssp', 'sspreq'), SHIFTS: set('ashr', 'lshr', 'shl'), @@ -207,8 +207,9 @@ var Types = { needAnalysis: {}, // Types noticed during parsing, that need analysis - preciseI64MathUsed: false // Set to true if we actually use precise i64 math: If PRECISE_I64_MATH is set, and also such math is actually - // needed (+,-,*,/,% - we do not need it for bitops) + // Set to true if we actually use precise i64 math: If PRECISE_I64_MATH is set, and also such math is actually + // needed (+,-,*,/,% - we do not need it for bitops), or PRECISE_I64_MATH is 2 (forced) + preciseI64MathUsed: (PRECISE_I64_MATH == 2) }; var Functions = { @@ -242,16 +243,14 @@ var Functions = { for (var ident in this.indexedFunctions) { vals[this.indexedFunctions[ident]] = ident; } - // Resolve multi-level aliases all the way down for (var i = 0; i < vals.length; i++) { while (1) { var varData = Variables.globals[vals[i]]; if (!(varData && varData.resolvedAlias)) break; - vals[i] = vals[varData.resolvedAlias]; + vals[i] = vals[+varData.resolvedAlias || eval(varData.resolvedAlias)]; // might need to eval to turn (6) into 6 } } - var indices = vals.toString().replace('"', ''); if (BUILD_AS_SHARED_LIB) { // Shared libraries reuse the parent's function table. @@ -302,11 +301,21 @@ function cDefine(key) { var PassManager = { serialize: function() { - print('\n//FORWARDED_DATA:' + JSON.stringify({ - Types: Types, - Variables: Variables, - Functions: Functions - })); + if (phase == 'pre') { + print('\n//FORWARDED_DATA:' + JSON.stringify({ + Types: Types, + Variables: Variables, + Functions: Functions + })); + } else if (phase == 'funcs') { + print('\n//FORWARDED_DATA:' + JSON.stringify({ + Types: { preciseI64MathUsed: Types.preciseI64MathUsed }, + Functions: { + blockAddresses: Functions.blockAddresses, + indexedFunctions: Functions.indexedFunctions + } + })); + } }, load: function(json) { var data = JSON.parse(json); diff --git a/src/parseTools.js b/src/parseTools.js index 6a10176c..c70b511a 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -167,7 +167,15 @@ function isFunctionDef(token, out) { return !fail; } + +function isPossiblyFunctionType(type) { + // A quick but unreliable way to see if something is a function type. Yes is just 'maybe', no is definite. + var len = type.length; + return type[len-2] == ')' && type[len-1] == '*'; +} + function isFunctionType(type, out) { + if (!isPossiblyFunctionType(type)) return false; type = type.replace(/"[^"]+"/g, '".."'); var parts; // hackish, but quick splitting of function def parts. this must be fast as it happens a lot @@ -184,14 +192,12 @@ function isFunctionType(type, out) { return isType(parts[0]) && isFunctionDef({ text: text, item: tokenize(text.substr(1, text.length-2), true) }, out); } -function isType(type) { // TODO! - return isVoidType(type) || Runtime.isNumberType(type) || isStructType(type) || isPointerType(type) || isFunctionType(type); -} - -function isPossiblyFunctionType(type) { - // A quick but unreliable way to see if something is a function type. Yes is just 'maybe', no is definite. - var suffix = ')*'; - return type.substr(-suffix.length) == suffix; +var isTypeCache = {}; // quite hot, optimize as much as possible +function isType(type) { + if (type in isTypeCache) return isTypeCache[type]; + var ret = isPointerType(type) || isVoidType(type) || Runtime.isNumberType(type) || isStructType(type) || isFunctionType(type); + isTypeCache[type] = ret; + return ret; } function isVarArgsFunctionType(type) { @@ -875,7 +881,6 @@ function correctRoundings() { } function checkSpecificSafeHeap() { - assert(!(SAFE_HEAP >= 2 && !Debugging.on), 'Need debugging for line-specific checks'); if (!Framework.currItem) return false; return (SAFE_HEAP === 2 && Debugging.getIdentifier() in SAFE_HEAP_LINES) || (SAFE_HEAP === 3 && !(Debugging.getIdentifier() in SAFE_HEAP_LINES)); @@ -1997,9 +2002,7 @@ function parseBlockAddress(segment) { } function finalizeBlockAddress(param) { - assert(param.func in Functions.blockAddresses); - assert(param.label in Functions.blockAddresses[param.func]); - return Functions.blockAddresses[param.func][param.label]; + return '{{{ BA_' + param.func + '|' + param.label + ' }}}'; // something python will replace later } function stripCorrections(param) { diff --git a/src/preamble.js b/src/preamble.js index 16865bd0..d2bbc6a4 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -544,6 +544,9 @@ function Pointer_stringify(ptr, /* optional */ length) { var i = 0; var t; while (1) { +#if ASSERTIONS + assert(i < TOTAL_MEMORY); +#endif t = {{{ makeGetValue('ptr', 'i', 'i8', 0, 1) }}}; if (nullTerminated && t == 0) break; ret += utf8.processCChar(t); @@ -602,26 +605,26 @@ function enlargeMemory() { } #if USE_TYPED_ARRAYS == 1 var oldIHEAP = IHEAP; - HEAP = IHEAP = new Int32Array(TOTAL_MEMORY); + Module['HEAP'] = Module['IHEAP'] = HEAP = IHEAP = new Int32Array(TOTAL_MEMORY); IHEAP.set(oldIHEAP); IHEAPU = new Uint32Array(IHEAP.buffer); #if USE_FHEAP var oldFHEAP = FHEAP; - FHEAP = new Float64Array(TOTAL_MEMORY); + Module['FHEAP'] = FHEAP = new Float64Array(TOTAL_MEMORY); FHEAP.set(oldFHEAP); #endif #endif #if USE_TYPED_ARRAYS == 2 var oldHEAP8 = HEAP8; var buffer = new ArrayBuffer(TOTAL_MEMORY); - HEAP8 = new Int8Array(buffer); - HEAP16 = new Int16Array(buffer); - HEAP32 = new Int32Array(buffer); - HEAPU8 = new Uint8Array(buffer); - HEAPU16 = new Uint16Array(buffer); - HEAPU32 = new Uint32Array(buffer); - HEAPF32 = new Float32Array(buffer); - HEAPF64 = new Float64Array(buffer); + Module['HEAP8'] = HEAP8 = new Int8Array(buffer); + Module['HEAP16'] = HEAP16 = new Int16Array(buffer); + Module['HEAP32'] = HEAP32 = new Int32Array(buffer); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(buffer); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(buffer); + Module['HEAPU32'] = HEAPU32 = new Uint32Array(buffer); + Module['HEAPF32'] = HEAPF32 = new Float32Array(buffer); + Module['HEAPF64'] = HEAPF64 = new Float64Array(buffer); HEAP8.set(oldHEAP8); #endif #endif @@ -750,7 +753,11 @@ function exitRuntime() { function String_len(ptr) { var i = ptr; - while ({{{ makeGetValue('i++', '0', 'i8') }}}) {}; // Note: should be |!= 0|, technically. But this helps catch bugs with undefineds + while ({{{ makeGetValue('i++', '0', 'i8') }}}) { // Note: should be |!= 0|, technically. But this helps catch bugs with undefineds +#if ASSERTIONS + assert(i < TOTAL_MEMORY); +#endif + } return i - ptr - 1; } Module['String_len'] = String_len; diff --git a/src/settings.js b/src/settings.js index 58635950..5dc1e2eb 100644 --- a/src/settings.js +++ b/src/settings.js @@ -88,6 +88,10 @@ var PRECISE_I64_MATH = 1; // If enabled, i64 addition etc. is emulated - which i // Note that we do not catch 32-bit multiplication by default (which must be done in // 64 bits for high values for full precision) - you must manually set PRECISE_I32_MUL // for that. + // If set to 2, we always include the i64 math code, which is necessary in the case + // that we can't know at compile time that 64-bit math is needed. For example, if you + // print 64-bit values with printf, but never add them, we can't know at compile time + // and you need to set this to 2. var PRECISE_I32_MUL = 0; // If enabled, i64 math is done in i32 multiplication. This is necessary if the values // exceed the JS double-integer limit of ~52 bits. This option can normally be disabled // because generally i32 multiplication works ok without it, and enabling it has a big @@ -192,6 +196,7 @@ var PGO = 0; // Profile-guided optimization. var PROFILE = 0; // Enables runtime profiling. See test_profiling for a usage example. +var EXPORT_ALL = 0; // If true, we export all the symbols var EXPORTED_FUNCTIONS = ['_main', '_malloc', '_free']; // Functions that are explicitly exported, so they are guaranteed to // be accessible outside of the generated code even after running closure compiler. // Note the necessary prefix of "_". diff --git a/src/utility.js b/src/utility.js index 632ee08c..f3ece90b 100644 --- a/src/utility.js +++ b/src/utility.js @@ -184,7 +184,7 @@ function dprint() { text = text(); // Allows deferred calculation, so dprints don't slow us down when not needed } text = DPRINT_INDENT + '// ' + text; - print(text); + printErr(text); } var PROF_ORIGIN = Date.now(); diff --git a/system/include/dlfcn.h b/system/include/dlfcn.h index 6854dab0..3a4edd00 100644 --- a/system/include/dlfcn.h +++ b/system/include/dlfcn.h @@ -3,10 +3,11 @@ extern "C" { #endif -#define RTLD_LAZY 1 -#define RTLD_NOW 2 -#define RTLD_GLOBAL 4 -#define RTLD_LOCAL 8 +#define RTLD_DEFAULT 0 +#define RTLD_LAZY 1 +#define RTLD_NOW 2 +#define RTLD_GLOBAL 4 +#define RTLD_LOCAL 8 typedef struct { const char *dli_fname; diff --git a/system/include/emscripten/emscripten.h b/system/include/emscripten/emscripten.h index 871a7162..3eefe0b8 100644 --- a/system/include/emscripten/emscripten.h +++ b/system/include/emscripten/emscripten.h @@ -31,6 +31,7 @@ extern "C" { */ extern void emscripten_run_script(const char *script); extern int emscripten_run_script_int(const char *script); +extern char *emscripten_run_script_string(const char *script); // uses a single buffer - shared between calls! extern void emscripten_async_run_script(const char *script, int millis); /* diff --git a/system/include/gc.h b/system/include/gc.h index 996bc9ec..e0419dcb 100644 --- a/system/include/gc.h +++ b/system/include/gc.h @@ -8,7 +8,7 @@ extern "C" { #endif -void __attribute__((used)) __GC_KEEPALIVE__() { +static void __attribute__((used)) __GC_KEEPALIVE__() { // Force inclusion of necessary dlmalloc functions static int times = 1; void *x = malloc(times); @@ -44,6 +44,14 @@ void GC_MAYBE_COLLECT(); /* Forces a GC. Mainly useful for testing, but call it if you know a good time to GC in your app. */ void GC_FORCE_COLLECT(); +typedef void (*GC_finalization_proc)(void *func, void *arg); +extern void (*GC_finalizer_notifier)(); + +extern int GC_finalize_on_demand; +extern int GC_java_finalization; + +void GC_enable_incremental(); + #ifdef __cplusplus } #endif diff --git a/system/include/libc/limits.h b/system/include/libc/limits.h index 93362b88..3f1def0f 100644 --- a/system/include/libc/limits.h +++ b/system/include/libc/limits.h @@ -144,3 +144,12 @@ #define PATH_MAX 4096 #endif +/* XXX EMSCRIPTEN */ +#ifndef _LIBC_LIMITS2_H_ +#define _LIBC_LIMITS2_H_ 1 + +#define _POSIX_PATH_MAX PATH_MAX +#define PTHREAD_STACK_MIN 0 + +#endif + diff --git a/system/include/libc/sys/_default_fcntl.h b/system/include/libc/sys/_default_fcntl.h index 188e25c4..379147ed 100644 --- a/system/include/libc/sys/_default_fcntl.h +++ b/system/include/libc/sys/_default_fcntl.h @@ -207,6 +207,8 @@ extern int _open64 _PARAMS ((const char *, int, ...)); /* XXX Emscripten */ #define POSIX_FADV_DONTNEED 135 +#define POSIX_FADV_SEQUENTIAL 136 +#define POSIX_FADV_RANDOM 137 int posix_fadvise(int fd, off_t offset, off_t len, int advice); int posix_fallocate(int fd, off_t offset, off_t len); #define LOCK_SH 1 diff --git a/system/include/libc/sys/features.h b/system/include/libc/sys/features.h index 9faf2b89..87a520a0 100644 --- a/system/include/libc/sys/features.h +++ b/system/include/libc/sys/features.h @@ -25,6 +25,10 @@ extern "C" { #endif +#if EMSCRIPTEN +#define _POSIX_REALTIME_SIGNALS 1 +#endif + /* RTEMS adheres to POSIX -- 1003.1b with some features from annexes. */ #ifdef __rtems__ diff --git a/system/include/libc/sys/resource.h b/system/include/libc/sys/resource.h index f39dcb33..abcfd34d 100644 --- a/system/include/libc/sys/resource.h +++ b/system/include/libc/sys/resource.h @@ -27,6 +27,7 @@ struct rusage { }; /* XXX Emscripten */ +#define PRIO_PROCESS 0 int getrusage(int who, struct rusage *r_usage); /* XXX Emscripten */ diff --git a/system/include/libc/sys/signal.h b/system/include/libc/sys/signal.h index 543b626e..910ccf78 100644 --- a/system/include/libc/sys/signal.h +++ b/system/include/libc/sys/signal.h @@ -114,6 +114,7 @@ struct sigaction _sig_func_ptr sa_handler; sigset_t sa_mask; int sa_flags; + void (*sa_sigaction)(int, siginfo_t *, void *); /* XXX EMSCRIPTEN */ }; #endif /* defined(__rtems__) */ diff --git a/system/include/net/netinet/in.h b/system/include/net/netinet/in.h index 2e4e4e57..2ac98dfe 100644 --- a/system/include/net/netinet/in.h +++ b/system/include/net/netinet/in.h @@ -48,6 +48,26 @@ struct ip_mreq { struct in_addr imr_interface; }; +#define IP_MULTICAST_IF 32 +#define IP_MULTICAST_TTL 33 +#define IP_MULTICAST_LOOP 34 +#define IP_ADD_MEMBERSHIP 35 +#define IP_DROP_MEMBERSHIP 36 +#define IP_UNBLOCK_SOURCE 37 +#define IP_BLOCK_SOURCE 38 +#define IP_ADD_SOURCE_MEMBERSHIP 39 +#define IP_DROP_SOURCE_MEMBERSHIP 40 +#define IP_MSFILTER 41 +#define MCAST_JOIN_GROUP 42 +#define MCAST_BLOCK_SOURCE 43 +#define MCAST_UNBLOCK_SOURCE 44 +#define MCAST_LEAVE_GROUP 45 +#define MCAST_JOIN_SOURCE_GROUP 46 +#define MCAST_LEAVE_SOURCE_GROUP 47 +#define MCAST_MSFILTER 48 +#define IP_MULTICAST_ALL 49 +#define IP_UNICAST_IF 50 + #ifdef __cplusplus } #endif diff --git a/system/include/netdb.h b/system/include/netdb.h index 20f876df..6e069448 100644 --- a/system/include/netdb.h +++ b/system/include/netdb.h @@ -9,6 +9,7 @@ extern "C" { #define TRY_AGAIN 2 #define NO_RECOVERY 3 #define NO_DATA 4 +#define NO_ADDRESS 5 #define IP_TOS 1 #define IP_TTL 2 @@ -48,6 +49,8 @@ void endhostent(void); void herror(const char* s); const char* hstrerror(int err); +extern int h_errno; + #ifdef __cplusplus } #endif diff --git a/system/include/sys/ioctl.h b/system/include/sys/ioctl.h index 12857ae3..c54d4ccc 100644 --- a/system/include/sys/ioctl.h +++ b/system/include/sys/ioctl.h @@ -12,6 +12,7 @@ extern "C" { #define TIOCGWINSZ 80 // bogus #define FIONREAD 1 +#define FIONBIO 2 int ioctl(int d, int request, ...); diff --git a/system/include/sys/poll.h b/system/include/sys/poll.h index 55e85237..7521ed0e 100644 --- a/system/include/sys/poll.h +++ b/system/include/sys/poll.h @@ -11,6 +11,7 @@ extern "C" { #define POLLNVAL 4 #define POLLERR 8 #define POLLHUP 16 +#define POLLPRI 32 struct pollfd { int fd; diff --git a/system/include/sys/socket.h b/system/include/sys/socket.h index e33dbefd..10ba5ce8 100644 --- a/system/include/sys/socket.h +++ b/system/include/sys/socket.h @@ -11,8 +11,11 @@ extern "C" { // Note that the values of these constants are mostly arbitrary numbers. #define SOMAXCONN 128 +#define PF_LOCAL 1 +#define PF_UNIX PF_LOCAL #define PF_INET 2 #define SO_BROADCAST 6 +#define AF_UNIX PF_UNIX #define AF_UNSPEC 100 #define SOCK_STREAM 200 @@ -25,8 +28,14 @@ extern "C" { #define SO_LINGER 70 #define SO_NOSIGPIPE 80 #define SO_KEEPALIVE 90 +#define SO_OOBINLINE 100 +#define SO_NO_CHECK 110 +#define SO_PRIORITY 120 +#define SO_LINGER 130 +#define SO_BSDCOMPAT 140 -#define SHUT_RDWR 1 +#define SHUT_RD 1 +#define SHUT_RDWR 2 typedef unsigned int sa_family_t; #define AF_INET 1 @@ -72,6 +81,13 @@ struct linger { int l_linger; }; +#define SIOCATMARK 0x8905 + +#define SOCK_RAW 111 +#define SOCK_SEQPACKET 555 + +#define PF_APPLETALK 5 + #ifdef __cplusplus } #endif diff --git a/system/include/sys/statvfs.h b/system/include/sys/statvfs.h index 192be153..8e17f817 100644 --- a/system/include/sys/statvfs.h +++ b/system/include/sys/statvfs.h @@ -22,6 +22,8 @@ struct statvfs { int statvfs(const char *path, struct statvfs *s); +#define ST_RDONLY 0 + #ifdef __cplusplus } #endif diff --git a/system/include/sys/un.h b/system/include/sys/un.h new file mode 100644 index 00000000..b70425fb --- /dev/null +++ b/system/include/sys/un.h @@ -0,0 +1,65 @@ +/*- + * Copyright (c) 1982, 1986, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)un.h 8.3 (Berkeley) 2/19/95 + * $FreeBSD$ + */ + +#ifndef _SYS_UN_H_ +#define _SYS_UN_H_ + +#include <sys/cdefs.h> +#include <sys/_types.h> + +/* + * Definitions for UNIX IPC domain. + */ +struct sockaddr_un { + unsigned char sun_len; /* sockaddr len including null */ + sa_family_t sun_family; /* AF_UNIX */ + char sun_path[104]; /* path name (gag) */ +}; + +#if __BSD_VISIBLE + +/* Socket options. */ +#define LOCAL_PEERCRED 0x001 /* retrieve peer credentials */ +#define LOCAL_CREDS 0x002 /* pass credentials to receiver */ +#define LOCAL_CONNWAIT 0x004 /* connects block until accepted */ + +#ifndef _KERNEL + +/* actual length of an initialized sockaddr_un */ +#define SUN_LEN(su) \ + (sizeof(*(su)) - sizeof((su)->sun_path) + strlen((su)->sun_path)) + +#endif /* !_KERNEL */ + +#endif /* __BSD_VISIBLE */ + +#endif /* !_SYS_UN_H_ */ diff --git a/tests/cases/gepoverflow.txt b/tests/cases/gepoverflow.txt index 90772c33..bd3091fd 100644 --- a/tests/cases/gepoverflow.txt +++ b/tests/cases/gepoverflow.txt @@ -1,2 +1,2 @@ -*5246502,5247072* +*5246498,5247068* *-514,56* diff --git a/tests/gl_subdata.cpp b/tests/gl_subdata.cpp new file mode 100644 index 00000000..d159b2b2 --- /dev/null +++ b/tests/gl_subdata.cpp @@ -0,0 +1,141 @@ +#define GL_GLEXT_PROTOTYPES +#define EGL_EGLEXT_PROTOTYPES +#include <cmath> +#include <iostream> +#include <vector> +extern "C" { +#include <GL/gl.h> +#include <GL/glut.h> +} +static const char vertex_shader[] = + "#ifdef GL_ES\n" + "precision highp float;\n" + "#endif\n" + "attribute float indices;\n" + "uniform sampler2D nodeInfo;\n" + "varying vec4 color;" + "\n" + "void main(void)\n" + "{\n" + " float s = (indices + 0.5) / 512.; \n" + " vec4 v = texture2D(nodeInfo, vec2( s, 0.5));\n" + " gl_Position = vec4(v.x, v.y, 0.5, 1.);\n" + " gl_PointSize = v.z;\n" + " color = vec4(0.5 + v.w/2., 0.5 + 0.5 * v.w/2., 0.5, 1);\n" + "}\n"; +static const char fragment_shader[] = + "#ifdef GL_ES\n" + "precision highp float;\n" + "#endif\n" + "\n" + "varying vec4 color;\n" + "void main(void)\n" + "{\n" + " float dst = distance(vec2(0.5, 0.5), gl_PointCoord); \n" + " gl_FragColor = color;\n" + " if ( dst > 0.3) {" + " gl_FragColor = vec4(0., 0., 0.5, 0.2);\n" + "}\n" + "if ( dst > 0.5) discard;\n" + "}"; +struct NodeInfo { //structure that we want to transmit to our shaders + float x; + float y; + float s; + float c; +}; +GLuint nodeTexture; //texture id used to bind +GLuint nodeSamplerLocation; //shader sampler address +GLuint indicesAttributeLocation; //shader attribute address +GLuint indicesVBO; //Vertex Buffer Object Id; +const int nbNodes = 512; +NodeInfo data[nbNodes]; //our data that will be transmitted using float texture. +double alpha = 0; //use to make a simple funny effect; +static void updateFloatTexture() { + int count = 0; + for (float x=0; x < nbNodes; ++x ) { + data[count].x = 0.2*pow(cos(alpha), 3) + (sin(alpha)*3. + 3.5) * x/nbNodes * cos(alpha + x/nbNodes * 16. * M_PI); + data[count].y = 0.2*pow(sin(alpha), 3) + (sin(alpha)*3. + 3.5) * x/nbNodes * sin(alpha + x/nbNodes * 16. * M_PI); + data[count].s = (16. + 16. * cos(alpha + x/nbNodes * 32. * M_PI)) + 8.;// * fmod(x/nbNodes + alpha, 1.) + 5.; + data[count].c = 0.5 + 0.5 * sin(alpha + x/nbNodes * 32. * M_PI); + ++count; + } + glBindTexture(GL_TEXTURE_2D, nodeTexture); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, nbNodes, 1, 0, GL_RGBA, GL_FLOAT, data); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glBindTexture(GL_TEXTURE_2D, NULL); + alpha -= 0.001; +} +static void glut_draw_callback(void) { + glDisable(GL_CULL_FACE); + glDisable(GL_DEPTH_TEST); + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + glClearColor(1., 1., 1., 0.); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + glActiveTexture(GL_TEXTURE0); + updateFloatTexture(); //we change the texture each time to create the effect (it is just for the test) + glBindTexture(GL_TEXTURE_2D, nodeTexture); + glUniform1i(nodeSamplerLocation, GL_TEXTURE0); + glEnableVertexAttribArray(0); + glBindBuffer(GL_ARRAY_BUFFER, indicesVBO); + glVertexAttribPointer(0, 1, GL_FLOAT, GL_FALSE, 0, NULL); + glDrawArrays(GL_POINTS, 0, nbNodes); + glutSwapBuffers(); +} +GLuint createShader(const char source[], int type) { + char msg[512]; + GLuint shader = glCreateShader(type); + glShaderSource(shader, 1, (const GLchar**)(&source), NULL); + glCompileShader(shader); + glGetShaderInfoLog(shader, sizeof msg, NULL, msg); + std::cout << "Shader info: " << msg << std::endl; + return shader; +} +static void gl_init(void) { + GLuint program = glCreateProgram(); + glAttachShader(program, createShader(vertex_shader , GL_VERTEX_SHADER)); + glAttachShader(program, createShader(fragment_shader, GL_FRAGMENT_SHADER)); + glLinkProgram(program); + char msg[512]; + glGetProgramInfoLog(program, sizeof msg, NULL, msg); + std::cout << "info: " << msg << std::endl; + glUseProgram(program); + std::vector<float> elements(nbNodes); + int count = 0; + for (float x=0; x < nbNodes; ++x ) { + elements[count] = count; + ++count; + } + /*Create one texture to store all the needed information */ + glGenTextures(1, &nodeTexture); + /* Store the vertices in a vertex buffer object (VBO) */ + glGenBuffers(1, &indicesVBO); + glBindBuffer(GL_ARRAY_BUFFER, indicesVBO); + float zeroes[nbNodes]; + memset(zeroes, 0, sizeof(zeroes)); + glBufferData(GL_ARRAY_BUFFER, elements.size() * sizeof(float), zeroes, GL_STATIC_DRAW); + for (int x = 0; x < nbNodes; x++) { + glBufferSubData(GL_ARRAY_BUFFER, x * sizeof(float), elements.size() * sizeof(float), &elements[x]); + } + /* Get the locations of the uniforms so we can access them */ + nodeSamplerLocation = glGetUniformLocation(program, "nodeInfo"); + glBindAttribLocation(program, 0, "indices"); + //Enable glPoint size in shader, always enable in Open Gl ES 2. + glEnable(GL_VERTEX_PROGRAM_POINT_SIZE); + glEnable(GL_POINT_SPRITE); +} +int main(int argc, char *argv[]) { + glutInit(&argc, argv); + glutInitWindowSize(640, 480); + glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH); + glutCreateWindow("Simple FLOAT Texture Test"); + /* Set up glut callback functions */ + glutDisplayFunc(glut_draw_callback ); + gl_init(); + glutMainLoop(); + return 0; +} + + diff --git a/tests/hello_libcxx_mod1.cpp b/tests/hello_libcxx_mod1.cpp new file mode 100644 index 00000000..2389b1ef --- /dev/null +++ b/tests/hello_libcxx_mod1.cpp @@ -0,0 +1,9 @@ +#include <iostream> + +int main() +{ + std::cout << "hello, world!" << std::endl; + std::cout << "hello, world!" << std::endl; + return 0; +} + diff --git a/tests/runner.py b/tests/runner.py index b7e44c05..09fd7d49 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -970,7 +970,7 @@ m_divisor is 1091269979 Settings.PRECISE_I64_MATH = 0 self.do_run(src, 'unsigned') code = open(os.path.join(self.get_dir(), 'src.cpp.o.js')).read() - assert 'goog.math.Long' not in code and 'jsbn' not in code, 'i64 precise math should not have been included if not asked for' + assert 'goog.math.Long' not in code, 'i64 precise math should not have been included if not asked for' # Verify that even if we ask for precision, if it is not needed it is not included Settings.PRECISE_I64_MATH = 1 @@ -994,7 +994,28 @@ m_divisor is 1091269979 ''' self.do_run(src, '*4903566027370624, 153236438355333*') code = open(os.path.join(self.get_dir(), 'src.cpp.o.js')).read() - assert 'goog.math.Long' not in code and 'jsbn' not in code, 'i64 precise math should not have been included if not actually used' + assert 'goog.math.Long' not in code, 'i64 precise math should not have been included if not actually used' + + # But if we force it to be included, it is. First, a case where we don't need it + Settings.PRECISE_I64_MATH = 2 + self.do_run(open(path_from_root('tests', 'hello_world.c')).read(), 'hello') + code = open(os.path.join(self.get_dir(), 'src.cpp.o.js')).read() + assert 'goog.math.Long' in code, 'i64 precise math should be included if forced' + + # and now one where we do + self.do_run(r''' + #include <stdio.h> + + int main( int argc, char ** argv ) + { + unsigned long a = 0x60DD1695U; + unsigned long b = 0xCA8C4E7BU; + unsigned long long c = (unsigned long long)a * b; + printf( "c = %016llx\n", c ); + + return 0; + } + ''', 'c = 4ca38a6bd2973f97') def test_i64_zextneg(self): if Settings.USE_TYPED_ARRAYS != 2: return self.skip('full i64 stuff only in ta2') @@ -1347,6 +1368,7 @@ c5,de,15,8a def test_floatvars(self): src = ''' #include <stdio.h> + #include <math.h> int main() { float x = 1.234, y = 3.5, q = 0.00000001; @@ -1354,6 +1376,8 @@ c5,de,15,8a int z = x < y; printf("*%d,%d,%.1f,%d,%.4f,%.2f*\\n", z, int(y), y, (int)x, x, q); + printf("%.2f, %.2f, %.2f, %.2f\\n", fmin(0.5, 3.3), fmin(NAN, 3.3), fmax(0.5, 3.3), fmax(NAN, 3.3)); + /* // Rounding behavior float fs[6] = { -2.75, -2.50, -2.25, 2.25, 2.50, 2.75 }; @@ -1365,7 +1389,7 @@ c5,de,15,8a return 0; } ''' - self.do_run(src, '*1,10,10.5,1,1.2340,0.00*') + self.do_run(src, '*1,10,10.5,1,1.2340,0.00*\n0.50, 3.30, 3.30, 3.30\n') def test_globaldoubles(self): src = r''' @@ -1429,6 +1453,24 @@ c5,de,15,8a ''' self.do_run(src, '*3.14,-3.14,1,0,0,0,1,0,1,1,0,2,3,0.0,1.0,0.0,1.0*') + def test_erf(self): + src = ''' + #include <math.h> + #include <stdio.h> + int main() + { + printf("%1.6f, %1.6f, %1.6f, %1.6f, %1.6f, %1.6f\\n", + erf(1.0), + erf(3.0), + erf(-1.0), + erfc(1.0), + erfc(3.0), + erfc(-1.5)); + return 0; + } + ''' + self.do_run(src, '0.842701, 0.999978, -0.842701, 0.157299, 0.000022, 1.966105') + def test_math_hyperbolic(self): src = open(path_from_root('tests', 'hyperbolic', 'src.c'), 'r').read() expected = open(path_from_root('tests', 'hyperbolic', 'output.txt'), 'r').read() @@ -1948,6 +1990,52 @@ c5,de,15,8a else: self.do_run(src, 'second\nmain: 0\n') + def test_longjmp2(self): + src = r''' + #include <setjmp.h> + #include <stdio.h> + + typedef struct { + jmp_buf* jmp; + } jmp_state; + + void stack_manipulate_func(jmp_state* s, int level) { + jmp_buf buf; + + printf("Entering stack_manipulate_func, level: %d\n", level); + + if (level == 0) { + s->jmp = &buf; + if (setjmp(*(s->jmp)) == 0) { + printf("Setjmp normal execution path, level: %d\n", level); + stack_manipulate_func(s, level + 1); + } else { + printf("Setjmp error execution path, level: %d\n", level); + } + } else { + printf("Perform longjmp at level %d\n", level); + longjmp(*(s->jmp), 1); + } + + printf("Exiting stack_manipulate_func, level: %d\n", level); + } + + int main(int argc, char *argv[]) { + jmp_state s; + s.jmp = NULL; + stack_manipulate_func(&s, 0); + + return 0; + } + ''' + self.do_run(src, '''Entering stack_manipulate_func, level: 0 +Setjmp normal execution path, level: 0 +Entering stack_manipulate_func, level: 1 +Perform longjmp at level 1 +Setjmp error execution path, level: 0 +Exiting stack_manipulate_func, level: 0 +''') + def test_exceptions(self): if Settings.QUANTUM_SIZE == 1: return self.skip("we don't support libcxx in q1") @@ -2738,6 +2826,7 @@ c5,de,15,8a // EMSCRIPTEN_COMMENT("hello from the source"); emscripten_run_script("Module.print('hello world' + '!')"); printf("*%d*\n", emscripten_run_script_int("5*20")); + printf("*%s*\n", emscripten_run_script_string("'five'+'six'")); emscripten_run_script("_save_me_aimee()"); return 0; } @@ -2749,19 +2838,26 @@ def process(filename): # TODO: restore this (see comment in emscripten.h) assert '// hello from the source' in src ''' - self.do_run(src, 'hello world!\n*100*\nmann\n', post_build=check) + self.do_run(src, 'hello world!\n*100*\n*fivesix*\nmann\n', post_build=check) def test_inlinejs(self): src = r''' #include <stdio.h> + double get() { + double ret = 0; + __asm __volatile__("12/3.3":"=a"(ret)); + return ret; + } + int main() { asm("Module.print('Inline JS is very cool')"); + printf("%.2f\n", get()); return 0; } ''' - self.do_run(src, 'Inline JS is very cool') + self.do_run(src, 'Inline JS is very cool\n3.64') def test_memorygrowth(self): if Settings.USE_TYPED_ARRAYS == 0: return self.skip('memory growth is only supported with typed arrays') @@ -4467,11 +4563,15 @@ Pass: 0.000012 0.000012''') int x = sscanf("one %n two", "%s %n", word, &l); printf("%d,%s,%d\n", x, word, l); - + { + int a, b, c, count; + count = sscanf("12345 6789", "%d %n%d", &a, &b, &c); + printf("%i %i %i %i\n", count, a, b, c); + } return 0; } ''' - self.do_run(src, '''[DEBUG] word 1: version, l: 7\n1,one,4''') + self.do_run(src, '''[DEBUG] word 1: version, l: 7\n1,one,4\n2 12345 6 6789\n''') def test_sscanf_whitespace(self): src = r''' @@ -4521,7 +4621,22 @@ Pass: 0.000012 0.000012''') self.do_run(src, '3\n123,1073741823,1125899906842620\n' + '3\n-123,-1073741823,-1125899906842620\n') - + + def test_sscanf_4(self): + src = r''' + #include <stdio.h> + + int main() + { + char pYear[16], pMonth[16], pDay[16], pDate[64]; + printf("%d\n", sscanf("Nov 19 2012", "%s%s%s", pMonth, pDay, pYear)); + printf("day %s, month %s, year %s \n", pDay, pMonth, pYear); + return(0); + } + ''' + + self.do_run(src, '3\nday 19, month Nov, year 2012'); + def test_langinfo(self): src = open(path_from_root('tests', 'langinfo', 'test.c'), 'r').read() expected = open(path_from_root('tests', 'langinfo', 'output.txt'), 'r').read() @@ -6126,7 +6241,9 @@ def process(filename): finally: del os.environ['EMCC_DEBUG'] for debug in [1,2]: - self.assertIdentical(open('release.js').read().replace('\n\n', '\n').replace('\n\n', '\n'), open('debug%d.js' % debug).read().replace('\n\n', '\n').replace('\n\n', '\n')) # EMCC_DEBUG=1 mode must not generate different code! + def clean(text): + return text.replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n') + self.assertIdentical(clean(open('release.js').read()), clean(open('debug%d.js' % debug).read())) # EMCC_DEBUG=1 mode must not generate different code! print >> sys.stderr, 'debug check %d passed too' % debug try_delete(CANONICAL_TEMP_DIR) @@ -7654,6 +7771,22 @@ f.close() Popen(['python', EMCC, os.path.join(self.get_dir(), 'test.cpp'), '-s', 'UNALIGNED_MEMORY=1']).communicate() self.assertContained('data: 67452301\ndata[0,1] 16bit: 2301\ndata[1,2] 16bit: 4523', run_js(os.path.join(self.get_dir(), 'a.out.js'))) + def test_unaligned_memory_2(self): + open(os.path.join(self.get_dir(), 'test.cpp'), 'w').write(r''' + #include <string> + #include <stdio.h> + + int main( int argc, char ** argv ) + { + std::string testString( "Hello, World!" ); + + printf( "testString = %s\n", testString.c_str() ); + return 0; + } + ''') + Popen(['python', EMCC, os.path.join(self.get_dir(), 'test.cpp'), '-s', 'UNALIGNED_MEMORY=1']).communicate() + self.assertContained('testString = Hello, World!', run_js(os.path.join(self.get_dir(), 'a.out.js'))) + def test_l_link(self): # Linking with -lLIBNAME and -L/DIRNAME should work @@ -8100,6 +8233,33 @@ f.close() output = Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp')], stderr=PIPE).communicate() self.assertNotContained('Unresolved symbol: _something\n', output[1]) + def test_toobig(self): + open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(r''' + #include <stdio.h> + + #define BYTES 100*1024*1024 + + int main(int argc, char **argv) { + if (argc == 100) { + static char buf[BYTES]; + static char buf2[BYTES]; + for (int i = 0; i < BYTES; i++) { + buf[i] = i*i; + buf2[i] = i/3; + } + for (int i = 0; i < BYTES; i++) { + buf[i] = buf2[i/2]; + buf2[i] = buf[i/3]; + } + printf("%d\n", buf[10] + buf2[20]); + } + return 0; + } + ''') + output = Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp')], stderr=PIPE).communicate()[1] + assert 'Emscripten failed' in output, output + assert 'warning: very large fixed-size structural type' in output, output + def test_prepost(self): open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(''' #include <stdio.h> @@ -8203,7 +8363,7 @@ f.close() (path_from_root('tools', 'test-js-optimizer-t2.js'), open(path_from_root('tools', 'test-js-optimizer-t2-output.js')).read(), ['simplifyExpressionsPre', 'optimizeShiftsAggressive']), # Make sure that optimizeShifts handles functions with shift statements. - (path_from_root('tools', 'test-js-optimizer-t3.js'), open(path_from_root('tools', 'test-js-optimizer-t3.js')).read(), + (path_from_root('tools', 'test-js-optimizer-t3.js'), open(path_from_root('tools', 'test-js-optimizer-t3-output.js')).read(), ['optimizeShiftsAggressive']), (path_from_root('tools', 'test-js-optimizer-regs.js'), open(path_from_root('tools', 'test-js-optimizer-regs-output.js')).read(), ['registerize']), @@ -8222,6 +8382,17 @@ f.close() assert 'foo.o: ' in output, '-%s failed to produce the right output: %s' % (opt, output) assert 'error' not in err, 'Unexpected stderr: ' + err + def test_chunking(self): + if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode') + if multiprocessing.cpu_count() < 2: return self.skip('need multiple cores') + try: + os.environ['EMCC_DEBUG'] = '1' + output, err = Popen(['python', EMCC, path_from_root('tests', 'hello_libcxx.cpp'), '-O1'], stdout=PIPE, stderr=PIPE).communicate() + assert 'phase 2 working on 3 chunks' in err, err + assert 'splitting up js optimization into 2 chunks' in err, err + finally: + del os.environ['EMCC_DEBUG'] + def test_scons(self): # also incidentally tests c++11 integration in llvm 3.1 try_delete(os.path.join(self.get_dir(), 'test')) shutil.copytree(path_from_root('tests', 'scons'), os.path.join(self.get_dir(), 'test')) @@ -9477,6 +9648,9 @@ elif 'browser' in str(sys.argv): def test_float_tex(self): self.btest('float_tex.cpp', reference='float_tex.png') + def test_subdata(self): + self.btest('gl_subdata.cpp', reference='float_tex.png') + def test_runtimelink(self): main, supp = self.setup_runtimelink_test() @@ -9978,6 +10152,8 @@ elif 'sanity' in str(sys.argv): assert os.path.exists(CONFIG_FILE), 'To run these tests, we need a (working!) %s file to already exist' % EM_CONFIG + assert not os.environ.get('EMCC_DEBUG'), 'do not run sanity checks in debug mode!' + shutil.copyfile(CONFIG_FILE, CONFIG_FILE + '_backup') def restore(): shutil.copyfile(CONFIG_FILE + '_backup', CONFIG_FILE) @@ -10210,10 +10386,11 @@ fi self.assertContained(SANITY_MESSAGE, output) # but with EMCC_DEBUG=1 we should check - assert not os.environ.get('EMCC_DEBUG'), 'do not run sanity checks in debug mode!' - os.environ['EMCC_DEBUG'] = '1' - output = self.check_working(EMCC) - del os.environ['EMCC_DEBUG'] + try: + os.environ['EMCC_DEBUG'] = '1' + output = self.check_working(EMCC) + finally: + del os.environ['EMCC_DEBUG'] self.assertContained(SANITY_MESSAGE, output) output = self.check_working(EMCC) self.assertNotContained(SANITY_MESSAGE, output) @@ -10270,8 +10447,8 @@ fi assert not os.path.exists(EMCC_CACHE) try: - emcc_debug = os.environ.get('EMCC_DEBUG') os.environ['EMCC_DEBUG'] ='1' + self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp') # Building a file that doesn't need cached stuff should not trigger cache generation output = self.do([EMCC, path_from_root('tests', 'hello_world.cpp')]) @@ -10287,8 +10464,6 @@ fi ll_name1 = os.path.join(TEMP_DIR, 'emscripten_temp', 'emcc-2-ll.ll') ll_name2 = os.path.join(TEMP_DIR, 'emscripten_temp', 'emcc-3-ll.ll') - self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp') - # Building a file that *does* need dlmalloc *should* trigger cache generation, but only the first time for filename, libname in [('hello_malloc.cpp', 'dlmalloc'), ('hello_libcxx.cpp', 'libcxx')]: for i in range(3): @@ -10321,8 +10496,7 @@ fi print i, 'll metadata should be removed in -O1 and O2 by default', ll[-300:] assert False finally: - if emcc_debug: - os.environ['EMCC_DEBUG'] = emcc_debug + del os.environ['EMCC_DEBUG'] # Manual cache clearing assert os.path.exists(EMCC_CACHE) @@ -10350,6 +10524,79 @@ fi assert os.path.exists(RELOOPER) == (i >= 2), 'have relooper on O2: ' + output assert ('L2 : do {' in open('a.out.js').read()) == (i >= 2), 'reloop code on O2: ' + output + def test_jcache(self): + PRE_LOAD_MSG = 'loading pre from jcache' + PRE_SAVE_MSG = 'saving pre to jcache' + FUNC_CHUNKS_LOAD_MSG = ' funcchunks from jcache' + FUNC_CHUNKS_SAVE_MSG = ' funcchunks to jcache' + JSFUNC_CHUNKS_LOAD_MSG = 'jsfuncchunks from jcache' + JSFUNC_CHUNKS_SAVE_MSG = 'jsfuncchunks to jcache' + + restore() + Cache.erase() + + try: + os.environ['EMCC_DEBUG'] = '1' + self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp') + if not os.path.exists(self.working_dir): os.makedirs(self.working_dir) + + assert not os.path.exists(JCache.get_cachename('emscript_files')) + + srcs = {} + used_jcache = False + + for args, input_file, expect_pre_save, expect_pre_load, expect_funcs_save, expect_funcs_load, expect_jsfuncs_save, expect_jsfuncs_load, expected in [ + ([], 'hello_world_loop.cpp', False, False, False, False, False, False, []), + (['--jcache'], 'hello_world_loop.cpp', True, False, True, False, True, False, []), + (['--jcache'], 'hello_world_loop.cpp', False, True, False, True, False, True, []), + ([], 'hello_world_loop.cpp', False, False, False, False, False, False, []), + # new + ([], 'hello_world.cpp', False, False, False, False, False, False, []), + (['--jcache'], 'hello_world.cpp', True, False, True, False, True, False, []), + (['--jcache'], 'hello_world.cpp', False, True, False, True, False, True, []), + ([], 'hello_world.cpp', False, False, False, False, False, False, []), + # go back to old file, experience caching + (['--jcache'], 'hello_world_loop.cpp', False, True, False, True, False, True, []), + # new, large file + ([], 'hello_malloc.cpp', False, False, False, False, False, False, []), + (['--jcache'], 'hello_malloc.cpp', True, False, True, False, True, False, []), + (['--jcache'], 'hello_malloc.cpp', False, True, False, True, False, True, []), + ([], 'hello_malloc.cpp', False, False, False, False, False, False, []), + # new, huge file + ([], 'hello_libcxx.cpp', False, False, False, False, False, False, ('2 chunks', '3 chunks')), + (['--jcache'], 'hello_libcxx.cpp', True, False, True, False, True, False, []), + (['--jcache'], 'hello_libcxx.cpp', False, True, False, True, False, True, []), + ([], 'hello_libcxx.cpp', False, False, False, False, False, False, []), + # finally, build a file close to the previous, to see that some chunks are found in the cache and some not + (['--jcache'], 'hello_libcxx_mod1.cpp', False, True, True, True, True, False, []), # win on pre, mix on funcs, fail on jsfuncs + (['--jcache'], 'hello_libcxx_mod1.cpp', False, True, False, True, False, True, []), + ]: + print >> sys.stderr, args, input_file, expect_pre_save, expect_pre_load, expect_funcs_save, expect_funcs_load, expect_jsfuncs_save, expect_jsfuncs_load, expected + self.clear() + out, err = Popen(['python', EMCC, '-O2', '--closure', '0', path_from_root('tests', input_file)] + args, stdout=PIPE, stderr=PIPE).communicate() + errtail = err.split('emcc invocation')[-1] + self.assertContained('hello, world!', run_js('a.out.js'), errtail) + assert (PRE_SAVE_MSG in err) == expect_pre_save, errtail + assert (PRE_LOAD_MSG in err) == expect_pre_load, errtail + assert (FUNC_CHUNKS_SAVE_MSG in err) == expect_funcs_save, errtail + assert (FUNC_CHUNKS_LOAD_MSG in err) == expect_funcs_load, errtail + assert (JSFUNC_CHUNKS_SAVE_MSG in err) == expect_jsfuncs_save, errtail + assert (JSFUNC_CHUNKS_LOAD_MSG in err) == expect_jsfuncs_load, errtail + for expect in expected: assert expect in err, expect + ' ? ' + errtail + curr = open('a.out.js').read() + if input_file not in srcs: + srcs[input_file] = curr + else: + open('/home/alon/Dev/emscripten/a', 'w').write(srcs[input_file]) + open('/home/alon/Dev/emscripten/b', 'w').write(curr) + assert abs(len(curr)/float(len(srcs[input_file]))-1)<0.01, 'contents may shift in order, but must remain the same size %d vs %d' % (len(curr), len(srcs[input_file])) + '\n' + errtail + used_jcache = used_jcache or ('--jcache' in args) + assert used_jcache == os.path.exists(JCache.get_cachename('emscript_files')) + #print >> sys.stderr, errtail + + finally: + del os.environ['EMCC_DEBUG'] + else: raise Exception('Test runner is confused: ' + str(sys.argv)) diff --git a/third_party/closure-compiler/README b/third_party/closure-compiler/README index 77ab89db..270cfc27 100644 --- a/third_party/closure-compiler/README +++ b/third_party/closure-compiler/README @@ -171,7 +171,7 @@ lib/args4j.jar Args4j URL: https://args4j.dev.java.net/ -Version: 2.0.12 +Version: 2.0.16 License: MIT Description: @@ -187,7 +187,7 @@ lib/guava.jar Guava Libraries URL: http://code.google.com/p/guava-libraries/ -Version: r08 +Version: 13.0.1 License: Apache License 2.0 Description: Google's core Java libraries. @@ -230,7 +230,7 @@ lib/junit.jar JUnit URL: http://sourceforge.net/projects/junit/ -Version: 4.8.2 +Version: 4.10 License: Common Public License 1.0 Description: A framework for writing and running automated tests in Java. @@ -244,7 +244,7 @@ lib/protobuf-java.jar Protocol Buffers URL: http://code.google.com/p/protobuf/ -Version: 2.3.0 +Version: 2.4.1 License: New BSD License Description: Supporting libraries for protocol buffers, @@ -281,9 +281,9 @@ Local Modifications: None --- Code in: -tools/maven-ant-tasks-2.1.1.jar +tools/maven-ant-tasks-2.1.3.jar URL: http://maven.apache.org -Version 2.1.1 +Version 2.1.3 License: Apache License 2.0 Description: Maven Ant tasks are used to manage dependencies and to install/deploy to diff --git a/third_party/closure-compiler/compiler.jar b/third_party/closure-compiler/compiler.jar Binary files differindex da191b01..e23352ff 100644 --- a/third_party/closure-compiler/compiler.jar +++ b/third_party/closure-compiler/compiler.jar diff --git a/tools/eliminator/eliminator-test-output.js b/tools/eliminator/eliminator-test-output.js index 00647fca..ede34103 100644 --- a/tools/eliminator/eliminator-test-output.js +++ b/tools/eliminator/eliminator-test-output.js @@ -6132,5 +6132,4 @@ function _mallocNoU($bytes) { return $mem_0; return null; } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a", "b", "c", "f", "g", "h", "py", "r", "t", "f2", "f3", "llvm3_1", "_inflate", "_malloc", "_mallocNoU"] diff --git a/tools/eliminator/safe-eliminator-test-output.js b/tools/eliminator/safe-eliminator-test-output.js index bb3f17e6..57f0a743 100644 --- a/tools/eliminator/safe-eliminator-test-output.js +++ b/tools/eliminator/safe-eliminator-test-output.js @@ -82,5 +82,4 @@ function a($directory) { print(zzz1); } while (1); } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a"] diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js index a9a181b1..29aecdcf 100644 --- a/tools/js-optimizer.js +++ b/tools/js-optimizer.js @@ -214,6 +214,7 @@ function traverse(node, pre, post, stack) { // Only walk through the generated functions function traverseGenerated(ast, pre, post, stack) { + assert(generatedFunctions); traverse(ast, function(node) { if (node[0] == 'defun' && isGenerated(node[1])) { traverse(node, pre, post, stack); @@ -223,6 +224,7 @@ function traverseGenerated(ast, pre, post, stack) { } function traverseGeneratedFunctions(ast, callback) { + assert(generatedFunctions); traverse(ast, function(node) { if (node[0] == 'defun' && isGenerated(node[1])) { callback(node); @@ -1914,6 +1916,5 @@ do { js = js.replace(/\n *\n/g, '\n'); } while (js != old); print(js); -if (metadata && printMetadata) print(metadata); print('\n'); diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index 8681280a..5bed4cb7 100644 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -1,5 +1,8 @@ -import os, sys, subprocess, multiprocessing +import os, sys, subprocess, multiprocessing, re +import shared + +temp_files = shared.TempFiles() __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def path_from_root(*pathelems): @@ -17,13 +20,15 @@ def run_on_chunk(command): filename = command[2] # XXX hackish output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output - filename += '.jo.js' + filename = temp_files.get(os.path.basename(filename) + '.jo.js').name f = open(filename, 'w') f.write(output) f.close() return filename -def run(filename, passes, js_engine): +def run(filename, passes, js_engine, jcache): + if jcache: shared.JCache.ensure() + if type(passes) == str: passes = [passes] @@ -37,64 +42,129 @@ def run(filename, passes, js_engine): suffix = '' if suffix_start >= 0: suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n' + # if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything. + generated = set(eval(suffix[len(suffix_marker)+1:])) + + if not suffix and jcache: + # JCache cannot be used without metadata, since it might reorder stuff, and that's dangerous since only generated can be reordered + # This means jcache does not work after closure compiler runs, for example. But you won't get much benefit from jcache with closure + # anyhow (since closure is likely the longest part of the build). + if DEBUG: print >>sys.stderr, 'js optimizer: no metadata, so disabling jcache' + jcache = False + + # If we process only generated code, find that and save the rest on the side + func_sig = re.compile('function (_[\w$]+)\(') + if suffix: + pos = 0 + gen_start = 0 + gen_end = 0 + while 1: + m = func_sig.search(js, pos) + if not m: break + pos = m.end() + ident = m.group(1) + if ident in generated: + if not gen_start: + gen_start = m.start() + assert gen_start + gen_end = js.find('\n}\n', m.end()) + 3 + assert gen_end > gen_start + pre = js[:gen_start] + post = js[gen_end:] + js = js[gen_start:gen_end] + else: + pre = '' + post = '' # Pick where to split into chunks, so that (1) they do not oom in node/uglify, and (2) we can run them in parallel - chunks = [] - i = 0 - f_start = 0 - while True: - f_end = f_start - while f_end-f_start < BEST_JS_PROCESS_SIZE and f_end != -1: - f_end = js.find('\n}\n', f_end+1) - chunk = js[f_start:(-1 if f_end == -1 else f_end+3)] + suffix - temp_file = filename + '.p%d.js' % i - #if DEBUG: print >> sys.stderr, ' chunk %d: %d bytes' % (i, (f_end if f_end >= 0 else len(js)) - f_start) - i += 1 - f_start = f_end+3 - done = f_end == -1 or f_start >= len(js) - if done and len(chunks) == 0: break # do not write anything out, just use the input file - f = open(temp_file, 'w') - f.write(chunk) - f.close() - chunks.append(temp_file) - if done: break - - if len(chunks) == 0: - chunks.append(filename) - - # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok) - commands = map(lambda chunk: [js_engine, JS_OPTIMIZER, chunk] + passes, chunks) - - if len(chunks) > 1: - # We are splitting into chunks. Hopefully we can do that in parallel - commands = map(lambda command: command + ['noPrintMetadata'], commands) - filename += '.jo.js' - - fail = None - cores = min(multiprocessing.cpu_count(), chunks) - if cores < 2: - fail = 'python reports you have %d cores' % cores - #elif WINDOWS: - # fail = 'windows (see issue 663)' # This seems fixed with adding emcc.py that imports this file - - if not fail: + # If we have metadata, we split only the generated code, and save the pre and post on the side (and do not optimize them) + parts = map(lambda part: part, js.split('\n}\n')) + funcs = [] + for i in range(len(parts)): + func = parts[i] + if i < len(parts)-1: func += '\n}\n' # last part needs no } + m = func_sig.search(func) + if m: + ident = m.group(1) + else: + if suffix: continue # ignore whitespace + ident = 'anon_%d' % i + funcs.append((ident, func)) + parts = None + total_size = len(js) + js = None + + chunks = shared.JCache.chunkify(funcs, BEST_JS_PROCESS_SIZE, 'jsopt' if jcache else None) + + if jcache: + # load chunks from cache where we can # TODO: ignore small chunks + cached_outputs = [] + def load_from_cache(chunk): + keys = [chunk] + shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code + out = shared.JCache.get(shortkey, keys) + if out: + cached_outputs.append(out) + return False + return True + chunks = filter(load_from_cache, chunks) + if len(cached_outputs) > 0: + if DEBUG: print >> sys.stderr, ' loading %d jsfuncchunks from jcache' % len(cached_outputs) + else: + cached_outputs = [] + + if len(chunks) > 0: + def write_chunk(chunk, i): + temp_file = temp_files.get('.jsfunc_%d.ll' % i).name + f = open(temp_file, 'w') + f.write(chunk) + f.write(suffix) + f.close() + return temp_file + filenames = [write_chunk(chunks[i], i) for i in range(len(chunks))] + else: + filenames = [] + + if len(filenames) > 0: + # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok) + commands = map(lambda filename: [js_engine, JS_OPTIMIZER, filename, 'noPrintMetadata'] + passes, filenames) + + cores = min(multiprocessing.cpu_count(), filenames) + if len(chunks) > 1 and cores >= 2: # We can parallelize - if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores' % (len(chunks), cores) + if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.)) pool = multiprocessing.Pool(processes=cores) filenames = pool.map(run_on_chunk, commands, chunksize=1) else: # We can't parallize, but still break into chunks to avoid uglify/node memory issues - if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks (not in parallel because %s)' % (len(chunks), fail) + if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks)) filenames = [run_on_chunk(command) for command in commands] + else: + filenames = [] - f = open(filename, 'w') - for out_file in filenames: - f.write(open(out_file).read()) - f.write(suffix) + filename += '.jo.js' + f = open(filename, 'w') + f.write(pre); + for out_file in filenames: + f.write(open(out_file).read()) f.write('\n') - f.close() - return filename - else: - # one simple chunk, just do it - return run_on_chunk(commands[0]) + if jcache: + for cached in cached_outputs: + f.write(cached); # TODO: preserve order + f.write('\n') + f.write(post); + # No need to write suffix: if there was one, it is inside post which exists when suffix is there + f.write('\n') + f.close() + + if jcache: + # save chunks to cache + for i in range(len(chunks)): + chunk = chunks[i] + keys = [chunk] + shortkey = shared.JCache.get_shortkey(keys) + shared.JCache.set(shortkey, keys, open(filenames[i]).read()) + if DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d jsfuncchunks to jcache' % len(chunks) + + return filename diff --git a/tools/shared.py b/tools/shared.py index 04d0fe57..7bafd51d 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,4 +1,4 @@ -import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess +import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, md5, cPickle from subprocess import Popen, PIPE, STDOUT from tempfile import mkstemp @@ -59,8 +59,6 @@ class WindowsPopen: # Install our replacement Popen handler if we are running on Windows to avoid python spawn process function. if os.name == 'nt': Popen = WindowsPopen - -import js_optimizer __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def path_from_root(*pathelems): @@ -1006,8 +1004,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e return opts @staticmethod - def js_optimizer(filename, passes): - return js_optimizer.run(filename, passes, NODE_JS) + def js_optimizer(filename, passes, jcache): + return js_optimizer.run(filename, passes, NODE_JS, jcache) @staticmethod def closure_compiler(filename): @@ -1088,16 +1086,17 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e @staticmethod def ensure_relooper(): if os.path.exists(RELOOPER): return + curr = os.getcwd() try: ok = False print >> sys.stderr, '=======================================' print >> sys.stderr, 'bootstrapping relooper...' Cache.ensure() - RELOOPER_DIR = path_from_root('src', 'relooper') + os.chdir(path_from_root('src')) def make(opt_level): raw = RELOOPER + '.raw.js' - Building.emcc(os.path.join(RELOOPER_DIR, 'Relooper.cpp'), ['-I' + os.path.join(RELOOPER_DIR), '--post-js', os.path.join(RELOOPER_DIR, 'emscripten', 'glue.js'), '-s', 'TOTAL_MEMORY=52428800', '-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["memcpy", "memset", "malloc", "free", "puts"]', '-O' + str(opt_level), '--closure', '0'], raw) + Building.emcc(os.path.join('relooper', 'Relooper.cpp'), ['-I' + os.path.join('relooper'), '--post-js', os.path.join('relooper', 'emscripten', 'glue.js'), '-s', 'TOTAL_MEMORY=52428800', '-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["memcpy", "memset", "malloc", "free", "puts"]', '-O' + str(opt_level), '--closure', '0'], raw) f = open(RELOOPER, 'w') f.write("// Relooper, (C) 2012 Alon Zakai, MIT license, https://github.com/kripken/Relooper\n") f.write("var Relooper = (function() {\n"); @@ -1116,6 +1115,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e print >> sys.stderr, '=======================================' ok = True finally: + os.chdir(curr) if not ok: print >> sys.stderr, 'bootstrapping relooper failed. You may need to manually create src/relooper.js by compiling it, see src/relooper/emscripten' 1/0 @@ -1151,6 +1151,152 @@ class Cache: shutil.copyfile(creator(), cachename) return cachename +# JS-specific cache. We cache the results of compilation and optimization, +# so that in incremental builds we can just load from cache. +# We cache reasonably-large-sized chunks +class JCache: + dirname = os.path.join(Cache.dirname, 'jcache') + + @staticmethod + def ensure(): + Cache.ensure() + if not os.path.exists(JCache.dirname): + os.makedirs(JCache.dirname) + + @staticmethod + def get_shortkey(keys): + if type(keys) not in [list, tuple]: + keys = [keys] + ret = '' + for key in keys: + assert type(key) == str + ret += md5.md5(key).hexdigest() + return ret + + @staticmethod + def get_cachename(shortkey): + return os.path.join(JCache.dirname, shortkey) + + # Returns a cached value, if it exists. Make sure the full key matches + @staticmethod + def get(shortkey, keys): + #if DEBUG: print >> sys.stderr, 'jcache get?', shortkey + cachename = JCache.get_cachename(shortkey) + if not os.path.exists(cachename): + #if DEBUG: print >> sys.stderr, 'jcache none at all' + return + data = cPickle.Unpickler(open(cachename, 'rb')).load() + if len(data) != 2: + #if DEBUG: print >> sys.stderr, 'jcache error in get' + return + oldkeys = data[0] + if len(oldkeys) != len(keys): + #if DEBUG: print >> sys.stderr, 'jcache collision (a)' + return + for i in range(len(oldkeys)): + if oldkeys[i] != keys[i]: + #if DEBUG: print >> sys.stderr, 'jcache collision (b)' + return + #if DEBUG: print >> sys.stderr, 'jcache win' + return data[1] + + # Sets the cached value for a key (from get_key) + @staticmethod + def set(shortkey, keys, value): + cachename = JCache.get_cachename(shortkey) + cPickle.Pickler(open(cachename, 'wb')).dump([keys, value]) + #if DEBUG: + # for i in range(len(keys)): + # open(cachename + '.key' + str(i), 'w').write(keys[i]) + # open(cachename + '.value', 'w').write(value) + + # Given a set of functions of form (ident, text), and a preferred chunk size, + # generates a set of chunks for parallel processing and caching. + # It is very important to generate similar chunks in incremental builds, in + # order to maximize the chance of cache hits. To achieve that, we save the + # chunking used in the previous compilation of this phase, and we try to + # generate the same chunks, barring big differences in function sizes that + # violate our chunk size guideline. If caching is not used, chunking_file + # should be None + @staticmethod + def chunkify(funcs, chunk_size, chunking_file): + previous_mapping = None + if chunking_file: + chunking_file = JCache.get_cachename(chunking_file) + if os.path.exists(chunking_file): + try: + previous_mapping = cPickle.Unpickler(open(chunking_file, 'rb')).load() # maps a function identifier to the chunk number it will be in + except: + pass + chunks = [] + if previous_mapping: + # initialize with previous chunking + news = [] + for func in funcs: + ident, data = func + if not ident in previous_mapping: + news.append(func) + else: + n = previous_mapping[ident] + while n >= len(chunks): chunks.append([]) + chunks[n].append(func) + # add news and adjust for new sizes + spilled = news + for chunk in chunks: + size = sum([len(func[1]) for func in chunk]) + while size > 1.5*chunk_size and len(chunk) > 0: + spill = chunk.pop() + spilled.append(spill) + size -= len(spill[1]) + for chunk in chunks: + size = sum([len(func[1]) for func in chunk]) + while size < 0.66*chunk_size and len(spilled) > 0: + spill = spilled.pop() + chunk.append(spill) + size += len(spill[1]) + chunks = filter(lambda chunk: len(chunk) > 0, chunks) # might have empty ones, eliminate them + funcs = spilled # we will allocate these into chunks as if they were normal inputs + # initialize reasonably, the rest of the funcs we need to split out + curr = [] + total_size = 0 + for i in range(len(funcs)): + func = funcs[i] + curr_size = len(func[1]) + if total_size + curr_size < chunk_size: + curr.append(func) + total_size += curr_size + else: + chunks.append(curr) + curr = [func] + total_size = curr_size + if curr: + chunks.append(curr) + curr = None + if chunking_file: + # sort within each chunk, to keep the order identical + for chunk in chunks: + chunk.sort(key=lambda func: func[0]) + # save new mapping info + new_mapping = {} + for i in range(len(chunks)): + chunk = chunks[i] + for ident, data in chunk: + new_mapping[ident] = i + cPickle.Pickler(open(chunking_file, 'wb')).dump(new_mapping) + #if DEBUG: + # if previous_mapping: + # for ident in set(previous_mapping.keys() + new_mapping.keys()): + # if previous_mapping.get(ident) != new_mapping.get(ident): + # print >> sys.stderr, 'mapping inconsistency', ident, previous_mapping.get(ident), new_mapping.get(ident) + # for key, value in new_mapping.iteritems(): + # print >> sys.stderr, 'mapping:', key, value + return [''.join([func[1] for func in chunk]) for chunk in chunks] # remove function names + +class JS: + @staticmethod + def to_nice_ident(ident): # limited version of the JS function toNiceIdent + return ident.replace('%', '$').replace('@', '_'); + # Compression of code and data for smaller downloads class Compression: on = False @@ -1189,3 +1335,5 @@ def unsuffixed(name): def unsuffixed_basename(name): return os.path.basename(unsuffixed(name)) +import js_optimizer + diff --git a/tools/test-js-optimizer-output.js b/tools/test-js-optimizer-output.js index 9c9fa063..730426b1 100644 --- a/tools/test-js-optimizer-output.js +++ b/tools/test-js-optimizer-output.js @@ -279,5 +279,4 @@ function notComps() { function tricky() { var $conv642 = $conv6374 - (($132 << 16 >> 16 | 0) / 2 & -1) & 65535; } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["abc", "xyz", "xyz2", "expr", "loopy", "bits", "maths", "hoisting", "demangle", "lua", "moreLabels", "notComps", "tricky"] diff --git a/tools/test-js-optimizer-regs-output.js b/tools/test-js-optimizer-regs-output.js index 2f91f52b..90b67a47 100644 --- a/tools/test-js-optimizer-regs-output.js +++ b/tools/test-js-optimizer-regs-output.js @@ -225,5 +225,4 @@ function switchey(x) { r8 = x + 2; pp(r8); } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["test", "primes", "atomic", "fcntl_open", "ex", "switchey"] diff --git a/tools/test-js-optimizer-t2-output.js b/tools/test-js-optimizer-t2-output.js index 62cd6e0d..0ae66be5 100644 --- a/tools/test-js-optimizer-t2-output.js +++ b/tools/test-js-optimizer-t2-output.js @@ -88,5 +88,4 @@ function shifty($id2) { q(go() << 16); q(go() + 2 >> 2); } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["shifty"] diff --git a/tools/test-js-optimizer-t2c-output.js b/tools/test-js-optimizer-t2c-output.js index f41bb815..726112ec 100644 --- a/tools/test-js-optimizer-t2c-output.js +++ b/tools/test-js-optimizer-t2c-output.js @@ -14,5 +14,4 @@ function shifty() { q($13 + 13 << 2); q(h() >> 2 << 2); } -// EMSCRIPTEN_GENERATED_FUNCTIONS: ["shifty"] diff --git a/tools/test-js-optimizer-t3-output.js b/tools/test-js-optimizer-t3-output.js new file mode 100644 index 00000000..924868fa --- /dev/null +++ b/tools/test-js-optimizer-t3-output.js @@ -0,0 +1,49 @@ +function _png_create_write_struct_2($user_png_ver, $error_ptr, $error_fn, $warn_fn, $mem_ptr, $malloc_fn, $free_fn) { + var $png_ptr$s2; + var label; + label = 2; + var setjmpTable = { + "2": (function(value) { + label = 5; + $call1 = value; + }), + dummy: 0 + }; + while (1) try { + switch (label) { + case 2: + var $png_ptr; + var $call = _png_create_struct(1); + $png_ptr = $call; + var $call1 = (HEAP32[$png_ptr >> 2] = label, 0); + label = 5; + break; + case 5: + var $2 = $png_ptr; + if (($call1 | 0) == 0) { + label = 4; + break; + } else { + label = 3; + break; + } + case 3: + var $4 = HEAP32[($png_ptr >> 2) + (148 >> 2)]; + _png_free($2, $4); + HEAP32[($png_ptr >> 2) + (148 >> 2)] = 0; + _png_destroy_struct($png_ptr); + var $retval_0 = 0; + label = 4; + break; + case 4: + var $retval_0; + return $retval_0; + default: + assert(0, "bad label: " + label); + } + } catch (e) { + if (!e.longjmp) throw e; + setjmpTable[e.label](e.value); + } +} + |