summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xemcc31
-rwxr-xr-xemscripten.py113
-rw-r--r--src/jsifier.js3
-rw-r--r--src/library.js46
-rw-r--r--src/modules.js20
-rw-r--r--system/include/emscripten/emscripten.h1
-rw-r--r--tests/hello_libcxx_mod1.cpp9
-rwxr-xr-xtests/runner.py166
-rw-r--r--tools/eliminator/eliminator-test-output.js1
-rw-r--r--tools/eliminator/safe-eliminator-test-output.js1
-rw-r--r--tools/js-optimizer.js3
-rw-r--r--tools/js_optimizer.py176
-rw-r--r--tools/shared.py156
-rw-r--r--tools/test-js-optimizer-output.js1
-rw-r--r--tools/test-js-optimizer-regs-output.js1
-rw-r--r--tools/test-js-optimizer-t2-output.js1
-rw-r--r--tools/test-js-optimizer-t2c-output.js1
-rw-r--r--tools/test-js-optimizer-t3-output.js49
18 files changed, 660 insertions, 119 deletions
diff --git a/emcc b/emcc
index 1b948341..7e7efe6d 100755
--- a/emcc
+++ b/emcc
@@ -327,6 +327,16 @@ Options that are modified or new in %s include:
llvm-link's behavior is not as permissive
as ld is.
+ --jcache Use a JavaScript cache. This is disabled by
+ default. When enabled, emcc will store the
+ results of compilation in a cache and check
+ the cache when compiling later, something
+ like what ccache does. This allows incremental
+ builds - where you are compiling a large
+ program but only modified a small part of it -
+ to be much faster (at the cost of more disk
+ IO for cache accesses).
+
--clear-cache Manually clears the cache of compiled
emscripten system libraries (libc++,
libc++abi, dlmalloc). This is normally
@@ -336,7 +346,10 @@ Options that are modified or new in %s include:
mechanism can get confused. Clearing the
cache can fix weird problems related to
cache incompatibilities, like clang failing
- to link with library files.
+ to link with library files. This also clears
+ other cached data like the jcache and
+ the bootstrapped relooper. After the cache
+ is cleared, this process will exit.
The target file, if specified (-o <target>), defines what will
be generated:
@@ -571,6 +584,7 @@ try:
remove_duplicates = False
keep_debug = False
bind = False
+ jcache = False
def check_bad_eq(arg):
assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)'
@@ -678,10 +692,14 @@ try:
elif newargs[i] == '--remove-duplicates':
remove_duplicates = True
newargs[i] = ''
+ elif newargs[i] == '--jcache':
+ jcache = True
+ newargs[i] = ''
elif newargs[i] == '--clear-cache':
newargs[i] = ''
print >> sys.stderr, 'emcc: clearing cache'
shared.Cache.erase()
+ sys.exit(0)
elif newargs[i].startswith(('-I/', '-L/')):
if not absolute_warning_shown:
print >> sys.stderr, 'emcc: warning: -I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript)' # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not
@@ -695,6 +713,8 @@ try:
minify_whitespace = closure # if closure is run, minify whitespace
if opt_level <= 0: keep_debug = True # always keep debug in -O0
+ if DEBUG: start_time = time.time() # done after parsing arguments, which might affect debug state
+
if closure:
assert os.path.exists(shared.CLOSURE_COMPILER), 'emcc: fatal: Closure compiler (%s) does not exist' % shared.CLOSURE_COMPILER
@@ -1044,6 +1064,7 @@ try:
# Emscripten
if DEBUG: print >> sys.stderr, 'emcc: LLVM => JS'
extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))]
+ if jcache: extra_args.append('--jcache')
final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args)
if DEBUG: save_intermediate('original')
@@ -1100,12 +1121,12 @@ try:
if len(js_optimizer_queue) > 0:
if DEBUG < 2:
if DEBUG: print >> sys.stderr, 'emcc: applying js optimization passes:', js_optimizer_queue
- final = shared.Building.js_optimizer(final, js_optimizer_queue)
+ final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache)
if DEBUG: save_intermediate('js_opts')
else:
for name in js_optimizer_queue:
print >> sys.stderr, 'emcc: applying js optimization pass:', name
- final = shared.Building.js_optimizer(final, [name])
+ final = shared.Building.js_optimizer(final, [name], jcache)
save_intermediate(name)
js_optimizer_queue = []
@@ -1114,7 +1135,7 @@ try:
if DEBUG >= 2:
# Clean up the syntax a bit
- final = shared.Building.js_optimizer(final, [])
+ final = shared.Building.js_optimizer(final, [], jcache)
if DEBUG: save_intermediate('pretty')
def get_eliminate():
@@ -1221,6 +1242,8 @@ try:
# copy final JS to output
shutil.move(final, target)
+ if DEBUG: print >> sys.stderr, 'emcc: total time: %.2f seconds' % (time.time() - start_time)
+
finally:
if not TEMP_DIR:
try:
diff --git a/emscripten.py b/emscripten.py
index 1f6ae2fc..15beb4ee 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -32,6 +32,7 @@ def path_from_root(*pathelems):
temp_files = shared.TempFiles()
compiler_engine = None
+jcache = False
def scan(ll, settings):
# blockaddress(@main, %23)
@@ -47,12 +48,13 @@ MIN_CHUNK_SIZE = 1024*1024
MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes
def process_funcs(args):
- i, ll, settings_file, compiler, forwarded_file, libraries = args
+ i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args
+ ll = ''.join(funcs) + '\n' + meta
funcs_file = temp_files.get('.func_%d.ll' % i).name
open(funcs_file, 'w').write(ll)
out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
shared.try_delete(funcs_file)
- return out.split('//FORWARDED_DATA:')
+ return out
def emscript(infile, settings, outfile, libraries=[]):
"""Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible
@@ -73,6 +75,8 @@ def emscript(infile, settings, outfile, libraries=[]):
if DEBUG: print >> sys.stderr, 'emscript: ll=>js'
+ if jcache: shared.JCache.ensure()
+
# Pre-scan ll and alter settings as necessary
if DEBUG: t = time.time()
ll = open(infile).read()
@@ -84,6 +88,7 @@ def emscript(infile, settings, outfile, libraries=[]):
# Split input into the relevant parts for each phase
pre = []
funcs = [] # split up functions here, for parallelism later
+ func_idents = []
meta = [] # needed by each function XXX
if DEBUG: t = time.time()
@@ -91,15 +96,16 @@ def emscript(infile, settings, outfile, libraries=[]):
ll_lines = open(infile).readlines()
for line in ll_lines:
if in_func:
- funcs[-1].append(line)
+ funcs[-1][1].append(line)
if line.startswith('}'):
in_func = False
- funcs[-1] = ''.join(funcs[-1])
+ funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1]))
pre.append(line) # pre needs it to, so we know about all implemented functions
else:
+ if line.startswith(';'): continue
if line.startswith('define '):
in_func = True
- funcs.append([line])
+ funcs.append((line, [line])) # use the entire line as the identifier
pre.append(line) # pre needs it to, so we know about all implemented functions
elif line.find(' = type { ') > 0:
pre.append(line) # type
@@ -122,15 +128,27 @@ def emscript(infile, settings, outfile, libraries=[]):
# Save settings to a file to work around v8 issue 1579
settings_file = temp_files.get('.txt').name
+ settings_text = json.dumps(settings)
s = open(settings_file, 'w')
- s.write(json.dumps(settings))
+ s.write(settings_text)
s.close()
# Phase 1 - pre
if DEBUG: t = time.time()
pre_file = temp_files.get('.pre.ll').name
- open(pre_file, 'w').write(''.join(pre) + '\n' + meta)
- out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
+ pre_input = ''.join(pre) + '\n' + meta
+ out = None
+ if jcache:
+ keys = [pre_input, settings_text, ','.join(libraries)]
+ shortkey = shared.JCache.get_shortkey(keys)
+ out = shared.JCache.get(shortkey, keys)
+ if out and DEBUG: print >> sys.stderr, ' loading pre from jcache'
+ if not out:
+ open(pre_file, 'w').write(pre_input)
+ out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
+ if jcache:
+ if DEBUG: print >> sys.stderr, ' saving pre to jcache'
+ shared.JCache.set(shortkey, keys, out)
pre, forwarded_data = out.split('//FORWARDED_DATA:')
forwarded_file = temp_files.get('.json').name
open(forwarded_file, 'w').write(forwarded_data)
@@ -151,28 +169,58 @@ def emscript(infile, settings, outfile, libraries=[]):
if DEBUG: t = time.time()
forwarded_json = json.loads(forwarded_data)
indexed_functions = set()
- chunks = [] # bundles of functions
- curr = ''
- for i in range(len(funcs)):
- func = funcs[i]
- if len(curr) + len(func) < chunk_size:
- curr += func
+
+ chunks = shared.JCache.chunkify(funcs, chunk_size, 'emscript_files' if jcache else None)
+
+ if jcache:
+ # load chunks from cache where we can # TODO: ignore small chunks
+ cached_outputs = []
+ def load_from_cache(chunk):
+ keys = [settings_text, forwarded_data, chunk]
+ shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code
+ out = shared.JCache.get(shortkey, keys) # this is relatively expensive (pickling?)
+ if out:
+ cached_outputs.append(out)
+ return False
+ return True
+ chunks = filter(load_from_cache, chunks)
+ if len(cached_outputs) > 0:
+ if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs)
else:
- chunks.append(curr)
- curr = func
- if curr:
- chunks.append(curr)
- curr = ''
+ cached_outputs = []
+
+ # TODO: minimize size of forwarded data from funcs to what we actually need
+
if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores'
- if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
- commands = [(i, chunks[i] + '\n' + meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))]
+ if len(chunks) > 0:
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
- if len(chunks) > 1:
- pool = multiprocessing.Pool(processes=cores)
- outputs = pool.map(process_funcs, commands, chunksize=1)
+ commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))]
+
+ if len(chunks) > 1:
+ pool = multiprocessing.Pool(processes=cores)
+ outputs = pool.map(process_funcs, commands, chunksize=1)
+ elif len(chunks) == 1:
+ outputs = [process_funcs(commands[0])]
else:
- outputs = [process_funcs(commands[0])]
+ outputs = []
+
+ if jcache:
+ # save chunks to cache
+ for i in range(len(chunks)):
+ chunk = chunks[i]
+ keys = [settings_text, forwarded_data, chunk]
+ shortkey = shared.JCache.get_shortkey(keys)
+ shared.JCache.set(shortkey, keys, outputs[i])
+ if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks)
+
+ if jcache: outputs += cached_outputs # TODO: preserve order
+
+ outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
+
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t)
+ if DEBUG: t = time.time()
funcs_js = ''.join([output[0] for output in outputs])
@@ -185,7 +233,7 @@ def emscript(infile, settings, outfile, libraries=[]):
for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys():
indexed_functions.add(key)
outputs = None
- if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t)
if DEBUG: t = time.time()
# calculations on merged forwarded data
@@ -204,11 +252,11 @@ def emscript(infile, settings, outfile, libraries=[]):
def blockaddrsize(js):
return re.sub(r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js)
- if DEBUG: outfile.write('// pre\n')
+ #if DEBUG: outfile.write('// pre\n')
outfile.write(blockaddrsize(indexize(pre)))
pre = None
- if DEBUG: outfile.write('// funcs\n')
+ #if DEBUG: outfile.write('// funcs\n')
outfile.write(blockaddrsize(indexize(funcs_js)))
funcs_js = None
@@ -216,14 +264,14 @@ def emscript(infile, settings, outfile, libraries=[]):
forwarded_data = json.dumps(forwarded_json)
forwarded_file = temp_files.get('.2.json').name
open(forwarded_file, 'w').write(indexize(forwarded_data))
- if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t)
# Phase 3 - post
if DEBUG: t = time.time()
post_file = temp_files.get('.post.ll').name
open(post_file, 'w').write('\n') # no input, just processing of forwarded data
out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
- if DEBUG: outfile.write('// post\n')
+ #if DEBUG: outfile.write('// post\n')
outfile.write(indexize(out))
if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t)
@@ -335,6 +383,10 @@ if __name__ == '__main__':
metavar='FOO=BAR',
help=('Overrides for settings defined in settings.js. '
'May occur multiple times.'))
+ parser.add_option('-j', '--jcache',
+ action='store_true',
+ default=False,
+ help=('Enable jcache (ccache-like caching of compilation results, for faster incremental builds).'))
# Convert to the same format that argparse would have produced.
keywords, positional = parser.parse_args()
@@ -344,6 +396,7 @@ if __name__ == '__main__':
if isinstance(keywords.outfile, basestring):
keywords.outfile = open(keywords.outfile, 'w')
compiler_engine = keywords.compiler
+ jcache = keywords.jcache
temp_files.run_and_clean(lambda: main(keywords))
diff --git a/src/jsifier.js b/src/jsifier.js
index 2791c65b..595e057c 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -619,6 +619,7 @@ function JSify(data, functionsOnly, givenFunctions) {
} // otherwise, should have been set before!
if (func.setjmpTable) {
var setjmpTable = {};
+ ret += indent + 'var setjmped = false;'; // set to true if we setjmp in this invocation
ret += indent + 'var setjmpTable = {';
func.setjmpTable.forEach(function(triple) { // original label, label we created for right after the setjmp, variable setjmp result goes into
ret += '"' + getLabelId(triple[0]) + '": ' + 'function(value) { label = ' + getLabelId(triple[1]) + '; ' + triple[2] + ' = value },';
@@ -637,7 +638,7 @@ function JSify(data, functionsOnly, givenFunctions) {
}).join('\n');
ret += '\n' + indent + ' default: assert(0, "bad label: " + label);\n' + indent + '}';
if (func.setjmpTable) {
- ret += ' } catch(e) { if (!e.longjmp) throw(e); setjmpTable[e.label](e.value) }';
+ ret += ' } catch(e) { if (!setjmped) throw(e); if (!e.longjmp) throw(e); setjmpTable[e.label](e.value) }';
}
} else {
ret += (SHOW_LABELS ? indent + '/* ' + block.entries[0] + ' */' : '') + '\n' + getLabelLines(block.labels[0], indent);
diff --git a/src/library.js b/src/library.js
index 0580407d..5c2e298e 100644
--- a/src/library.js
+++ b/src/library.js
@@ -4039,6 +4039,10 @@ LibraryManager.library = {
return Math.floor(Math.random()*0x80000000);
},
+ drand48: function() {
+ return Math.random();
+ },
+
realpath__deps: ['$FS', '__setErrNo'],
realpath: function(file_name, resolved_name) {
// char *realpath(const char *restrict file_name, char *restrict resolved_name);
@@ -5113,6 +5117,34 @@ LibraryManager.library = {
llvm_objectsize_i32: function() { return -1 }, // TODO: support this
// ==========================================================================
+ // llvm-mono integration
+ // ==========================================================================
+
+ llvm_mono_load_i8_p0i8: function(ptr) {
+ return {{{ makeGetValue('ptr', 0, 'i8') }}};
+ },
+
+ llvm_mono_store_i8_p0i8: function(value, ptr) {
+ {{{ makeSetValue('ptr', 0, 'value', 'i8') }}};
+ },
+
+ llvm_mono_load_i16_p0i16: function(ptr) {
+ return {{{ makeGetValue('ptr', 0, 'i16') }}};
+ },
+
+ llvm_mono_store_i16_p0i16: function(value, ptr) {
+ {{{ makeSetValue('ptr', 0, 'value', 'i16') }}};
+ },
+
+ llvm_mono_load_i32_p0i32: function(ptr) {
+ return {{{ makeGetValue('ptr', 0, 'i32') }}};
+ },
+
+ llvm_mono_store_i32_p0i32: function(value, ptr) {
+ {{{ makeSetValue('ptr', 0, 'value', 'i32') }}};
+ },
+
+ // ==========================================================================
// math.h
// ==========================================================================
@@ -5907,7 +5939,7 @@ LibraryManager.library = {
setjmp__inline: function(env) {
// Save the label
- return '(' + makeSetValue(env, '0', 'label', 'i32') + ', 0)';
+ return '(setjmped = true, ' + makeSetValue(env, '0', 'label', 'i32') + ', 0)';
},
longjmp: function(env, value) {
@@ -6910,6 +6942,18 @@ LibraryManager.library = {
return eval(Pointer_stringify(ptr));
},
+ emscripten_run_script_string: function(ptr) {
+ var s = eval(Pointer_stringify(ptr));
+ var me = _emscripten_run_script_string;
+ if (!me.bufferSize || me.bufferSize < s.length+1) {
+ if (me.bufferSize) _free(me.buffer);
+ me.bufferSize = s.length+1;
+ me.buffer = _malloc(me.bufferSize);
+ }
+ writeStringToMemory(s, me.buffer);
+ return me.buffer;
+ },
+
emscripten_random: function() {
return Math.random();
},
diff --git a/src/modules.js b/src/modules.js
index 06677936..9ef87691 100644
--- a/src/modules.js
+++ b/src/modules.js
@@ -301,11 +301,21 @@ function cDefine(key) {
var PassManager = {
serialize: function() {
- print('\n//FORWARDED_DATA:' + JSON.stringify({
- Types: Types,
- Variables: Variables,
- Functions: Functions
- }));
+ if (phase == 'pre') {
+ print('\n//FORWARDED_DATA:' + JSON.stringify({
+ Types: Types,
+ Variables: Variables,
+ Functions: Functions
+ }));
+ } else if (phase == 'funcs') {
+ print('\n//FORWARDED_DATA:' + JSON.stringify({
+ Types: { preciseI64MathUsed: Types.preciseI64MathUsed },
+ Functions: {
+ blockAddresses: Functions.blockAddresses,
+ indexedFunctions: Functions.indexedFunctions
+ }
+ }));
+ }
},
load: function(json) {
var data = JSON.parse(json);
diff --git a/system/include/emscripten/emscripten.h b/system/include/emscripten/emscripten.h
index 871a7162..3eefe0b8 100644
--- a/system/include/emscripten/emscripten.h
+++ b/system/include/emscripten/emscripten.h
@@ -31,6 +31,7 @@ extern "C" {
*/
extern void emscripten_run_script(const char *script);
extern int emscripten_run_script_int(const char *script);
+extern char *emscripten_run_script_string(const char *script); // uses a single buffer - shared between calls!
extern void emscripten_async_run_script(const char *script, int millis);
/*
diff --git a/tests/hello_libcxx_mod1.cpp b/tests/hello_libcxx_mod1.cpp
new file mode 100644
index 00000000..2389b1ef
--- /dev/null
+++ b/tests/hello_libcxx_mod1.cpp
@@ -0,0 +1,9 @@
+#include <iostream>
+
+int main()
+{
+ std::cout << "hello, world!" << std::endl;
+ std::cout << "hello, world!" << std::endl;
+ return 0;
+}
+
diff --git a/tests/runner.py b/tests/runner.py
index 6404a211..a4bc4944 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -1987,6 +1987,52 @@ c5,de,15,8a
else:
self.do_run(src, 'second\nmain: 0\n')
+ def test_longjmp2(self):
+ src = r'''
+ #include <setjmp.h>
+ #include <stdio.h>
+
+ typedef struct {
+ jmp_buf* jmp;
+ } jmp_state;
+
+ void stack_manipulate_func(jmp_state* s, int level) {
+ jmp_buf buf;
+
+ printf("Entering stack_manipulate_func, level: %d\n", level);
+
+ if (level == 0) {
+ s->jmp = &buf;
+ if (setjmp(*(s->jmp)) == 0) {
+ printf("Setjmp normal execution path, level: %d\n", level);
+ stack_manipulate_func(s, level + 1);
+ } else {
+ printf("Setjmp error execution path, level: %d\n", level);
+ }
+ } else {
+ printf("Perform longjmp at level %d\n", level);
+ longjmp(*(s->jmp), 1);
+ }
+
+ printf("Exiting stack_manipulate_func, level: %d\n", level);
+ }
+
+ int main(int argc, char *argv[]) {
+ jmp_state s;
+ s.jmp = NULL;
+ stack_manipulate_func(&s, 0);
+
+ return 0;
+ }
+ '''
+ self.do_run(src, '''Entering stack_manipulate_func, level: 0
+Setjmp normal execution path, level: 0
+Entering stack_manipulate_func, level: 1
+Perform longjmp at level 1
+Setjmp error execution path, level: 0
+Exiting stack_manipulate_func, level: 0
+''')
+
def test_exceptions(self):
if Settings.QUANTUM_SIZE == 1: return self.skip("we don't support libcxx in q1")
@@ -2777,6 +2823,7 @@ c5,de,15,8a
// EMSCRIPTEN_COMMENT("hello from the source");
emscripten_run_script("Module.print('hello world' + '!')");
printf("*%d*\n", emscripten_run_script_int("5*20"));
+ printf("*%s*\n", emscripten_run_script_string("'five'+'six'"));
emscripten_run_script("_save_me_aimee()");
return 0;
}
@@ -2788,19 +2835,26 @@ def process(filename):
# TODO: restore this (see comment in emscripten.h) assert '// hello from the source' in src
'''
- self.do_run(src, 'hello world!\n*100*\nmann\n', post_build=check)
+ self.do_run(src, 'hello world!\n*100*\n*fivesix*\nmann\n', post_build=check)
def test_inlinejs(self):
src = r'''
#include <stdio.h>
+ double get() {
+ double ret = 0;
+ __asm __volatile__("12/3.3":"=a"(ret));
+ return ret;
+ }
+
int main() {
asm("Module.print('Inline JS is very cool')");
+ printf("%.2f\n", get());
return 0;
}
'''
- self.do_run(src, 'Inline JS is very cool')
+ self.do_run(src, 'Inline JS is very cool\n3.64')
def test_memorygrowth(self):
if Settings.USE_TYPED_ARRAYS == 0: return self.skip('memory growth is only supported with typed arrays')
@@ -6180,7 +6234,9 @@ def process(filename):
finally:
del os.environ['EMCC_DEBUG']
for debug in [1,2]:
- self.assertIdentical(open('release.js').read().replace('\n\n', '\n').replace('\n\n', '\n'), open('debug%d.js' % debug).read().replace('\n\n', '\n').replace('\n\n', '\n')) # EMCC_DEBUG=1 mode must not generate different code!
+ def clean(text):
+ return text.replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n').replace('\n\n', '\n')
+ self.assertIdentical(clean(open('release.js').read()), clean(open('debug%d.js' % debug).read())) # EMCC_DEBUG=1 mode must not generate different code!
print >> sys.stderr, 'debug check %d passed too' % debug
try_delete(CANONICAL_TEMP_DIR)
@@ -8287,7 +8343,7 @@ f.close()
(path_from_root('tools', 'test-js-optimizer-t2.js'), open(path_from_root('tools', 'test-js-optimizer-t2-output.js')).read(),
['simplifyExpressionsPre', 'optimizeShiftsAggressive']),
# Make sure that optimizeShifts handles functions with shift statements.
- (path_from_root('tools', 'test-js-optimizer-t3.js'), open(path_from_root('tools', 'test-js-optimizer-t3.js')).read(),
+ (path_from_root('tools', 'test-js-optimizer-t3.js'), open(path_from_root('tools', 'test-js-optimizer-t3-output.js')).read(),
['optimizeShiftsAggressive']),
(path_from_root('tools', 'test-js-optimizer-regs.js'), open(path_from_root('tools', 'test-js-optimizer-regs-output.js')).read(),
['registerize']),
@@ -8306,6 +8362,17 @@ f.close()
assert 'foo.o: ' in output, '-%s failed to produce the right output: %s' % (opt, output)
assert 'error' not in err, 'Unexpected stderr: ' + err
+ def test_chunking(self):
+ if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
+ if multiprocessing.cpu_count() < 2: return self.skip('need multiple cores')
+ try:
+ os.environ['EMCC_DEBUG'] = '1'
+ output, err = Popen(['python', EMCC, path_from_root('tests', 'hello_libcxx.cpp'), '-O1'], stdout=PIPE, stderr=PIPE).communicate()
+ assert 'phase 2 working on 3 chunks' in err, err
+ assert 'splitting up js optimization into 2 chunks' in err, err
+ finally:
+ del os.environ['EMCC_DEBUG']
+
def test_scons(self): # also incidentally tests c++11 integration in llvm 3.1
try_delete(os.path.join(self.get_dir(), 'test'))
shutil.copytree(path_from_root('tests', 'scons'), os.path.join(self.get_dir(), 'test'))
@@ -10062,6 +10129,8 @@ elif 'sanity' in str(sys.argv):
assert os.path.exists(CONFIG_FILE), 'To run these tests, we need a (working!) %s file to already exist' % EM_CONFIG
+ assert not os.environ.get('EMCC_DEBUG'), 'do not run sanity checks in debug mode!'
+
shutil.copyfile(CONFIG_FILE, CONFIG_FILE + '_backup')
def restore():
shutil.copyfile(CONFIG_FILE + '_backup', CONFIG_FILE)
@@ -10294,10 +10363,11 @@ fi
self.assertContained(SANITY_MESSAGE, output)
# but with EMCC_DEBUG=1 we should check
- assert not os.environ.get('EMCC_DEBUG'), 'do not run sanity checks in debug mode!'
- os.environ['EMCC_DEBUG'] = '1'
- output = self.check_working(EMCC)
- del os.environ['EMCC_DEBUG']
+ try:
+ os.environ['EMCC_DEBUG'] = '1'
+ output = self.check_working(EMCC)
+ finally:
+ del os.environ['EMCC_DEBUG']
self.assertContained(SANITY_MESSAGE, output)
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
@@ -10354,8 +10424,8 @@ fi
assert not os.path.exists(EMCC_CACHE)
try:
- emcc_debug = os.environ.get('EMCC_DEBUG')
os.environ['EMCC_DEBUG'] ='1'
+ self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp')
# Building a file that doesn't need cached stuff should not trigger cache generation
output = self.do([EMCC, path_from_root('tests', 'hello_world.cpp')])
@@ -10371,8 +10441,6 @@ fi
ll_name1 = os.path.join(TEMP_DIR, 'emscripten_temp', 'emcc-2-ll.ll')
ll_name2 = os.path.join(TEMP_DIR, 'emscripten_temp', 'emcc-3-ll.ll')
- self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp')
-
# Building a file that *does* need dlmalloc *should* trigger cache generation, but only the first time
for filename, libname in [('hello_malloc.cpp', 'dlmalloc'), ('hello_libcxx.cpp', 'libcxx')]:
for i in range(3):
@@ -10405,8 +10473,7 @@ fi
print i, 'll metadata should be removed in -O1 and O2 by default', ll[-300:]
assert False
finally:
- if emcc_debug:
- os.environ['EMCC_DEBUG'] = emcc_debug
+ del os.environ['EMCC_DEBUG']
# Manual cache clearing
assert os.path.exists(EMCC_CACHE)
@@ -10434,6 +10501,79 @@ fi
assert os.path.exists(RELOOPER) == (i >= 2), 'have relooper on O2: ' + output
assert ('L2 : do {' in open('a.out.js').read()) == (i >= 2), 'reloop code on O2: ' + output
+ def test_jcache(self):
+ PRE_LOAD_MSG = 'loading pre from jcache'
+ PRE_SAVE_MSG = 'saving pre to jcache'
+ FUNC_CHUNKS_LOAD_MSG = ' funcchunks from jcache'
+ FUNC_CHUNKS_SAVE_MSG = ' funcchunks to jcache'
+ JSFUNC_CHUNKS_LOAD_MSG = 'jsfuncchunks from jcache'
+ JSFUNC_CHUNKS_SAVE_MSG = 'jsfuncchunks to jcache'
+
+ restore()
+ Cache.erase()
+
+ try:
+ os.environ['EMCC_DEBUG'] = '1'
+ self.working_dir = os.path.join(TEMP_DIR, 'emscripten_temp')
+ if not os.path.exists(self.working_dir): os.makedirs(self.working_dir)
+
+ assert not os.path.exists(JCache.get_cachename('emscript_files'))
+
+ srcs = {}
+ used_jcache = False
+
+ for args, input_file, expect_pre_save, expect_pre_load, expect_funcs_save, expect_funcs_load, expect_jsfuncs_save, expect_jsfuncs_load, expected in [
+ ([], 'hello_world_loop.cpp', False, False, False, False, False, False, []),
+ (['--jcache'], 'hello_world_loop.cpp', True, False, True, False, True, False, []),
+ (['--jcache'], 'hello_world_loop.cpp', False, True, False, True, False, True, []),
+ ([], 'hello_world_loop.cpp', False, False, False, False, False, False, []),
+ # new
+ ([], 'hello_world.cpp', False, False, False, False, False, False, []),
+ (['--jcache'], 'hello_world.cpp', True, False, True, False, True, False, []),
+ (['--jcache'], 'hello_world.cpp', False, True, False, True, False, True, []),
+ ([], 'hello_world.cpp', False, False, False, False, False, False, []),
+ # go back to old file, experience caching
+ (['--jcache'], 'hello_world_loop.cpp', False, True, False, True, False, True, []),
+ # new, large file
+ ([], 'hello_malloc.cpp', False, False, False, False, False, False, []),
+ (['--jcache'], 'hello_malloc.cpp', True, False, True, False, True, False, []),
+ (['--jcache'], 'hello_malloc.cpp', False, True, False, True, False, True, []),
+ ([], 'hello_malloc.cpp', False, False, False, False, False, False, []),
+ # new, huge file
+ ([], 'hello_libcxx.cpp', False, False, False, False, False, False, ('2 chunks', '3 chunks')),
+ (['--jcache'], 'hello_libcxx.cpp', True, False, True, False, True, False, []),
+ (['--jcache'], 'hello_libcxx.cpp', False, True, False, True, False, True, []),
+ ([], 'hello_libcxx.cpp', False, False, False, False, False, False, []),
+ # finally, build a file close to the previous, to see that some chunks are found in the cache and some not
+ (['--jcache'], 'hello_libcxx_mod1.cpp', False, True, True, True, True, False, []), # win on pre, mix on funcs, fail on jsfuncs
+ (['--jcache'], 'hello_libcxx_mod1.cpp', False, True, False, True, False, True, []),
+ ]:
+ print >> sys.stderr, args, input_file, expect_pre_save, expect_pre_load, expect_funcs_save, expect_funcs_load, expect_jsfuncs_save, expect_jsfuncs_load, expected
+ self.clear()
+ out, err = Popen(['python', EMCC, '-O2', '--closure', '0', path_from_root('tests', input_file)] + args, stdout=PIPE, stderr=PIPE).communicate()
+ errtail = err.split('emcc invocation')[-1]
+ self.assertContained('hello, world!', run_js('a.out.js'), errtail)
+ assert (PRE_SAVE_MSG in err) == expect_pre_save, errtail
+ assert (PRE_LOAD_MSG in err) == expect_pre_load, errtail
+ assert (FUNC_CHUNKS_SAVE_MSG in err) == expect_funcs_save, errtail
+ assert (FUNC_CHUNKS_LOAD_MSG in err) == expect_funcs_load, errtail
+ assert (JSFUNC_CHUNKS_SAVE_MSG in err) == expect_jsfuncs_save, errtail
+ assert (JSFUNC_CHUNKS_LOAD_MSG in err) == expect_jsfuncs_load, errtail
+ for expect in expected: assert expect in err, expect + ' ? ' + errtail
+ curr = open('a.out.js').read()
+ if input_file not in srcs:
+ srcs[input_file] = curr
+ else:
+ open('/home/alon/Dev/emscripten/a', 'w').write(srcs[input_file])
+ open('/home/alon/Dev/emscripten/b', 'w').write(curr)
+ assert abs(len(curr)/float(len(srcs[input_file]))-1)<0.01, 'contents may shift in order, but must remain the same size %d vs %d' % (len(curr), len(srcs[input_file])) + '\n' + errtail
+ used_jcache = used_jcache or ('--jcache' in args)
+ assert used_jcache == os.path.exists(JCache.get_cachename('emscript_files'))
+ #print >> sys.stderr, errtail
+
+ finally:
+ del os.environ['EMCC_DEBUG']
+
else:
raise Exception('Test runner is confused: ' + str(sys.argv))
diff --git a/tools/eliminator/eliminator-test-output.js b/tools/eliminator/eliminator-test-output.js
index 00647fca..ede34103 100644
--- a/tools/eliminator/eliminator-test-output.js
+++ b/tools/eliminator/eliminator-test-output.js
@@ -6132,5 +6132,4 @@ function _mallocNoU($bytes) {
return $mem_0;
return null;
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a", "b", "c", "f", "g", "h", "py", "r", "t", "f2", "f3", "llvm3_1", "_inflate", "_malloc", "_mallocNoU"]
diff --git a/tools/eliminator/safe-eliminator-test-output.js b/tools/eliminator/safe-eliminator-test-output.js
index bb3f17e6..57f0a743 100644
--- a/tools/eliminator/safe-eliminator-test-output.js
+++ b/tools/eliminator/safe-eliminator-test-output.js
@@ -82,5 +82,4 @@ function a($directory) {
print(zzz1);
} while (1);
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["a"]
diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js
index a9a181b1..29aecdcf 100644
--- a/tools/js-optimizer.js
+++ b/tools/js-optimizer.js
@@ -214,6 +214,7 @@ function traverse(node, pre, post, stack) {
// Only walk through the generated functions
function traverseGenerated(ast, pre, post, stack) {
+ assert(generatedFunctions);
traverse(ast, function(node) {
if (node[0] == 'defun' && isGenerated(node[1])) {
traverse(node, pre, post, stack);
@@ -223,6 +224,7 @@ function traverseGenerated(ast, pre, post, stack) {
}
function traverseGeneratedFunctions(ast, callback) {
+ assert(generatedFunctions);
traverse(ast, function(node) {
if (node[0] == 'defun' && isGenerated(node[1])) {
callback(node);
@@ -1914,6 +1916,5 @@ do {
js = js.replace(/\n *\n/g, '\n');
} while (js != old);
print(js);
-if (metadata && printMetadata) print(metadata);
print('\n');
diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py
index 8681280a..5bed4cb7 100644
--- a/tools/js_optimizer.py
+++ b/tools/js_optimizer.py
@@ -1,5 +1,8 @@
-import os, sys, subprocess, multiprocessing
+import os, sys, subprocess, multiprocessing, re
+import shared
+
+temp_files = shared.TempFiles()
__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
def path_from_root(*pathelems):
@@ -17,13 +20,15 @@ def run_on_chunk(command):
filename = command[2] # XXX hackish
output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
- filename += '.jo.js'
+ filename = temp_files.get(os.path.basename(filename) + '.jo.js').name
f = open(filename, 'w')
f.write(output)
f.close()
return filename
-def run(filename, passes, js_engine):
+def run(filename, passes, js_engine, jcache):
+ if jcache: shared.JCache.ensure()
+
if type(passes) == str:
passes = [passes]
@@ -37,64 +42,129 @@ def run(filename, passes, js_engine):
suffix = ''
if suffix_start >= 0:
suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n'
+ # if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
+ generated = set(eval(suffix[len(suffix_marker)+1:]))
+
+ if not suffix and jcache:
+ # JCache cannot be used without metadata, since it might reorder stuff, and that's dangerous since only generated can be reordered
+ # This means jcache does not work after closure compiler runs, for example. But you won't get much benefit from jcache with closure
+ # anyhow (since closure is likely the longest part of the build).
+ if DEBUG: print >>sys.stderr, 'js optimizer: no metadata, so disabling jcache'
+ jcache = False
+
+ # If we process only generated code, find that and save the rest on the side
+ func_sig = re.compile('function (_[\w$]+)\(')
+ if suffix:
+ pos = 0
+ gen_start = 0
+ gen_end = 0
+ while 1:
+ m = func_sig.search(js, pos)
+ if not m: break
+ pos = m.end()
+ ident = m.group(1)
+ if ident in generated:
+ if not gen_start:
+ gen_start = m.start()
+ assert gen_start
+ gen_end = js.find('\n}\n', m.end()) + 3
+ assert gen_end > gen_start
+ pre = js[:gen_start]
+ post = js[gen_end:]
+ js = js[gen_start:gen_end]
+ else:
+ pre = ''
+ post = ''
# Pick where to split into chunks, so that (1) they do not oom in node/uglify, and (2) we can run them in parallel
- chunks = []
- i = 0
- f_start = 0
- while True:
- f_end = f_start
- while f_end-f_start < BEST_JS_PROCESS_SIZE and f_end != -1:
- f_end = js.find('\n}\n', f_end+1)
- chunk = js[f_start:(-1 if f_end == -1 else f_end+3)] + suffix
- temp_file = filename + '.p%d.js' % i
- #if DEBUG: print >> sys.stderr, ' chunk %d: %d bytes' % (i, (f_end if f_end >= 0 else len(js)) - f_start)
- i += 1
- f_start = f_end+3
- done = f_end == -1 or f_start >= len(js)
- if done and len(chunks) == 0: break # do not write anything out, just use the input file
- f = open(temp_file, 'w')
- f.write(chunk)
- f.close()
- chunks.append(temp_file)
- if done: break
-
- if len(chunks) == 0:
- chunks.append(filename)
-
- # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok)
- commands = map(lambda chunk: [js_engine, JS_OPTIMIZER, chunk] + passes, chunks)
-
- if len(chunks) > 1:
- # We are splitting into chunks. Hopefully we can do that in parallel
- commands = map(lambda command: command + ['noPrintMetadata'], commands)
- filename += '.jo.js'
-
- fail = None
- cores = min(multiprocessing.cpu_count(), chunks)
- if cores < 2:
- fail = 'python reports you have %d cores' % cores
- #elif WINDOWS:
- # fail = 'windows (see issue 663)' # This seems fixed with adding emcc.py that imports this file
-
- if not fail:
+ # If we have metadata, we split only the generated code, and save the pre and post on the side (and do not optimize them)
+ parts = map(lambda part: part, js.split('\n}\n'))
+ funcs = []
+ for i in range(len(parts)):
+ func = parts[i]
+ if i < len(parts)-1: func += '\n}\n' # last part needs no }
+ m = func_sig.search(func)
+ if m:
+ ident = m.group(1)
+ else:
+ if suffix: continue # ignore whitespace
+ ident = 'anon_%d' % i
+ funcs.append((ident, func))
+ parts = None
+ total_size = len(js)
+ js = None
+
+ chunks = shared.JCache.chunkify(funcs, BEST_JS_PROCESS_SIZE, 'jsopt' if jcache else None)
+
+ if jcache:
+ # load chunks from cache where we can # TODO: ignore small chunks
+ cached_outputs = []
+ def load_from_cache(chunk):
+ keys = [chunk]
+ shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code
+ out = shared.JCache.get(shortkey, keys)
+ if out:
+ cached_outputs.append(out)
+ return False
+ return True
+ chunks = filter(load_from_cache, chunks)
+ if len(cached_outputs) > 0:
+ if DEBUG: print >> sys.stderr, ' loading %d jsfuncchunks from jcache' % len(cached_outputs)
+ else:
+ cached_outputs = []
+
+ if len(chunks) > 0:
+ def write_chunk(chunk, i):
+ temp_file = temp_files.get('.jsfunc_%d.ll' % i).name
+ f = open(temp_file, 'w')
+ f.write(chunk)
+ f.write(suffix)
+ f.close()
+ return temp_file
+ filenames = [write_chunk(chunks[i], i) for i in range(len(chunks))]
+ else:
+ filenames = []
+
+ if len(filenames) > 0:
+ # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok)
+ commands = map(lambda filename: [js_engine, JS_OPTIMIZER, filename, 'noPrintMetadata'] + passes, filenames)
+
+ cores = min(multiprocessing.cpu_count(), filenames)
+ if len(chunks) > 1 and cores >= 2:
# We can parallelize
- if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores' % (len(chunks), cores)
+ if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (len(chunks), cores, total_size/(1024*1024.))
pool = multiprocessing.Pool(processes=cores)
filenames = pool.map(run_on_chunk, commands, chunksize=1)
else:
# We can't parallize, but still break into chunks to avoid uglify/node memory issues
- if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks (not in parallel because %s)' % (len(chunks), fail)
+ if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks' % (len(chunks))
filenames = [run_on_chunk(command) for command in commands]
+ else:
+ filenames = []
- f = open(filename, 'w')
- for out_file in filenames:
- f.write(open(out_file).read())
- f.write(suffix)
+ filename += '.jo.js'
+ f = open(filename, 'w')
+ f.write(pre);
+ for out_file in filenames:
+ f.write(open(out_file).read())
f.write('\n')
- f.close()
- return filename
- else:
- # one simple chunk, just do it
- return run_on_chunk(commands[0])
+ if jcache:
+ for cached in cached_outputs:
+ f.write(cached); # TODO: preserve order
+ f.write('\n')
+ f.write(post);
+ # No need to write suffix: if there was one, it is inside post which exists when suffix is there
+ f.write('\n')
+ f.close()
+
+ if jcache:
+ # save chunks to cache
+ for i in range(len(chunks)):
+ chunk = chunks[i]
+ keys = [chunk]
+ shortkey = shared.JCache.get_shortkey(keys)
+ shared.JCache.set(shortkey, keys, open(filenames[i]).read())
+ if DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d jsfuncchunks to jcache' % len(chunks)
+
+ return filename
diff --git a/tools/shared.py b/tools/shared.py
index 4992badc..7bafd51d 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1,4 +1,4 @@
-import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess
+import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, md5, cPickle
from subprocess import Popen, PIPE, STDOUT
from tempfile import mkstemp
@@ -59,8 +59,6 @@ class WindowsPopen:
# Install our replacement Popen handler if we are running on Windows to avoid python spawn process function.
if os.name == 'nt':
Popen = WindowsPopen
-
-import js_optimizer
__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
def path_from_root(*pathelems):
@@ -1006,8 +1004,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
return opts
@staticmethod
- def js_optimizer(filename, passes):
- return js_optimizer.run(filename, passes, NODE_JS)
+ def js_optimizer(filename, passes, jcache):
+ return js_optimizer.run(filename, passes, NODE_JS, jcache)
@staticmethod
def closure_compiler(filename):
@@ -1153,6 +1151,152 @@ class Cache:
shutil.copyfile(creator(), cachename)
return cachename
+# JS-specific cache. We cache the results of compilation and optimization,
+# so that in incremental builds we can just load from cache.
+# We cache reasonably-large-sized chunks
+class JCache:
+ dirname = os.path.join(Cache.dirname, 'jcache')
+
+ @staticmethod
+ def ensure():
+ Cache.ensure()
+ if not os.path.exists(JCache.dirname):
+ os.makedirs(JCache.dirname)
+
+ @staticmethod
+ def get_shortkey(keys):
+ if type(keys) not in [list, tuple]:
+ keys = [keys]
+ ret = ''
+ for key in keys:
+ assert type(key) == str
+ ret += md5.md5(key).hexdigest()
+ return ret
+
+ @staticmethod
+ def get_cachename(shortkey):
+ return os.path.join(JCache.dirname, shortkey)
+
+ # Returns a cached value, if it exists. Make sure the full key matches
+ @staticmethod
+ def get(shortkey, keys):
+ #if DEBUG: print >> sys.stderr, 'jcache get?', shortkey
+ cachename = JCache.get_cachename(shortkey)
+ if not os.path.exists(cachename):
+ #if DEBUG: print >> sys.stderr, 'jcache none at all'
+ return
+ data = cPickle.Unpickler(open(cachename, 'rb')).load()
+ if len(data) != 2:
+ #if DEBUG: print >> sys.stderr, 'jcache error in get'
+ return
+ oldkeys = data[0]
+ if len(oldkeys) != len(keys):
+ #if DEBUG: print >> sys.stderr, 'jcache collision (a)'
+ return
+ for i in range(len(oldkeys)):
+ if oldkeys[i] != keys[i]:
+ #if DEBUG: print >> sys.stderr, 'jcache collision (b)'
+ return
+ #if DEBUG: print >> sys.stderr, 'jcache win'
+ return data[1]
+
+ # Sets the cached value for a key (from get_key)
+ @staticmethod
+ def set(shortkey, keys, value):
+ cachename = JCache.get_cachename(shortkey)
+ cPickle.Pickler(open(cachename, 'wb')).dump([keys, value])
+ #if DEBUG:
+ # for i in range(len(keys)):
+ # open(cachename + '.key' + str(i), 'w').write(keys[i])
+ # open(cachename + '.value', 'w').write(value)
+
+ # Given a set of functions of form (ident, text), and a preferred chunk size,
+ # generates a set of chunks for parallel processing and caching.
+ # It is very important to generate similar chunks in incremental builds, in
+ # order to maximize the chance of cache hits. To achieve that, we save the
+ # chunking used in the previous compilation of this phase, and we try to
+ # generate the same chunks, barring big differences in function sizes that
+ # violate our chunk size guideline. If caching is not used, chunking_file
+ # should be None
+ @staticmethod
+ def chunkify(funcs, chunk_size, chunking_file):
+ previous_mapping = None
+ if chunking_file:
+ chunking_file = JCache.get_cachename(chunking_file)
+ if os.path.exists(chunking_file):
+ try:
+ previous_mapping = cPickle.Unpickler(open(chunking_file, 'rb')).load() # maps a function identifier to the chunk number it will be in
+ except:
+ pass
+ chunks = []
+ if previous_mapping:
+ # initialize with previous chunking
+ news = []
+ for func in funcs:
+ ident, data = func
+ if not ident in previous_mapping:
+ news.append(func)
+ else:
+ n = previous_mapping[ident]
+ while n >= len(chunks): chunks.append([])
+ chunks[n].append(func)
+ # add news and adjust for new sizes
+ spilled = news
+ for chunk in chunks:
+ size = sum([len(func[1]) for func in chunk])
+ while size > 1.5*chunk_size and len(chunk) > 0:
+ spill = chunk.pop()
+ spilled.append(spill)
+ size -= len(spill[1])
+ for chunk in chunks:
+ size = sum([len(func[1]) for func in chunk])
+ while size < 0.66*chunk_size and len(spilled) > 0:
+ spill = spilled.pop()
+ chunk.append(spill)
+ size += len(spill[1])
+ chunks = filter(lambda chunk: len(chunk) > 0, chunks) # might have empty ones, eliminate them
+ funcs = spilled # we will allocate these into chunks as if they were normal inputs
+ # initialize reasonably, the rest of the funcs we need to split out
+ curr = []
+ total_size = 0
+ for i in range(len(funcs)):
+ func = funcs[i]
+ curr_size = len(func[1])
+ if total_size + curr_size < chunk_size:
+ curr.append(func)
+ total_size += curr_size
+ else:
+ chunks.append(curr)
+ curr = [func]
+ total_size = curr_size
+ if curr:
+ chunks.append(curr)
+ curr = None
+ if chunking_file:
+ # sort within each chunk, to keep the order identical
+ for chunk in chunks:
+ chunk.sort(key=lambda func: func[0])
+ # save new mapping info
+ new_mapping = {}
+ for i in range(len(chunks)):
+ chunk = chunks[i]
+ for ident, data in chunk:
+ new_mapping[ident] = i
+ cPickle.Pickler(open(chunking_file, 'wb')).dump(new_mapping)
+ #if DEBUG:
+ # if previous_mapping:
+ # for ident in set(previous_mapping.keys() + new_mapping.keys()):
+ # if previous_mapping.get(ident) != new_mapping.get(ident):
+ # print >> sys.stderr, 'mapping inconsistency', ident, previous_mapping.get(ident), new_mapping.get(ident)
+ # for key, value in new_mapping.iteritems():
+ # print >> sys.stderr, 'mapping:', key, value
+ return [''.join([func[1] for func in chunk]) for chunk in chunks] # remove function names
+
+class JS:
+ @staticmethod
+ def to_nice_ident(ident): # limited version of the JS function toNiceIdent
+ return ident.replace('%', '$').replace('@', '_');
+
# Compression of code and data for smaller downloads
class Compression:
on = False
@@ -1191,3 +1335,5 @@ def unsuffixed(name):
def unsuffixed_basename(name):
return os.path.basename(unsuffixed(name))
+import js_optimizer
+
diff --git a/tools/test-js-optimizer-output.js b/tools/test-js-optimizer-output.js
index 9c9fa063..730426b1 100644
--- a/tools/test-js-optimizer-output.js
+++ b/tools/test-js-optimizer-output.js
@@ -279,5 +279,4 @@ function notComps() {
function tricky() {
var $conv642 = $conv6374 - (($132 << 16 >> 16 | 0) / 2 & -1) & 65535;
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["abc", "xyz", "xyz2", "expr", "loopy", "bits", "maths", "hoisting", "demangle", "lua", "moreLabels", "notComps", "tricky"]
diff --git a/tools/test-js-optimizer-regs-output.js b/tools/test-js-optimizer-regs-output.js
index 2f91f52b..90b67a47 100644
--- a/tools/test-js-optimizer-regs-output.js
+++ b/tools/test-js-optimizer-regs-output.js
@@ -225,5 +225,4 @@ function switchey(x) {
r8 = x + 2;
pp(r8);
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["test", "primes", "atomic", "fcntl_open", "ex", "switchey"]
diff --git a/tools/test-js-optimizer-t2-output.js b/tools/test-js-optimizer-t2-output.js
index 62cd6e0d..0ae66be5 100644
--- a/tools/test-js-optimizer-t2-output.js
+++ b/tools/test-js-optimizer-t2-output.js
@@ -88,5 +88,4 @@ function shifty($id2) {
q(go() << 16);
q(go() + 2 >> 2);
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["shifty"]
diff --git a/tools/test-js-optimizer-t2c-output.js b/tools/test-js-optimizer-t2c-output.js
index f41bb815..726112ec 100644
--- a/tools/test-js-optimizer-t2c-output.js
+++ b/tools/test-js-optimizer-t2c-output.js
@@ -14,5 +14,4 @@ function shifty() {
q($13 + 13 << 2);
q(h() >> 2 << 2);
}
-// EMSCRIPTEN_GENERATED_FUNCTIONS: ["shifty"]
diff --git a/tools/test-js-optimizer-t3-output.js b/tools/test-js-optimizer-t3-output.js
new file mode 100644
index 00000000..924868fa
--- /dev/null
+++ b/tools/test-js-optimizer-t3-output.js
@@ -0,0 +1,49 @@
+function _png_create_write_struct_2($user_png_ver, $error_ptr, $error_fn, $warn_fn, $mem_ptr, $malloc_fn, $free_fn) {
+ var $png_ptr$s2;
+ var label;
+ label = 2;
+ var setjmpTable = {
+ "2": (function(value) {
+ label = 5;
+ $call1 = value;
+ }),
+ dummy: 0
+ };
+ while (1) try {
+ switch (label) {
+ case 2:
+ var $png_ptr;
+ var $call = _png_create_struct(1);
+ $png_ptr = $call;
+ var $call1 = (HEAP32[$png_ptr >> 2] = label, 0);
+ label = 5;
+ break;
+ case 5:
+ var $2 = $png_ptr;
+ if (($call1 | 0) == 0) {
+ label = 4;
+ break;
+ } else {
+ label = 3;
+ break;
+ }
+ case 3:
+ var $4 = HEAP32[($png_ptr >> 2) + (148 >> 2)];
+ _png_free($2, $4);
+ HEAP32[($png_ptr >> 2) + (148 >> 2)] = 0;
+ _png_destroy_struct($png_ptr);
+ var $retval_0 = 0;
+ label = 4;
+ break;
+ case 4:
+ var $retval_0;
+ return $retval_0;
+ default:
+ assert(0, "bad label: " + label);
+ }
+ } catch (e) {
+ if (!e.longjmp) throw e;
+ setjmpTable[e.label](e.value);
+ }
+}
+
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-09-09 05:14:36 +0000