aboutsummaryrefslogtreecommitdiff
path: root/emscripten.py
diff options
context:
space:
mode:
Diffstat (limited to 'emscripten.py')
-rwxr-xr-xemscripten.py196
1 files changed, 182 insertions, 14 deletions
diff --git a/emscripten.py b/emscripten.py
index 800ca8d7..687adb66 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -9,12 +9,7 @@ header files (so that the JS compiler can see the constants in those
headers, for the libc implementation in JS).
'''
-import json
-import optparse
-import os
-import subprocess
-import re
-import sys
+import os, sys, json, optparse, subprocess, re, time, multiprocessing
if not os.environ.get('EMSCRIPTEN_SUPPRESS_USAGE_WARNING'):
print >> sys.stderr, '''
@@ -25,6 +20,8 @@ WARNING: You should normally never use this! Use emcc instead.
from tools import shared
+DEBUG = os.environ.get('EMCC_DEBUG')
+
__rootpath__ = os.path.abspath(os.path.dirname(__file__))
def path_from_root(*pathelems):
"""Returns the absolute path for which the given path elements are
@@ -34,22 +31,189 @@ def path_from_root(*pathelems):
temp_files = shared.TempFiles()
+compiler_engine = None
+
+def scan(ll, settings):
+ # blockaddress(@main, %23)
+ blockaddrs = []
+ for blockaddr in re.findall('blockaddress\([^)]*\)', ll):
+ b = blockaddr.split('(')[1][:-1].split(', ')
+ blockaddrs.append(b)
+ if len(blockaddrs) > 0:
+ settings['NECESSARY_BLOCKADDRS'] = blockaddrs
+
+NUM_CHUNKS_PER_CORE = 4
+MIN_CHUNK_SIZE = 1024*1024
+MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes
+
+def process_funcs(args):
+ i, ll, settings_file, compiler, forwarded_file, libraries = args
+ funcs_file = temp_files.get('.func_%d.ll' % i).name
+ open(funcs_file, 'w').write(ll)
+ out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
+ return out.split('//FORWARDED_DATA:')
def emscript(infile, settings, outfile, libraries=[]):
- """Runs the emscripten LLVM-to-JS compiler.
+ """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible
Args:
infile: The path to the input LLVM assembly file.
- settings: JSON-formatted string of settings that overrides the values
+ settings: JSON-formatted settings that override the values
defined in src/settings.js.
outfile: The file where the output is written.
"""
- settings_file = temp_files.get('.txt').name # Save settings to a file to work around v8 issue 1579
+
+ compiler = path_from_root('src', 'compiler.js')
+
+ # Parallelization: We run 3 phases:
+ # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble.
+ # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently.
+ # 3 aka 'post' : Process globals, generate postamble and finishing touches.
+
+ if DEBUG: print >> sys.stderr, 'emscript: ll=>js'
+
+ # Pre-scan ll and alter settings as necessary
+ if DEBUG: t = time.time()
+ ll = open(infile).read()
+ scan(ll, settings)
+ total_ll_size = len(ll)
+ ll = None # allow collection
+ if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t)
+
+ # Split input into the relevant parts for each phase
+ pre = []
+ funcs = [] # split up functions here, for parallelism later
+ meta = [] # needed by each function XXX
+ post = []
+
+ if DEBUG: t = time.time()
+ in_func = False
+ ll_lines = open(infile).readlines()
+ for line in ll_lines:
+ if in_func:
+ funcs[-1].append(line)
+ if line.startswith('}'):
+ in_func = False
+ funcs[-1] = ''.join(funcs[-1])
+ pre.append(line) # pre needs it to, so we know about all implemented functions
+ else:
+ if line.startswith('define '):
+ in_func = True
+ funcs.append([line])
+ pre.append(line) # pre needs it to, so we know about all implemented functions
+ elif line.find(' = type { ') > 0:
+ pre.append(line) # type
+ elif line.startswith('!'):
+ meta.append(line) # metadata
+ else:
+ post.append(line) # global
+ pre.append(line) # pre needs it to, so we know about globals in pre and funcs
+ ll_lines = None
+ meta = ''.join(meta)
+ if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down'
+ if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t)
+
+ #if DEBUG:
+ # print >> sys.stderr, '========= pre ================\n'
+ # print >> sys.stderr, ''.join(pre)
+ # print >> sys.stderr, '========== funcs ===============\n'
+ # for func in funcs:
+ # print >> sys.stderr, '\n// ===\n\n', ''.join(func)
+ # print >> sys.stderr, '========== post ==============\n'
+ # print >> sys.stderr, ''.join(post)
+ # print >> sys.stderr, '=========================\n'
+
+ # Save settings to a file to work around v8 issue 1579
+ settings_file = temp_files.get('.txt').name
s = open(settings_file, 'w')
- s.write(settings)
+ s.write(json.dumps(settings))
s.close()
- compiler = path_from_root('src', 'compiler.js')
- shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, infile] + libraries, stdout=outfile, cwd=path_from_root('src'))
+
+ # Phase 1 - pre
+ if DEBUG: t = time.time()
+ pre_file = temp_files.get('.pre.ll').name
+ open(pre_file, 'w').write(''.join(pre) + '\n' + meta)
+ out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
+ js, forwarded_data = out.split('//FORWARDED_DATA:')
+ #print 'js', js
+ #print >> sys.stderr, 'FORWARDED_DATA 1:', forwarded_data, type(forwarded_data)
+ forwarded_file = temp_files.get('.json').name
+ open(forwarded_file, 'w').write(forwarded_data)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t)
+
+ # Phase 2 - func
+
+ cores = multiprocessing.cpu_count()
+ assert cores >= 1
+ intended_num_chunks = cores * NUM_CHUNKS_PER_CORE
+ chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
+ chunk_size += 3*len(meta) # keep ratio of lots of function code to meta
+ chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
+
+ if DEBUG: t = time.time()
+ forwarded_json = json.loads(forwarded_data)
+ indexed_functions = set()
+ chunks = [] # bundles of functions
+ curr = ''
+ for i in range(len(funcs)):
+ func = funcs[i]
+ if len(curr) + len(func) < chunk_size:
+ curr += func
+ else:
+ chunks.append(curr)
+ curr = func
+ if curr:
+ chunks.append(curr)
+ curr = ''
+ if cores == 1: assert len(chunks) == 1, 'no point in splitting up without multiple cores'
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.))
+
+ commands = [(i, chunk + '\n' + meta, settings_file, compiler, forwarded_file, libraries) for chunk in chunks]
+
+ if len(chunks) > 1:
+ pool = multiprocessing.Pool(processes=cores)
+ outputs = pool.map(process_funcs, commands, chunksize=1)
+ else:
+ outputs = [process_funcs(commands[0])]
+
+ for funcs_js, curr_forwarded_data in outputs:
+ js += funcs_js
+ # merge forwarded data
+ curr_forwarded_json = json.loads(curr_forwarded_data)
+ forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed']
+ for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems():
+ forwarded_json['Functions']['blockAddresses'][key] = value
+ for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys():
+ indexed_functions.add(key)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t)
+ if DEBUG: t = time.time()
+
+ # calculations on merged forwarded data
+ forwarded_json['Functions']['indexedFunctions'] = {}
+ i = 2
+ for indexed in indexed_functions:
+ forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize
+ i += 2
+ forwarded_json['Functions']['nextIndex'] = i
+ indexing = forwarded_json['Functions']['indexedFunctions']
+ def indexize(js):
+ return re.sub(r'{{{ FI_([\w\d_$]+) }}}', lambda m: str(indexing[m.groups(0)[0]]), js)
+ # forward
+ forwarded_data = json.dumps(forwarded_json)
+ forwarded_file = temp_files.get('.2.json').name
+ open(forwarded_file, 'w').write(forwarded_data)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t)
+
+ # Phase 3 - post
+ if DEBUG: t = time.time()
+ post_file = temp_files.get('.post.ll').name
+ open(post_file, 'w').write(''.join(post) + '\n' + meta)
+ out = shared.run_js(compiler, shared.COMPILER_ENGINE, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
+ js += out
+ js = indexize(js)
+ if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t)
+
+ outfile.write(js) # TODO: write in parts (see previous line though)
outfile.close()
@@ -127,11 +291,11 @@ def main(args):
libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else []
# Compile the assembly to Javascript.
- emscript(args.infile, json.dumps(settings), args.outfile, libraries)
+ emscript(args.infile, settings, args.outfile, libraries)
if __name__ == '__main__':
parser = optparse.OptionParser(
- usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-s FOO=BAR]* infile',
+ usage='usage: %prog [-h] [-H HEADERS] [-o OUTFILE] [-c COMPILER_ENGINE] [-s FOO=BAR]* infile',
description=('You should normally never use this! Use emcc instead. '
'This is a wrapper around the JS compiler, converting .ll to .js.'),
epilog='')
@@ -146,6 +310,9 @@ if __name__ == '__main__':
parser.add_option('-o', '--outfile',
default=sys.stdout,
help='Where to write the output; defaults to stdout.')
+ parser.add_option('-c', '--compiler',
+ default=shared.COMPILER_ENGINE,
+ help='Which JS engine to use to run the compiler; defaults to the one in ~/.emscripten.')
parser.add_option('-s', '--setting',
dest='settings',
default=[],
@@ -161,6 +328,7 @@ if __name__ == '__main__':
keywords.infile = os.path.abspath(positional[0])
if isinstance(keywords.outfile, basestring):
keywords.outfile = open(keywords.outfile, 'w')
+ compiler_engine = keywords.compiler
temp_files.run_and_clean(lambda: main(keywords))