aboutsummaryrefslogtreecommitdiff
path: root/tools/shared.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/shared.py')
-rw-r--r--tools/shared.py132
1 files changed, 61 insertions, 71 deletions
diff --git a/tools/shared.py b/tools/shared.py
index 37552b09..a75a1de6 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1,4 +1,4 @@
-import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess
+import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, multiprocessing
from subprocess import Popen, PIPE, STDOUT
from tempfile import mkstemp
@@ -190,7 +190,6 @@ LLVM_NM=os.path.expanduser(build_llvm_tool_path('llvm-nm'))
LLVM_INTERPRETER=os.path.expanduser(build_llvm_tool_path('lli'))
LLVM_COMPILER=os.path.expanduser(build_llvm_tool_path('llc'))
LLVM_EXTRACT=os.path.expanduser(build_llvm_tool_path('llvm-extract'))
-COFFEESCRIPT = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee')
EMSCRIPTEN = path_from_root('emscripten.py')
DEMANGLER = path_from_root('third_party', 'demangler.py')
@@ -205,7 +204,6 @@ EMMAKEN = path_from_root('tools', 'emmaken.py')
AUTODEBUGGER = path_from_root('tools', 'autodebugger.py')
BINDINGS_GENERATOR = path_from_root('tools', 'bindings_generator.py')
EXEC_LLVM = path_from_root('tools', 'exec_llvm.py')
-VARIABLE_ELIMINATOR = path_from_root('tools', 'eliminator', 'eliminator.coffee')
JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js')
FILE_PACKAGER = path_from_root('tools', 'file_packager.py')
@@ -430,6 +428,16 @@ def read_pgo_data(filename):
'overflows_lines': overflows_lines
}
+def run_js_optimizer(command):
+ filename = command[2] # XXX hackish
+ output = Popen(command, stdout=PIPE).communicate()[0]
+ assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
+ filename += '.jo.js'
+ f = open(filename, 'w')
+ f.write(output)
+ f.close()
+ return filename
+
# Settings. A global singleton. Not pretty, but nicer than passing |, settings| everywhere
class Settings:
@@ -926,83 +934,65 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
Building.LLVM_OPT_OPTS = opts
return opts
- MAX_JS_PROCESS_SIZE = 12*1024*1024
- BEST_JS_PROCESS_SIZE = 8*1024*1024
+ BEST_JS_PROCESS_SIZE = 1024*1024
@staticmethod
- def splitter(filename, addendum, func, args=[]):
- # Split up huge files into pieces small enough for node/uglify/etc. to handle
- js = open(filename).read()
- if len(js) > Building.MAX_JS_PROCESS_SIZE:
- if os.environ.get('EMCC_DEBUG'): print >> sys.stderr, 'splitting up js file'
+ def js_optimizer(filename, passes):
+ if type(passes) == str:
+ passes = [passes]
- suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS'
- suffix_start = js.find(suffix_marker)
- suffix = ''
- if suffix_start >= 0:
- suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n'
+ js = open(filename).read()
- filename += addendum
- f = open(filename, 'w')
+ # Find suffix
+ suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS'
+ suffix_start = js.find(suffix_marker)
+ suffix = ''
+ if suffix_start >= 0:
+ suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n'
+
+ # Pick where to split into chunks, so that (1) they do not oom in node/uglify, and (2) we can run them in parallel
+ chunks = []
+ i = 0
+ f_start = 0
+ while True:
+ f_end = f_start
+ while f_end-f_start < Building.BEST_JS_PROCESS_SIZE and f_end != -1:
+ f_end = js.find('\n}\n', f_end+1)
+ chunk = js[f_start:(-1 if f_end == -1 else f_end+3)] + suffix
+ temp_file = filename + '.p%d.js' % i
+ i += 1
+ f_start = f_end+3
+ done = f_end == -1 or f_start >= len(js)
+ if done and len(chunks) == 0: break # do not write anything out, just use the input file
+ f = open(temp_file, 'w')
+ f.write(chunk)
+ f.close()
+ chunks.append(temp_file)
+ if done: break
- i = 0
- f_start = 0
- while True:
- f_end = f_start
- while f_end-f_start < Building.BEST_JS_PROCESS_SIZE and f_end != -1:
- f_end = js.find('\n}\n', f_end+1)
- chunk = js[f_start:(-1 if f_end == -1 else f_end+3)] + suffix
- temp_in_file = filename + '.p%d.js' % i
- if os.environ.get('EMCC_DEBUG'): print >> sys.stderr, ' split %d, size %d' % (i, len(chunk))
- i += 1
- open(temp_in_file, 'w').write(chunk)
- temp_out_file = func(*([temp_in_file] + args + [False])) # maybe_big is now false
- f.write(
- ''.join(filter(lambda line: not line.startswith(suffix_marker), open(temp_out_file).readlines()))
- )
- if f_end == -1: break
- f_start = f_end+3
- if f_start >= len(js): break
+ if len(chunks) == 0:
+ chunks.append(filename)
+ # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok)
+ commands = map(lambda chunk: [NODE_JS, JS_OPTIMIZER, chunk] + passes, chunks)
+
+ if len(chunks) > 1:
+ cores = min(multiprocessing.cpu_count(), chunks)
+ if os.environ.get('EMCC_DEBUG'): print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores' % (len(chunks), cores)
+ pool = multiprocessing.Pool(processes=cores)
+ commands = map(lambda command: command + ['noPrintMetadata'], commands)
+ filenames = pool.map(run_js_optimizer, commands, chunksize=1)
+ filename += '.jo.js'
+ f = open(filename, 'w')
+ for out_file in filenames:
+ f.write(open(out_file).read())
f.write(suffix)
+ f.write('\n')
f.close()
return filename
-
- @staticmethod
- def js_optimizer(filename, passes, maybe_big=True):
- if maybe_big:
- # When we split up, we cannot do unGlobalize, the only pass which is *not* function-local
- args = [filter(lambda p: p != 'unGlobalize', passes)]
- ret = Building.splitter(filename, addendum='.jo.js', func=Building.js_optimizer, args=args)
- if ret: return ret
-
- if type(passes) == str:
- passes = [passes]
- # XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok)
- output = Popen([NODE_JS, JS_OPTIMIZER, filename] + passes, stdout=PIPE).communicate()[0]
- assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
- filename += '.jo.js'
- f = open(filename, 'w')
- f.write(output)
- f.close()
- return filename
-
- @staticmethod
- def eliminator(filename, maybe_big=True):
- if maybe_big:
- ret = Building.splitter(filename, addendum='.el.js', func=Building.eliminator)
- if ret: return ret
-
- coffee = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee')
- eliminator = path_from_root('tools', 'eliminator', 'eliminator.coffee')
- input = open(filename, 'r').read()
- output = Popen([NODE_JS, coffee, eliminator, filename], stdout=PIPE).communicate()[0]
- assert len(output) > 0, 'Error in eliminator: ' + output
- filename += '.el.js'
- f = open(filename, 'w')
- f.write(output)
- f.close()
- return filename
+ else:
+ # one simple chunk, just do it
+ return run_js_optimizer(commands[0])
@staticmethod
def closure_compiler(filename):
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-07-25 00:46:17 +0000