diff options
Diffstat (limited to 'tools/shared.py')
| -rw-r--r-- | tools/shared.py | 350 |
1 files changed, 264 insertions, 86 deletions
diff --git a/tools/shared.py b/tools/shared.py index 6a63d4ec..997c0ad9 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,4 +1,4 @@ -import shutil, time, os, sys, json, tempfile, copy +import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess from subprocess import Popen, PIPE, STDOUT from tempfile import mkstemp @@ -6,15 +6,22 @@ __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) -# Config file +# Emscripten configuration is done through the EM_CONFIG environment variable. +# If the string value contained in this environment variable contains newline +# separated definitions, then these definitions will be used to configure +# Emscripten. Otherwise, the string is understood to be a path to a settings +# file that contains the required definitions. EM_CONFIG = os.environ.get('EM_CONFIG') if not EM_CONFIG: EM_CONFIG = '~/.emscripten' -CONFIG_FILE = os.path.expanduser(EM_CONFIG) -if not os.path.exists(CONFIG_FILE): - shutil.copy(path_from_root('settings.py'), CONFIG_FILE) - print >> sys.stderr, ''' +if '\n' in EM_CONFIG: + CONFIG_FILE = None +else: + CONFIG_FILE = os.path.expanduser(EM_CONFIG) + if not os.path.exists(CONFIG_FILE): + shutil.copy(path_from_root('settings.py'), CONFIG_FILE) + print >> sys.stderr, ''' ============================================================================== Welcome to Emscripten! @@ -28,11 +35,12 @@ make sure LLVM_ROOT and NODE_JS are correct. This command will now exit. When you are done editing those paths, re-run it. ============================================================================== ''' % (EM_CONFIG, CONFIG_FILE) - sys.exit(0) + sys.exit(0) try: - exec(open(CONFIG_FILE, 'r').read()) + config_text = open(CONFIG_FILE, 'r').read() if CONFIG_FILE else EM_CONFIG + exec(config_text) except Exception, e: - print >> sys.stderr, 'Error in evaluating %s (at %s): %s' % (EM_CONFIG, CONFIG_FILE, str(e)) + print >> sys.stderr, 'Error in evaluating %s (at %s): %s, text: %s' % (EM_CONFIG, CONFIG_FILE, str(e), config_text) sys.exit(1) # Check that basic stuff we need (a JS engine to compile, Node.js, and Clang and LLVM) @@ -43,6 +51,8 @@ except Exception, e: def check_sanity(force=False): try: if not force: + if not CONFIG_FILE: + return # config stored directly in EM_CONFIG => skip sanity checks settings_mtime = os.stat(CONFIG_FILE).st_mtime sanity_file = CONFIG_FILE + '_sanity' try: @@ -52,6 +62,9 @@ def check_sanity(force=False): except: pass + print >> sys.stderr, '(Emscripten: Config file changed, clearing cache)' # LLVM may have changed, etc. + Cache.erase() + print >> sys.stderr, '(Emscripten: Running sanity checks)' if not check_engine(COMPILER_ENGINE): @@ -68,6 +81,11 @@ def check_sanity(force=False): print >> sys.stderr, 'FATAL: Cannot find %s, check the paths in %s' % (cmd, EM_CONFIG) sys.exit(0) + try: + subprocess.call([JAVA, '-version'], stdout=PIPE, stderr=PIPE) + except: + print >> sys.stderr, 'WARNING: java does not seem to exist, required for closure compiler. -O2 and above will fail. You need to define JAVA in ~/.emscripten (see settings.py)' + if not os.path.exists(CLOSURE_COMPILER): print >> sys.stderr, 'WARNING: Closure compiler (%s) does not exist, check the paths in %s. -O2 and above will fail' % (CLOSURE_COMPILER, EM_CONFIG) @@ -89,7 +107,6 @@ CLANG_CC=os.path.expanduser(os.path.join(LLVM_ROOT, 'clang')) CLANG_CPP=os.path.expanduser(os.path.join(LLVM_ROOT, 'clang++')) CLANG=CLANG_CPP LLVM_LINK=os.path.join(LLVM_ROOT, 'llvm-link') -LLVM_LD=os.path.join(LLVM_ROOT, 'llvm-ld') LLVM_AR=os.path.join(LLVM_ROOT, 'llvm-ar') LLVM_OPT=os.path.expanduser(os.path.join(LLVM_ROOT, 'opt')) LLVM_AS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-as')) @@ -97,6 +114,7 @@ LLVM_DIS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-dis')) LLVM_NM=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-nm')) LLVM_INTERPRETER=os.path.expanduser(os.path.join(LLVM_ROOT, 'lli')) LLVM_COMPILER=os.path.expanduser(os.path.join(LLVM_ROOT, 'llc')) +LLVM_EXTRACT=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-extract')) COFFEESCRIPT = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee') EMSCRIPTEN = path_from_root('emscripten.py') @@ -105,28 +123,41 @@ NAMESPACER = path_from_root('tools', 'namespacer.py') EMCC = path_from_root('emcc') EMXX = path_from_root('em++') EMAR = path_from_root('emar') -EMLD = path_from_root('emld') EMRANLIB = path_from_root('emranlib') EMLIBTOOL = path_from_root('emlibtool') +EMCONFIG = path_from_root('em-config') EMMAKEN = path_from_root('tools', 'emmaken.py') AUTODEBUGGER = path_from_root('tools', 'autodebugger.py') BINDINGS_GENERATOR = path_from_root('tools', 'bindings_generator.py') EXEC_LLVM = path_from_root('tools', 'exec_llvm.py') VARIABLE_ELIMINATOR = path_from_root('tools', 'eliminator', 'eliminator.coffee') JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js') +FILE_PACKAGER = path_from_root('tools', 'file_packager.py') -# Temp dir +# Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use TEMP_DIR/emscripten_temp try: - EMSCRIPTEN_TEMP_DIR = os.path.join(TEMP_DIR, 'emscripten_temp') - if not os.path.exists(EMSCRIPTEN_TEMP_DIR): - try: - os.makedirs(EMSCRIPTEN_TEMP_DIR) - except Exception, e: - print >> sys.stderr, 'Warning: Could not create temp dir (%s): %s' % (EMSCRIPTEN_TEMP_DIR, str(e)) + TEMP_DIR except: + print >> sys.stderr, 'TEMP_DIR not defined in ~/.emscripten, using /tmp' + TEMP_DIR = '/tmp' + +CANONICAL_TEMP_DIR = os.path.join(TEMP_DIR, 'emscripten_temp') +EMSCRIPTEN_TEMP_DIR = None + +if os.environ.get('EMCC_DEBUG'): + try: + EMSCRIPTEN_TEMP_DIR = CANONICAL_TEMP_DIR + if not os.path.exists(EMSCRIPTEN_TEMP_DIR): + os.makedirs(EMSCRIPTEN_TEMP_DIR) + except: + print >> sys.stderr, 'Could not create canonical temp dir. Check definition of TEMP_DIR in ~/.emscripten' + +if not EMSCRIPTEN_TEMP_DIR: EMSCRIPTEN_TEMP_DIR = tempfile.mkdtemp(prefix='emscripten_temp_') - print >> sys.stderr, 'Warning: TEMP_DIR not defined in %s, using %s' % (EM_CONFIG, EMSCRIPTEN_TEMP_DIR) + def clean_temp(): + try_delete(EMSCRIPTEN_TEMP_DIR) + atexit.register(clean_temp) # EM_CONFIG stuff @@ -144,6 +175,12 @@ try: except: CLOSURE_COMPILER = path_from_root('third_party', 'closure-compiler', 'compiler.jar') +try: + JAVA +except: + print >> sys.stderr, 'JAVA not defined in ~/.emscripten, using "java"' + JAVA = 'java' + # Additional compiler options try: @@ -153,7 +190,7 @@ except: # Force a simple, standard target as much as possible: target 32-bit linux, and disable various flags that hint at other platforms COMPILER_OPTS = COMPILER_OPTS + ['-m32', '-U__i386__', '-U__x86_64__', '-U__i386', '-U__x86_64', '-U__SSE__', '-U__SSE2__', '-U__MMX__', '-UX87_DOUBLE_ROUNDING', '-UHAVE_GCC_ASM_FOR_X87', '-DEMSCRIPTEN', '-U__STRICT_ANSI__', '-U__CYGWIN__', - '-D__STDC__', '-Xclang', '-triple=i386-pc-linux-gnu'] + '-D__STDC__', '-Xclang', '-triple=i386-pc-linux-gnu', '-D__IEEE_LITTLE_ENDIAN'] USE_EMSDK = not os.environ.get('EMMAKEN_NO_SDK') @@ -161,8 +198,10 @@ USE_EMSDK = not os.environ.get('EMMAKEN_NO_SDK') if USE_EMSDK: # Disable system C and C++ include directories, and add our own (using -idirafter so they are last, like system dirs, which # allows projects to override them) - EMSDK_OPTS = ['-nostdinc', '-nostdinc++', '-Xclang', '-nobuiltininc', '-Xclang', '-nostdinc++', '-Xclang', '-nostdsysteminc', + # Note that -nostdinc++ is not needed, since -nostdinc implies that! + EMSDK_OPTS = ['-nostdinc', '-Xclang', '-nobuiltininc', '-Xclang', '-nostdsysteminc', '-Xclang', '-isystem' + path_from_root('system', 'include'), + '-Xclang', '-isystem' + path_from_root('system', 'include', 'emscripten'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'bsd'), # posix stuff '-Xclang', '-isystem' + path_from_root('system', 'include', 'libc'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'libcxx'), @@ -170,7 +209,7 @@ if USE_EMSDK: '-Xclang', '-isystem' + path_from_root('system', 'include', 'net'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'SDL'), ] + [ - '-U__APPLE__' + '-U__APPLE__', '-U__linux__' ] COMPILER_OPTS += EMSDK_OPTS else: @@ -178,11 +217,25 @@ else: # Engine tweaks -#if 'strict' not in str(SPIDERMONKEY_ENGINE): # XXX temporarily disable strict mode until we sort out some stuff -# SPIDERMONKEY_ENGINE += ['-e', "options('strict')"] # Strict mode in SpiderMonkey. With V8 we check that fallback to non-strict works too - -if 'gcparam' not in str(SPIDERMONKEY_ENGINE): - SPIDERMONKEY_ENGINE += ['-e', "gcparam('maxBytes', 1024*1024*1024);"] # Our very large files need lots of gc heap +try: + if 'gcparam' not in str(SPIDERMONKEY_ENGINE): + if type(SPIDERMONKEY_ENGINE) is str: + SPIDERMONKEY_ENGINE = [SPIDERMONKEY_ENGINE] + SPIDERMONKEY_ENGINE += ['-e', "gcparam('maxBytes', 1024*1024*1024);"] # Our very large files need lots of gc heap +except NameError: + pass + +WINDOWS = sys.platform.startswith('win') + +# If we have 'env', we should use that to find python, because |python| may fail while |env python| may work +# (For example, if system python is 3.x while we need 2.x, and env gives 2.x if told to do so.) +ENV_PREFIX = [] +if not WINDOWS: + try: + assert 'Python' in Popen(['env', 'python', '-V'], stdout=PIPE, stderr=STDOUT).communicate()[0] + ENV_PREFIX = ['env'] + except: + pass # Temp file utilities @@ -190,7 +243,10 @@ def try_delete(filename): try: os.unlink(filename) except: - pass + try: + shutil.rmtree(filename) + except: + pass class TempFiles: def __init__(self): @@ -221,6 +277,8 @@ class TempFiles: def check_engine(engine): # TODO: we call this several times, perhaps cache the results? try: + if not CONFIG_FILE: + return True # config stored directly in EM_CONFIG => skip engine check return 'hello, world!' in run_js(path_from_root('tests', 'hello_world.js'), engine) except Exception, e: print 'Checking JS engine %s failed. Check %s. Details: %s' % (str(engine), EM_CONFIG, str(e)) @@ -236,11 +294,14 @@ def timeout_run(proc, timeout, note='unnamed process'): raise Exception("Timed out: " + note) return proc.communicate()[0] +EM_DEBUG = os.environ.get('EM_DEBUG') + def run_js(filename, engine=None, args=[], check_timeout=False, stdout=PIPE, stderr=None, cwd=None): if engine is None: engine = JS_ENGINES[0] if type(engine) is not list: engine = [engine] - return timeout_run(Popen(engine + [filename] + (['--'] if 'd8' in engine[0] else []) + args, - stdout=stdout, stderr=stderr, cwd=cwd), 15*60 if check_timeout else None, 'Execution') + command = engine + [filename] + (['--'] if 'd8' in engine[0] else []) + args + if EM_DEBUG: print >> sys.stderr, 'run_js: ' + ' '.join(command) + return timeout_run(Popen(command, stdout=stdout, stderr=stderr, cwd=cwd), 15*60 if check_timeout else None, 'Execution') def to_cc(cxx): # By default, LLVM_GCC and CLANG are really the C++ versions. This gets an explicit C version @@ -272,7 +333,7 @@ def read_pgo_data(filename): ''' signs_lines = [] overflows_lines = [] - + for line in open(filename, 'r'): try: if line.rstrip() == '': continue @@ -335,10 +396,12 @@ class Settings: if opt_level >= 2: Settings.RELOOP = 1 if opt_level >= 3: + Settings.INLINING_LIMIT = 0 + Settings.DOUBLE_MODE = 0 + Settings.PRECISE_I64_MATH = 0 Settings.CORRECT_SIGNS = 0 Settings.CORRECT_OVERFLOWS = 0 Settings.CORRECT_ROUNDINGS = 0 - Settings.DOUBLE_MODE = 0 if noisy: print >> sys.stderr, 'Warning: Applying some potentially unsafe optimizations! (Use -O2 if this fails.)' global Settings @@ -357,11 +420,11 @@ class Building: @staticmethod def get_building_env(): env = os.environ.copy() - env['CC'] = EMCC - env['CXX'] = EMXX - env['AR'] = EMAR - env['RANLIB'] = EMRANLIB - env['LIBTOOL'] = EMLIBTOOL + env['CC'] = EMCC if not WINDOWS else 'python %r' % EMCC + env['CXX'] = EMXX if not WINDOWS else 'python %r' % EMXX + env['AR'] = EMAR if not WINDOWS else 'python %r' % EMAR + env['RANLIB'] = EMRANLIB if not WINDOWS else 'python %r' % EMRANLIB + env['LIBTOOL'] = EMLIBTOOL if not WINDOWS else 'python %r' % EMLIBTOOL env['EMMAKEN_COMPILER'] = Building.COMPILER env['EMSCRIPTEN_TOOLS'] = path_from_root('tools') env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(Building.COMPILER_TEST_OPTS) @@ -373,14 +436,14 @@ class Building: @staticmethod def handle_CMake_toolchain(args, env): - CMakeToolchain = '''# the name of the target operating system + CMakeToolchain = ('''# the name of the target operating system SET(CMAKE_SYSTEM_NAME Linux) # which C and C++ compiler to use -SET(CMAKE_C_COMPILER $EMSCRIPTEN_ROOT/emcc) -SET(CMAKE_CXX_COMPILER $EMSCRIPTEN_ROOT/em++) -SET(CMAKE_AR $EMSCRIPTEN_ROOT/emar) -SET(CMAKE_RANLIB $EMSCRIPTEN_ROOT/emranlib) +SET(CMAKE_C_COMPILER %(winfix)s$EMSCRIPTEN_ROOT/emcc) +SET(CMAKE_CXX_COMPILER %(winfix)s$EMSCRIPTEN_ROOT/em++) +SET(CMAKE_AR %(winfix)s$EMSCRIPTEN_ROOT/emar) +SET(CMAKE_RANLIB %(winfix)s$EMSCRIPTEN_ROOT/emranlib) SET(CMAKE_C_FLAGS $CFLAGS) SET(CMAKE_CXX_FLAGS $CXXFLAGS) @@ -393,8 +456,8 @@ SET(CMAKE_FIND_ROOT_PATH $EMSCRIPTEN_ROOT/system/include ) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ - .replace('$EMSCRIPTEN_ROOT', path_from_root('')) \ +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS else 'python ' }) \ + .replace('$EMSCRIPTEN_ROOT', path_from_root('').replace('\\', '/')) \ .replace('$CFLAGS', env['CFLAGS']) \ .replace('$CXXFLAGS', env['CFLAGS']) toolchainFile = mkstemp(suffix='.txt')[1] @@ -409,17 +472,17 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ env['EMMAKEN_JUST_CONFIGURE'] = '1' if 'cmake' in args[0]: args = Building.handle_CMake_toolchain(args, env) - Popen(args, stdout=stdout, stderr=stderr, env=env).communicate()[0] + Popen(args, stdout=stdout, stderr=stderr, env=env).communicate() del env['EMMAKEN_JUST_CONFIGURE'] @staticmethod def make(args, stdout=None, stderr=None, env=None): if env is None: env = Building.get_building_env() - Popen(args, stdout=stdout, stderr=stderr, env=env).communicate()[0] + Popen(args, stdout=stdout, stderr=stderr, env=env).communicate() @staticmethod - def build_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None): + def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None): ''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in memory since the test directory is destroyed and recreated for each test. Note that we cache separately for different compilers). @@ -453,22 +516,95 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |link| call) Building.configure(configure + configure_args, stdout=open(os.path.join(project_dir, 'configure_'), 'w'), stderr=open(os.path.join(project_dir, 'configure_err'), 'w'), env=env) - Building.make(make + make_args, stdout=open(os.path.join(project_dir, 'make_'), 'w'), - stderr=open(os.path.join(project_dir, 'make_err'), 'w'), env=env) - if cache is not None: - cache[cache_name] = [] - for f in generated_libs: - basename = os.path.basename(f) - cache[cache_name].append((basename, open(f, 'rb').read())) + for i in range(2): # workaround for some build systems that need to be run twice to succeed (e.g. poppler) + Building.make(make + make_args, stdout=open(os.path.join(project_dir, 'make_' + str(i)), 'w'), + stderr=open(os.path.join(project_dir, 'make_err' + str(i)), 'w'), env=env) + try: + if cache is not None: + cache[cache_name] = [] + for f in generated_libs: + basename = os.path.basename(f) + cache[cache_name].append((basename, open(f, 'rb').read())) + break + except: + if i > 0: raise Exception('could not build library ' + name) if old_dir: os.chdir(old_dir) return generated_libs @staticmethod - def link(files, target): + def link(files, target, remove_duplicates=False): + actual_files = [] + unresolved_symbols = set() # necessary for .a linking, see below + resolved_symbols = set() + temp_dir = None + for f in files: + if not Building.is_ar(f): + if Building.is_bitcode(f): + new_symbols = Building.llvm_nm(f) + resolved_symbols = resolved_symbols.union(new_symbols.defs) + unresolved_symbols = unresolved_symbols.union(new_symbols.undefs.difference(resolved_symbols)).difference(new_symbols.defs) + actual_files.append(f) + else: + # Extract object files from ar archives, and link according to gnu ld semantics + # (link in an entire .o from the archive if it supplies symbols still unresolved) + cwd = os.getcwd() + try: + temp_dir = os.path.join(EMSCRIPTEN_TEMP_DIR, 'ar_output_' + str(os.getpid())) + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + os.chdir(temp_dir) + contents = filter(lambda x: len(x) > 0, Popen([LLVM_AR, 't', f], stdout=PIPE).communicate()[0].split('\n')) + if len(contents) == 0: + print >> sys.stderr, 'Warning: Archive %s appears to be empty (recommendation: link an .so instead of .a)' % f + else: + for content in contents: # ar will silently fail if the directory for the file does not exist, so make all the necessary directories + dirname = os.path.dirname(content) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + Popen([LLVM_AR, 'x', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory + contents = map(lambda content: os.path.join(temp_dir, content), contents) + contents = filter(os.path.exists, map(os.path.abspath, contents)) + needed = False # We add or do not add the entire archive. We let llvm dead code eliminate parts we do not need, instead of + # doing intra-dependencies between archive contents + for content in contents: + new_symbols = Building.llvm_nm(content) + # Link in the .o if it provides symbols, *or* this is a singleton archive (which is apparently an exception in gcc ld) + if new_symbols.defs.intersection(unresolved_symbols) or len(files) == 1: + needed = True + if needed: + for content in contents: + if Building.is_bitcode(content): + new_symbols = Building.llvm_nm(content) + resolved_symbols = resolved_symbols.union(new_symbols.defs) + unresolved_symbols = unresolved_symbols.union(new_symbols.undefs.difference(resolved_symbols)).difference(new_symbols.defs) + actual_files.append(content) + finally: + os.chdir(cwd) try_delete(target) - output = Popen([LLVM_LD, '-disable-opt'] + files + ['-b', target], stdout=PIPE).communicate()[0] - assert os.path.exists(target) and (output is None or 'Could not open input file' not in output), 'Linking error: ' + output + + if remove_duplicates: + # Remove duplicate symbols. This is a workaround for how we compile .a files, we try to + # emulate ld behavior which is permissive TODO: cache llvm-nm results + seen_symbols = set() + print >> sys.stderr, actual_files + for actual in actual_files: + symbols = Building.llvm_nm(actual) + dupes = seen_symbols.intersection(symbols.defs) + if len(dupes) > 0: + print >> sys.stderr, 'emcc: warning: removing duplicates in', actual + for dupe in dupes: + print >> sys.stderr, 'emcc: warning: removing duplicate', dupe + Popen([LLVM_EXTRACT, actual, '-delete', '-glob=' + dupe, '-o', actual], stderr=PIPE).communicate() + Popen([LLVM_EXTRACT, actual, '-delete', '-func=' + dupe, '-o', actual], stderr=PIPE).communicate() + Popen([LLVM_EXTRACT, actual, '-delete', '-glob=.str', '-o', actual], stderr=PIPE).communicate() # garbage that appears here + seen_symbols = seen_symbols.union(symbols.defs) + + # Finish link + output = Popen([LLVM_LINK] + actual_files + ['-o', target], stdout=PIPE).communicate()[0] + assert os.path.exists(target) and (output is None or 'Could not open input file' not in output), 'Linking error: ' + output + '\nemcc: If you get duplicate symbol errors, try --remove-duplicates' + if temp_dir: + try_delete(temp_dir) # Emscripten optimizations that we run on the .ll file @staticmethod @@ -497,12 +633,6 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ shutil.move(filename + '.o', filename + '.o.pre') output = Popen([LLVM_OPT, filename + '.o.pre'] + Building.LLVM_OPT_OPTS + ['-o=' + filename + '.o'], stdout=PIPE).communicate()[0] assert os.path.exists(filename + '.o'), 'Failed to run llvm optimizations: ' + output - #if Building.LLVM_OPTS == 2: - # print 'Unsafe LD!' - # shutil.move(filename + '.o', filename + '.o.pre') - # output = Popen([LLVM_LD, filename + '.o.pre', '-o=' + filename + '.tmp'], stdout=PIPE).communicate()[0] - # assert os.path.exists(filename + '.tmp.bc'), 'Failed to run llvm optimizations: ' + output - # shutil.move(filename + '.tmp.bc', filename + '.o') @staticmethod def llvm_dis(input_filename, output_filename=None): @@ -538,13 +668,15 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ commons = [] for line in output.split('\n'): if len(line) == 0: continue - status, symbol = filter(lambda seg: len(seg) > 0, line.split(' ')) - if status == 'U': - ret.undefs.append(symbol) - elif status != 'C': - ret.defs.append(symbol) - else: - ret.commons.append(symbol) + parts = filter(lambda seg: len(seg) > 0, line.split(' ')) + if len(parts) == 2: # ignore lines with absolute offsets, these are not bitcode anyhow (e.g. |00000630 t d_source_name|) + status, symbol = parts + if status == 'U': + ret.undefs.append(symbol) + elif status != 'C': + ret.defs.append(symbol) + else: + ret.commons.append(symbol) ret.defs = set(ret.defs) ret.undefs = set(ret.undefs) ret.commons = set(ret.commons) @@ -555,13 +687,13 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ if output_filename is None: output_filename = filename + '.o' try_delete(output_filename) - Popen([EMCC, filename] + args + ['-o', output_filename], stdout=stdout, stderr=stderr, env=env).communicate() + Popen(ENV_PREFIX + ['python', EMCC, filename] + args + ['-o', output_filename], stdout=stdout, stderr=stderr, env=env).communicate() assert os.path.exists(output_filename), 'emcc could not create output file' @staticmethod def emar(action, output_filename, filenames, stdout=None, stderr=None, env=None): try_delete(output_filename) - Popen([EMAR, action, output_filename] + filenames, stdout=stdout, stderr=stderr, env=env).communicate() + Popen(ENV_PREFIX + ['python', EMAR, action, output_filename] + filenames, stdout=stdout, stderr=stderr, env=env).communicate() if 'c' in action: assert os.path.exists(output_filename), 'emar could not create output file' @@ -572,7 +704,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ # Run Emscripten settings = Settings.serialize() - compiler_output = timeout_run(Popen(['python', EMSCRIPTEN, filename + ('.o.ll' if append_ext else ''), '-o', filename + '.o.js'] + settings + extra_args, stdout=PIPE), None, 'Compiling') + compiler_output = timeout_run(Popen(ENV_PREFIX + ['python', EMSCRIPTEN, filename + ('.o.ll' if append_ext else ''), '-o', filename + '.o.js'] + settings + extra_args, stdout=PIPE), None, 'Compiling') #print compiler_output # Detect compilation crashes and errors @@ -703,8 +835,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ if type(passes) == str: passes = [passes] # XXX Disable crankshaft to work around v8 bug 1895 - output, err = Popen([NODE_JS, '--nocrankshaft', JS_OPTIMIZER, filename] + passes, stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate() - assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + err + '\n\n' + output + output = Popen([NODE_JS, '--nocrankshaft', JS_OPTIMIZER, filename] + passes, stdout=PIPE).communicate()[0] + assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output filename += '.jo.js' f = open(filename, 'w') f.write(output) @@ -719,8 +851,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ coffee = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee') eliminator = path_from_root('tools', 'eliminator', 'eliminator.coffee') input = open(filename, 'r').read() - output, err = Popen([NODE_JS, coffee, eliminator], stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate(input) - assert len(output) > 0, 'Error in eliminator: ' + err + '\n\n' + output + output = Popen([NODE_JS, coffee, eliminator, filename], stdout=PIPE).communicate()[0] + assert len(output) > 0, 'Error in eliminator: ' + output filename += '.el.js' f = open(filename, 'w') f.write(output) @@ -734,13 +866,19 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ # Something like this (adjust memory as needed): # java -Xmx1024m -jar CLOSURE_COMPILER --compilation_level ADVANCED_OPTIMIZATIONS --variable_map_output_file src.cpp.o.js.vars --js src.cpp.o.js --js_output_file src.cpp.o.cc.js - cc_output = Popen(['java', '-jar', CLOSURE_COMPILER, - '--compilation_level', 'ADVANCED_OPTIMIZATIONS', - '--formatting', 'PRETTY_PRINT', - #'--variable_map_output_file', filename + '.vars', - '--js', filename, '--js_output_file', filename + '.cc.js'], stdout=PIPE, stderr=STDOUT).communicate()[0] - if 'ERROR' in cc_output or not os.path.exists(filename + '.cc.js'): - raise Exception('closure compiler error: ' + cc_output) + args = [JAVA, + '-Xmx1024m', + '-jar', CLOSURE_COMPILER, + '--compilation_level', 'ADVANCED_OPTIMIZATIONS', + '--formatting', 'PRETTY_PRINT', + #'--variable_map_output_file', filename + '.vars', + '--js', filename, '--js_output_file', filename + '.cc.js'] + if os.environ.get('EMCC_CLOSURE_ARGS'): + args += shlex.split(os.environ.get('EMCC_CLOSURE_ARGS')) + process = Popen(args, stdout=PIPE, stderr=STDOUT) + cc_output = process.communicate()[0] + if process.returncode != 0 or not os.path.exists(filename + '.cc.js'): + raise Exception('closure compiler error: ' + cc_output + ' (rc: %d)' % process.returncode) return filename + '.cc.js' @@ -789,12 +927,14 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' \ elif ord(b[0]) == 222 and ord(b[1]) == 192 and ord(b[2]) == 23 and ord(b[3]) == 11: b = open(filename, 'r').read(24) return b[20] == 'B' and b[21] == 'C' - + return False # Permanent cache for dlmalloc and stdlibc++ class Cache: - dirname = os.path.expanduser(os.path.join('~', '.emscripten_cache')) + dirname = os.environ.get('EM_CACHE') + if not dirname: + dirname = os.path.expanduser(os.path.join('~', '.emscripten_cache')) @staticmethod def erase(): @@ -816,3 +956,41 @@ class Cache: shutil.copyfile(creator(), cachename) return cachename +# Compression of code and data for smaller downloads +class Compression: + on = False + + @staticmethod + def compressed_name(filename): + return filename + '.compress' + + @staticmethod + def compress(filename): + execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb')) + + @staticmethod + def worth_it(original, compressed): + return compressed < original - 1500 # save at least one TCP packet or so + +def execute(cmd, *args, **kw): + try: + return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that) + except: + if not isinstance(cmd, str): + cmd = ' '.join(cmd) + print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>' + raise + +def suffix(name): + parts = name.split('.') + if len(parts) > 1: + return parts[-1] + else: + return None + +def unsuffixed(name): + return '.'.join(name.split('.')[:-1]) + +def unsuffixed_basename(name): + return os.path.basename(unsuffixed(name)) + |