diff options
Diffstat (limited to 'tools/shared.py')
| -rw-r--r-- | tools/shared.py | 791 |
1 files changed, 581 insertions, 210 deletions
diff --git a/tools/shared.py b/tools/shared.py index 799a5bbe..764877ba 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,8 +1,9 @@ -import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re +import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re, errno from subprocess import Popen, PIPE, STDOUT from tempfile import mkstemp +from distutils.spawn import find_executable import jsrun, cache, tempfiles -from response_file import create_response_file +import response_file import logging, platform def listify(x): @@ -40,12 +41,13 @@ class WindowsPopen: # emscripten.py supports reading args from a response file instead of cmdline. # Use .rsp to avoid cmdline length limitations on Windows. if len(args) >= 2 and args[1].endswith("emscripten.py"): - self.response_filename = create_response_file(args[2:], TEMP_DIR) - args = args[0:2] + ['@' + self.response_filename] + response_filename = response_file.create_response_file(args[2:], TEMP_DIR) + args = args[0:2] + ['@' + response_filename] try: # Call the process with fixed streams. self.process = subprocess.Popen(args, bufsize, executable, self.stdin_, self.stdout_, self.stderr_, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags) + self.pid = self.process.pid except Exception, e: logging.error('\nsubprocess.Popen(args=%s) failed! Exception %s\n' % (' '.join(args), str(e))) raise e @@ -76,17 +78,6 @@ class WindowsPopen: def kill(self): return self.process.kill() - def __del__(self): - try: - # Clean up the temporary response file that was used to spawn this process, so that we don't leave temp files around. - tempfiles.try_delete(self.response_filename) - except: - pass # Mute all exceptions in dtor, particularly if we didn't use a response file, self.response_filename doesn't exist. - -# Install our replacement Popen handler if we are running on Windows to avoid python spawn process function. -if os.name == 'nt': - Popen = WindowsPopen - __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def path_from_root(*pathelems): return os.path.join(__rootpath__, *pathelems) @@ -185,13 +176,28 @@ if WINDOWS: else: logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit) -# Emscripten configuration is done through the EM_CONFIG environment variable. -# If the string value contained in this environment variable contains newline -# separated definitions, then these definitions will be used to configure +# Emscripten configuration is done through the --em-config command line option or +# the EM_CONFIG environment variable. If the specified string value contains newline +# or semicolon-separated definitions, then these definitions will be used to configure # Emscripten. Otherwise, the string is understood to be a path to a settings # file that contains the required definitions. -EM_CONFIG = os.environ.get('EM_CONFIG') +try: + EM_CONFIG = sys.argv[sys.argv.index('--em-config')+1] + # Emscripten compiler spawns other processes, which can reimport shared.py, so make sure that + # those child processes get the same configuration file by setting it to the currently active environment. + os.environ['EM_CONFIG'] = EM_CONFIG +except: + EM_CONFIG = os.environ.get('EM_CONFIG') + +if EM_CONFIG and not os.path.isfile(EM_CONFIG): + if EM_CONFIG.startswith('-'): + raise Exception('Passed --em-config without an argument. Usage: --em-config /path/to/.emscripten or --em-config EMSCRIPTEN_ROOT=/path/;LLVM_ROOT=/path;...') + if not '=' in EM_CONFIG: + raise Exception('File ' + EM_CONFIG + ' passed to --em-config does not exist!') + else: + EM_CONFIG = EM_CONFIG.replace(';', '\n') + '\n' + if not EM_CONFIG: EM_CONFIG = '~/.emscripten' if '\n' in EM_CONFIG: @@ -199,31 +205,25 @@ if '\n' in EM_CONFIG: else: CONFIG_FILE = os.path.expanduser(EM_CONFIG) if not os.path.exists(CONFIG_FILE): + # Note: repr is used to ensure the paths are escaped correctly on Windows. + # The full string is replaced so that the template stays valid Python. config_file = open(path_from_root('tools', 'settings_template_readonly.py')).read().split('\n') config_file = config_file[1:] # remove "this file will be copied..." config_file = '\n'.join(config_file) # autodetect some default paths - config_file = config_file.replace('{{{ EMSCRIPTEN_ROOT }}}', __rootpath__) - llvm_root = '/usr/bin' - try: - llvm_root = os.path.dirname(Popen(['which', 'llvm-dis'], stdout=PIPE).communicate()[0].replace('\n', '')) - except: - pass - config_file = config_file.replace('{{{ LLVM_ROOT }}}', llvm_root) - node = 'node' - try: - node = Popen(['which', 'node'], stdout=PIPE).communicate()[0].replace('\n', '') or \ - Popen(['which', 'nodejs'], stdout=PIPE).communicate()[0].replace('\n', '') or node - except: - pass - config_file = config_file.replace('{{{ NODE }}}', node) - python = sys.executable or 'python' - try: - python = Popen(['which', 'python2'], stdout=PIPE).communicate()[0].replace('\n', '') or \ - Popen(['which', 'python'], stdout=PIPE).communicate()[0].replace('\n', '') or python - except: - pass - config_file = config_file.replace('{{{ PYTHON }}}', python) + config_file = config_file.replace('\'{{{ EMSCRIPTEN_ROOT }}}\'', repr(__rootpath__)) + llvm_root = os.path.dirname(find_executable('llvm-dis') or '/usr/bin/llvm-dis') + config_file = config_file.replace('\'{{{ LLVM_ROOT }}}\'', repr(llvm_root)) + node = find_executable('node') or find_executable('nodejs') or 'node' + config_file = config_file.replace('\'{{{ NODE }}}\'', repr(node)) + python = find_executable('python2') or find_executable('python') or \ + sys.executable or 'python' + config_file = config_file.replace('\'{{{ PYTHON }}}\'', repr(python)) + if WINDOWS: + tempdir = os.environ.get('TEMP') or os.environ.get('TMP') or 'c:\\temp' + else: + tempdir = '/tmp' + config_file = config_file.replace('\'{{{ TEMP }}}\'', repr(tempdir)) # write open(CONFIG_FILE, 'w').write(config_file) @@ -255,13 +255,34 @@ except Exception, e: logging.error('Error in evaluating %s (at %s): %s, text: %s' % (EM_CONFIG, CONFIG_FILE, str(e), config_text)) sys.exit(1) +try: + EM_POPEN_WORKAROUND +except: + EM_POPEN_WORKAROUND = os.environ.get('EM_POPEN_WORKAROUND') + +# Install our replacement Popen handler if we are running on Windows to avoid python spawn process function. +# nb. This is by default disabled since it has the adverse effect of buffering up all logging messages, which makes +# builds look unresponsive (messages are printed only after the whole build finishes). Whether this workaround is needed +# seems to depend on how the host application that invokes emcc has set up its stdout and stderr. +if EM_POPEN_WORKAROUND and os.name == 'nt': + logging.debug('Installing Popen workaround handler to avoid bug http://bugs.python.org/issue3905') + Popen = WindowsPopen + # Expectations EXPECTED_LLVM_VERSION = (3,2) +actual_clang_version = None + +def get_clang_version(): + global actual_clang_version + if actual_clang_version is None: + actual_clang_version = Popen([CLANG, '-v'], stderr=PIPE).communicate()[1].split('\n')[0].split(' ')[2] + return actual_clang_version + def check_clang_version(): - expected = 'clang version ' + '.'.join(map(str, EXPECTED_LLVM_VERSION)) - actual = Popen([CLANG, '-v'], stderr=PIPE).communicate()[1].split('\n')[0] + expected = '.'.join(map(str, EXPECTED_LLVM_VERSION)) + actual = get_clang_version() if expected in actual: return True logging.warning('LLVM version appears incorrect (seeing "%s", expected "%s")' % (actual, expected)) @@ -269,17 +290,32 @@ def check_clang_version(): def check_llvm_version(): try: - check_clang_version(); + check_clang_version() except Exception, e: logging.warning('Could not verify LLVM version: %s' % str(e)) -EXPECTED_NODE_VERSION = (0,6,8) +def check_fastcomp(): + try: + llc_version_info = Popen([LLVM_COMPILER, '--version'], stdout=PIPE).communicate()[0] + pre, targets = llc_version_info.split('Registered Targets:') + if 'js' not in targets or 'JavaScript (asm.js, emscripten) backend' not in targets: + logging.critical('fastcomp in use, but LLVM has not been built with the JavaScript backend as a target, llc reports:') + print >> sys.stderr, '===========================================================================' + print >> sys.stderr, llc_version_info, + print >> sys.stderr, '===========================================================================' + return False + return True + except Exception, e: + logging.warning('cound not check fastcomp: %s' % str(e)) + return True + +EXPECTED_NODE_VERSION = (0,8,0) def check_node_version(): try: node = listify(NODE_JS) actual = Popen(node + ['--version'], stdout=PIPE).communicate()[0].strip() - version = tuple(map(int, actual.replace('v', '').split('.'))) + version = tuple(map(int, actual.replace('v', '').replace('-pre', '').split('.'))) if version >= EXPECTED_NODE_VERSION: return True logging.warning('node version appears too old (seeing "%s", expected "%s")' % (actual, 'v' + ('.'.join(map(str, EXPECTED_NODE_VERSION))))) @@ -288,6 +324,20 @@ def check_node_version(): logging.warning('cannot check node version: %s', e) return False +# Finds the system temp directory without resorting to using the one configured in .emscripten +def find_temp_directory(): + if WINDOWS: + if os.getenv('TEMP') and os.path.isdir(os.getenv('TEMP')): + return os.getenv('TEMP') + elif os.getenv('TMP') and os.path.isdir(os.getenv('TMP')): + return os.getenv('TMP') + elif os.path.isdir('C:\\temp'): + return os.getenv('C:\\temp') + else: + return None # No luck! + else: + return '/tmp' + # Check that basic stuff we need (a JS engine to compile, Node.js, and Clang and LLVM) # exists. # The test runner always does this check (through |force|). emcc does this less frequently, @@ -295,16 +345,16 @@ def check_node_version(): # we re-check sanity when the settings are changed) # We also re-check sanity and clear the cache when the version changes -EMSCRIPTEN_VERSION = '1.4.7' +EMSCRIPTEN_VERSION = '1.10.4' def generate_sanity(): - return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + '|' + LLVM_ROOT + '|' + get_clang_version() def check_sanity(force=False): try: reason = None if not CONFIG_FILE: - if not force: return # config stored directly in EM_CONFIG => skip sanity checks + return # config stored directly in EM_CONFIG => skip sanity checks else: settings_mtime = os.stat(CONFIG_FILE).st_mtime sanity_file = CONFIG_FILE + '_sanity' @@ -324,10 +374,13 @@ def check_sanity(force=False): if reason: logging.warning('(Emscripten: %s, clearing cache)' % reason) Cache.erase() + force = False # the check actually failed, so definitely write out the sanity file, to avoid others later seeing failures too - # some warning, not fatal checks - do them even if EM_IGNORE_SANITY is on + # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on check_llvm_version() check_node_version() + if os.environ.get('EMCC_FAST_COMPILER') == '1': + fastcomp_ok = check_fastcomp() if os.environ.get('EM_IGNORE_SANITY'): logging.info('EM_IGNORE_SANITY set, ignoring sanity checks') @@ -344,11 +397,16 @@ def check_sanity(force=False): logging.critical('Node.js (%s) does not seem to work, check the paths in %s' % (NODE_JS, EM_CONFIG)) sys.exit(1) - for cmd in [CLANG, LLVM_LINK, LLVM_AR, LLVM_OPT, LLVM_AS, LLVM_DIS, LLVM_NM]: + for cmd in [CLANG, LINK_CMD[0], LLVM_AR, LLVM_OPT, LLVM_AS, LLVM_DIS, LLVM_NM, LLVM_INTERPRETER]: if not os.path.exists(cmd) and not os.path.exists(cmd + '.exe'): # .exe extension required for Windows logging.critical('Cannot find %s, check the paths in %s' % (cmd, EM_CONFIG)) sys.exit(1) + if os.environ.get('EMCC_FAST_COMPILER') == '1': + if not fastcomp_ok: + logging.critical('failing sanity checks due to previous fastcomp failure') + sys.exit(1) + try: subprocess.call([JAVA, '-version'], stdout=PIPE, stderr=PIPE) except: @@ -371,12 +429,31 @@ def check_sanity(force=False): # Tools/paths -LLVM_ADD_VERSION = os.getenv('LLVM_ADD_VERSION') -CLANG_ADD_VERSION = os.getenv('CLANG_ADD_VERSION') +try: + LLVM_ADD_VERSION +except NameError: + LLVM_ADD_VERSION = os.getenv('LLVM_ADD_VERSION') + +try: + CLANG_ADD_VERSION +except NameError: + CLANG_ADD_VERSION = os.getenv('CLANG_ADD_VERSION') + +USING_PNACL_TOOLCHAIN = os.path.exists(os.path.join(LLVM_ROOT, 'pnacl-clang')) + +def modify_prefix(tool): + if USING_PNACL_TOOLCHAIN: + if tool.startswith('llvm-'): + tool = tool[5:] + tool = 'pnacl-' + tool + if WINDOWS: + tool += '.bat' + return tool # Some distributions ship with multiple llvm versions so they add # the version to the binaries, cope with that def build_llvm_tool_path(tool): + tool = modify_prefix(tool) if LLVM_ADD_VERSION: return os.path.join(LLVM_ROOT, tool + "-" + LLVM_ADD_VERSION) else: @@ -385,6 +462,7 @@ def build_llvm_tool_path(tool): # Some distributions ship with multiple clang versions so they add # the version to the binaries, cope with that def build_clang_tool_path(tool): + tool = modify_prefix(tool) if CLANG_ADD_VERSION: return os.path.join(LLVM_ROOT, tool + "-" + CLANG_ADD_VERSION) else: @@ -393,7 +471,11 @@ def build_clang_tool_path(tool): CLANG_CC=os.path.expanduser(build_clang_tool_path('clang')) CLANG_CPP=os.path.expanduser(build_clang_tool_path('clang++')) CLANG=CLANG_CPP -LLVM_LINK=build_llvm_tool_path('llvm-link') +if USING_PNACL_TOOLCHAIN: + # The PNaCl toolchain doesn't have llvm-link, but we can fake it + LINK_CMD = [build_llvm_tool_path('llvm-ld'), '-nostdlib', '-r'] +else: + LINK_CMD = [build_llvm_tool_path('llvm-link')] LLVM_AR=build_llvm_tool_path('llvm-ar') LLVM_OPT=os.path.expanduser(build_llvm_tool_path('opt')) LLVM_AS=os.path.expanduser(build_llvm_tool_path('llvm-as')) @@ -409,8 +491,8 @@ EMCC = path_from_root('emcc') EMXX = path_from_root('em++') EMAR = path_from_root('emar') EMRANLIB = path_from_root('emranlib') -EMLIBTOOL = path_from_root('emlibtool') EMCONFIG = path_from_root('em-config') +EMLINK = path_from_root('emlink.py') EMMAKEN = path_from_root('tools', 'emmaken.py') AUTODEBUGGER = path_from_root('tools', 'autodebugger.py') BINDINGS_GENERATOR = path_from_root('tools', 'bindings_generator.py') @@ -419,8 +501,20 @@ FILE_PACKAGER = path_from_root('tools', 'file_packager.py') # Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use TEMP_DIR/emscripten_temp +def safe_ensure_dirs(dirname): + try: + os.makedirs(dirname) + except os.error, e: + # Ignore error for already existing dirname + if e.errno != errno.EEXIST: + raise e + # FIXME: Notice that this will result in a false positive, + # should the dirname be a file! There seems to no way to + # handle this atomically in Python 2.x. + # There is an additional option for Python 3.x, though. + class Configuration: - def __init__(self, environ): + def __init__(self, environ=os.environ): self.DEBUG = environ.get('EMCC_DEBUG') if self.DEBUG == "0": self.DEBUG = None @@ -430,38 +524,51 @@ class Configuration: try: self.TEMP_DIR = TEMP_DIR except NameError: - logging.debug('TEMP_DIR not defined in ~/.emscripten, using /tmp') - self.TEMP_DIR = '/tmp' + self.TEMP_DIR = find_temp_directory() + if self.TEMP_DIR == None: + logging.critical('TEMP_DIR not defined in ' + os.path.expanduser('~\\.emscripten') + ", and could not detect a suitable directory! Please configure .emscripten to contain a variable TEMP_DIR='/path/to/temp/dir'.") + logging.debug('TEMP_DIR not defined in ~/.emscripten, using ' + self.TEMP_DIR) + + if not os.path.isdir(self.TEMP_DIR): + logging.critical("The temp directory TEMP_DIR='" + self.TEMP_DIR + "' doesn't seem to exist! Please make sure that the path is correct.") self.CANONICAL_TEMP_DIR = os.path.join(self.TEMP_DIR, 'emscripten_temp') if self.DEBUG: try: self.EMSCRIPTEN_TEMP_DIR = self.CANONICAL_TEMP_DIR - if not os.path.exists(self.EMSCRIPTEN_TEMP_DIR): - os.makedirs(self.EMSCRIPTEN_TEMP_DIR) + safe_ensure_dirs(self.EMSCRIPTEN_TEMP_DIR) except Exception, e: - logging.debug(e + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ~/.emscripten') + logging.error(str(e) + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ~/.emscripten') def get_temp_files(self): return tempfiles.TempFiles( tmp=self.TEMP_DIR if not self.DEBUG else self.EMSCRIPTEN_TEMP_DIR, save_debug_files=os.environ.get('EMCC_DEBUG_SAVE')) -configuration = Configuration(environ=os.environ) -DEBUG = configuration.DEBUG -EMSCRIPTEN_TEMP_DIR = configuration.EMSCRIPTEN_TEMP_DIR -DEBUG_CACHE = configuration.DEBUG_CACHE -CANONICAL_TEMP_DIR = configuration.CANONICAL_TEMP_DIR +def apply_configuration(): + global configuration, DEBUG, EMSCRIPTEN_TEMP_DIR, DEBUG_CACHE, CANONICAL_TEMP_DIR, TEMP_DIR + configuration = Configuration() + DEBUG = configuration.DEBUG + EMSCRIPTEN_TEMP_DIR = configuration.EMSCRIPTEN_TEMP_DIR + DEBUG_CACHE = configuration.DEBUG_CACHE + CANONICAL_TEMP_DIR = configuration.CANONICAL_TEMP_DIR + TEMP_DIR = configuration.TEMP_DIR +apply_configuration() + +logging.basicConfig(format='%(levelname)-8s %(name)s: %(message)s') +def set_logging(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG if os.environ.get('EMCC_DEBUG') else logging.INFO) +set_logging() -level = logging.DEBUG if os.environ.get('EMCC_DEBUG') else logging.INFO -logging.basicConfig(level=level, format='%(levelname)-8s %(name)s: %(message)s') - if not EMSCRIPTEN_TEMP_DIR: EMSCRIPTEN_TEMP_DIR = tempfile.mkdtemp(prefix='emscripten_temp_', dir=configuration.TEMP_DIR) - def clean_temp(): - try_delete(EMSCRIPTEN_TEMP_DIR) - atexit.register(clean_temp) + def prepare_to_clean_temp(d): + def clean_temp(): + try_delete(d) + atexit.register(clean_temp) + prepare_to_clean_temp(EMSCRIPTEN_TEMP_DIR) # this global var might change later # EM_CONFIG stuff @@ -498,27 +605,33 @@ def get_llvm_target(): return os.environ.get('EMCC_LLVM_TARGET') or 'le32-unknown-nacl' # 'i386-pc-linux-gnu' LLVM_TARGET = get_llvm_target() +# COMPILER_OPTS: options passed to clang when generating bitcode for us try: COMPILER_OPTS # Can be set in EM_CONFIG, optionally except: COMPILER_OPTS = [] -# Force a simple, standard target as much as possible: target 32-bit linux, and disable various flags that hint at other platforms -COMPILER_OPTS = COMPILER_OPTS + ['-m32', '-U__i386__', '-U__i386', '-Ui386', - '-U__SSE__', '-U__SSE_MATH__', '-U__SSE2__', '-U__SSE2_MATH__', '-U__MMX__', - '-DEMSCRIPTEN', '-D__EMSCRIPTEN__', '-U__STRICT_ANSI__', - '-D__IEEE_LITTLE_ENDIAN', '-fno-math-errno', '-fno-threadsafe-statics', +COMPILER_OPTS = COMPILER_OPTS + ['-m32', '-DEMSCRIPTEN', '-D__EMSCRIPTEN__', + '-fno-math-errno', + #'-fno-threadsafe-statics', # disabled due to issue 1289 '-target', LLVM_TARGET] if LLVM_TARGET == 'le32-unknown-nacl': + COMPILER_OPTS = filter(lambda opt: opt != '-m32', COMPILER_OPTS) # le32 target is 32-bit anyhow, no need for -m32 COMPILER_OPTS += ['-U__native_client__', '-U__pnacl__', '-U__ELF__'] # The nacl target is originally used for Google Native Client. Emscripten is not NaCl, so remove the platform #define, when using their triple. +# Remove various platform specific defines, and set little endian +COMPILER_STANDARDIZATION_OPTS = ['-U__i386__', '-U__i386', '-Ui386', '-U__STRICT_ANSI__', '-D__IEEE_LITTLE_ENDIAN', + '-U__SSE__', '-U__SSE_MATH__', '-U__SSE2__', '-U__SSE2_MATH__', '-U__MMX__', + '-U__APPLE__', '-U__linux__'] + USE_EMSDK = not os.environ.get('EMMAKEN_NO_SDK') if USE_EMSDK: # Disable system C and C++ include directories, and add our own (using -idirafter so they are last, like system dirs, which # allows projects to override them) - EMSDK_OPTS = ['-nostdinc', '-nostdinc++', '-Xclang', '-nobuiltininc', '-Xclang', '-nostdsysteminc', + EMSDK_OPTS = ['-nostdinc', '-Xclang', '-nobuiltininc', '-Xclang', '-nostdsysteminc', '-Xclang', '-isystem' + path_from_root('system', 'local', 'include'), + '-Xclang', '-isystem' + path_from_root('system', 'include', 'compat'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'libcxx'), '-Xclang', '-isystem' + path_from_root('system', 'include'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'emscripten'), @@ -527,12 +640,17 @@ if USE_EMSDK: '-Xclang', '-isystem' + path_from_root('system', 'include', 'gfx'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'net'), '-Xclang', '-isystem' + path_from_root('system', 'include', 'SDL'), - ] + [ - '-U__APPLE__', '-U__linux__' ] + EMSDK_OPTS += COMPILER_STANDARDIZATION_OPTS + if LLVM_TARGET != 'le32-unknown-nacl': + EMSDK_CXX_OPTS = ['-nostdinc++'] # le32 target does not need -nostdinc++ + else: + EMSDK_CXX_OPTS = [] COMPILER_OPTS += EMSDK_OPTS else: EMSDK_OPTS = [] + EMSDK_CXX_OPTS = [] + COMPILER_OPTS += COMPILER_STANDARDIZATION_OPTS #print >> sys.stderr, 'SDK opts', ' '.join(EMSDK_OPTS) #print >> sys.stderr, 'Compiler opts', ' '.join(COMPILER_OPTS) @@ -564,7 +682,7 @@ def check_engine(engine): try: if not CONFIG_FILE: return True # config stored directly in EM_CONFIG => skip engine check - return 'hello, world!' in run_js(path_from_root('tests', 'hello_world.js'), engine) + return 'hello, world!' in run_js(path_from_root('src', 'hello_world.js'), engine) except Exception, e: print 'Checking JS engine %s failed. Check %s. Details: %s' % (str(engine), EM_CONFIG, str(e)) return False @@ -594,7 +712,7 @@ def line_splitter(data): return out -def limit_size(string, MAX=80*20): +def limit_size(string, MAX=800*20): if len(string) < MAX: return string return string[0:MAX/2] + '\n[..]\n' + string[-MAX/2:] @@ -638,62 +756,85 @@ def expand_response(data): # Settings. A global singleton. Not pretty, but nicer than passing |, settings| everywhere -class Settings: - @classmethod - def reset(self): - class Settings2: - QUANTUM_SIZE = 4 - reset = Settings.reset - - # Given some emcc-type args (-O3, -s X=Y, etc.), fill Settings with the right settings - @classmethod - def load(self, args=[]): - # Load the JS defaults into python - settings = open(path_from_root('src', 'settings.js')).read().replace('var ', 'Settings.').replace('//', '#') - exec settings in globals() - - # Apply additional settings. First -O, then -s - for i in range(len(args)): - if args[i].startswith('-O'): - level = eval(args[i][2]) - Settings.apply_opt_level(level) - for i in range(len(args)): - if args[i] == '-s': - exec 'Settings.' + args[i+1] in globals() # execute the setting - - # Transforms the Settings information into emcc-compatible args (-s X=Y, etc.). Basically - # the reverse of load_settings, except for -Ox which is relevant there but not here - @classmethod - def serialize(self): - ret = [] - for key, value in Settings.__dict__.iteritems(): - if key == key.upper(): # this is a hack. all of our settings are ALL_CAPS, python internals are not - jsoned = json.dumps(value, sort_keys=True) - ret += ['-s', key + '=' + jsoned] - return ret - - @classmethod - def apply_opt_level(self, opt_level, noisy=False): - if opt_level >= 1: - Settings.ASM_JS = 1 - Settings.ASSERTIONS = 0 - Settings.DISABLE_EXCEPTION_CATCHING = 1 - Settings.EMIT_GENERATED_FUNCTIONS = 1 - if opt_level >= 2: - Settings.RELOOP = 1 - Settings.ALIASING_FUNCTION_POINTERS = 1 - if opt_level >= 3: - # Aside from these, -O3 also runs closure compiler and llvm lto - Settings.FORCE_ALIGNED_MEMORY = 1 - Settings.DOUBLE_MODE = 0 - Settings.PRECISE_I64_MATH = 0 - if noisy: logging.warning('Applying some potentially unsafe optimizations! (Use -O2 if this fails.)') - - global Settings - Settings = Settings2 - Settings.load() # load defaults - -Settings.reset() +class Settings2(type): + class __impl: + attrs = {} + + def __init__(self): + self.reset() + + @classmethod + def reset(self): + self.attrs = { 'QUANTUM_SIZE': 4 } + self.load() + + # Given some emcc-type args (-O3, -s X=Y, etc.), fill Settings with the right settings + @classmethod + def load(self, args=[]): + # Load the JS defaults into python + settings = open(path_from_root('src', 'settings.js')).read().replace('//', '#') + settings = re.sub(r'var ([\w\d]+)', r'self.attrs["\1"]', settings) + exec settings + + # Apply additional settings. First -O, then -s + for i in range(len(args)): + if args[i].startswith('-O'): + level = eval(args[i][2]) + self.apply_opt_level(level) + for i in range(len(args)): + if args[i] == '-s': + declare = re.sub(r'([\w\d]+)\s*=\s*(.+)', r'self.attrs["\1"]=\2;', args[i+1]) + exec declare + + # Transforms the Settings information into emcc-compatible args (-s X=Y, etc.). Basically + # the reverse of load_settings, except for -Ox which is relevant there but not here + @classmethod + def serialize(self): + ret = [] + for key, value in self.attrs.iteritems(): + if key == key.upper(): # this is a hack. all of our settings are ALL_CAPS, python internals are not + jsoned = json.dumps(value, sort_keys=True) + ret += ['-s', key + '=' + jsoned] + return ret + + @classmethod + def copy(self, values): + self.attrs = values + + @classmethod + def apply_opt_level(self, opt_level, noisy=False): + if opt_level >= 1: + self.attrs['ASM_JS'] = 1 + self.attrs['ASSERTIONS'] = 0 + self.attrs['DISABLE_EXCEPTION_CATCHING'] = 1 + self.attrs['RELOOP'] = 1 + self.attrs['ALIASING_FUNCTION_POINTERS'] = 1 + + def __getattr__(self, attr): + if attr in self.attrs: + return self.attrs[attr] + else: + raise AttributeError + + def __setattr__(self, attr, value): + self.attrs[attr] = value + + __instance = None + + @staticmethod + def instance(): + if Settings2.__instance is None: + Settings2.__instance = Settings2.__impl() + return Settings2.__instance + + def __getattr__(self, attr): + return getattr(self.instance(), attr) + + def __setattr__(self, attr, value): + return setattr(self.instance(), attr, value) + +class Settings(object): + __metaclass__ = Settings2 # Building @@ -701,6 +842,7 @@ class Building: COMPILER = CLANG LLVM_OPTS = False COMPILER_TEST_OPTS = [] # For use of the test runner + JS_ENGINE_OVERRIDE = None # Used to pass the JS engine override from runner.py -> test_benchmark.py @staticmethod def get_building_env(native=False): @@ -717,7 +859,6 @@ class Building: env['LD'] = EMCC if not WINDOWS else 'python %r' % EMCC env['LDSHARED'] = EMCC if not WINDOWS else 'python %r' % EMCC env['RANLIB'] = EMRANLIB if not WINDOWS else 'python %r' % EMRANLIB - #env['LIBTOOL'] = EMLIBTOOL if not WINDOWS else 'python %r' % EMLIBTOOL env['EMMAKEN_COMPILER'] = Building.COMPILER env['EMSCRIPTEN_TOOLS'] = path_from_root('tools') env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(Building.COMPILER_TEST_OPTS) @@ -727,38 +868,55 @@ class Building: env['HOST_CXXFLAGS'] = "-W" #if set to nothing, CXXFLAGS is used, which we don't want env['PKG_CONFIG_LIBDIR'] = path_from_root('system', 'local', 'lib', 'pkgconfig') + os.path.pathsep + path_from_root('system', 'lib', 'pkgconfig') env['PKG_CONFIG_PATH'] = os.environ.get ('EM_PKG_CONFIG_PATH') or '' - env['EMSCRIPTEN'] = '1' + env['EMSCRIPTEN'] = path_from_root() return env + # Finds the given executable 'program' in PATH. Operates like the Unix tool 'which'. + @staticmethod + def which(program): + import os + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + if WINDOWS and not '.' in fname: + if is_exe(exe_file + '.exe'): + return exe_file + '.exe' + if is_exe(exe_file + '.cmd'): + return exe_file + '.cmd' + if is_exe(exe_file + '.bat'): + return exe_file + '.bat' + + return None + @staticmethod def handle_CMake_toolchain(args, env): - CMakeToolchain = ('''# the name of the target operating system -SET(CMAKE_SYSTEM_NAME Linux) - -# which C and C++ compiler to use -SET(CMAKE_C_COMPILER %(winfix)s$EMSCRIPTEN_ROOT/emcc) -SET(CMAKE_CXX_COMPILER %(winfix)s$EMSCRIPTEN_ROOT/em++) -SET(CMAKE_AR %(winfix)s$EMSCRIPTEN_ROOT/emar) -SET(CMAKE_RANLIB %(winfix)s$EMSCRIPTEN_ROOT/emranlib) -SET(CMAKE_C_FLAGS $CFLAGS) -SET(CMAKE_CXX_FLAGS $CXXFLAGS) - -# here is the target environment located -SET(CMAKE_FIND_ROOT_PATH $EMSCRIPTEN_ROOT/system/include ) - -# adjust the default behaviour of the FIND_XXX() commands: -# search headers and libraries in the target environment, search -# programs in the host environment -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS else 'python ' }) \ - .replace('$EMSCRIPTEN_ROOT', path_from_root('').replace('\\', '/')) \ - .replace('$CFLAGS', env['CFLAGS']) \ - .replace('$CXXFLAGS', env['CFLAGS']) - toolchainFile = mkstemp(suffix='.cmaketoolchain.txt', dir=configuration.TEMP_DIR)[1] - open(toolchainFile, 'w').write(CMakeToolchain) - args.append('-DCMAKE_TOOLCHAIN_FILE=%s' % os.path.abspath(toolchainFile)) + + def has_substr(array, substr): + for arg in array: + if substr in arg: + return True + return False + + # Append the Emscripten toolchain file if the user didn't specify one. + if not has_substr(args, '-DCMAKE_TOOLCHAIN_FILE'): + args.append('-DCMAKE_TOOLCHAIN_FILE=' + path_from_root('cmake', 'Platform', 'Emscripten.cmake')) + + # On Windows specify MinGW Makefiles if we have MinGW and no other toolchain was specified, to avoid CMake + # pulling in a native Visual Studio, or Unix Makefiles. + if WINDOWS and not '-G' in args and Building.which('mingw32-make'): + args += ['-G', 'MinGW Makefiles'] + return args @staticmethod @@ -778,6 +936,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e raise del env['EMMAKEN_JUST_CONFIGURE'] if process.returncode is not 0: + logging.error('Configure step failed with non-zero return code ' + str(process.returncode) + '! Command line: ' + str(args)) raise subprocess.CalledProcessError(cmd=args, returncode=process.returncode) @staticmethod @@ -788,6 +947,13 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e logging.error('Executable to run not specified.') sys.exit(1) #args += ['VERBOSE=1'] + + # On Windows prefer building with mingw32-make instead of make, if it exists. + if WINDOWS and args[0] == 'make': + mingw32_make = Building.which('mingw32-make') + if mingw32_make: + args[0] = mingw32_make + try: process = Popen(args, stdout=stdout, stderr=stderr, env=env) process.communicate() @@ -799,7 +965,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e @staticmethod - def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None, native=False): + def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args='help', cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None, native=False): ''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in memory since the test directory is destroyed and recreated for each test. Note that we cache separately for different compilers). @@ -807,6 +973,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e if type(generated_libs) is not list: generated_libs = [generated_libs] if source_dir is None: source_dir = path_from_root('tests', name.replace('_native', '')) + if make_args == 'help': + make_args = ['-j', str(multiprocessing.cpu_count())] temp_dir = build_dir if copy_project: @@ -828,12 +996,13 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e # except: # pass env = Building.get_building_env(native) + verbose_level = int(os.getenv('EM_BUILD_VERBOSE')) if os.getenv('EM_BUILD_VERBOSE') != None else 0 for k, v in env_init.iteritems(): env[k] = v if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |link| call) try: - Building.configure(configure + configure_args, stdout=open(os.path.join(project_dir, 'configure_'), 'w'), - stderr=open(os.path.join(project_dir, 'configure_err'), 'w'), env=env) + Building.configure(configure + configure_args, env=env, stdout=open(os.path.join(project_dir, 'configure_'), 'w') if verbose_level < 2 else None, + stderr=open(os.path.join(project_dir, 'configure_err'), 'w') if verbose_level < 1 else None) except subprocess.CalledProcessError, e: pass # Ignore exit code != 0 def open_make_out(i, mode='r'): @@ -841,13 +1010,16 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e def open_make_err(i, mode='r'): return open(os.path.join(project_dir, 'make_err' + str(i)), mode) - + + if verbose_level >= 3: + make_args += ['VERBOSE=1'] + for i in range(2): # FIXME: Sad workaround for some build systems that need to be run twice to succeed (e.g. poppler) with open_make_out(i, 'w') as make_out: with open_make_err(i, 'w') as make_err: try: - Building.make(make + make_args, stdout=make_out, - stderr=make_err, env=env) + Building.make(make + make_args, stdout=make_out if verbose_level < 2 else None, + stderr=make_err if verbose_level < 1 else None, env=env) except subprocess.CalledProcessError, e: pass # Ignore exit code != 0 try: @@ -859,19 +1031,23 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e break except Exception, e: if i > 0: - # Due to the ugly hack above our best guess is to output the first run - with open_make_err(0) as ferr: - for line in ferr: - sys.stderr.write(line) + if verbose_level == 0: + # Due to the ugly hack above our best guess is to output the first run + with open_make_err(0) as ferr: + for line in ferr: + sys.stderr.write(line) raise Exception('could not build library ' + name + ' due to exception ' + str(e)) if old_dir: os.chdir(old_dir) return generated_libs @staticmethod - def link(files, target): + def link(files, target, force_archive_contents=False): actual_files = [] - unresolved_symbols = set(['main']) # tracking unresolveds is necessary for .a linking, see below. (and main is always a necessary symbol) + # Tracking unresolveds is necessary for .a linking, see below. + # Specify all possible entry points to seed the linking process. + # For a simple application, this would just be "main". + unresolved_symbols = set([func[1:] for func in Settings.EXPORTED_FUNCTIONS]) resolved_symbols = set() temp_dirs = [] files = map(os.path.abspath, files) @@ -893,8 +1069,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e try: temp_dir = os.path.join(EMSCRIPTEN_TEMP_DIR, 'ar_output_' + str(os.getpid()) + '_' + str(len(temp_dirs))) temp_dirs.append(temp_dir) - if not os.path.exists(temp_dir): - os.makedirs(temp_dir) + safe_ensure_dirs(temp_dir) os.chdir(temp_dir) contents = filter(lambda x: len(x) > 0, Popen([LLVM_AR, 't', f], stdout=PIPE).communicate()[0].split('\n')) #print >> sys.stderr, ' considering archive', f, ':', contents @@ -903,9 +1078,9 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e else: for content in contents: # ar will silently fail if the directory for the file does not exist, so make all the necessary directories dirname = os.path.dirname(content) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - Popen([LLVM_AR, 'x', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory + if dirname: + safe_ensure_dirs(dirname) + Popen([LLVM_AR, 'xo', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory contents = map(lambda content: os.path.join(temp_dir, content), contents) contents = filter(os.path.exists, map(os.path.abspath, contents)) added_contents = set() @@ -920,7 +1095,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e # Link in the .o if it provides symbols, *or* this is a singleton archive (which is apparently an exception in gcc ld) #print >> sys.stderr, 'need', content, '?', unresolved_symbols, 'and we can supply', new_symbols.defs #print >> sys.stderr, content, 'DEF', new_symbols.defs, '\n' - if new_symbols.defs.intersection(unresolved_symbols) or len(files) == 1: + |