diff options
53 files changed, 6443 insertions, 101 deletions
diff --git a/cmake/Platform/Emscripten.cmake b/cmake/Platform/Emscripten.cmake index 73f2c8ad..7c8e83fa 100644 --- a/cmake/Platform/Emscripten.cmake +++ b/cmake/Platform/Emscripten.cmake @@ -104,8 +104,8 @@ set(CMAKE_C_RESPONSE_FILE_LINK_FLAG "@") set(CMAKE_CXX_RESPONSE_FILE_LINK_FLAG "@") # Specify the program to use when building static libraries. Force Emscripten-related command line options to clang. -set(CMAKE_CXX_ARCHIVE_CREATE "${CMAKE_CXX_COMPILER} ${CMAKE_START_TEMP_FILE} -o <TARGET> -emit-llvm <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") -set(CMAKE_C_ARCHIVE_CREATE "${CMAKE_C_COMPILER} ${CMAKE_START_TEMP_FILE} -o <TARGET> -emit-llvm <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") +set(CMAKE_CXX_ARCHIVE_CREATE "${CMAKE_AR} rc <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") +set(CMAKE_C_ARCHIVE_CREATE "${CMAKE_AR} rc <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") # Set a global EMSCRIPTEN variable that can be used in client CMakeLists.txt to detect when building using Emscripten. # There seems to be some kind of bug with CMake, so you might need to define this manually on the command line with "-DEMSCRIPTEN=1". @@ -720,6 +720,13 @@ else: def in_temp(name): return os.path.join(temp_dir, os.path.basename(name)) +# Parses the essential suffix of a filename, discarding Unix-style version numbers in the name. For example for 'libz.so.1.2.8' returns '.so' +def filename_type_suffix(filename): + for i in reversed(filename.split('.')[1:]): + if not i.isdigit(): + return '.' + i + return '' + try: call = CXX if use_cxx else CC @@ -974,35 +981,45 @@ try: if i > 0: prev = newargs[i-1] - if prev in ['-MT', '-MF', '-MQ', '-D', '-U', '-o', '-x', '-Xpreprocessor', '-include', '-imacros', '-idirafter', '-iprefix', '-iwithprefix', '-iwithprefixbefore', '-isysroot', '-imultilib', '-A', '-isystem', '-iquote', '-install_name', '-I', '-L']: continue # ignore this gcc-style argument + if prev in ['-MT', '-MF', '-MQ', '-D', '-U', '-o', '-x', '-Xpreprocessor', '-include', '-imacros', '-idirafter', '-iprefix', '-iwithprefix', '-iwithprefixbefore', '-isysroot', '-imultilib', '-A', '-isystem', '-iquote', '-install_name', '-compatibility_version', '-current_version', '-I', '-L']: continue # ignore this gcc-style argument if (os.path.islink(arg) and os.path.realpath(arg).endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES)): arg = os.path.realpath(arg) - if not arg.startswith('-') and (arg.endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES) or shared.Building.is_ar(arg)): # we already removed -o <target>, so all these should be inputs - newargs[i] = '' - if os.path.exists(arg): - if arg.endswith(SOURCE_SUFFIXES): + if not arg.startswith('-'): + if not os.path.exists(arg): + logging.error(arg + ': No such file or directory') + exit(1) + + arg_suffix = filename_type_suffix(arg) + if arg_suffix.endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES) or shared.Building.is_ar(arg): # we already removed -o <target>, so all these should be inputs + newargs[i] = '' + if arg_suffix.endswith(SOURCE_SUFFIXES): input_files.append(arg) has_source_inputs = True + elif arg_suffix.endswith(ASSEMBLY_SUFFIXES) or shared.Building.is_bitcode(arg): # this should be bitcode, make sure it is valid + input_files.append(arg) + elif arg_suffix.endswith(STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES): + # if it's not, and it's a library, just add it to libs to find later + l = unsuffixed_basename(arg) + for prefix in LIB_PREFIXES: + if not prefix: continue + if l.startswith(prefix): + l = l[len(prefix):] + break + libs.append(l) + newargs[i] = '' else: - # this should be bitcode, make sure it is valid - if arg.endswith(ASSEMBLY_SUFFIXES) or shared.Building.is_bitcode(arg): - input_files.append(arg) - elif arg.endswith(STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES): - # if it's not, and it's a library, just add it to libs to find later - l = unsuffixed_basename(arg) - for prefix in LIB_PREFIXES: - if not prefix: continue - if l.startswith(prefix): - l = l[len(prefix):] - break - libs.append(l) - newargs[i] = '' + logging.warning(arg + ' is not valid LLVM bitcode') + elif arg_suffix.endswith(STATICLIB_SUFFIXES): + if not shared.Building.is_ar(arg): + if shared.Building.is_bitcode(arg): + logging.error(arg + ': File has a suffix of a static library ' + str(STATICLIB_SUFFIXES) + ', but instead is an LLVM bitcode file! When linking LLVM bitcode files, use one of the suffixes ' + str(BITCODE_SUFFIXES)) else: - logging.warning(arg + ' is not valid LLVM bitcode') + logging.error(arg + ': Unknown format, not a static library!') + exit(1) else: - logging.error(arg + ': No such file or directory') + logging.error(arg + ": Input file has an unknown suffix, don't know what to do with it!") exit(1) elif arg.startswith('-L'): lib_dirs.append(arg[2:]) @@ -1151,13 +1168,14 @@ try: # First, generate LLVM bitcode. For each input file, we get base.o with bitcode for input_file in input_files: - if input_file.endswith(SOURCE_SUFFIXES): + file_suffix = filename_type_suffix(input_file) + if file_suffix.endswith(SOURCE_SUFFIXES): logging.debug('compiling source file: ' + input_file) input_file = shared.Building.preprocess(input_file, in_temp(uniquename(input_file))) output_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') temp_files.append(output_file) args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file] - if input_file.endswith(CXX_SUFFIXES): + if file_suffix.endswith(CXX_SUFFIXES): args += shared.EMSDK_CXX_OPTS logging.debug("running: " + call + ' ' + ' '.join(args)) execute([call] + args) # let compiler frontend print directly, so colors are saved (PIPE kills that) @@ -1165,17 +1183,17 @@ try: logging.error('compiler frontend failed to generate LLVM bitcode, halting') sys.exit(1) else: # bitcode - if input_file.endswith(BITCODE_SUFFIXES): + if file_suffix.endswith(BITCODE_SUFFIXES): logging.debug('copying bitcode file: ' + input_file) temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') shutil.copyfile(input_file, temp_file) temp_files.append(temp_file) - elif input_file.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file): + elif file_suffix.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file): logging.debug('copying library file: ' + input_file) temp_file = in_temp(uniquename(input_file)) shutil.copyfile(input_file, temp_file) temp_files.append(temp_file) - else: #.ll + elif file_suffix.endswith(ASSEMBLY_SUFFIXES): if not LEAVE_INPUTS_RAW: # Note that by assembling the .ll file, then disassembling it later, we will # remove annotations which is a good thing for compilation time @@ -1183,6 +1201,9 @@ try: temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') shared.Building.llvm_as(input_file, temp_file) temp_files.append(temp_file) + else: + logging.error(input_file + ': Unknown file suffix when compiling to LLVM bitcode!') + sys.exit(1) if not LEAVE_INPUTS_RAW: assert len(temp_files) == len(input_files) @@ -1190,7 +1211,8 @@ try: # Optimize source files if llvm_opts > 0: for i, input_file in enumerate(input_files): - if input_file.endswith(SOURCE_SUFFIXES): + file_suffix = filename_type_suffix(input_file) + if file_suffix.endswith(SOURCE_SUFFIXES): temp_file = temp_files[i] logging.debug('optimizing %s with -O%d' % (input_file, llvm_opts)) shared.Building.llvm_opt(temp_file, llvm_opts) @@ -1282,6 +1304,10 @@ try: os.path.join('libc', 'gen', 'vwarnx.c'), os.path.join('libc', 'stdlib', 'strtod.c'), ] + musl_files = [ + ] + for directory, sources in musl_files: + libc_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources] return build_libc('libc.bc', libc_files) def apply_libc(need): @@ -1319,6 +1345,33 @@ try: 'wctrans.c', 'wcwidth.c', ]], + ['locale', [ + 'iconv.c', + 'iswalnum_l.c', + 'iswalpha_l.c', + 'iswblank_l.c', + 'iswcntrl_l.c', + 'iswctype_l.c', + 'iswdigit_l.c', + 'iswgraph_l.c', + 'iswlower_l.c', + 'iswprint_l.c', + 'iswpunct_l.c', + 'iswspace_l.c', + 'iswupper_l.c', + 'iswxdigit_l.c', + 'strfmon.c', + 'strxfrm.c', + 'towctrans_l.c', + 'towlower_l.c', + 'towupper_l.c', + 'wcscoll.c', + 'wcscoll_l.c', + 'wcsxfrm.c', + 'wcsxfrm_l.c', + 'wctrans_l.c', + 'wctype_l.c', + ]], ['multibyte', [ 'btowc.c', 'mblen.c', @@ -1336,6 +1389,14 @@ try: 'wctob.c', 'wctomb.c', ]], + ['stdio', [ + 'fwprintf.c', + 'swprintf.c', + 'vfwprintf.c', + 'vswprintf.c', + 'vwprintf.c', + 'wprintf.c', + ]], ['stdlib', [ 'ecvt.c', 'fcvt.c', @@ -1345,7 +1406,7 @@ try: 'wcpcpy.c', 'wcpncpy.c', 'wcscasecmp.c', - # 'wcscasecmp_l.c', # XXX: alltypes.h issue + 'wcscasecmp_l.c', 'wcscat.c', 'wcschr.c', 'wcscmp.c', @@ -1354,7 +1415,7 @@ try: 'wcsdup.c', 'wcslen.c', 'wcsncasecmp.c', - # 'wcsncasecmp_l.c', # XXX: alltypes.h issue + 'wcsncasecmp_l.c', 'wcsncat.c', 'wcsncmp.c', 'wcsncpy.c', diff --git a/emscripten.py b/emscripten.py index 19e2160d..abff6b1d 100755 --- a/emscripten.py +++ b/emscripten.py @@ -9,7 +9,7 @@ header files (so that the JS compiler can see the constants in those headers, for the libc implementation in JS). ''' -import os, sys, json, optparse, subprocess, re, time, multiprocessing, string +import os, sys, json, optparse, subprocess, re, time, multiprocessing, string, logging from tools import jsrun, cache as cache_module, tempfiles from tools.response_file import read_response_file @@ -46,7 +46,7 @@ MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # con STDERR_FILE = os.environ.get('EMCC_STDERR_FILE') if STDERR_FILE: STDERR_FILE = os.path.abspath(STDERR_FILE) - print >> sys.stderr, 'logging stderr in js compiler phase into %s' % STDERR_FILE + logging.info('logging stderr in js compiler phase into %s' % STDERR_FILE) STDERR_FILE = open(STDERR_FILE, 'w') def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): @@ -58,6 +58,8 @@ def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libr f.write('\n') f.write(meta) f.close() + #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments, + # # just copy temp dir to under this one out = jsrun.run_js( compiler, engine=compiler_engine, @@ -68,9 +70,7 @@ def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libr except KeyboardInterrupt: # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt raise Exception() - finally: - tempfiles.try_delete(funcs_file) - if DEBUG: print >> sys.stderr, '.' + if DEBUG: logging.debug('.') return out def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, @@ -91,7 +91,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. - if DEBUG: print >> sys.stderr, 'emscript: ll=>js' + if DEBUG: logging.debug('emscript: ll=>js') if jcache: jcache.ensure() @@ -101,7 +101,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection - if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: scan took %s seconds' % (time.time() - t)) # Split input into the relevant parts for each phase pre = [] @@ -136,19 +136,19 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) - if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' - if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) + if DEBUG and len(meta) > 1024*1024: logging.debug('emscript warning: large amounts of metadata, will slow things down') + if DEBUG: logging.debug(' emscript: split took %s seconds' % (time.time() - t)) if len(funcs) == 0: - print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' + logging.error('No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)') #if DEBUG: - # print >> sys.stderr, '========= pre ================\n' - # print >> sys.stderr, ''.join(pre) - # print >> sys.stderr, '========== funcs ===============\n' + # logging.debug('========= pre ================\n') + # logging.debug(''.join(pre)) + # logging.debug('========== funcs ===============\n') # for func in funcs: - # print >> sys.stderr, '\n// ===\n\n', ''.join(func) - # print >> sys.stderr, '=========================\n' + # logging.debug('\n// ===\n\n', ''.join(func)) + # logging.debug('=========================\n') # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name @@ -168,7 +168,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) - if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey + if DEBUG_CACHE: logging.debug('shortkey', shortkey) out = jcache.get(shortkey, keys) @@ -181,21 +181,21 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() - print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath + logging.debug(' cache miss, key data dumped to %s' % dfpath) - if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' + if out and DEBUG: logging.debug(' loading pre from jcache') if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: - if DEBUG: print >> sys.stderr, ' saving pre to jcache' + if DEBUG: logging.debug(' saving pre to jcache') jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) - if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 1 took %s seconds' % (time.time() - t)) indexed_functions = set() forwarded_json = json.loads(forwarded_data) @@ -223,6 +223,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) + #chunks = [chunks[0]] # pick specific chunks for debugging/profiling + funcs = None if jcache: @@ -238,7 +240,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: - if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) + if out and DEBUG: logging.debug(' loading %d funcchunks from jcache' % len(cached_outputs)) else: cached_outputs = [] @@ -248,10 +250,11 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' - if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) + if DEBUG: logging.debug(' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))) commands = [ - (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) + (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine,# + ['--prof'], + temp_files, DEBUG) for i, chunk in enumerate(chunks) ] @@ -273,7 +276,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) - if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) + if out and DEBUG and len(chunks) > 0: logging.debug(' saving %d funcchunks to jcache' % len(chunks)) chunks = None @@ -283,7 +286,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] - if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2 took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # merge forwarded data @@ -319,7 +322,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, funcs_js = [output[0] for output in outputs] outputs = None - if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2b took %s seconds' % (time.time() - t)) if DEBUG: t = time.time() # calculations on merged forwarded data @@ -340,7 +343,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, else: curr = i i += 2 - #print >> sys.stderr, 'function indexing', indexed, curr, sig + #logging.debug('function indexing', indexed, curr, sig) forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): @@ -378,7 +381,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) - if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 2c took %s seconds' % (time.time() - t)) # Phase 3 - post if DEBUG: t = time.time() @@ -539,7 +542,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, # finalize - if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) + if DEBUG: logging.debug('asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)) funcs_js = [''' %s @@ -687,7 +690,7 @@ Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; funcs_js = None outfile.write(indexize(post)) - if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) + if DEBUG: logging.debug(' emscript: phase 3 took %s seconds' % (time.time() - t)) outfile.close() @@ -779,11 +782,11 @@ def _main(environ): keywords, positional = parser.parse_args() if not keywords.suppressUsageWarning: - print >> sys.stderr, ''' + logging.warning(''' ============================================================== WARNING: You should normally never use this! Use emcc instead. ============================================================== - ''' + ''') if len(positional) != 1: raise RuntimeError('Must provide e |