diff options
68 files changed, 7866 insertions, 3436 deletions
@@ -113,3 +113,4 @@ a license to everyone to use it as detailed in LICENSE.) * Heidi Pan <heidi.pan@intel.com> (copyright owned by Intel) * Vasilis Kalintiris <ehostunreach@gmail.com> * Adam C. Clifton <adam@hulkamaniac.com> +* Volo Zyko <volo.zyko@gmail.com> diff --git a/cmake/Platform/Emscripten.cmake b/cmake/Platform/Emscripten.cmake index c30632ca..ef813eb8 100644 --- a/cmake/Platform/Emscripten.cmake +++ b/cmake/Platform/Emscripten.cmake @@ -109,6 +109,8 @@ set(CMAKE_CXX_RESPONSE_FILE_LINK_FLAG "@") # Specify the program to use when building static libraries. Force Emscripten-related command line options to clang. set(CMAKE_CXX_ARCHIVE_CREATE "${CMAKE_AR} rc <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") set(CMAKE_C_ARCHIVE_CREATE "${CMAKE_AR} rc <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") +set(CMAKE_CXX_ARCHIVE_APPEND "${CMAKE_AR} r <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") +set(CMAKE_C_ARCHIVE_APPEND "${CMAKE_AR} r <TARGET> ${CMAKE_START_TEMP_FILE} <LINK_FLAGS> <OBJECTS>${CMAKE_END_TEMP_FILE}") # Set a global EMSCRIPTEN variable that can be used in client CMakeLists.txt to detect when building using Emscripten. # There seems to be some kind of bug with CMake, so you might need to define this manually on the command line with "-DEMSCRIPTEN=1". @@ -53,14 +53,17 @@ from tools import shared, jsrun from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename, WINDOWS from tools.response_file import read_response_file -C_SUFFIXES = ('.c', '.C') -CXX_SUFFIXES = ('.cpp', '.cxx', '.cc', '.CPP', '.CXX', '.CC') -SOURCE_SUFFIXES = C_SUFFIXES + CXX_SUFFIXES + ('.m', '.mm') -BITCODE_SUFFIXES = ('.bc', '.o', '.obj') -DYNAMICLIB_SUFFIXES = ('.dylib', '.so', '.dll') -STATICLIB_SUFFIXES = ('.a',) -ASSEMBLY_SUFFIXES = ('.ll',) +# endings = dot + a suffix, safe to test by filename.endswith(endings) +C_ENDINGS = ('.c', '.C') +CXX_ENDINGS = ('.cpp', '.cxx', '.cc', '.CPP', '.CXX', '.CC') +SOURCE_ENDINGS = C_ENDINGS + CXX_ENDINGS + ('.m', '.mm') +BITCODE_ENDINGS = ('.bc', '.o', '.obj') +DYNAMICLIB_ENDINGS = ('.dylib', '.so', '.dll') +STATICLIB_ENDINGS = ('.a',) +ASSEMBLY_ENDINGS = ('.ll',) + LIB_PREFIXES = ('', 'lib') + JS_CONTAINING_SUFFIXES = ('js', 'html') # Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt @@ -341,6 +344,10 @@ Options that are modified or new in %s include: For more docs on the options --preload-file accepts, see https://github.com/kripken/emscripten/wiki/Filesystem-Guide + --exclude-file <name> Files and directories to be excluded from + --embed-file and --preload-file + wildcard is supported + --compression <codec> Compress both the compiled code and embedded/ preloaded files. <codec> should be a triple, @@ -758,9 +765,13 @@ def in_temp(name): def filename_type_suffix(filename): for i in reversed(filename.split('.')[1:]): if not i.isdigit(): - return '.' + i + return i return '' +def filename_type_ending(filename): + suffix = filename_type_suffix(filename) + return '' if not suffix else ('.' + suffix) + try: call = CXX if use_cxx else CC @@ -780,6 +791,7 @@ try: split_js_file = None preload_files = [] embed_files = [] + exclude_files = [] compression = None ignore_dynamic_linking = False shell_path = shared.path_from_root('src', 'shell.html') @@ -896,6 +908,11 @@ try: preload_files.append(newargs[i+1]) newargs[i] = '' newargs[i+1] = '' + elif newargs[i].startswith('--exclude-file'): + check_bad_eq(newargs[i]) + exclude_files.append(newargs[i+1]) + newargs[i] = '' + newargs[i+1] = '' elif newargs[i].startswith('--compression'): check_bad_eq(newargs[i]) parts = newargs[i+1].split(',') @@ -990,6 +1007,7 @@ try: newargs = newargs + [default_cxx_std] if emrun: + pre_js += open(shared.path_from_root('src', 'emrun_prejs.js')).read() + '\n' post_js += open(shared.path_from_root('src', 'emrun_postjs.js')).read() + '\n' if js_opts is None: js_opts = opt_level >= 2 @@ -1042,23 +1060,23 @@ try: prev = newargs[i-1] if prev in ['-MT', '-MF', '-MQ', '-D', '-U', '-o', '-x', '-Xpreprocessor', '-include', '-imacros', '-idirafter', '-iprefix', '-iwithprefix', '-iwithprefixbefore', '-isysroot', '-imultilib', '-A', '-isystem', '-iquote', '-install_name', '-compatibility_version', '-current_version', '-I', '-L']: continue # ignore this gcc-style argument - if (os.path.islink(arg) and os.path.realpath(arg).endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES)): + if (os.path.islink(arg) and os.path.realpath(arg).endswith(SOURCE_ENDINGS + BITCODE_ENDINGS + DYNAMICLIB_ENDINGS + ASSEMBLY_ENDINGS)): arg = os.path.realpath(arg) if not arg.startswith('-'): if not os.path.exists(arg): - logging.error(arg + ': No such file or directory') + logging.error('%s: No such file or directory ("%s" was expected to be an input file, based on the commandline arguments provided)' % (arg, arg)) exit(1) - arg_suffix = filename_type_suffix(arg) - if arg_suffix.endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES) or shared.Building.is_ar(arg): # we already removed -o <target>, so all these should be inputs + arg_ending = filename_type_ending(arg) + if arg_ending.endswith(SOURCE_ENDINGS + BITCODE_ENDINGS + DYNAMICLIB_ENDINGS + ASSEMBLY_ENDINGS) or shared.Building.is_ar(arg): # we already removed -o <target>, so all these should be inputs newargs[i] = '' - if arg_suffix.endswith(SOURCE_SUFFIXES): + if arg_ending.endswith(SOURCE_ENDINGS): input_files.append(arg) has_source_inputs = True - elif arg_suffix.endswith(ASSEMBLY_SUFFIXES) or shared.Building.is_bitcode(arg): # this should be bitcode, make sure it is valid + elif arg_ending.endswith(ASSEMBLY_ENDINGS) or shared.Building.is_bitcode(arg): # this should be bitcode, make sure it is valid input_files.append(arg) - elif arg_suffix.endswith(STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES): + elif arg_ending.endswith(STATICLIB_ENDINGS + DYNAMICLIB_ENDINGS): # if it's not, and it's a library, just add it to libs to find later l = unsuffixed_basename(arg) for prefix in LIB_PREFIXES: @@ -1070,10 +1088,10 @@ try: newargs[i] = '' else: logging.warning(arg + ' is not valid LLVM bitcode') - elif arg_suffix.endswith(STATICLIB_SUFFIXES): + elif arg_ending.endswith(STATICLIB_ENDINGS): if not shared.Building.is_ar(arg): if shared.Building.is_bitcode(arg): - logging.error(arg + ': File has a suffix of a static library ' + str(STATICLIB_SUFFIXES) + ', but instead is an LLVM bitcode file! When linking LLVM bitcode files, use one of the suffixes ' + str(BITCODE_SUFFIXES)) + logging.error(arg + ': File has a suffix of a static library ' + str(STATICLIB_ENDINGS) + ', but instead is an LLVM bitcode file! When linking LLVM bitcode files, use one of the suffixes ' + str(BITCODE_ENDINGS)) else: logging.error(arg + ': Unknown format, not a static library!') exit(1) @@ -1098,17 +1116,12 @@ try: target = target_basename + '.o' final_suffix = 'o' - # do not link in libs when just generating object code (not an 'executable', i.e. JS, or a library) - if ('.' + final_suffix) in BITCODE_SUFFIXES and len(libs) > 0: - logging.warning('not linking against libraries since only compiling to bitcode') - libs = [] - # Find library files for lib in libs: logging.debug('looking for library "%s"' % lib) found = False for prefix in LIB_PREFIXES: - for suff in STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES: + for suff in STATICLIB_ENDINGS + DYNAMICLIB_ENDINGS: name = prefix + lib + suff for lib_dir in lib_dirs: path = os.path.join(lib_dir, name) @@ -1119,12 +1132,21 @@ try: break if found: break if found: break - - if ignore_dynamic_linking: - input_files = filter(lambda input_file: not input_file.endswith(DYNAMICLIB_SUFFIXES), input_files) + if not found: logging.warning('emcc: cannot find library "%s"' % lib) + + # If not compiling to JS, then we are compiling to an intermediate bitcode objects or library, so + # ignore dynamic linking, since multiple dynamic linkings can interfere with each other + if not filename_type_suffix(target) in JS_CONTAINING_SUFFIXES or ignore_dynamic_linking: + def check(input_file): + if filename_type_ending(input_file) in DYNAMICLIB_ENDINGS: + if not ignore_dynamic_linking: logging.warning('ignoring dynamic library %s because not compiling to JS or HTML, remember to link it when compiling to JS or HTML at the end' % os.path.basename(input_file)) + return False + else: + return True + input_files = filter(lambda input_file: check(input_file), input_files) if len(input_files) == 0: - logging.error('no input files\nnote that input files without a known suffix are ignored, make sure your input files end with one of: ' + str(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + STATICLIB_SUFFIXES + ASSEMBLY_SUFFIXES)) + logging.error('no input files\nnote that input files without a known suffix are ignored, make sure your input files end with one of: ' + str(SOURCE_ENDINGS + BITCODE_ENDINGS + DYNAMICLIB_ENDINGS + STATICLIB_ENDINGS + ASSEMBLY_ENDINGS)) exit(0) newargs = CC_ADDITIONAL_ARGS + newargs @@ -1153,6 +1175,23 @@ try: if os.environ.get('EMCC_FAST_COMPILER'): shared.Settings.ASM_JS = 1 + if shared.Settings.DISABLE_EXCEPTION_CATCHING == 0: + logging.warning('disabling exception catching since not supported in fastcomp yet') + shared.Settings.DISABLE_EXCEPTION_CATCHING = 1 + assert shared.Settings.ALLOW_MEMORY_GROWTH == 0, 'memory growth not supported in fastcomp yet' + assert shared.Settings.UNALIGNED_MEMORY == 0, 'forced unaligned memory not supported in fastcomp' + assert shared.Settings.SAFE_HEAP == 0, 'safe heap not supported in fastcomp yet' + assert shared.Settings.CHECK_HEAP_ALIGN == 0, 'check heap align not supported in fastcomp yet' + assert shared.Settings.SAFE_DYNCALLS == 0, 'safe dyncalls not supported in fastcomp' + assert shared.Settings.RESERVED_FUNCTION_POINTERS == 0, 'reserved function pointers not supported in fastcomp' + assert shared.Settings.ASM_HEAP_LOG == 0, 'asm heap log not supported in fastcomp' + assert shared.Settings.LABEL_DEBUG == 0, 'label debug not supported in fastcomp' + assert shared.Settings.LEGACY_GL_EMULATION == 0, 'legacy gl emulation not supported in fastcomp' + assert shared.Settings.EXECUTION_TIMEOUT == -1, 'execution timeout not supported in fastcomp' + assert shared.Settings.NAMED_GLOBALS == 0, 'named globals not supported in fastcomp' + assert shared.Settings.PGO == 0, 'pgo not supported in fastcomp' + assert shared.Settings.TARGET_LE32 == 1, 'fastcomp requires le32' + assert not bind, 'embind not supported in fastcomp yet' if shared.Settings.ASM_JS: assert opt_level >= 1 or os.environ.get('EMCC_FAST_COMPILER'), 'asm.js requires -O1 or above' @@ -1244,14 +1283,14 @@ try: # First, generate LLVM bitcode. For each input file, we get base.o with bitcode for input_file in input_files: - file_suffix = filename_type_suffix(input_file) - if file_suffix.endswith(SOURCE_SUFFIXES): + file_ending = filename_type_ending(input_file) + if file_ending.endswith(SOURCE_ENDINGS): logging.debug('compiling source file: ' + input_file) input_file = shared.Building.preprocess(input_file, in_temp(uniquename(input_file))) output_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') temp_files.append(output_file) args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file] - if file_suffix.endswith(CXX_SUFFIXES): + if file_ending.endswith(CXX_ENDINGS): args += shared.EMSDK_CXX_OPTS logging.debug("running: " + call + ' ' + ' '.join(args)) execute([call] + args) # let compiler frontend print directly, so colors are saved (PIPE kills that) @@ -1259,17 +1298,17 @@ try: logging.error('compiler frontend failed to generate LLVM bitcode, halting') sys.exit(1) else: # bitcode - if file_suffix.endswith(BITCODE_SUFFIXES): + if file_ending.endswith(BITCODE_ENDINGS): logging.debug('copying bitcode file: ' + input_file) temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o') shutil.copyfile(input_file, temp_file) temp_files.append(temp_file) - elif file_suffix.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file): + elif file_ending.endswith(DYNAMICLIB_ENDINGS) or shared.Building.is_ar(input_file): logging.debug('copying library file: ' + input_file) temp_file = in_temp(uniquename(input_file)) shutil.copyfile(input_file, temp_file) temp_files.append(temp_file) - elif file_suffix.endswith(ASSEMBLY_SUFFIXES): + elif file_ending.endswith(ASSEMBLY_ENDINGS): if not LEAVE_INPUTS_RAW: # Note that by assembling the .ll file, then disassembling it later, we will # remove annotations which is a good thing for compilation time @@ -1287,8 +1326,8 @@ try: # Optimize source files if llvm_opts > 0: for i, input_file in enumerate(input_files): - file_suffix = filename_type_suffix(input_file) - if file_suffix.endswith(SOURCE_SUFFIXES): + file_ending = filename_type_ending(input_file) + if file_ending.endswith(SOURCE_ENDINGS): temp_file = temp_files[i] logging.debug('optimizing %s with -O%s' % (input_file, llvm_opts)) shared.Building.llvm_opt(temp_file, llvm_opts) @@ -1633,11 +1672,11 @@ try: # First, combine the bitcode files if there are several. We must also link if we have a singleton .a if len(input_files) + len(extra_files_to_link) > 1 or \ - (not LEAVE_INPUTS_RAW and not (suffix(temp_files[0]) in BITCODE_SUFFIXES or suffix(temp_files[0]) in DYNAMICLIB_SUFFIXES) and shared.Building.is_ar(temp_files[0])): + (not LEAVE_INPUTS_RAW and not (suffix(temp_files[0]) in BITCODE_ENDINGS or suffix(temp_files[0]) in DYNAMICLIB_ENDINGS) and shared.Building.is_ar(temp_files[0])): linker_inputs = temp_files + extra_files_to_link logging.debug('linking: ' + str(linker_inputs)) t0 = time.time() - shared.Building.link(linker_inputs, in_temp(target_basename + '.bc'), force_archive_contents = len(filter(lambda temp: not temp.endswith(STATICLIB_SUFFIXES), temp_files)) == 0) + shared.Building.link(linker_inputs, in_temp(target_basename + '.bc'), force_archive_contents = len(filter(lambda temp: not temp.endswith(STATICLIB_ENDINGS), temp_files)) == 0) t1 = time.time() logging.debug(' linking took %.2f seconds' % (t1 - t0)) final = in_temp(target_basename + '.bc') @@ -1682,7 +1721,12 @@ try: else: # At minimum remove dead functions etc., this potentially saves a lot in the size of the generated code (and the time to compile it) link_opts += shared.Building.get_safe_internalize() + ['-globaldce'] - if not save_bc: + + # Simplify LLVM bitcode for fastcomp + if os.environ.get('EMCC_FAST_COMPILER') and not AUTODEBUG: + link_opts += ['-pnacl-abi-simplify-preopt', '-pnacl-abi-simplify-postopt'] + + if (not save_bc and not os.environ.get('EMCC_FAST_COMPILER')) or AUTODEBUG: # let llvm opt directly emit ll, to skip writing and reading all the bitcode link_opts += ['-S'] shared.Building.llvm_opt(final, link_opts, final + '.link.ll') @@ -1709,6 +1753,12 @@ try: final += '.ad.ll' if DEBUG: save_intermediate('autodebug', 'll') + # Simplify bitcode after autodebug + if os.environ.get('EMCC_FAST_COMPILER') and (AUTODEBUG or LEAVE_INPUTS_RAW): + shared.Building.llvm_opt(final, ['-pnacl-abi-simplify-preopt', '-pnacl-abi-simplify-postopt'], final + '.adsimp.bc') + final += '.adsimp.bc' + if DEBUG: save_intermediate('adsimp', 'bc') + # Emscripten logging.debug('LLVM => JS') extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))] @@ -1719,13 +1769,16 @@ try: # Embed and preload files if len(preload_files) + len(embed_files) > 0: logging.debug('setting up files') - file_args = ['--pre-run'] + file_args = [] if len(preload_files) > 0: file_args.append('--preload') file_args += preload_files if len(embed_files) > 0: file_args.append('--embed') file_args += embed_files + if len(exclude_files) > 0: + file_args.append('--exclude') + file_args += exclude_files if Compression.on: file_args += ['--compress', Compression.encoder, Compression.decoder, Compression.js_name] if use_preload_cache: @@ -8,6 +8,7 @@ import os, platform, optparse, logging, re, pprint, atexit, urlparse, subprocess from operator import itemgetter from urllib import unquote from Queue import PriorityQueue +from threading import Thread, RLock # Populated from cmdline params emrun_options = None @@ -41,6 +42,9 @@ processname_killed_atexit = "" # If user does not specify a --port parameter, this port is used to launch the server. default_webserver_port = 6931 +# Location of Android Debug Bridge executable +ADB = '' + # Host OS detection to autolocate browsers and other OS-specific support needs. WINDOWS = False LINUX = False @@ -73,7 +77,7 @@ last_message_time = time.clock() page_start_time = time.clock() # Stores the time of most recent http page serve. -page_last_served_time = time.clock() +page_last_served_time = None # Returns given log message formatted to be outputted on a HTML page. def format_html(msg): @@ -82,21 +86,14 @@ def format_html(msg): msg = cgi.escape(msg) msg = msg.replace('\r\n', '<br />').replace('\n', '<br />') return msg - + +# HTTP requests are handled from separate threads - synchronize them to avoid race conditions +http_mutex = RLock() + # Prints a log message to 'info' stdout channel. Always printed. def logi(msg): global last_message_time - if emrun_options.log_html: - sys.stdout.write |