diff options
Diffstat (limited to 'tools/shared.py')
| -rw-r--r-- | tools/shared.py | 162 |
1 files changed, 155 insertions, 7 deletions
diff --git a/tools/shared.py b/tools/shared.py index 04d0fe57..7bafd51d 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,4 +1,4 @@ -import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess +import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, md5, cPickle from subprocess import Popen, PIPE, STDOUT from tempfile import mkstemp @@ -59,8 +59,6 @@ class WindowsPopen: # Install our replacement Popen handler if we are running on Windows to avoid python spawn process function. if os.name == 'nt': Popen = WindowsPopen - -import js_optimizer __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) def path_from_root(*pathelems): @@ -1006,8 +1004,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e return opts @staticmethod - def js_optimizer(filename, passes): - return js_optimizer.run(filename, passes, NODE_JS) + def js_optimizer(filename, passes, jcache): + return js_optimizer.run(filename, passes, NODE_JS, jcache) @staticmethod def closure_compiler(filename): @@ -1088,16 +1086,17 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e @staticmethod def ensure_relooper(): if os.path.exists(RELOOPER): return + curr = os.getcwd() try: ok = False print >> sys.stderr, '=======================================' print >> sys.stderr, 'bootstrapping relooper...' Cache.ensure() - RELOOPER_DIR = path_from_root('src', 'relooper') + os.chdir(path_from_root('src')) def make(opt_level): raw = RELOOPER + '.raw.js' - Building.emcc(os.path.join(RELOOPER_DIR, 'Relooper.cpp'), ['-I' + os.path.join(RELOOPER_DIR), '--post-js', os.path.join(RELOOPER_DIR, 'emscripten', 'glue.js'), '-s', 'TOTAL_MEMORY=52428800', '-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["memcpy", "memset", "malloc", "free", "puts"]', '-O' + str(opt_level), '--closure', '0'], raw) + Building.emcc(os.path.join('relooper', 'Relooper.cpp'), ['-I' + os.path.join('relooper'), '--post-js', os.path.join('relooper', 'emscripten', 'glue.js'), '-s', 'TOTAL_MEMORY=52428800', '-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["memcpy", "memset", "malloc", "free", "puts"]', '-O' + str(opt_level), '--closure', '0'], raw) f = open(RELOOPER, 'w') f.write("// Relooper, (C) 2012 Alon Zakai, MIT license, https://github.com/kripken/Relooper\n") f.write("var Relooper = (function() {\n"); @@ -1116,6 +1115,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e print >> sys.stderr, '=======================================' ok = True finally: + os.chdir(curr) if not ok: print >> sys.stderr, 'bootstrapping relooper failed. You may need to manually create src/relooper.js by compiling it, see src/relooper/emscripten' 1/0 @@ -1151,6 +1151,152 @@ class Cache: shutil.copyfile(creator(), cachename) return cachename +# JS-specific cache. We cache the results of compilation and optimization, +# so that in incremental builds we can just load from cache. +# We cache reasonably-large-sized chunks +class JCache: + dirname = os.path.join(Cache.dirname, 'jcache') + + @staticmethod + def ensure(): + Cache.ensure() + if not os.path.exists(JCache.dirname): + os.makedirs(JCache.dirname) + + @staticmethod + def get_shortkey(keys): + if type(keys) not in [list, tuple]: + keys = [keys] + ret = '' + for key in keys: + assert type(key) == str + ret += md5.md5(key).hexdigest() + return ret + + @staticmethod + def get_cachename(shortkey): + return os.path.join(JCache.dirname, shortkey) + + # Returns a cached value, if it exists. Make sure the full key matches + @staticmethod + def get(shortkey, keys): + #if DEBUG: print >> sys.stderr, 'jcache get?', shortkey + cachename = JCache.get_cachename(shortkey) + if not os.path.exists(cachename): + #if DEBUG: print >> sys.stderr, 'jcache none at all' + return + data = cPickle.Unpickler(open(cachename, 'rb')).load() + if len(data) != 2: + #if DEBUG: print >> sys.stderr, 'jcache error in get' + return + oldkeys = data[0] + if len(oldkeys) != len(keys): + #if DEBUG: print >> sys.stderr, 'jcache collision (a)' + return + for i in range(len(oldkeys)): + if oldkeys[i] != keys[i]: + #if DEBUG: print >> sys.stderr, 'jcache collision (b)' + return + #if DEBUG: print >> sys.stderr, 'jcache win' + return data[1] + + # Sets the cached value for a key (from get_key) + @staticmethod + def set(shortkey, keys, value): + cachename = JCache.get_cachename(shortkey) + cPickle.Pickler(open(cachename, 'wb')).dump([keys, value]) + #if DEBUG: + # for i in range(len(keys)): + # open(cachename + '.key' + str(i), 'w').write(keys[i]) + # open(cachename + '.value', 'w').write(value) + + # Given a set of functions of form (ident, text), and a preferred chunk size, + # generates a set of chunks for parallel processing and caching. + # It is very important to generate similar chunks in incremental builds, in + # order to maximize the chance of cache hits. To achieve that, we save the + # chunking used in the previous compilation of this phase, and we try to + # generate the same chunks, barring big differences in function sizes that + # violate our chunk size guideline. If caching is not used, chunking_file + # should be None + @staticmethod + def chunkify(funcs, chunk_size, chunking_file): + previous_mapping = None + if chunking_file: + chunking_file = JCache.get_cachename(chunking_file) + if os.path.exists(chunking_file): + try: + previous_mapping = cPickle.Unpickler(open(chunking_file, 'rb')).load() # maps a function identifier to the chunk number it will be in + except: + pass + chunks = [] + if previous_mapping: + # initialize with previous chunking + news = [] + for func in funcs: + ident, data = func + if not ident in previous_mapping: + news.append(func) + else: + n = previous_mapping[ident] + while n >= len(chunks): chunks.append([]) + chunks[n].append(func) + # add news and adjust for new sizes + spilled = news + for chunk in chunks: + size = sum([len(func[1]) for func in chunk]) + while size > 1.5*chunk_size and len(chunk) > 0: + spill = chunk.pop() + spilled.append(spill) + size -= len(spill[1]) + for chunk in chunks: + size = sum([len(func[1]) for func in chunk]) + while size < 0.66*chunk_size and len(spilled) > 0: + spill = spilled.pop() + chunk.append(spill) + size += len(spill[1]) + chunks = filter(lambda chunk: len(chunk) > 0, chunks) # might have empty ones, eliminate them + funcs = spilled # we will allocate these into chunks as if they were normal inputs + # initialize reasonably, the rest of the funcs we need to split out + curr = [] + total_size = 0 + for i in range(len(funcs)): + func = funcs[i] + curr_size = len(func[1]) + if total_size + curr_size < chunk_size: + curr.append(func) + total_size += curr_size + else: + chunks.append(curr) + curr = [func] + total_size = curr_size + if curr: + chunks.append(curr) + curr = None + if chunking_file: + # sort within each chunk, to keep the order identical + for chunk in chunks: + chunk.sort(key=lambda func: func[0]) + # save new mapping info + new_mapping = {} + for i in range(len(chunks)): + chunk = chunks[i] + for ident, data in chunk: + new_mapping[ident] = i + cPickle.Pickler(open(chunking_file, 'wb')).dump(new_mapping) + #if DEBUG: + # if previous_mapping: + # for ident in set(previous_mapping.keys() + new_mapping.keys()): + # if previous_mapping.get(ident) != new_mapping.get(ident): + # print >> sys.stderr, 'mapping inconsistency', ident, previous_mapping.get(ident), new_mapping.get(ident) + # for key, value in new_mapping.iteritems(): + # print >> sys.stderr, 'mapping:', key, value + return [''.join([func[1] for func in chunk]) for chunk in chunks] # remove function names + +class JS: + @staticmethod + def to_nice_ident(ident): # limited version of the JS function toNiceIdent + return ident.replace('%', '$').replace('@', '_'); + # Compression of code and data for smaller downloads class Compression: on = False @@ -1189,3 +1335,5 @@ def unsuffixed(name): def unsuffixed_basename(name): return os.path.basename(unsuffixed(name)) +import js_optimizer + |