cache emscript function chunks

3 files changed, 53 insertions, 9 deletions

diff --git a/emscripten.py b/emscripten.py
index 99c55174..c91814a4 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -173,18 +173,52 @@ def emscript(infile, settings, outfile, libraries=[]):
 
   chunks = shared.JCache.chunkify(funcs, chunk_size, 'emscript_files' if jcache else None)
 
+  if jcache:
+    # load chunks from cache where we can # TODO: ignore small chunks
+    cached_funcs = []
+    def load_from_cache(chunk):
+      keys = [settings_text, forwarded_data, chunk]
+      shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code
+      out = shared.JCache.get(shortkey, keys)
+      if out:
+        cached_funcs.append(out)
+        return False
+      return True
+    chunks = filter(load_from_cache, chunks)
+    if len(cached_funcs) > 0:
+      if out and DEBUG: print >> sys.stderr, '  loading %d funcchunks from jcache' % len(cached_funcs)
+      cached_funcs_js = ''.join(cached_funcs)
+      cached_funcs = None
+    else:
+      cached_funcs_js = ''
+
   if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores'
-  if DEBUG: print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
 
-  commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))]
+  if len(chunks) > 0:
+    if DEBUG: print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
+
+    commands = [(i, chunks[i], meta, settings_file, compiler, forwarded_file, libraries) for i in range(len(chunks))]
 
-  if len(chunks) > 1:
-    pool = multiprocessing.Pool(processes=cores)
-    outputs = pool.map(process_funcs, commands, chunksize=1)
+    if len(chunks) > 1:
+      pool = multiprocessing.Pool(processes=cores)
+      outputs = pool.map(process_funcs, commands, chunksize=1)
+    elif len(chunks) == 1:
+      outputs = [process_funcs(commands[0])]
   else:
-    outputs = [process_funcs(commands[0])]
+    outputs = []
+
+  if jcache:
+    # save chunks to cache
+    for i in range(len(chunks)):
+      chunk = chunks[i]
+      keys = [settings_text, forwarded_data, chunk]
+      shortkey = shared.JCache.get_shortkey(keys)
+      shared.JCache.set(shortkey, keys, outputs[i])
+    if out and DEBUG and len(chunks) > 0: print >> sys.stderr, '  saving %d funcchunks to jcache' % len(chunks)
 
   funcs_js = ''.join([output[0] for output in outputs])
+  if jcache:
+    funcs_js += cached_funcs_js # TODO insert them in the original order
 
   for func_js, curr_forwarded_data in outputs:
     # merge forwarded data
diff --git a/tests/runner.py b/tests/runner.py
index f844483e..0b0a41ca 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -10449,6 +10449,8 @@ fi
     def test_jcache(self):
       PRE_LOAD_MSG = 'loading pre from jcache'
       PRE_SAVE_MSG = 'saving pre to jcache'
+      FUNC_CHUNKS_LOAD_MSG = 'funcchunks from jcache'
+      FUNC_CHUNKS_SAVE_MSG = 'funcchunks to jcache'
 
       restore()
       Cache.erase()
@@ -10475,7 +10477,9 @@ fi
           self.clear()
           out, err = Popen(['python', EMCC, path_from_root('tests', input_file)] + args, stdout=PIPE, stderr=PIPE).communicate()
           assert (PRE_SAVE_MSG in err) == expect_save, err
-          assert (PRE_LOAD_MSG in err) == expect_load, errr
+          assert (PRE_LOAD_MSG in err) == expect_load, err
+          assert (FUNC_CHUNKS_SAVE_MSG in err) == expect_save, err
+          assert (FUNC_CHUNKS_LOAD_MSG in err) == expect_load, err
           curr = open('a.out.js').read()
           if src is None:
             src = None
diff --git a/tools/shared.py b/tools/shared.py
index 8cbafcfa..0f22bb8c 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1182,8 +1182,11 @@ class JCache:
   # Returns a cached value, if it exists. Make sure the full key matches
   @staticmethod
   def get(shortkey, keys):
+    #if DEBUG: print >> sys.stderr, 'jcache get?'
     cachename = JCache.get_cachename(shortkey)
-    if not os.path.exists(cachename): return
+    if not os.path.exists(cachename):
+      #if DEBUG: print >> sys.stderr, 'jcache none at all'
+      return
     data = cPickle.Unpickler(open(cachename, 'rb')).load()
     if len(data) != 2:
       #if DEBUG: print >> sys.stderr, 'jcache error in get'
@@ -1233,7 +1236,7 @@ class JCache:
           news.append(func)
         else:
           n = previous_mapping[ident]
-          while n > len(chunks): chunks.append([])
+          while n >= len(chunks): chunks.append([])
           chunks[n].append(func)
       # add news and adjust for new sizes
       spilled = news
@@ -1268,6 +1271,9 @@ class JCache:
       chunks.append(curr)
       curr = None
     if chunking_file:
+      # sort within each chunk, to keep the order identical
+      for chunk in chunks:
+        chunk.sort(key=lambda func: func[0])
       # save new mapping info
       new_mapping = {}
       for i in range(len(chunks)):