cache for dlmalloc and soon libc++

3 files changed, 69 insertions, 8 deletions

diff --git a/emcc b/emcc
index 2a575559..56b70c8d 100755
--- a/emcc
+++ b/emcc
@@ -458,10 +458,11 @@ try:
     if need_dlmalloc and not has_dlmalloc:
       # We need to build and link dlmalloc in
       if DEBUG: print >> sys.stderr, 'emcc: including dlmalloc'
-      Popen([shared.EMCC, shared.path_from_root('src', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')], stdout=PIPE, stderr=PIPE).communicate()
-      if llvm_opt_level > 0:
-        shared.Building.llvm_opt(in_temp('dlmalloc.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2)
-      extra_files_to_link.append(in_temp('dlmalloc.o'))
+      def create_dlmalloc():
+        print >> sys.stderr, 'emcc: building dlmalloc for cache'
+        Popen([shared.EMCC, shared.path_from_root('src', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')], stdout=PIPE, stderr=PIPE).communicate()
+        return in_temp('dlmalloc.o')
+      extra_files_to_link.append(shared.Cache.get('dlmalloc', create_dlmalloc))
 
       # dlmalloc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
       try:
diff --git a/tests/runner.py b/tests/runner.py
index b03bc196..77deeee7 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -5866,6 +5866,9 @@ elif 'sanity' in str(sys.argv):
 
   commands = [[EMCC], ['python', path_from_root('tests', 'runner.py'), 'blahblah']]
 
+  def mtime(filename):
+    return os.stat(filename).st_mtime
+
   class sanity(RunnerCore):
     def setUp(self):
       wipe()
@@ -5953,9 +5956,6 @@ elif 'sanity' in str(sys.argv):
       assert os.path.exists('a.out.js')
 
     def test_emcc(self):
-      def mtime(filename):
-        return os.stat(filename).st_mtime
-
       SANITY_MESSAGE = 'Emscripten: Running sanity checks'
       SANITY_FAIL_MESSAGE = 'sanity check failed to run'
 
@@ -5993,6 +5993,41 @@ elif 'sanity' in str(sys.argv):
       assert mtime(SANITY_FILE) >= mtime(CONFIG_FILE)
       self.assertNotContained(SANITY_FAIL_MESSAGE, output)
 
+    def test_emcc_caching(self):
+      INCLUDING_DLMALLOC_MESSAGE = 'emcc: including dlmalloc'
+      BUILDING_DLMALLOC_MESSAGE = 'emcc: building dlmalloc for cache'
+
+      EMCC_CACHE = Cache.dirname
+
+      restore()
+
+      Cache.erase()
+      assert not os.path.exists(EMCC_CACHE)
+
+      try:
+        emcc_debug = os.environ.get('EMCC_DEBUG')
+        os.environ['EMCC_DEBUG'] ='1'
+
+        # Building a file that doesn't need cached stuff should not trigger cache generation
+        output = self.do([EMCC, path_from_root('tests', 'hello_world.cpp')])
+        assert INCLUDING_DLMALLOC_MESSAGE not in output
+        assert BUILDING_DLMALLOC_MESSAGE not in output
+        self.assertContained('hello, world!', run_js('a.out.js'))
+        assert not os.path.exists(EMCC_CACHE)
+        try_delete('a.out.js')
+
+        # Building a file that *does* need dlmalloc *should* trigger cache generation, but only the first time
+        for i in range(3):
+          output = self.do([EMCC, path_from_root('tests', 'hello_malloc.cpp')])
+          assert INCLUDING_DLMALLOC_MESSAGE in output
+          assert (BUILDING_DLMALLOC_MESSAGE in output) == (i == 0), 'Must only build the first time'
+          self.assertContained('hello, world!', run_js('a.out.js'))
+          assert os.path.exists(EMCC_CACHE)
+          assert os.path.exists(os.path.join(EMCC_CACHE, 'dlmalloc.bc'))
+      finally:
+        if emcc_debug:
+          os.environ['EMCC_DEBUG'] = emcc_debug
+
 else:
   raise Exception('Test runner is confused: ' + str(sys.argv))
 
diff --git a/tools/shared.py b/tools/shared.py
index aee7de27..0f117fad 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -381,7 +381,8 @@ class Building:
   def build_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}):
     ''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in
         memory since the test directory is destroyed and recreated for each test. Note that we cache separately
-        for different compilers) '''
+        for different compilers).
+        This cache is just during the test runner. There is a different concept of caching as well, see |Cache|. '''
 
     if type(generated_libs) is not list: generated_libs = [generated_libs]
 
@@ -677,3 +678,27 @@ class Building:
     
     return False
 
+# Permanent cache for dlmalloc and stdlibc++
+class Cache:
+  dirname = os.path.expanduser(os.path.join('~', '.emscripten_cache'))
+
+  @staticmethod
+  def erase():
+    try:
+      shutil.rmtree(Cache.dirname)
+    except:
+      pass
+
+  # Request a cached file. If it isn't in the cache, it will be created with
+  # the given creator function
+  @staticmethod
+  def get(shortname, creator):
+    if not shortname.endswith('.bc'): shortname += '.bc'
+    cachename = os.path.join(Cache.dirname, shortname)
+    if os.path.exists(cachename):
+      return cachename
+    if not os.path.exists(Cache.dirname):
+      os.makedirs(Cache.dirname)
+    shutil.copyfile(creator(), cachename)
+    return cachename
+