preload/embed file compression option in emcc

2 files changed, 47 insertions, 6 deletions

diff --git a/emcc b/emcc
index 984e4014..b0ed1862 100755
--- a/emcc
+++ b/emcc
@@ -807,16 +807,13 @@ try:
   # Embed and preload files
   if len(embed_files) + len(preload_files) > 0:
     if DEBUG: print >> sys.stderr, 'emcc: setting up files'
-
-    assert not Compression.on
-
     code = ''
 
     # Sanity checks
     for filename in embed_files:
       assert filename not in preload_files, 'Cannot both embed and preload file %s' % filename
 
-    # Expand directories
+    # Expand directories into individual files
     def add(filename, dirname, names):
       for name in names:
         combined = os.path.join(dirname, name)
@@ -831,6 +828,13 @@ try:
           if filename in source:
             source.remove(filename)
 
+    if Compression.on:
+      # Compress each file
+      for filename in embed_files + preload_files:
+        Compression.compress(filename)
+      embed_files = map(lambda filename: Compression.compressed_name(filename), embed_files)
+      preload_files = map(lambda filename: Compression.compressed_name(filename), preload_files)
+
     # Set up folders
     partial_dirs = []
     for filename in embed_files + preload_files:
@@ -864,12 +868,19 @@ try:
         var arrayBuffer = %(varname)s.response; // Note: not X.responseText
         assert(arrayBuffer, 'Loading file %(filename)s failed.');
         var byteArray = new Uint8Array(arrayBuffer);
+        %(decompress)s;
         FS.createDataFile('/%(dirname)s', '%(basename)s', byteArray, true, true);
         removeRunDependency();
       };
       addRunDependency();
       %(varname)s.send(null);
-  ''' % { 'varname': varname, 'filename': filename, 'dirname': os.path.dirname(filename), 'basename': os.path.basename(filename) }
+  ''' % {
+          'varname': varname,
+          'filename': filename,
+          'dirname': os.path.dirname(filename),
+          'basename': os.path.basename(filename) if not Compression.on else os.path.basename(filename)[:-len(Compression.compressed_name(''))],
+          'decompress': '' if not Compression.on else 'byteArray = Module["decompress"](byteArray)'
+        }
       else:
         print >> sys.stderr, 'emcc: warning: preloading file %s as image because of suffix. it will not be available to load as a normal file' % filename
         code += '''
@@ -983,6 +994,7 @@ try:
       shutil.move(final, js_target)
       Compression.compress(js_target)
       decoding = open(Compression.decoder).read()
+      decoding += 'Module["decompress"] = function(data) { return %s(data) };\n' % Compression.js_name
       decoding += '''
         var compiledCodeXHR = new XMLHttpRequest();
         compiledCodeXHR.open('GET', '%s', true);
@@ -992,7 +1004,7 @@ try:
           if (!arrayBuffer) throw('Loading compressed code failed.');
           var byteArray = new Uint8Array(arrayBuffer);
           var decompressed = %s(byteArray);
-          var source = Array.prototype.slice.apply(decompressed).map(function(x) { return String.fromCharCode(x) }).join('');
+          var source = Array.prototype.slice.apply(decompressed).map(function(x) { return String.fromCharCode(x) }).join(''); // createObjectURL instead?
           var scriptTag = document.createElement('script');
           scriptTag.setAttribute('type', 'text/javascript');
           scriptTag.innerHTML = source;
diff --git a/tests/runner.py b/tests/runner.py
index 28e918b7..3c593af1 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -6305,6 +6305,35 @@ f.close()
       Popen([EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', 'subdirr', '-o', 'page.html']).communicate()
       self.run_browser('page.html', 'You should see two cool numbers', '/report_result?1')
 
+    def test_emcc_compressed_file(self):
+      open(os.path.join(self.get_dir(), 'datafile.txt'), 'w').write('''compress this please''')
+      open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
+        #include <stdio.h>
+        #include <string.h>
+        #include <emscripten.h>
+        int main() {
+          char buf[21];
+          FILE *f = fopen("datafile.txt", "r");
+          fread(buf, 1, 20, f);
+          buf[20] = 0;
+          fclose(f);
+          printf("file says: |%s|\n", buf);
+          int result = !strcmp("compress this please", buf);
+          REPORT_RESULT();
+          return 0;
+        }
+      '''))
+
+      Popen([EMCC, os.path.join(self.get_dir(), 'main.cpp'), '-o', 'page.html', '--preload-file', 'datafile.txt',
+             '--compression', '%s,%s,%s' % (path_from_root('third_party', 'lzma.js', 'lzma-native'),
+                                            path_from_root('third_party', 'lzma.js', 'lzma-decoder.js'),
+                                            'LZMA.decompress')]).communicate()
+      assert os.path.exists(os.path.join(self.get_dir(), 'datafile.txt')), 'must be data file'
+      assert os.path.exists(os.path.join(self.get_dir(), 'datafile.txt.compress')), 'must be data file in compressed form'
+      assert os.stat(os.path.join(self.get_dir(), 'page.js')).st_size != os.stat(os.path.join(self.get_dir(), 'page.js.compress')).st_size, 'compressed file must be different'
+      shutil.move(os.path.join(self.get_dir(), 'datafile.txt'), 'datafile.txt.renamedsoitcannotbefound');
+      self.run_browser('page.html', '', '/report_result?1')
+
     def test_emcc_sdl_image(self):
       # load an image file, get pixel data
       shutil.copyfile(path_from_root('tests', 'screenshot.jpg'), os.path.join(self.get_dir(), 'screenshot.jpg'))