refactor file packaging into standalone tool

1 files changed, 16 insertions, 240 deletions

diff --git a/emcc b/emcc
index 859a67dc..57fc1263 100755
--- a/emcc
+++ b/emcc
@@ -77,15 +77,7 @@ emcc can be influenced by a few environment variables:
 import os, sys, shutil, tempfile, subprocess, shlex
 from subprocess import PIPE, STDOUT
 from tools import shared
-
-def execute(cmd, *args, **kw):
-  try:
-    return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that)
-  except:
-    if not isinstance(cmd, str):
-      cmd = ' '.join(cmd)
-    print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>'
-    raise
+from tools.shared import Compression, execute
 
 # Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt
 # levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get
@@ -343,10 +335,6 @@ STATICLIB_SUFFIXES = ('.a',)
 ASSEMBLY_SUFFIXES = ('.ll',)
 LIB_PREFIXES = ('', 'lib')
 
-IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
-AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
-AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' }
-
 def suffix(name):
   return name.split('.')[-1]
 
@@ -418,21 +406,6 @@ else:
 def in_temp(name):
   return os.path.join(temp_dir, name)
 
-class Compression:
-  on = False
-
-  @staticmethod
-  def compressed_name(filename):
-    return filename + '.compress'
-
-  @staticmethod
-  def compress(filename):
-    execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb'))
-
-  @staticmethod
-  def worth_it(original, compressed):
-    return compressed < original - 1500 # save at least one TCP packet or so
-
 try:
   call = CXX if use_cxx else CC
 
@@ -448,8 +421,8 @@ try:
   pre_js = None
   post_js = None
   minify_whitespace = None
-  data_files = []
-  has_preloaded_files = False
+  preload_files = []
+  embed_files = []
   compression = None
   ignore_dynamic_linking = False
   shell_path = shared.path_from_root('src', 'shell.html')
@@ -503,13 +476,12 @@ try:
       newargs[i+1] = ''
     elif newargs[i].startswith('--embed-file'):
       check_bad_eq(newargs[i])
-      data_files.append({ 'name': newargs[i+1], 'mode': 'embed' })
+      embed_files.append(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
     elif newargs[i].startswith('--preload-file'):
       check_bad_eq(newargs[i])
-      data_files.append({ 'name': newargs[i+1], 'mode': 'preload' })
-      has_preloaded_files = True
+      preload_files.append(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
     elif newargs[i].startswith('--compression'):
@@ -875,214 +847,18 @@ try:
   if DEBUG: save_intermediate('original')
 
   # Embed and preload files
-  if len(data_files) > 0:
+  if len(preload_files) + len(embed_files) > 0:
     if DEBUG: print >> sys.stderr, 'emcc: setting up files'
-    code = ''
-
-    if has_preloaded_files:
-      code += '''
-      var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));
-      var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");
-      var hasBlobConstructor;
-      try {
-        new Blob();
-        hasBlobConstructor = true;
-      } catch(e) {
-        hasBlobConstructor = false;
-        console.log("warning: no blob constructor, cannot create blobs with mimetypes");
-      }
-  '''
-
-      code += 'var preloadedImages = {}; // maps url to image data\n'
-      code += 'var preloadedAudios = {}; // maps url to audio data\n'
-
-    # Expand directories into individual files
-    def add(mode, dirname, names):
-      for name in names:
-        fullname = os.path.join(dirname, name)
-        if not os.path.isdir(fullname):
-          data_files.append({ 'name': fullname, 'mode': mode })
-
-    for file_ in data_files:
-      if os.path.isdir(file_['name']):
-        os.path.walk(file_['name'], add, file_['mode'])
-    data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)
-
-    for file_ in data_files:
-      file_['name'] = file_['name'].replace(os.path.sep, '/')
-      file_['net_name'] = file_['name']
-
-    data_target = unsuffixed(target) + '.data'
-
-    # Set up folders
-    partial_dirs = []
-    for file_ in data_files:
-      dirname = os.path.dirname(file_['name'])
-      dirname = dirname.lstrip('/') # absolute paths start with '/', remove that
-      if dirname != '':
-        parts = dirname.split('/')
-        for i in range(len(parts)):
-          partial = '/'.join(parts[:i+1])
-          if partial not in partial_dirs:
-            code += '''Module['FS_createFolder']('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i])
-            partial_dirs.append(partial)
-
-    if has_preloaded_files:
-      # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead.
-      data = open(data_target, 'wb')
-      start = 0
-      for file_ in data_files:
-        file_['data_start'] = start
-        curr = open(file_['name'], 'rb').read()
-        file_['data_end'] = start + len(curr)
-        start += len(curr)
-        data.write(curr)
-      data.close()
-      if Compression.on:
-        Compression.compress(data_target)
-
-      # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs
-      code += '''
-        function DataRequest() {}
-        DataRequest.prototype = {
-          requests: {},
-          open: function(mode, name) {
-            this.requests[name] = this;
-          },
-          send: function() {}
-        };
-      '''
-
-    counter = 0
-    for file_ in data_files:
-      filename = file_['name']
-      if file_['mode'] == 'embed':
-        # Embed
-        code += '''Module['FS_createDataFile']('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
-      elif file_['mode'] == 'preload':
-        # Preload
-        varname = 'filePreload%d' % counter
-        counter += 1
-        image = filename.endswith(IMAGE_SUFFIXES)
-        audio = filename.endswith(AUDIO_SUFFIXES)
-
-        if image:
-          finish =  '''
-            var bb = new BlobBuilder();
-            bb.append(byteArray.buffer);
-            var b = bb.getBlob();
-            var url = URLObject.createObjectURL(b);
-            var img = new Image();
-            img.onload = function() {
-              assert(img.complete, 'Image %(filename)s could not be decoded');
-              var canvas = document.createElement('canvas');
-              canvas.width = img.width;
-              canvas.height = img.height;
-              var ctx = canvas.getContext('2d');
-              ctx.drawImage(img, 0, 0);
-              preloadedImages['%(filename)s'] = canvas;
-              URLObject.revokeObjectURL(url);
-              removeRunDependency();
-            };
-            img.onerror = function(event) {
-              console.log('Image %(filename)s could not be decoded');
-            };
-            img.src = url;
-''' % { 'filename': filename }
-        elif audio:
-          # Need actual blob constructor here, to set the mimetype or else audios fail to decode
-          finish =  '''
-            if (hasBlobConstructor) {
-              var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' });
-              var url = URLObject.createObjectURL(b); // XXX we never revoke this!
-              var audio = new Audio();
-              audio.removedDependency = false;
-              audio['oncanplaythrough'] = function() { // XXX string for closure
-                audio['oncanplaythrough'] = null;
-                preloadedAudios['%(filename)s'] = audio;
-                if (!audio.removedDependency) {
-                  removeRunDependency();
-                  audio.removedDependency = true;
-                }
-              };
-              audio.onerror = function(event) {
-                if (!audio.removedDependency) {
-                  console.log('Audio %(filename)s could not be decoded or timed out trying to decode');
-                  removeRunDependency();
-                  audio.removedDependency = true;
-                }
-              };
-              setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
-              audio.src = url;
-            } else {
-              preloadedAudios['%(filename)s'] = new Audio(); // empty shim
-              removeRunDependency();
-            }
-''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] }
-        else:
-          finish = 'removeRunDependency();\n'
-
-        code += '''
-        var %(varname)s = new %(request)s();
-        %(varname)s.open('GET', '%(netname)s', true);
-        %(varname)s.responseType = 'arraybuffer';
-        %(varname)s.onload = function() {
-          var arrayBuffer = %(varname)s.response;
-          assert(arrayBuffer, 'Loading file %(filename)s failed.');
-          var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
-          Module['FS_createDataFile']('/%(dirname)s', '%(basename)s', byteArray, true, true);
-          %(finish)s
-        };
-        addRunDependency();
-        %(varname)s.send(null);
-''' % {
-            'request': 'DataRequest', # In the past we also supported XHRs here
-            'varname': varname,
-            'filename': filename,
-            'netname': file_['net_name'],
-            'dirname': os.path.dirname(filename),
-            'basename': os.path.basename(filename),
-            'finish': finish
-      }
-      else:
-        assert 0
-
-    if has_preloaded_files:
-      # Get the big archive and split it up
-      use_data = ''
-      for file_ in data_files:
-        if file_['mode'] == 'preload':
-          use_data += '''
-            curr = DataRequest.prototype.requests['%s'];
-            curr.response = byteArray.subarray(%d,%d);
-            curr.onload();
-          ''' % (file_['name'], file_['data_start'], file_['data_end'])
-      use_data += '          removeRunDependency();\n'
-
-      if Compression.on:
-        use_data = '''
-          Module["decompress"](byteArray, function(decompressed) {
-            byteArray = new Uint8Array(decompressed);
-            %s
-          });
-  ''' % use_data
-
-      code += '''
-        var dataFile = new XMLHttpRequest();
-        dataFile.open('GET', '%s', true);
-        dataFile.responseType = 'arraybuffer';
-        dataFile.onload = function() {
-          var arrayBuffer = dataFile.response;
-          assert(arrayBuffer, 'Loading data file failed.');
-          var byteArray = new Uint8Array(arrayBuffer);
-          var curr;
-          %s
-        };
-        addRunDependency();
-        dataFile.send(null);
-        if (Module['setStatus']) Module['setStatus']('Downloading...');
-      ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data)
-
+    file_args = []
+    if len(preload_files) > 0:
+      file_args.append('--preload')
+      file_args += preload_files
+    if len(embed_files) > 0:
+      file_args.append('--embed')
+      file_args += embed_files
+    if Compression.on:
+      file_args += ['--compress', Compression.encoder, Compression.decoder, Compression.js_name]
+    code = execute(shared.ENV_PREFIX + ['python', shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0]
     src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code)
     final += '.files.js'
     open(final, 'w').write(src)