refactor file packaging into standalone tool

1 files changed, 264 insertions, 0 deletions

diff --git a/tools/file_packager.py b/tools/file_packager.py
new file mode 100644
index 00000000..ebd3d37a
--- /dev/null
+++ b/tools/file_packager.py
@@ -0,0 +1,264 @@
+'''
+A tool that generates FS API calls to generate a filesystem, and packages the files
+to work with that.
+
+This is called by emcc. You can also call it yourself.
+
+Usage:
+
+  file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--compress COMPRESSION_DATA]
+
+'''
+
+import os, sys
+
+from shared import Compression, execute
+
+def suffix(name):
+  return name.split('.')[-1]
+
+IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
+AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
+AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' }
+
+data_files = []
+in_preload = False
+in_embed = False
+has_preloaded = False
+in_compress = 0
+for arg in sys.argv[1:]:
+  if arg == '--preload':
+    in_preload = True
+    in_embed = False
+    has_preloaded = True
+  elif arg == '--embed':
+    in_embed = True
+    in_preload = False
+  elif arg == '--compress':
+    Compression.on = True
+    in_compress = 1
+    in_preload = False
+    in_embed = False
+  elif in_preload:
+    data_files.append({ 'name': arg, 'mode': 'preload' })
+  elif in_embed:
+    data_files.append({ 'name': arg, 'mode': 'embed' })
+  elif in_compress:
+    if in_compress == 1:
+      Compression.encoder = arg
+      in_compress = 2
+    elif in_compress == 2:
+      Compression.decoder = arg
+      in_compress = 3
+    elif in_compress == 3:
+      Compression.js_name = arg
+      in_compress = 0
+
+code = ''
+
+if has_preloaded:
+  code += '''
+  var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));
+  var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");
+  var hasBlobConstructor;
+  try {
+    new Blob();
+    hasBlobConstructor = true;
+  } catch(e) {
+    hasBlobConstructor = false;
+    console.log("warning: no blob constructor, cannot create blobs with mimetypes");
+  }
+'''
+
+  code += 'var preloadedImages = {}; // maps url to image data\n'
+  code += 'var preloadedAudios = {}; // maps url to audio data\n'
+
+# Expand directories into individual files
+def add(mode, dirname, names):
+  for name in names:
+    fullname = os.path.join(dirname, name)
+    if not os.path.isdir(fullname):
+      data_files.append({ 'name': fullname, 'mode': mode })
+
+for file_ in data_files:
+  if os.path.isdir(file_['name']):
+    os.path.walk(file_['name'], add, file_['mode'])
+data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)
+
+for file_ in data_files:
+  file_['name'] = file_['name'].replace(os.path.sep, '/')
+  file_['net_name'] = file_['name']
+
+data_target = sys.argv[1]
+
+# Set up folders
+partial_dirs = []
+for file_ in data_files:
+  dirname = os.path.dirname(file_['name'])
+  dirname = dirname.lstrip('/') # absolute paths start with '/', remove that
+  if dirname != '':
+    parts = dirname.split('/')
+    for i in range(len(parts)):
+      partial = '/'.join(parts[:i+1])
+      if partial not in partial_dirs:
+        code += '''Module['FS_createFolder']('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i])
+        partial_dirs.append(partial)
+
+if has_preloaded:
+  # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead.
+  data = open(data_target, 'wb')
+  start = 0
+  for file_ in data_files:
+    file_['data_start'] = start
+    curr = open(file_['name'], 'rb').read()
+    file_['data_end'] = start + len(curr)
+    start += len(curr)
+    data.write(curr)
+  data.close()
+  if Compression.on:
+    Compression.compress(data_target)
+
+  # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs
+  code += '''
+    function DataRequest() {}
+    DataRequest.prototype = {
+      requests: {},
+      open: function(mode, name) {
+        this.requests[name] = this;
+      },
+      send: function() {}
+    };
+  '''
+
+counter = 0
+for file_ in data_files:
+  filename = file_['name']
+  if file_['mode'] == 'embed':
+    # Embed
+    code += '''Module['FS_createDataFile']('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
+  elif file_['mode'] == 'preload':
+    # Preload
+    varname = 'filePreload%d' % counter
+    counter += 1
+    image = filename.endswith(IMAGE_SUFFIXES)
+    audio = filename.endswith(AUDIO_SUFFIXES)
+
+    if image:
+      finish =  '''
+        var bb = new BlobBuilder();
+        bb.append(byteArray.buffer);
+        var b = bb.getBlob();
+        var url = URLObject.createObjectURL(b);
+        var img = new Image();
+        img.onload = function() {
+          assert(img.complete, 'Image %(filename)s could not be decoded');
+          var canvas = document.createElement('canvas');
+          canvas.width = img.width;
+          canvas.height = img.height;
+          var ctx = canvas.getContext('2d');
+          ctx.drawImage(img, 0, 0);
+          preloadedImages['%(filename)s'] = canvas;
+          URLObject.revokeObjectURL(url);
+          removeRunDependency();
+        };
+        img.onerror = function(event) {
+          console.log('Image %(filename)s could not be decoded');
+        };
+        img.src = url;
+''' % { 'filename': filename }
+    elif audio:
+      # Need actual blob constructor here, to set the mimetype or else audios fail to decode
+      finish =  '''
+        if (hasBlobConstructor) {
+          var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' });
+          var url = URLObject.createObjectURL(b); // XXX we never revoke this!
+          var audio = new Audio();
+          audio.removedDependency = false;
+          audio['oncanplaythrough'] = function() { // XXX string for closure
+            audio['oncanplaythrough'] = null;
+            preloadedAudios['%(filename)s'] = audio;
+            if (!audio.removedDependency) {
+              removeRunDependency();
+              audio.removedDependency = true;
+            }
+          };
+          audio.onerror = function(event) {
+            if (!audio.removedDependency) {
+              console.log('Audio %(filename)s could not be decoded or timed out trying to decode');
+              removeRunDependency();
+              audio.removedDependency = true;
+            }
+          };
+          setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
+          audio.src = url;
+        } else {
+          preloadedAudios['%(filename)s'] = new Audio(); // empty shim
+          removeRunDependency();
+        }
+''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] }
+    else:
+      finish = 'removeRunDependency();\n'
+
+    code += '''
+    var %(varname)s = new %(request)s();
+    %(varname)s.open('GET', '%(netname)s', true);
+    %(varname)s.responseType = 'arraybuffer';
+    %(varname)s.onload = function() {
+      var arrayBuffer = %(varname)s.response;
+      assert(arrayBuffer, 'Loading file %(filename)s failed.');
+      var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
+      Module['FS_createDataFile']('/%(dirname)s', '%(basename)s', byteArray, true, true);
+      %(finish)s
+    };
+    addRunDependency();
+    %(varname)s.send(null);
+''' % {
+        'request': 'DataRequest', # In the past we also supported XHRs here
+        'varname': varname,
+        'filename': filename,
+        'netname': file_['net_name'],
+        'dirname': os.path.dirname(filename),
+        'basename': os.path.basename(filename),
+        'finish': finish
+  }
+  else:
+    assert 0
+
+if has_preloaded:
+  # Get the big archive and split it up
+  use_data = ''
+  for file_ in data_files:
+    if file_['mode'] == 'preload':
+      use_data += '''
+        curr = DataRequest.prototype.requests['%s'];
+        curr.response = byteArray.subarray(%d,%d);
+        curr.onload();
+      ''' % (file_['name'], file_['data_start'], file_['data_end'])
+  use_data += '          removeRunDependency();\n'
+
+  if Compression.on:
+    use_data = '''
+      Module["decompress"](byteArray, function(decompressed) {
+        byteArray = new Uint8Array(decompressed);
+        %s
+      });
+''' % use_data
+
+  code += '''
+    var dataFile = new XMLHttpRequest();
+    dataFile.open('GET', '%s', true);
+    dataFile.responseType = 'arraybuffer';
+    dataFile.onload = function() {
+      var arrayBuffer = dataFile.response;
+      assert(arrayBuffer, 'Loading data file failed.');
+      var byteArray = new Uint8Array(arrayBuffer);
+      var curr;
+      %s
+    };
+    addRunDependency();
+    dataFile.send(null);
+    if (Module['setStatus']) Module['setStatus']('Downloading...');
+  ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data)
+
+print code
+