merge

1 files changed, 462 insertions, 57 deletions

diff --git a/emcc b/emcc
index 630c2504..a54b805b 100755
--- a/emcc
+++ b/emcc
@@ -74,10 +74,19 @@ emcc can be influenced by a few environment variables:
   EMMAKEN_COMPILER - The compiler to be used, if you don't want the default clang.
 '''
 
-import os, sys, shutil, tempfile
-from subprocess import Popen, PIPE, STDOUT
+import os, sys, shutil, tempfile, subprocess, shlex
+from subprocess import PIPE, STDOUT
 from tools import shared
 
+def execute(cmd, *args, **kw):
+  try:
+    return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that)
+  except:
+    if not isinstance(cmd, str):
+      cmd = ' '.join(cmd)
+    print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>'
+    raise
+
 # Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt
 # levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get
 # llvm opt level 3, and speed-wise emcc level 2 is already the slowest/most optimizing
@@ -151,18 +160,29 @@ Options that are modified or new in %s include:
                            break the generated code! If that happens, try
                            -O2 and then adding dangerous optimizations one
                            by one.
+
   -s OPTION=VALUE          JavaScript code generation option passed
                            into the emscripten compiler. For the
                            available options, see src/settings.js
+
   --typed-arrays <mode>    0: No typed arrays
                            1: Parallel typed arrays
                            2: Shared (C-like) typed arrays (default)
+
   --llvm-opts <level>      0: No LLVM optimizations (default in -O0)
                            1: -O1 LLVM optimizations (default in -O1)
                            2: -O2 LLVM optimizations
                            3: -O3 LLVM optimizations (default in -O2+)
+
+  --llvm-lto <level>       0: No LLVM LTO (default in -O0)
+                           1: LLVM LTO (default in -O1+)
+                           Note: If LLVM optimizations are not run
+                           (see --llvm-opts), setting this to 1 has no
+                           effect.
+
   --closure <on>           0: No closure compiler (default in -O0, -O1)
                            1: Run closure compiler (default in -O2, -O3)
+
   --js-transform <cmd>     <cmd> will be called on the generated code
                            before it is optimized. This lets you modify
                            the JavaScript, for example adding some code
@@ -177,23 +197,61 @@ Options that are modified or new in %s include:
                            list of arguments, for example, <cmd> of
                            "python processor.py" will cause a python
                            script to be run.
+
   --pre-js <file>          A file whose contents are added before the
-                           generated code
+                           generated code. This is done *before*
+                           optimization, so it will be minified
+                           properly if closure compiler is run.
+
   --post-js <file>         A file whose contents are added after the
-                           generated code
-  --compress <on>          0: Do not compress the generated JavaScript's
+                           generated code This is done *before*
+                           optimization, so it will be minified
+                           properly if closure compiler is run.
+
+  --embed-file <file>      A file to embed inside the generated
+                           JavaScript. The compiled code will be able
+                           to access the file in the current directory
+                           with the same basename as given here (that is,
+                           just the filename, without a path to it).
+                           If a directory is passed here, its entire
+                           contents will be embedded.
+
+  --preload-file <name>    A file to preload before running the
+                           compiled code asynchronously. Otherwise
+                           similar to --embed-file, except that this
+                           option is only relevant when generating
+                           HTML (it uses asynchronous binary XHRs).
+                           If a directory is passed here, its entire
+                           contents will be preloaded.
+
+  --compression <codec>    Compress both the compiled code and embedded/
+                           preloaded files. <codec> should be a triple,
+
+                              <native_encoder>,<js_decoder>,<js_name>
+
+                           where native_encoder is a native executable
+                           that compresses stdin to stdout (the simplest
+                           possible interface), js_decoder is a
+                           JavaScript file that implements a decoder,
+                           and js_name is the name of the function to
+                           call in the decoder file (which should
+                           receive an array/typed array and return
+                           an array/typed array.
+                           Compression only works when generating HTML.
+                           When compression is on, all filed specified
+                           to be preloaded are compressed in one big
+                           archive, which is given the same name as the
+                           output HTML but with suffix .data.compress
+
+  --minify <on>            0: Do not minify the generated JavaScript's
                               whitespace (default if closure compiler
                               will not be run)
-                           1: Compress the generated JavaScript's
+                           1: Minify the generated JavaScript's
                               whitespace (default if closure compiler
                               will be run). Note that this by itself
                               will not minify the code (closure does
                               that)
-  --embed-file <filename>  A file to embed inside the generated
-                           JavaScript. The compiled code will be able
-                           to access the file in the current directory
-                           with the same basename as given here (that is,
-                           just the filename, without a path to it).
+
   --ignore-dynamic-linking Normally emcc will treat dynamic linking like
                            static linking, by linking in the code from
                            the dynamic library. This fails if the same
@@ -202,6 +260,7 @@ Options that are modified or new in %s include:
                            which allows the build system to proceed without
                            errors. However, you will need to manually
                            link to the shared libraries later on yourself.
+
   --shell-file <path>      The path name to a skeleton HTML file used
                            when generating HTML output. The shell file
                            used needs to have this token inside it:
@@ -210,13 +269,16 @@ Options that are modified or new in %s include:
                            target other than HTML is specified using
                            the -o option.
 
+  --js-library <lib>       A JavaScript library to use in addition to
+                           those in Emscripten's src/library_*
+
 The target file, if specified (-o <target>), defines what will
 be generated:
 
   <name>.js                JavaScript (default)
   <name>.html              HTML with embedded JavaScript
   <name>.bc                LLVM bitcode
-  <name>.o                 LLVM bitcode
+  <name>.o                 LLVM bitcode (same as .bc)
 
 The -c option (which tells gcc not to run the linker) will
 cause LLVM bitcode to be generated, as %s only generates
@@ -234,15 +296,15 @@ the source of emcc (search for 'os.environ').
 
 # If this is a configure-type thing, do not compile to JavaScript, instead use clang
 # to compile to a native binary (using our headers, so things make sense later)
-CONFIGURE_CONFIG = os.environ.get('EMMAKEN_JUST_CONFIGURE')
+CONFIGURE_CONFIG = os.environ.get('EMMAKEN_JUST_CONFIGURE') or 'conftest.c' in sys.argv
 CMAKE_CONFIG = 'CMakeFiles/cmTryCompileExec.dir' in ' '.join(sys.argv)# or 'CMakeCCompilerId' in ' '.join(sys.argv)
 if CONFIGURE_CONFIG or CMAKE_CONFIG:
   compiler = shared.CLANG
   if not ('CXXCompiler' in ' '.join(sys.argv) or os.environ.get('EMMAKEN_CXX')):
     compiler = shared.to_cc(compiler)
-  cmd = [compiler] + shared.EMSDK_OPTS + sys.argv[1:]
-  if DEBUG: print >> sys.stderr, 'emcc, just configuring: ', compiler, cmd
-  exit(os.execvp(compiler, cmd))
+  cmd = [compiler] + shared.EMSDK_OPTS + ['-DEMSCRIPTEN'] + sys.argv[1:]
+  if DEBUG: print >> sys.stderr, 'emcc, just configuring: ', ' '.join(cmd)
+  exit(subprocess.call(cmd))
 
 if os.environ.get('EMMAKEN_COMPILER'):
   CXX = os.environ['EMMAKEN_COMPILER']
@@ -258,7 +320,7 @@ if os.environ.get('EMMAKEN_CXX'):
 CC_ADDITIONAL_ARGS = shared.COMPILER_OPTS # + ['-g']?
 
 EMMAKEN_CFLAGS = os.environ.get('EMMAKEN_CFLAGS')
-if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += EMMAKEN_CFLAGS.split(' ')
+if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += shlex.split(EMMAKEN_CFLAGS)
 
 # ---------------- Utilities ---------------
 
@@ -269,8 +331,12 @@ STATICLIB_SUFFIXES = ('.a',)
 ASSEMBLY_SUFFIXES = ('.ll',)
 LIB_PREFIXES = ('', 'lib')
 
+IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
+AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
+AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' }
+
 def suffix(name):
-  return name.split('.')[:-1]
+  return name.split('.')[-1]
 
 def unsuffixed(name):
   return '.'.join(name.split('.')[:-1])
@@ -278,6 +344,13 @@ def unsuffixed(name):
 def unsuffixed_basename(name):
   return os.path.basename(unsuffixed(name))
 
+seen_names = {}
+def unsuffixed_uniquename(name):
+  ret = unsuffixed_basename(name)
+  if name not in seen_names:
+    seen_names[name] = str(len(seen_names))
+  return ret + '_' + seen_names[name]
+
 # ---------------- End configs -------------
 
 if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
@@ -296,6 +369,12 @@ for i in range(1, len(sys.argv)):
     if arg.endswith('.h') and sys.argv[i-1] != '-include':
       header = True
 
+if '-M' in sys.argv or '-MM' in sys.argv:
+  # Just output dependencies, do not compile. Warning: clang and gcc behave differently with -MF! (clang seems to not recognize it)
+  cmd = [CC] + shared.COMPILER_OPTS + sys.argv[1:]
+  if DEBUG: print >> sys.stderr, 'emcc, just dependencies: ', ' '.join(cmd)
+  exit(subprocess.call(cmd))
+
 # Check if a target is specified
 target = None
 for i in range(len(sys.argv)-1):
@@ -308,8 +387,12 @@ for i in range(len(sys.argv)-1):
     break
 
 if header: # header or such
-  if DEBUG: print >> sys.stderr, 'Just copy.'
-  shutil.copy(sys.argv[-1], sys.argv[-2])
+  if len(sys.argv) >= 3: # if there is a source and a target, then copy, otherwise do nothing
+    sys.argv = filter(lambda arg: not arg.startswith('-I'), sys.argv)
+    if DEBUG: print >> sys.stderr, 'Just copy:', sys.argv[-1], target
+    shutil.copy(sys.argv[-1], target)
+  else:
+    if DEBUG: print >> sys.stderr, 'No-op.'
   exit(0)
 
 if TEMP_DIR:
@@ -323,6 +406,21 @@ else:
 def in_temp(name):
   return os.path.join(temp_dir, name)
 
+class Compression:
+  on = False
+
+  @staticmethod
+  def compressed_name(filename):
+    return filename + '.compress'
+
+  @staticmethod
+  def compress(filename):
+    execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb'))
+
+  @staticmethod
+  def worth_it(original, compressed):
+    return compressed < original - 1500 # save at least one TCP packet or so
+
 try:
   call = CXX if use_cxx else CC
 
@@ -332,14 +430,17 @@ try:
 
   opt_level = 0
   llvm_opts = None
+  llvm_lto = None
   closure = None
   js_transform = None
   pre_js = None
   post_js = None
-  compress_whitespace = None
-  embed_files = []
+  minify_whitespace = None
+  data_files = []
+  compression = None
   ignore_dynamic_linking = False
   shell_path = shared.path_from_root('src', 'shell.html')
+  js_libraries = []
 
   def check_bad_eq(arg):
     assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)'
@@ -357,6 +458,11 @@ try:
       llvm_opts = eval(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i].startswith('--llvm-lto'):
+      check_bad_eq(newargs[i])
+      llvm_lto = eval(newargs[i+1])
+      newargs[i] = ''
+      newargs[i+1] = ''
     elif newargs[i].startswith('--closure'):
       check_bad_eq(newargs[i])
       closure = int(newargs[i+1])
@@ -377,20 +483,31 @@ try:
       post_js = open(newargs[i+1]).read()
       newargs[i] = ''
       newargs[i+1] = ''
-    elif newargs[i].startswith('--compress'):
+    elif newargs[i].startswith('--minify'):
       check_bad_eq(newargs[i])
-      compress_whitespace = int(newargs[i+1])
+      minify_whitespace = int(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
     elif newargs[i].startswith('--embed-file'):
       check_bad_eq(newargs[i])
-      embed_files.append(newargs[i+1])
+      data_files.append({ 'name': newargs[i+1], 'mode': 'embed' })
       newargs[i] = ''
       newargs[i+1] = ''
-    elif newargs[i] == '-MF': # clang cannot handle this, so we fake it
-      f = open(newargs[i+1], 'w')
-      f.write('\n')
-      f.close()
+    elif newargs[i].startswith('--preload-file'):
+      check_bad_eq(newargs[i])
+      data_files.append({ 'name': newargs[i+1], 'mode': 'preload' })
+      newargs[i] = ''
+      newargs[i+1] = ''
+    elif newargs[i].startswith('--compression'):
+      check_bad_eq(newargs[i])
+      parts = newargs[i+1].split(',')
+      assert len(parts) == 3, '--compression requires specifying    native_encoder,js_decoder,js_name  - see emcc --help. got: %s' % newargs[i+1]
+      Compression.encoder = parts[0]
+      Compression.decoder = parts[1]
+      Compression.js_name = parts[2]
+      assert os.path.exists(Compression.encoder), 'native encoder %s does not exist' % Compression.encoder
+      assert os.path.exists(Compression.decoder), 'js decoder %s does not exist' % Compression.decoder
+      Compression.on = True
       newargs[i] = ''
       newargs[i+1] = ''
     elif newargs[i] == '--ignore-dynamic-linking':
@@ -401,12 +518,18 @@ try:
       shell_path = newargs[i+1]
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i].startswith('--js-library'):
+      check_bad_eq(newargs[i])
+      js_libraries.append(newargs[i+1])
+      newargs[i] = ''
+      newargs[i+1] = ''
   newargs = [ arg for arg in newargs if arg is not '' ]
 
   if llvm_opts is None: llvm_opts = LLVM_OPT_LEVEL[opt_level]
+  if llvm_lto is None: llvm_lto = llvm_opts > 0
   if closure is None: closure = 1 if opt_level >= 2 else 0
-  if compress_whitespace is None:
-    compress_whitespace = closure # if closure is run, compress whitespace
+  if minify_whitespace is None:
+    minify_whitespace = closure # if closure is run, minify whitespace
 
   if closure:
     assert os.path.exists(shared.CLOSURE_COMPILER), 'emcc: fatal: Closure compiler (%s) does not exist' % shared.CLOSURE_COMPILER
@@ -428,11 +551,16 @@ try:
 
   input_files = []
   has_source_inputs = False
-  lib_dirs = []
+  lib_dirs = [shared.path_from_root('system', 'lib')]
   libs = []
   for i in range(len(newargs)): # find input files XXX this a simple heuristic. we should really analyze based on a full understanding of gcc params,
                                 # right now we just assume that what is left contains no more |-x OPT| things
     arg = newargs[i]
+
+    if i > 0:
+      prev = newargs[i-1]
+      if prev in ['-MT', '-install_name']: continue # ignore this gcc-style argument
+
     if arg.endswith(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + ASSEMBLY_SUFFIXES) or shared.Building.is_ar(arg): # we already removed -o <target>, so all these should be inputs
       newargs[i] = ''
       if os.path.exists(arg):
@@ -443,6 +571,16 @@ try:
           # this should be bitcode, make sure it is valid
           if arg.endswith(ASSEMBLY_SUFFIXES) or shared.Building.is_bitcode(arg):
             input_files.append(arg)
+          elif arg.endswith(STATICLIB_SUFFIXES + DYNAMICLIB_SUFFIXES):
+            # if it's not, and it's a library, just add it to libs to find later
+            l = unsuffixed_basename(arg)
+            for prefix in LIB_PREFIXES:
+              if not prefix: continue
+              if l.startswith(prefix):
+                l = l[len(prefix):]
+                break;
+            libs.append(l)
+            newargs[i] = ''
           else:
             print >> sys.stderr, 'emcc: %s: warning: Not valid LLVM bitcode' % arg
       else:
@@ -453,6 +591,7 @@ try:
     elif arg.startswith('-l'):
       libs.append(arg[2:])
       newargs[i] = ''
+
   newargs = [ arg for arg in newargs if arg is not '' ]
 
   # Find library files
@@ -475,7 +614,10 @@ try:
   if ignore_dynamic_linking:
     input_files = filter(lambda input_file: not input_file.endswith(DYNAMICLIB_SUFFIXES), input_files)
 
-  assert len(input_files) > 0, 'emcc: no input files'
+  if len(input_files) == 0:
+    print >> sys.stderr, 'emcc: no input files'
+    print >> sys.stderr, 'note that input files without a known suffix are ignored, make sure your input files end with one of: ' + str(SOURCE_SUFFIXES + BITCODE_SUFFIXES + DYNAMICLIB_SUFFIXES + STATICLIB_SUFFIXES + ASSEMBLY_SUFFIXES)
+    exit(0)
 
   newargs += CC_ADDITIONAL_ARGS
 
@@ -495,6 +637,8 @@ try:
   else:
     final_suffix = ''
 
+  assert not (Compression.on and final_suffix != 'html'), 'Compression only works when generating HTML'
+
   # Apply optimization level settings
   shared.Settings.apply_opt_level(opt_level, noisy=True)
 
@@ -513,18 +657,18 @@ try:
   for input_file in input_files:
     if input_file.endswith(SOURCE_SUFFIXES):
       if DEBUG: print >> sys.stderr, 'emcc: compiling source file: ', input_file
-      output_file = in_temp(unsuffixed_basename(input_file) + '.o')
+      output_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
       temp_files.append(output_file)
       args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file]
       if DEBUG: print >> sys.stderr, "emcc running:", call, ' '.join(args)
-      Popen([call] + args).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that)
+      execute([call] + args) # let compiler frontend print directly, so colors are saved (PIPE kills that)
       if not os.path.exists(output_file):
         print >> sys.stderr, 'emcc: compiler frontend failed to generate LLVM bitcode, halting'
         sys.exit(1)
     else: # bitcode
       if input_file.endswith(BITCODE_SUFFIXES):
         if DEBUG: print >> sys.stderr, 'emcc: copying bitcode file: ', input_file
-        temp_file = in_temp(unsuffixed_basename(input_file) + '.o')
+        temp_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
         shutil.copyfile(input_file, temp_file)
         temp_files.append(temp_file)
       elif input_file.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file):
@@ -537,7 +681,7 @@ try:
           # Note that by assembling the .ll file, then disassembling it later, we will
           # remove annotations which is a good thing for compilation time
           if DEBUG: print >> sys.stderr, 'emcc: assembling assembly file: ', input_file
-          temp_file = in_temp(unsuffixed_basename(input_file) + '.o')
+          temp_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
           shared.Building.llvm_as(input_file, temp_file)
           temp_files.append(temp_file)
 
@@ -549,10 +693,10 @@ try:
       print >> sys.stderr, 'emcc: warning: -Ox flags ignored, since not generating JavaScript'
     if not specified_target:
       for input_file in input_files:
-        shutil.move(in_temp(unsuffixed_basename(input_file) + '.o'), unsuffixed_basename(input_file) + '.' + final_suffix)
+        shutil.move(in_temp(unsuffixed_uniquename(input_file) + '.o'), unsuffixed_basename(input_file) + '.' + final_suffix)
     else:
       if len(input_files) == 1:
-        shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), specified_target)
+        shutil.move(in_temp(unsuffixed_uniquename(input_files[0]) + '.o'), specified_target)
       else:
         assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv)
         # We have a specified target (-o <target>), which is not JavaScript or HTML, and
@@ -560,7 +704,7 @@ try:
         ld_args = temp_files + ['-b', specified_target]
                   #[arg.split('-Wl,')[1] for arg in filter(lambda arg: arg.startswith('-Wl,'), sys.argv)]
         if DEBUG: print >> sys.stderr, 'emcc: link: ' + str(ld_args)
-        Popen([shared.LLVM_LD, '-disable-opt'] + ld_args).communicate()
+        execute([shared.LLVM_LD, '-disable-opt'] + ld_args)
     exit(0)
 
   ## Continue on to create JavaScript
@@ -578,9 +722,9 @@ try:
     # dlmalloc
     def create_dlmalloc():
       if DEBUG: print >> sys.stderr, 'emcc: building dlmalloc for cache'
-      Popen([shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')], stdout=stdout, stderr=stderr).communicate()
+      execute(['python', shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')], stdout=stdout, stderr=stderr)
       # we include the libc++ new stuff here, so that the common case of using just new/delete is quick to link
-      Popen([shared.EMXX, shared.path_from_root('system', 'lib', 'libcxx', 'new.cpp'), '-g', '-o', in_temp('new.o')], stdout=stdout, stderr=stderr).communicate()
+      execute(['python', shared.EMXX, shared.path_from_root('system', 'lib', 'libcxx', 'new.cpp'), '-g', '-o', in_temp('new.o')], stdout=stdout, stderr=stderr)
       shared.Building.link([in_temp('dlmalloc.o'), in_temp('new.o')], in_temp('dlmalloc_full.o'))
       return in_temp('dlmalloc_full.o')
     def fix_dlmalloc():
@@ -679,7 +823,7 @@ try:
     shared.Building.llvm_opt(in_temp(target_basename + '.bc'), llvm_opts)
     if DEBUG: save_intermediate('opt', 'bc')
     # Do LTO in a separate pass to work around LLVM bug XXX (see failure e.g. in cubescript)
-    if shared.Building.can_use_unsafe_opts() and shared.Building.can_build_standalone():
+    if llvm_lto and shared.Building.can_use_unsafe_opts() and shared.Building.can_build_standalone():
       lto_opts = []
       if not shared.Building.can_inline(): lto_opts.append('-disable-inlining')
       lto_opts.append('-std-link-opts')
@@ -702,25 +846,221 @@ try:
 
   if AUTODEBUG:
     if DEBUG: print >> sys.stderr, 'emcc: autodebug'
-    Popen(['python', shared.AUTODEBUGGER, final, final + '.ad.ll']).communicate()[0]
+    execute(['python', shared.AUTODEBUGGER, final, final + '.ad.ll'])
     final += '.ad.ll'
     if DEBUG: save_intermediate('autodebug', 'll')
 
   # Emscripten
   if DEBUG: print >> sys.stderr, 'emcc: LLVM => JS'
-  final = shared.Building.emscripten(final, append_ext=False)
+  extra_args = [] if not js_libraries else ['--libraries', ','.join(map(os.path.abspath, js_libraries))]
+  final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args)
   if DEBUG: save_intermediate('original')
 
-  # Embed files
-  if len(embed_files) > 0:
-    if DEBUG: print >> sys.stderr, 'emcc: embedding files'
-    src = open(final).read().replace(
-      '// {{PRE_RUN_ADDITIONS}}',
-      '\n'.join(map(lambda embed_file: "FS.createDataFile('/', '%s', %s, true, false);" % (os.path.basename(embed_file), str(map(ord, open(embed_file, 'rb').read()))), embed_files))
-    )
-    final += '.ef.js'
+  # Embed and preload files
+  if len(data_files) > 0:
+    if DEBUG: print >> sys.stderr, 'emcc: setting up files'
+    code = ''
+
+    if final_suffix == 'html':
+      code += '''
+      var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));
+      var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");
+      var hasBlobConstructor;
+      try {
+        new Blob();
+        hasBlobConstructor = true;
+      } catch(e) {
+        hasBlobConstructor = false;
+        console.log("warning: no blob constructor, cannot create blobs with mimetypes");
+      }
+'''
+
+      code += 'var preloadedImages = {}; // maps url to image data\n'
+      code += 'var preloadedAudios = {}; // maps url to audio data\n'
+
+    # Expand directories into individual files
+    def add(mode, dirname, names):
+      for name in names:
+        fullname = os.path.join(dirname, name)
+        if not os.path.isdir(fullname):
+          data_files.append({ 'name': fullname, 'mode': mode })
+
+    for file_ in data_files:
+      if os.path.isdir(file_['name']):
+        os.path.walk(file_['name'], add, file_['mode'])
+    data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)
+
+    for file_ in data_files:
+      file_['name'] = file_['name'].replace(os.path.sep, '/')
+      file_['net_name'] = file_['name']
+
+    data_target = unsuffixed(target) + '.data'
+
+    # Set up folders
+    partial_dirs = []
+    for file_ in data_files:
+      dirname = os.path.dirname(file_['name'])
+      if dirname != '' and dirname != '/':
+        parts = dirname.split('/')
+        for i in range(len(parts)):
+          partial = '/'.join(parts[:i+1])
+          if partial not in partial_dirs:
+            code += '''FS.createFolder('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i])
+            partial_dirs.append(partial)
+
+    if final_suffix == 'html':
+      # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead.
+      data = open(data_target, 'wb')
+      start = 0
+      for file_ in data_files:
+        file_['data_start'] = start
+        curr = open(file_['name'], 'rb').read()
+        file_['data_end'] = start + len(curr)
+        start += len(curr)
+        data.write(curr)
+      data.close()
+      if Compression.on:
+        Compression.compress(data_target)
+
+      # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs
+      code += '''
+        function DataRequest() {}
+        DataRequest.prototype = {
+          requests: {},
+          open: function(mode, name) {
+            this.requests[name] = this;
+          },
+          send: function() {}
+        };
+      '''
+
+    counter = 0
+    for file_ in data_files:
+      filename = file_['name']
+      if file_['mode'] == 'embed':
+        # Embed
+        code += '''FS.createDataFile('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
+      elif file_['mode'] == 'preload':
+        # Preload
+        assert final_suffix == 'html', 'Can only preload files when generating HTML'
+
+        varname = 'filePreload%d' % counter
+        counter += 1
+        image = filename.endswith(IMAGE_SUFFIXES)
+        audio = filename.endswith(AUDIO_SUFFIXES)
+
+        if image:
+          finish =  '''
+            var bb = new BlobBuilder();
+            bb.append(byteArray.buffer);
+            var b = bb.getBlob();
+            var url = URLObject.createObjectURL(b);
+            var img = new Image();
+            img.onload = function() {
+              assert(img.complete, 'Image %(filename)s could not be decoded');
+              var canvas = document.createElement('canvas');
+              canvas.width = img.width;
+              canvas.height = img.height;
+              var ctx = canvas.getContext('2d');
+              ctx.drawImage(img, 0, 0);
+              preloadedImages['%(filename)s'] = canvas;
+              URLObject.revokeObjectURL(url);
+              removeRunDependency();
+            };
+            img.onerror = function(event) {
+              console.log('Image %(filename)s could not be decoded');
+            };
+            img.src = url;
+''' % { 'filename': filename }
+        elif audio:
+          # Need actual blob constructor here, to set the mimetype or else audios fail to decode
+          finish =  '''
+            if (hasBlobConstructor) {
+              var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' });
+              var url = URLObject.createObjectURL(b); // XXX we never revoke this!
+              var audio = new Audio();
+              audio['oncanplaythrough'] = function() { // XXX string for closure
+                audio['oncanplaythrough'] = null;
+                preloadedAudios['%(filename)s'] = audio;
+                removeRunDependency();
+              };
+              audio.onerror = function(event) {
+                console.log('Audio %(filename)s could not be decoded');
+              };
+              audio.src = url;
+            } else {
+              preloadedAudios['%(filename)s'] = new Audio(); // empty shim
+              removeRunDependency();
+            }
+''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] }
+        else:
+          finish = 'removeRunDependency();\n'
+
+        code += '''
+        var %(varname)s = new %(request)s();
+        %(varname)s.open('GET', '%(netname)s', true);
+        %(varname)s.responseType = 'arraybuffer';
+        %(varname)s.onload = function() {
+          var arrayBuffer = %(varname)s.response;
+          assert(arrayBuffer, 'Loading file %(filename)s failed.');
+          var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
+          FS.createDataFile('/%(dirname)s', '%(basename)s', byteArray, true, true);
+          %(finish)s
+        };
+        addRunDependency();
+        %(varname)s.send(null);
+''' % {
+            'request': 'DataRequest', # In the past we also supported XHRs here
+            'varname': varname,
+            'filename': filename,
+            'netname': file_['net_name'],
+            'dirname': os.path.dirname(filename),
+            'basename': os.path.basename(filename),
+            'finish': finish
+      }
+      else:
+        assert 0
+
+    if final_suffix == 'html':
+      # Get the big archive and split it up
+      use_data = ''
+      for file_ in data_files:
+        if file_['mode'] == 'preload':
+          use_data += '''
+            curr = DataRequest.prototype.requests['%s'];
+            curr.response = byteArray.subarray(%d,%d);
+            curr.onload();
+          ''' % (file_['name'], file_['data_start'], file_['data_end'])
+      use_data += '          removeRunDependency();\n'
+
+      if Compression.on:
+        use_data = '''
+          Module["decompress"](byteArray, function(decompressed) {
+            byteArray = new Uint8Array(decompressed);
+            %s
+          });
+  ''' % use_data
+
+      code += '''
+        var dataFile = new XMLHttpRequest();
+        dataFile.open('GET', '%s', true);
+        dataFile.responseType = 'arraybuffer';
+        dataFile.onload = function() {
+          var arrayBuffer = dataFile.response;
+          assert(arrayBuffer, 'Loading data file failed.');
+          var byteArray = new Uint8Array(arrayBuffer);
+          var curr;
+          %s
+        };
+        addRunDependency();
+        dataFile.send(null);
+        if (Module['setStatus']) Module['setStatus']('Downloading...');
+      ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data)
+
+    src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code)
+    final += '.files.js'
     open(final, 'w').write(src)
-    if DEBUG: save_intermediate('embedded_files')
+    if DEBUG: save_intermediate('files')
 
   # Apply pre and postjs files
   if pre_js or post_js:
@@ -734,8 +1074,9 @@ try:
   if js_transform:
     shutil.copyfile(final, final + '.tr.js')
     final += '.tr.js'
+    posix = True if not shared.WINDOWS else False
     if DEBUG: print >> sys.stderr, 'emcc: applying transform: %s' % js_transform
-    Popen(js_transform.split(' ') + [os.path.abspath(final)]).communicate()
+    execute(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)])
     if DEBUG: save_intermediate('transformed')
 
   # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
@@ -790,7 +1131,7 @@ try:
     if DEBUG: print >> sys.stderr, 'emcc: running post-closure post-opts'
     js_optimizer_queue += ['simplifyExpressionsPost']
 
-  if compress_whitespace:
+  if minify_whitespace:
     js_optimizer_queue += ['compress']
 
   flush_js_optimizer_queue()
@@ -800,7 +1141,71 @@ try:
     if DEBUG: print >> sys.stderr, 'emcc: generating HTML'
     shell = open(shell_path).read()
     html = open(target, 'w')
-    html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
+    if not Compression.on:
+      html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
+    else:
+      # Compress the main code
+      js_target = unsuffixed(target) + '.js'
+      shutil.move(final, js_target)
+      Compression.compress(js_target)
+
+      # Run the decompressor in a worker, and add code to
+      #   1. download the compressed file
+      #   2. decompress to a typed array
+      #   3. convert to a string of source code
+      #   4. insert a script element with that source code (more effective than eval)
+      decoding = '''
+        var decompressWorker = new Worker('decompress.js');
+        var decompressCallbacks = [];
+        var decompressions = 0;
+        Module["decompress"] = function(data, callback) {
+          var id = decompressCallbacks.length;
+          decompressCallbacks.push(callback);
+          decompressWorker.postMessage({ data: data, id: id });
+          if (Module['setStatus']) {
+            decompressions++;
+            Module['setStatus']('Decompressing...');
+          }
+        };
+        decompressWorker.onmessage = function(event) {
+          decompressCallbacks[event.data.id](event.data.data);
+          decompressCallbacks[event.data.id] = null;
+          if (Module['setStatus']) {
+            decompressions--;
+            if (decompressions == 0) {
+              Module['setStatus']('');
+            }
+          }
+        };
+        var compiledCodeXHR = new XMLHttpRequest();
+        compiledCodeXHR.open('GET', '%s', true);
+        compiledCodeXHR.responseType = 'arraybuffer';
+        compiledCodeXHR.onload = function() {
+          var arrayBuffer = compiledCodeXHR.response;
+          if (!arrayBuffer) throw('Loading compressed code failed.');
+          var byteArray = new Uint8Array(arrayBuffer);
+          Module.decompress(byteArray, function(decompressed) {
+            var source = Array.prototype.slice.apply(decompressed).map(function(x) { return String.fromCharCode(x) }).join(''); // createObjectURL instead?
+            var scriptTag = document.createElement('script');
+            scriptTag.setAttribute('type', 'text/javascript');
+            scriptTag.innerHTML = source;
+            document.body.appendChild(scriptTag);
+          });
+        };
+        compiledCodeXHR.send(null);
+''' % Compression.compressed_name(js_target)
+      html.write(shell.replace('{{{ SCRIPT_CODE }}}', decoding))
+
+      # Add decompressor with web worker glue code
+      decompressor = open('decompress.js', 'w')
+      decompressor.write(open(Compression.decoder).read())
+      decompressor.write('''
+        onmessage = function(event) {
+          postMessage({ data: %s(event.data.data), id: event.data.id });
+        };
+''' % Compression.js_name)
+      decompressor.close()
+
     html.close()
   else:
     # copy final JS to output