Merge branch 'incoming'

81 files changed, 10815 insertions, 881 deletions

diff --git a/AUTHORS b/AUTHORS
index 296d4150..0eb8f7df 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -52,4 +52,6 @@ a license to everyone to use it as detailed in LICENSE.)
 * Roger Braun <roger@rogerbraun.net>
 * Vladimir Vukicevic <vladimir@pobox.com> (copyright owned by Mozilla Foundation)
 * Lorant Pinter <lorant.pinter@prezi.com>
+* Tobias Doerffel <tobias.doerffel@gmail.com>
+* Martin von Gagern <martin@von-gagern.net>
 
diff --git a/emcc b/emcc
index f644b924..488f0788 100755
--- a/emcc
+++ b/emcc
@@ -75,7 +75,7 @@ emcc can be influenced by a few environment variables:
   EMMAKEN_COMPILER - The compiler to be used, if you don't want the default clang.
 '''
 
-import os, sys, shutil, tempfile, subprocess, shlex, time
+import os, sys, shutil, tempfile, subprocess, shlex, time, re
 from subprocess import PIPE, STDOUT
 from tools import shared
 from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename
@@ -129,8 +129,8 @@ while response_file:
   for index in range(1, len(sys.argv)):
     if sys.argv[index][0] == '@':
       # found one, loop again next time
-      print >>sys.stderr, 'emcc: using response file: %s' % response_file
       response_file = sys.argv[index][1:]
+      print >>sys.stderr, 'emcc: using response file: %s' % response_file
       if not os.path.exists(response_file):
         print >>sys.stderr, 'emcc: error: Response file not found: %s' % response_file
         exit(1)
@@ -215,6 +215,20 @@ Options that are modified or new in %s include:
                            (without the external "s in either of those,
                            you would get an error)
 
+                           You can also specify a file from which the
+                           value would be read, for example,
+
+                            -s DEAD_FUNCTIONS=@/path/to/file
+
+                           The contents of /path/to/file will be read,
+                           JSON.parsed and set into DEAD_FUNCTIONS (so
+                           the file could contain
+
+                            ["_func1", "func2"]
+
+                           ). Note that the path must be absolute, not
+                           relative.
+
   -g                       Use debug info. Note that you need this during
                            the last compilation phase from bitcode to
                            JavaScript, or else we will remove it by
@@ -332,13 +346,11 @@ Options that are modified or new in %s include:
                            output HTML but with suffix .data.compress
 
   --minify <on>            0: Do not minify the generated JavaScript's
-                              whitespace (default if closure compiler
-                              will not be run)
+                              whitespace (default in -O0, -O1, or if
+                              -g is used)
                            1: Minify the generated JavaScript's
-                              whitespace (default if closure compiler
-                              will be run). Note that this by itself
-                              will not minify the code (closure does
-                              that)
+                              whitespace (default in -O2+, assuming
+                              -g is not used)
 
   --split <size>           Splits the resulting javascript file into pieces
                            to ease debugging. This option only works if
@@ -451,6 +463,13 @@ Options that are modified or new in %s include:
                            the bootstrapped relooper. After the cache
                            is cleared, this process will exit.
 
+  --save-bc PATH           When compiling to JavaScript or HTML, this
+                           option will save a copy of the bitcode to
+                           the specified path. The bitcode will include
+                           all files being linked, including standard
+                           libraries, and after any link-time optimizations
+                           (if any).
+
 The target file, if specified (-o <target>), defines what will
 be generated:
 
@@ -696,6 +715,8 @@ try:
   keep_js_debug = False
   bind = False
   jcache = False
+  save_bc = False
+
   if use_cxx:
     default_cxx_std = '-std=c++03' # Enforce a consistent C++ standard when compiling .cpp files, if user does not specify one on the cmdline.
   else:
@@ -706,19 +727,21 @@ try:
 
   absolute_warning_shown = False
 
+  settings_changes = []
+
   for i in range(len(newargs)):
     newargs[i] = newargs[i].strip() # On Windows Vista (and possibly others), excessive spaces in the command line leak into the items in this array, so trim e.g. 'foo.cpp ' -> 'foo.cpp'
     if newargs[i].startswith('-O'):
       # Let -O default to -O2, which is what gcc does.
       requested_level = newargs[i][2:] or '2'
       if requested_level == 's':
-        print >> sys.stderr, 'emcc: warning: -Os is ignored (use -O0, -O1, -O2)'
-      else:
-        try:
-          opt_level = int(requested_level)
-          assert 0 <= opt_level <= 3
-        except:
-          raise Exception('Invalid optimization level: ' + newargs[i])
+        requested_level = 2
+        settings_changes.append('INLINING_LIMIT=50')
+      try:
+        opt_level = int(requested_level)
+        assert 0 <= opt_level <= 3
+      except:
+        raise Exception('Invalid optimization level: ' + newargs[i])
       newargs[i] = ''
     elif newargs[i].startswith('--llvm-opts'):
       check_bad_eq(newargs[i])
@@ -821,6 +844,11 @@ try:
       print >> sys.stderr, 'emcc: clearing cache'
       shared.Cache.erase()
       sys.exit(0)
+    elif newargs[i] == '--save-bc':
+      check_bad_eq(newargs[i])
+      save_bc = newargs[i+1]
+      newargs[i] = ''
+      newargs[i+1] = ''
     elif newargs[i].startswith(('-I/', '-L/')):
       if not absolute_warning_shown:
         print >> sys.stderr, 'emcc: warning: -I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript)' # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not
@@ -842,7 +870,6 @@ try:
   if closure:
     assert os.path.exists(shared.CLOSURE_COMPILER), 'emcc: fatal: Closure compiler (%s) does not exist' % shared.CLOSURE_COMPILER
 
-  settings_changes = []
   for i in range(len(newargs)):
     if newargs[i] == '-s':
       if is_minus_s_for_emcc(newargs, i):
@@ -960,6 +987,8 @@ try:
   # Apply -s settings in newargs here (after optimization levels, so they can override them)
   for change in settings_changes:
     key, value = change.split('=')
+    if value[0] == '@':
+      value = '"' + value + '"'
     exec('shared.Settings.' + key + ' = ' + value)
 
   # Apply effects from settings
@@ -973,6 +1002,7 @@ try:
     if shared.Settings.CORRECT_OVERFLOWS != 1: 
       print >> sys.stderr, 'emcc: warning: setting CORRECT_OVERFLOWS to 1 for asm.js code generation'
       shared.Settings.CORRECT_OVERFLOWS = 1
+    assert not shared.Settings.PGO, 'cannot run PGO in ASM_JS mode'
 
   if shared.Settings.CORRECT_SIGNS >= 2 or shared.Settings.CORRECT_OVERFLOWS >= 2 or shared.Settings.CORRECT_ROUNDINGS >= 2:
     keep_llvm_debug = True # must keep debug info to do line-by-line operations 
@@ -982,7 +1012,7 @@ try:
     closure = False
 
   if minify_whitespace is None:
-    minify_whitespace = closure # if closure is run, minify whitespace
+    minify_whitespace = opt_level >= 2 and not keep_js_debug
 
   ## Compile source code to bitcode
 
@@ -1068,10 +1098,29 @@ try:
     def create_libc():
       if DEBUG: print >> sys.stderr, 'emcc: building libc for cache'
       o_s = []
-      for src in ['dlmalloc.c', os.path.join('libcxx', 'new.cpp')]:
+      libc_files = [
+        'dlmalloc.c',
+        os.path.join('libcxx', 'new.cpp'),
+        os.path.join('libc', 'stdlib', 'getopt_long.c'),
+        os.path.join('libc', 'gen', 'err.c'),
+        os.path.join('libc', 'gen', 'errx.c'),
+        os.path.join('libc', 'gen', 'warn.c'),
+        os.path.join('libc', 'gen', 'warnx.c'),
+        os.path.join('libc', 'gen', 'verr.c'),
+        os.path.join('libc', 'gen', 'verrx.c'),
+        os.path.join('libc', 'gen', 'vwarn.c'),
+        os.path.join('libc', 'gen', 'vwarnx.c'),
+        os.path.join('libc', 'stdlib', 'strtod.c'),
+      ];
+
+      prev_cxx = os.environ.get('EMMAKEN_CXX')
+      if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
+      for src in libc_files:
         o = in_temp(os.path.basename(src) + '.o')
         execute([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o], stdout=stdout, stderr=stderr)
         o_s.append(o)
+      if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
+
       shared.Building.link(o_s, in_temp('libc.bc'))
       return in_temp('libc.bc')
 
@@ -1219,6 +1268,9 @@ try:
       shared.Building.llvm_opt(in_temp(target_basename + '.bc'), link_opts)
       if DEBUG: save_intermediate('linktime', 'bc')
 
+  if save_bc:
+    shutil.copyfile(final, save_bc)
+
   # Prepare .ll for Emscripten
   if not LEAVE_INPUTS_RAW:
     final = shared.Building.llvm_dis(final, final + '.ll')
@@ -1331,14 +1383,15 @@ try:
     if DEBUG: print >> sys.stderr, 'emcc: running closure'
     final = shared.Building.closure_compiler(final)
     if DEBUG: save_intermediate('closure')
-  elif shared.Settings.RELOOP and not closure and not keep_js_debug:
-    # do this if closure is not enabled (it gives similar speedups), and we do not need to keep debug info around
-    js_optimizer_queue += ['registerize']
 
   if opt_level >= 1:
     if DEBUG: print >> sys.stderr, 'emcc: running post-closure post-opts'
     js_optimizer_queue += ['simplifyExpressionsPost']
 
+  if not closure and shared.Settings.RELOOP and not keep_js_debug:
+    # do this if closure is not enabled (it gives similar speedups), and we do not need to keep debug info around
+    js_optimizer_queue += ['registerize']
+
   if minify_whitespace:
     js_optimizer_queue += ['compress']
 
@@ -1346,6 +1399,11 @@ try:
 
   flush_js_optimizer_queue()
 
+  # Remove some trivial whitespace # TODO: do not run when compress has already been done on all parts of the code
+  src = open(final).read()
+  src = re.sub(r'\n+[ \n]*\n+', '\n', src)
+  open(final, 'w').write(src)
+
   # If we were asked to also generate HTML, do that
   if final_suffix == 'html':
     if DEBUG: print >> sys.stderr, 'emcc: generating HTML'
diff --git a/emscripten.py b/emscripten.py
index 0b9244c2..0698c783 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -33,16 +33,22 @@ NUM_CHUNKS_PER_CORE = 1.25
 MIN_CHUNK_SIZE = 1024*1024
 MAX_CHUNK_SIZE = float(os.environ.get('EMSCRIPT_MAX_CHUNK_SIZE') or 'inf') # configuring this is just for debugging purposes
 
-def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files)):
-  ll = ''.join(funcs) + '\n' + meta
+def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)):
   funcs_file = temp_files.get('.func_%d.ll' % i).name
-  open(funcs_file, 'w').write(ll)
+  f = open(funcs_file, 'w')
+  f.write(funcs)
+  funcs = None
+  f.write('\n')
+  f.write(meta)
+  f.close()
   out = jsrun.run_js(
     compiler,
     engine=compiler_engine,
     args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
-    stdout=subprocess.PIPE)
+    stdout=subprocess.PIPE,
+    cwd=path_from_root('src'))
   tempfiles.try_delete(funcs_file)
+  if DEBUG: print >> sys.stderr, '.'
   return out
 
 def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
@@ -153,7 +159,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     if out and DEBUG: print >> sys.stderr, '  loading pre from jcache'
   if not out:
     open(pre_file, 'w').write(pre_input)
-    out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE)
+    out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE,
+                       cwd=path_from_root('src'))
     if jcache:
       if DEBUG: print >> sys.stderr, '  saving pre to jcache'
       jcache.set(shortkey, keys, out)
@@ -185,6 +192,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     funcs, chunk_size,
     jcache.get_cachename('emscript_files') if jcache else None)
 
+  funcs = None
+
   if jcache:
     # load chunks from cache where we can # TODO: ignore small chunks
     cached_outputs = []
@@ -211,7 +220,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     if DEBUG: print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))
 
     commands = [
-      (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files)
+      (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)
       for i, chunk in enumerate(chunks)
     ]
 
@@ -220,6 +229,9 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
       outputs = pool.map(process_funcs, commands, chunksize=1)
     elif len(chunks) == 1:
       outputs = [process_funcs(commands[0])]
+
+    commands = None
+
   else:
     outputs = []
 
@@ -232,11 +244,13 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
       jcache.set(shortkey, keys, outputs[i])
     if out and DEBUG and len(chunks) > 0: print >> sys.stderr, '  saving %d funcchunks to jcache' % len(chunks)
 
+  chunks = None
+
   if jcache: outputs += cached_outputs # TODO: preserve order
 
   outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
   for output in outputs:
-    assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0]
+    assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:]
 
   if DEBUG: print >> sys.stderr, '  emscript: phase 2 took %s seconds' % (time.time() - t)
   if DEBUG: t = time.time()
@@ -265,7 +279,7 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     if len(parts) > 1:
       pre = parts[0]
       outputs.append([parts[1]])
-  funcs_js = ''.join([output[0] for output in outputs])
+  funcs_js = [''.join([output[0] for output in outputs])] # this will be a list of things, so we do not do string appending as we add more
 
   outputs = None
   if DEBUG: print >> sys.stderr, '  emscript: phase 2b took %s seconds' % (time.time() - t)
@@ -304,11 +318,16 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
   if DEBUG: t = time.time()
   post_file = temp_files.get('.post.ll').name
   open(post_file, 'w').write('\n') # no input, just processing of forwarded data
-  out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE)
+  out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE,
+                     cwd=path_from_root('src'))
   post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data?
   last_forwarded_json = json.loads(last_forwarded_data)
 
   if settings.get('ASM_JS'):
+    post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n')
+    post = post_rest
+    funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n']
+
     simple = os.environ.get('EMCC_SIMPLE_ASM')
     class Counter:
       i = 0
@@ -316,11 +335,12 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     del last_forwarded_json['Fu