Merge branch 'source-maps' of github.com:int3/emscripten into int3-source-maps

Conflicts:
	tools/js-optimizer.js

51 files changed, 5507 insertions, 425 deletions

diff --git a/emcc b/emcc
index 3bc35aa4..0c151575 100755
--- a/emcc
+++ b/emcc
@@ -49,7 +49,7 @@ emcc can be influenced by a few environment variables:
 
 import os, sys, shutil, tempfile, subprocess, shlex, time, re, logging
 from subprocess import PIPE, STDOUT
-from tools import shared
+from tools import shared, jsrun
 from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename
 from tools.response_file import read_response_file
 
@@ -203,10 +203,18 @@ Options that are modified or new in %s include:
                             -g2  Preserve function names
                             -g3  Preserve variable names
                             -g4  Preserve LLVM debug info (if -g was
-                                 used when compiling the C/C++ sources)
-                                 and show line number debug comments.
-                                 This is the highest level of debuggability.
-                                 (default in -O0)
+                                 used when compiling the C/C++ sources),
+                                 show line number debug comments, and
+                                 generate source maps. This is the highest
+                                 level of debuggability. Note that this
+                                 may make -O1 and above significantly
+                                 slower because JS optimization will be
+                                 limited to 1 core.  (default in -O0)
+
+  -g2                      Like -g1, but we generate source maps as well,
+                           and we preserve comments even with -O1 and above.
+                           Note that this may be considerably slower because
+                           JS optimization is limited to a single core.
 
   --typed-arrays <mode>    0: No typed arrays
                            1: Parallel typed arrays
@@ -731,6 +739,14 @@ try:
 
   settings_changes = []
 
+  def validate_arg_level(level_string, max_level, err_msg):
+    try:
+      level = int(level_string)
+      assert 0 <= level <= max_level
+    except:
+      raise Exception(err_msg)
+    return level
+
   for i in range(len(newargs)):
     newargs[i] = newargs[i].strip() # On Windows Vista (and possibly others), excessive spaces in the command line leak into the items in this array, so trim e.g. 'foo.cpp ' -> 'foo.cpp'
     if newargs[i].startswith('-O'):
@@ -739,11 +755,7 @@ try:
       if requested_level == 's':
         requested_level = 2
         settings_changes.append('INLINING_LIMIT=50')
-      try:
-        opt_level = int(requested_level)
-        assert 0 <= opt_level <= 3
-      except:
-        raise Exception('Invalid optimization level: ' + newargs[i])
+      opt_level = validate_arg_level(requested_level, 3, 'Invalid optimization level: ' + newargs[i])
       newargs[i] = ''
     elif newargs[i].startswith('--llvm-opts'):
       check_bad_eq(newargs[i])
@@ -787,12 +799,8 @@ try:
       newargs[i+1] = ''
     elif newargs[i].startswith('-g'):
       requested_level = newargs[i][2:] or '3'
-      try:
-        debug_level = int(requested_level)
-        assert 0 <= debug_level <= 4
-      except:
-        raise Exception('Invalid debug level: ' + newargs[i])
-      newargs[i] = '-g' # discard level for clang args
+      debug_level = validate_arg_level(requested_level, 4, 'Invalid debug level: ' + newargs[i])
+      newargs[i] = '-g' # we'll need this to get LLVM debug info
     elif newargs[i] == '--bind':
       bind = True
       newargs[i] = ''
@@ -885,6 +893,11 @@ try:
   if opt_level == 0: debug_level = 4
   if closure is None and opt_level == 3: closure = True
 
+  # TODO: support source maps with js_transform
+  if js_transform and debug_level >= 4:
+    logging.warning('disabling source maps because a js transform is being done')
+    debug_level = 3
+
   if DEBUG: start_time = time.time() # done after parsing arguments, which might affect debug state
 
   if closure:
@@ -1409,6 +1422,7 @@ try:
   # Optimize, if asked to
   if not LEAVE_INPUTS_RAW:
     link_opts = [] if debug_level >= 4 else ['-strip-debug'] # remove LLVM debug if we are not asked for it
+
     if llvm_opts > 0:
       if not os.environ.get('EMCC_OPTIMIZE_NORMALLY'):
         shared.Building.llvm_opt(in_temp(target_basename + '.bc'), llvm_opts)
@@ -1496,9 +1510,11 @@ try:
     final += '.tr.js'
     posix = True if not shared.WINDOWS else False
     logging.debug('applying transform: %s' % js_transform)
-    execute(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)])
+    subprocess.check_call(shlex.split(js_transform, posix=posix) + [os.path.abspath(final)])
     if DEBUG: save_intermediate('transformed')
 
+  js_transform_tempfiles = [final]
+
   # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
   js_optimizer_queue = []
   def flush_js_optimizer_queue():
@@ -1508,7 +1524,8 @@ try:
         if shared.Settings.ASM_JS:
           js_optimizer_queue = ['asm'] + js_optimizer_queue
         logging.debug('applying js optimization passes: %s', js_optimizer_queue)
-        final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache)
+        final = shared.Building.js_optimizer(final, js_optimizer_queue, jcache, debug_level >= 4)
+        js_transform_tempfiles.append(final)
         if DEBUG: save_intermediate('js_opts')
       else:
         for name in js_optimizer_queue:
@@ -1516,7 +1533,8 @@ try:
           if shared.Settings.ASM_JS:
             passes = ['asm'] + passes
           logging.debug('applying js optimization pass: %s', passes)
-          final = shared.Building.js_optimizer(final, passes, jcache)
+          final = shared.Building.js_optimizer(final, passes, jcache, debug_level >= 4)
+          js_transform_tempfiles.append(final)
           save_intermediate(name)
       js_optimizer_queue = []
 
@@ -1525,7 +1543,8 @@ try:
 
     if DEBUG == '2':
       # Clean up the syntax a bit
-      final = shared.Building.js_optimizer(final, [], jcache)
+      final = shared.Building.js_optimizer(final, [], jcache, debug_level >= 4)
+      js_transform_tempfiles.append(final)
       if DEBUG: save_intermediate('pretty')
 
     def get_eliminate():
@@ -1543,6 +1562,8 @@ try:
     flush_js_optimizer_queue()
 
     logging.debug('running closure')
+    # no need to add this to js_transform_tempfiles, because closure and
+    # debug_level > 0 are never simultaneously true
     final = shared.Building.closure_compiler(final)
     if DEBUG: save_intermediate('closure')
 
@@ -1590,6 +1611,7 @@ try:
       src = re.sub('/\* memory initializer \*/ allocate\(([\d,\.concat\(\)\[\]\\n ]+)"i8", ALLOC_NONE, Runtime\.GLOBAL_BASE\)', repl, src, count=1)
       open(final + '.mem.js', 'w').write(src)
       final += '.mem.js'
+      js_transform_tempfiles[-1] = final # simple text substitution preserves comment line number mappings
       if DEBUG:
         if os.path.exists(memfile):
           save_intermediate('meminit')
@@ -1597,12 +1619,25 @@ try:
         else:
           logging.debug('did not see memory initialization')
 
+  def generate_source_map(map_file_base_name, offset=0):
+    jsrun.run_js(shared.path_from_root('tools', 'source-maps', 'sourcemapper.js'),
+      shared.NODE_JS, js_transform_tempfiles +
+        ['--sourceRoot', os.getcwd(),
+         '--mapFileBaseName', map_file_base_name,
+         '--offset', str(offset)])
+
   # If we were asked to also generate HTML, do that
   if final_suffix == 'html':
     logging.debug('generating HTML')
     shell = open(shell_path).read()
     html = open(target, 'w')
     if not Compression.on:
+      if debug_level >= 4:
+        match = re.match('.*?<script[^>]*>{{{ SCRIPT_CODE }}}</script>', shell,
+            re.DOTALL)
+        if match is None:
+          raise RuntimeError('Could not find script insertion point')
+        generate_source_map(target, match.group().count('\n'))
       html.write(shell.replace('{{{ SCRIPT_CODE }}}', open(final).read()))
     else:
       # Compress the main code
@@ -1673,6 +1708,8 @@ try:
         from tools.split import split_javascript_file
         split_javascript_file(final, unsuffixed(target), split_js_file)
     else:
+        if debug_level >= 4: generate_source_map(target)
+
         # copy final JS to output
         shutil.move(final, target)
 
diff --git a/src/jsifier.js b/src/jsifier.js
index ac6c259b..d6cd188c 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -695,7 +695,9 @@ function JSify(data, functionsOnly, givenFunctions) {
               }
             }
             i++;
-            return JS + (Debugging.on ? Debugging.getComment(line.lineNum) : '');
+            // invoke instructions span two lines, and the debug info is located
+            // on the second line, hence the +1
+            return JS + (Debugging.on ? Debugging.getComment(line.lineNum + (line.intertype === 'invoke' ? 1 : 0)) : '');
           })
                                   .join('\n')
                                   .split('\n') // some lines include line breaks
diff --git a/src/shell.html b/src/shell.html
index f7eb9e1f..00765271 100644
--- a/src/shell.html
+++ b/src/shell.html
@@ -87,10 +87,6 @@
       };
       Module.setStatus('Downloading...');
     </script>      
-    <script type='text/javascript'>
-
-      {{{ SCRIPT_CODE }}}
-
-    </script>
+    <script type='text/javascript'>{{{ SCRIPT_CODE }}}</script>
   </body>
 </html>
diff --git a/tests/runner.py b/tests/runner.py
index 2c459f6f..c55c96fd 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -170,16 +170,13 @@ class RunnerCore(unittest.TestCase):
       post1 = post_build
       post2 = None
 
-    def run_post(post):
-      if not post: return
-      exec post in locals()
-      shutil.copyfile(filename + '.o.js', filename + '.o.js.prepost.js')
-      process(filename + '.o.js')
-
     if self.emcc_args is None:
       Building.emscripten(filename, append_ext=True, extra_args=extra_emscripten_args)
-      run_post(post1)
-      run_post(post2)
+      if post1:
+        exec post1 in locals()
+        shutil.copyfile(filename + '.o.js', filename + '.o.js.prepost.js')
+        process(filename + '.o.js')
+      if post2: post2(filename + '.o.js')
     else:
       transform_args = []
       if post1:
@@ -196,7 +193,7 @@ process(sys.argv[1])
         transform.close()
         transform_args = ['--js-transform', "%s %s" % (PYTHON, transform_filename)]
       Building.emcc(filename + '.o.ll', Settings.serialize() + self.emcc_args + transform_args + Building.COMPILER_TEST_OPTS, filename + '.o.js')
-      run_post(post2)
+      if post2: post2(filename + '.o.js')
 
   # Build JavaScript code from source code
   def build(self, src, dirname, filename, output_processor=None, main_file=None, additional_files=[], libraries=[], includes=[], build_ll_hook=None, extra_emscripten_args=[], post_build=None):
@@ -9215,97 +9212,92 @@ def process(filename):
   src.close()
 '''
 
-        post3 = '''
-def process(filename):
-  script_src_2 = \'\'\'
-          var sme = new Module.Parent(42);
-          sme.mulVal(2);
-          Module.print('*')
-          Module.print(sme.getVal());
-
-          Module.print('c1');
-
-          var c1 = new Module.Child1();
-          Module.print(c1.getVal());
-          c1.mulVal(2);
-          Module.print(c1.getVal());
-          Module.print(c1.getValSqr());
-          Module.print(c1.getValSqr(3));
-          Module.print(c1.getValTimes()); // default argument should be 1
-          Module.print(c1.getValTimes(2));
-
-          Module.print('c1 v2');
-
-          c1 = new Module.Child1(8); // now with a parameter, we should handle the overloading automatically and properly and use constructor #2
-          Module.print(c1.getVal());
-          c1.mulVal(2);
-          Module.print(c1.getVal());
-          Module.print(c1.getValSqr());
-          Module.print(c1.getValSqr(3));
-
-          Module.print('c2')
-
-          var c2 = new Module.Child2();
-          Module.print(c2.getVal());
-          c2.mulVal(2);
-          Module.print(c2.getVal());
-          Module.print(c2.getValCube());
-          var succeeded;
-          try {
-            succeeded = 0;
-            Module.print(c2.doSomethingSecret()); // should fail since private
-            succeeded = 1;
-          } catch(e) {}
-          Module.print(succeeded);
-          try {
-            succeeded = 0;
-            Module.print(c2.getValSqr()); // function from the other class
-            succeeded = 1;
-          } catch(e) {}
-          Module.print(succeeded);
-          try {
-            succeeded = 0;
-            c2.getValCube(); // sanity
-            succeeded = 1;
-          } catch(e) {}
-          Module.print(succeeded);
-
-          Module.Child2.prototype.printStatic(); // static calls go through the prototype
-
-          // virtual function
-          c2.virtualFunc();
-          Module.Child2.prototype.runVirtualFunc(c2);
-          c2.virtualFunc2();
-
-''' + ('''
-          // extend the class from JS
-          var c3 = new Module.Child2;
-          Module.customizeVTable(c3, [{
-            original: Module.Child2.prototype.virtualFunc,
-            replacement: function() {
-              Module.print('*js virtualf replacement*');
-            }
-          }, {
-            original: Module.Child2.prototype.virtualFunc2,
-            replacement: function() {
-              Module.print('*js virtualf2 replacement*');
-            }
-          }]);
-          c3.virtualFunc();
-          Module.Child2.prototype.runVirtualFunc(c3);
-          c3.virtualFunc2();
-
-          c2.virtualFunc(); // original should remain the same
-          Module.Child2.prototype.runVirtualFunc(c2);
-          c2.virtualFunc2();
-''') + '''
-
-          Module.print('*ok*');
-        \'\'\'
-  src = open(filename, 'a')
-  src.write(script_src_2 + '\\n')
-  src.close()
-'''
+        def post3(filename):
+          script_src_2 = '''
+            var sme = new Module.Parent(42);
+            sme.mulVal(2);
+            Module.print('*')
+            Module.print(sme.getVal());
+
+            Module.print('c1');
+
+            var c1 = new Module.Child1();
+            Module.print(c1.getVal());
+            c1.mulVal(2);
+            Module.print(c1.getVal());
+            Module.print(c1.getValSqr());
+            Module.print(c1.getValSqr(3));
+            Module.print(c1.getValTimes()); // default argument should be 1
+            Module.print(c1.getValTimes(2));
+
+            Module.print('c1 v2');
+
+            c1 = new Module.Child1(8); // now with a parameter, we should handle the overloading automatically and properly and use constructor #2
+            Module.print(c1.getVal());
+            c1.mulVal(2);
+            Module.print(c1.getVal());
+            Module.print(c1.getValSqr());
+            Module.print(c1.getValSqr(3));
+
+            Module.print('c2')
+
+            var c2 = new Module.Child2();
+            Module.print(c2.getVal());
+            c2.mulVal(2);
+            Module.print(c2.getVal());
+            Module.print(c2.getValCube());
+            var succeeded;
+            try {
+              succeeded = 0;
+              Module.print(c2.doSomethingSecret()); // should fail since private
+              succeeded = 1;
+            } catch(e) {}
+            Module.print(succeeded);
+            try {
+              succeeded = 0;
+              Module.print(c2.getValSqr()); // function from the other class
+              succeeded = 1;
+            } catch(e) {}
+            Module.print(succeeded);
+            try {
+              succeeded = 0;
+              c2.getValCube(); // sanity
+              succeeded = 1;
+            } catch(e) {}
+            Module.print(succeeded);
+
+            Module.Child2.prototype.printStatic(); // static calls go through the prototype
+
+            // virtual function
+            c2.virtualFunc();
+            Module.Child2.prototype.runVirtualFunc(c2);
+            c2.virtualFunc2();
+
+            // extend the class from JS
+            var c3 = new Module.Child2;
+            Module.customizeVTable(c3, [{
+              original: Module.Child2.prototype.virtualFunc,
+              replacement: function() {
+                Module.print('*js virtualf replacement*');
+              }
+            }, {
+              original: Module.Child2.prototype.virtualFunc2,
+              replacement: function() {
+                Module.print('*js virtualf2 replacement*');
+              }
+            }]);
+            c3.virtualFunc();
+            Module.Child2.prototype.runVirtualFunc(c3);
+            c3.virtualFunc2();
+
+            c2.virtualFunc(); // original should remain the same
+            Module.Child2.prototype.runVirtualFunc(c2);
+            c2.virtualFunc2();
+            Module.print('*ok*');
+          '''
+          src = open(filename, 'a')
+          src.write(script_src_2 + '\n')
+          src.close()
 
         Settings.RESERVED_FUNCTION_POINTERS = 20
 
@@ -9349,7 +9341,7 @@ Child2:9
 *virtualf*
 *virtualf2*''') + '''
 *ok*
-''', post_build=[post2, post3])
+''', post_build=(post2, post3))
 
     def test_scriptaclass_2(self):
         if self.emcc_args is None: return self.skip('requires emcc')
@@ -9590,7 +9582,120 @@ def process(filename):
         self.do_run(src, '*nothingatall*', post_build=post)
       except Exception, e:
         # This test *should* fail
-        assert 'Assertion failed' in str(e), str(e)
+        assert 'Assertion failed: x < 15' in str(e), str(e)
+
+    def test_source_map(self):
+      if Settings.USE_TYPED_ARRAYS != 2: return self.skip("doesn't pass without typed arrays")
+      if NODE_JS not in JS_ENGINES: return self.skip('sourcemapper requires Node to run')
+      if '-g' not in Building.COMPILER_TEST_OPTS: Building.COMPILER_TEST_OPTS.append('-g')
+
+      src = '''
+        #include <stdio.h>
+        #include <assert.h>
+
+        __attribute__((noinline)) int foo() {
+          printf("hi"); // line 6
+          return 1; // line 7
+        }
+
+        int main() {
+          printf("%d", foo()); // line 11
+          return 0; // line 12
+        }
+      '''
+
+      dirname = self.get_dir()
+      src_filename = os.path.join(dirname, 'src.cpp')
+      out_filename = os.path.join(dirname, 'a.out.js')
+      no_maps_filename = os.path.join(dirname, 'no-maps.out.js')
+
+      with open(src_filename, 'w') as f: f.write(src)
+      assert '-g4' not in Building.COMPILER_TEST_OPTS
+      Building.emcc(src_filename, Settings.serialize() + self.emcc_args +
+          Building.COMPILER_TEST_OPTS, out_filename)
+      # the file name may find its way into the generated code, so make sure we
+      # can do an apples-to-apples comparison by compiling with the same file name
+      shutil.move(out_filename, no_maps_filename)
+      with open(no_maps_filename) as f: no_maps_file = f.read()
+      no_maps_file = re.sub(' *//@.*$', '', no_maps_file, flags=re.MULTILINE)
+      Building.COMPILER_TEST_OPTS.append('-g4')
+
+      def build_and_check():
+        import json
+        Building.emcc(src_filename, Settings.serialize() + self.emcc_args +
+            Building.COMPILER_TEST_OPTS, out_filename, stderr=PIPE)
+        with open(out_filename) as f: out_file = f.read()
+        # after removing the @line and @sourceMappingURL comments, the build
+        # result should be identical to the non-source-mapped debug version.
+        # this is worth checking because the parser AST swaps strings for token
+        # objects when generating source maps, so we want to make sure the
+        # optimizer can deal with both types.
+        out_file = re.sub(' *//@.*$', '', out_file, flags=re.MULTILINE)
+        self.assertIdentical(no_maps_file, out_file)
+        map_filename = out_filename + '.map'
+        data = json.load(open(map_filename, 'r'))
+        self.assertIdentical(out_filename, data['file'])
+        self.assertIdentical(src_filename, data['sources'][0])
+        self.assertIdentical(src, data['sourcesContent'][0])
+        mappings = json.loads(jsrun.run_js(
+          path_from_root('tools', 'source-maps', 'sourcemap2json.js'),
+          tools.shared.NODE_JS, [map_filename]))
+        seen_lines = set()
+        for m in mappings:
+          self.assertIdentical(src_filename, m['source'])
+          seen_lines.add(m['originalLine'])
+        # ensure that all the 'meaningful' lines in the original code get mapped
+        assert seen_lines.issuperset([6, 7, 11, 12])
+
+      # EMCC_DEBUG=2 causes lots of intermediate files to be written, and so
+      # serves as a stress test for source maps because it needs to correlate
+      # line numbers across all those files.
+      old_emcc_debug = os.environ.get('EMCC_DEBUG', None)
+      os.environ.pop('EMCC_DEBUG', None)
+      try:
+        build_and_check()
+        os.environ['EMCC_DEBUG'] = '2'
+        build_and_check()
+      finally:
+        if old_emcc_debug is not None:
+          os.environ['EMCC_DEBUG'] = old_emcc_debug
+        else:
+          os.environ.pop('EMCC_DEBUG', None)
+
+    def test_exception_source_map(self):
+      if Settings.USE_TYPED_ARRAYS != 2: return self.skip("doesn't pass without typed arrays")
+      if '-g4' not in Building.COMPILER_TEST_OPTS: Building.COMPILER_TEST_OPTS.append('-g4')
+      if NODE_JS not in JS_ENGINES: return self.skip('sourcemapper requires Node to run')
+
+      src = '''
+        #include <stdio.h>
+
+        __attribute__((noinline)) void foo(int i) {
+            if (i < 10) throw i; // line 5
+        }
+
+        int main() {
+          int i;
+          scanf("%d", &i);
+          foo(i);
+          return 0;
+        }
+