refactoring and fixes for running more tests in new o1 and o2 modes

6 files changed, 141 insertions, 74 deletions

diff --git a/emcc b/emcc
index 2d21ce87..b14810bb 100755
--- a/emcc
+++ b/emcc
@@ -443,6 +443,14 @@ try:
 
   final = shared.Building.emscripten(in_temp(target_basename + '.bc'), append_ext=False)
 
+  # Apply a source code transformation, if requested
+  source_transform = os.environ.get('EMCC_JS_PROCESSOR')
+  if source_transform:
+    exec source_transform in locals()
+    shutil.copyfile(final, final + '.tr.js')
+    final += '.tr.js'
+    process(final)
+
   if opt_level >= 1:
     # js optimizer
     if DEBUG: print >> sys.stderr, 'emcc: running pre-closure post-opts'
diff --git a/tests/runner.py b/tests/runner.py
index eb95a516..3a104609 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -91,7 +91,7 @@ class RunnerCore(unittest.TestCase):
         Building.llvm_dis(filename)
 
   # Build JavaScript code from source code
-  def build(self, src, dirname, filename, output_processor=None, main_file=None, additional_files=[], libraries=[], includes=[], build_ll_hook=None, extra_emscripten_args=[]):
+  def build(self, src, dirname, filename, output_processor=None, main_file=None, additional_files=[], libraries=[], includes=[], build_ll_hook=None, extra_emscripten_args=[], post_build=None):
     # Copy over necessary files for compiling the source
     if main_file is None:
       f = open(filename, 'w')
@@ -141,8 +141,19 @@ class RunnerCore(unittest.TestCase):
     # BC => JS
     if self.emcc_args is None:
       Building.emscripten(filename, append_ext=True, extra_args=extra_emscripten_args)
+      if post_build is not None:
+        exec post_build in locals()
+        shutil.copyfile(filename + '.o.js', filename + '.o.js.prepost.js')
+        process(filename + '.o.js')
     else:
-      Building.emcc(filename + '.o.ll', self.emcc_args, filename + '.o.js')
+      if post_build is not None:
+        os.environ['EMCC_JS_PROCESSOR'] = post_build
+      else:
+        try:
+          del os.environ['EMCC_JS_PROCESSOR']
+        except:
+          pass
+      Building.emcc(filename + '.o.ll', Settings.serialize() + self.emcc_args, filename + '.o.js')
 
     if output_processor is not None:
       output_processor(open(filename + '.o.js').read())
@@ -240,10 +251,7 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
         filename = os.path.join(dirname, basename)
         if not no_build:
           self.build(src, dirname, filename, main_file=main_file, additional_files=additional_files, libraries=libraries, includes=includes,
-                     build_ll_hook=build_ll_hook, extra_emscripten_args=extra_emscripten_args)
-
-        if post_build is not None:
-          post_build(filename + '.o.js')
+                     build_ll_hook=build_ll_hook, extra_emscripten_args=extra_emscripten_args, post_build=post_build)
 
         # If not provided with expected output, then generate it right now, using lli
         if expected_output is None:
@@ -1631,9 +1639,11 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
           }
           '''
 
-        def check(filename):
-          src = open(filename, 'r').read()
-          # TODO: restore this (see comment in emscripten.h) assert '// hello from the source' in src
+        check = '''
+def process(filename):
+  src = open(filename, 'r').read()
+  # TODO: restore this (see comment in emscripten.h) assert '// hello from the source' in src
+        '''
 
         self.do_run(src, 'hello world!\n*100*', post_build=check)
 
@@ -2348,12 +2358,14 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
         }
         '''
       Settings.BUILD_AS_SHARED_LIB = 0
-      def add_pre_run_and_checks(filename):
-        src = open(filename, 'r').read().replace(
-          '// {{PRE_RUN_ADDITIONS}}',
-          '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
-        )
-        open(filename, 'w').write(src)
+      add_pre_run_and_checks = '''
+def process(filename):
+  src = open(filename, 'r').read().replace(
+    '// {{PRE_RUN_ADDITIONS}}',
+    "FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);"
+  )
+  open(filename, 'w').write(src)
+'''
       self.do_run(src, 'Constructing main object.\nConstructing lib object.\n',
                    post_build=add_pre_run_and_checks)
 
@@ -4824,7 +4836,7 @@ class %s(T):
 
     self.emcc_args = %s
     if self.emcc_args is not None:
-      Settings.load_settings(self.emcc_args)
+      Settings.load(self.emcc_args)
       Building.LLVM_OPTS = 0
       return
 
@@ -4855,8 +4867,7 @@ class %s(T):
     Settings.BUILD_AS_SHARED_LIB = 0
     Settings.RUNTIME_LINKED_LIBS = []
     Settings.CATCH_EXIT_CODE = 0
-    Settings.TOTAL_MEMORY = Settings.FAST_MEMORY = None
-    Settings.EMULATE_UNALIGNED_ACCESSES = Settings.USE_TYPED_ARRAYS == 2 and Building.LLVM_OPTS == 2
+    Settings.EMULATE_UNALIGNED_ACCESSES = int(Settings.USE_TYPED_ARRAYS == 2 and Building.LLVM_OPTS == 2)
     Settings.DOUBLE_MODE = 1 if Settings.USE_TYPED_ARRAYS and Building.LLVM_OPTS == 0 else 0
     if Settings.USE_TYPED_ARRAYS == 2:
       Settings.I64_MODE = 1
diff --git a/tools/js-optimizer.js b/tools/js-optimizer.js
index bc5d5d8c..665ba0db 100644
--- a/tools/js-optimizer.js
+++ b/tools/js-optimizer.js
@@ -433,52 +433,57 @@ function hoistMultiples(ast) {
   var more = true;
   while (more) {
     more = false;
-    traverse(ast, function(node, type) {
-      if (type == 'if' && node[2][0] == 'block' && node[2][1].length == 0) {
-        more = true;
-        if (node[2][2]) { // if there is an else, return that
-          return node[2][2];
-        } else {
-          return emptyNode();
-        }
-      } else if (type == 'block' && !node[1]) {
-        return emptyNode();
-      } else if (type == 'block' && (node[1].length == 0 || (node[1].length == 1 && jsonCompare(node[1][0], emptyNode())))) {
-        more = true;
-        return emptyNode();
-      } else if (type == 'block' && node[1].length == 1 && node[1][0][0] == 'block') {
-        more = true;
-        return node[1][0];
-      } else if (type == 'stat' && node[1][0] == 'block') {
-        more = true;
-        return node[1];
-      } else if (type == 'block') {
-        var pre = node[1].length;
-        node[1] = node[1].filter(function(blockItem) { return !jsonCompare(blockItem, emptyNode()) });
-        if (node[1].length < pre) {
-          more = true;
-          return node;
-        }
-      } else if (type == 'defun' && node[3].length == 1 && node[3][0][0] == 'block') {
-        more = true;
-        node[3] = node[3][0][1];
-        return node;
-      } else if (type == 'defun') {
-        var pre = node[3].length;
-        node[3] = node[3].filter(function(blockItem) { return !jsonCompare(blockItem, emptyNode()) });
-        if (node[3].length < pre) {
-          more = true;
-          return node;
-        }
-      } else if (type == 'do' && node[1][0] == 'num' && jsonCompare(node[2], emptyNode())) {
-        more = true;
-        return emptyNode();
-      } else if (type == 'label' && jsonCompare(node[2], emptyNode())) {
-        more = true;
-        return emptyNode();
-      } else if (type == 'if' && jsonCompare(node[3], emptyNode())) { // empty else clauses
-        node[3] = null;
-        return node;
+    ast[1].forEach(function(node, i) {
+      var type = node[0];
+      if (type == 'defun') {
+        traverse(node, function(node, type) {
+          if (type == 'if' && node[2][0] == 'block' && node[2][1].length == 0) {
+            more = true;
+            if (node[2][2]) { // if there is an else, return that
+              return node[2][2];
+            } else {
+              return emptyNode();
+            }
+          } else if (type == 'block' && !node[1]) {
+            return emptyNode();
+          } else if (type == 'block' && (node[1].length == 0 || (node[1].length == 1 && jsonCompare(node[1][0], emptyNode())))) {
+            more = true;
+            return emptyNode();
+          } else if (type == 'block' && node[1].length == 1 && node[1][0][0] == 'block') {
+            more = true;
+            return node[1][0];
+          } else if (type == 'stat' && node[1][0] == 'block') {
+            more = true;
+            return node[1];
+          } else if (type == 'block') {
+            var pre = node[1].length;
+            node[1] = node[1].filter(function(blockItem) { return !jsonCompare(blockItem, emptyNode()) });
+            if (node[1].length < pre) {
+              more = true;
+              return node;
+            }
+          } else if (type == 'defun' && node[3].length == 1 && node[3][0][0] == 'block') {
+            more = true;
+            node[3] = node[3][0][1];
+            return node;
+          } else if (type == 'defun') {
+            var pre = node[3].length;
+            node[3] = node[3].filter(function(blockItem) { return !jsonCompare(blockItem, emptyNode()) });
+            if (node[3].length < pre) {
+              more = true;
+              return node;
+            }
+          } else if (type == 'do' && node[1][0] == 'num' && jsonCompare(node[2], emptyNode())) {
+            more = true;
+            return emptyNode();
+          } else if (type == 'label' && jsonCompare(node[2], emptyNode())) {
+            more = true;
+            return emptyNode();
+          } else if (type == 'if' && jsonCompare(node[3], emptyNode())) { // empty else clauses
+            node[3] = null;
+            return node;
+          }
+        });
       }
     });
   }
diff --git a/tools/shared.py b/tools/shared.py
index 3567e28e..adf3004e 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -273,8 +273,9 @@ class Settings:
       QUANTUM_SIZE = 4
       reset = Settings.reset
 
+      # Given some emcc-type args (-O3, -s X=Y, etc.), fill Settings with the right settings
       @classmethod
-      def load_settings(self, args):
+      def load(self, args):
         # Load the JS defaults into python
         settings = open(path_from_root('src', 'settings.js')).read().replace('var ', 'Settings.').replace('//', '#')
         exec settings in globals()
@@ -288,6 +289,16 @@ class Settings:
           if args[i] == '-s':
             exec 'Settings.' + args[i+1] in globals() # execute the setting
 
+      # Transforms the Settings information into emcc-compatible args (-s X=Y, etc.). Basically
+      # the reverse of load_settings, except for -Ox which is relevant there but not here
+      @classmethod
+      def serialize(self):
+        ret = []
+        for key, value in Settings.__dict__.iteritems():
+          if key == key.upper(): # this is a hack. all of our settings are ALL_CAPS, python internals are not
+            ret += ['-s', key + '=' + json.dumps(value)]
+        return ret
+
       @classmethod
       def apply_opt_level(self, opt_level, noisy=False):
         if opt_level >= 1:
@@ -469,15 +480,7 @@ class Building:
       extra_args += ['-H', 'libc/fcntl.h,libc/sys/unistd.h,poll.h,libc/math.h,libc/langinfo.h,libc/time.h']
 
     # Run Emscripten
-    exported_settings = {}
-    for setting in ['QUANTUM_SIZE', 'RELOOP', 'MICRO_OPTS', 'ASSERTIONS', 'USE_TYPED_ARRAYS', 'SAFE_HEAP', 'CHECK_OVERFLOWS', 'CORRECT_OVERFLOWS', 'CORRECT_SIGNS', 'CHECK_SIGNS', 'CORRECT_OVERFLOWS_LINES', 'CORRECT_SIGNS_LINES', 'CORRECT_ROUNDINGS', 'CORRECT_ROUNDINGS_LINES', 'INVOKE_RUN', 'SAFE_HEAP_LINES', 'INIT_STACK', 'PGO', 'EXPORTED_FUNCTIONS', 'EXPORTED_GLOBALS', 'BUILD_AS_SHARED_LIB', 'RUNTIME_LINKED_LIBS', 'INCLUDE_FULL_LIBRARY', 'RUNTIME_TYPE_INFO', 'DISABLE_EXCEPTION_CATCHING', 'TOTAL_MEMORY', 'FAST_MEMORY', 'EXCEPTION_DEBUG', 'PROFILE', 'I64_MODE', 'EMULATE_UNALIGNED_ACCESSES', 'CATCH_EXIT_CODE', 'USE_FHEAP']:
-      try:
-        value = eval('Settings.' + setting)
-        if value is not None:
-          exported_settings[setting] = value
-      except:
-        pass
-    settings = ['-s %s=%s' % (k, json.dumps(v)) for k, v in exported_settings.items()]
+    settings = Settings.serialize()
     compiler_output = timeout_run(Popen(['python', EMSCRIPTEN, filename + ('.o.ll' if append_ext else ''), '-o', filename + '.o.js'] + settings + extra_args, stdout=PIPE), None, 'Compiling')
     #print compiler_output
 
diff --git a/tools/test-js-optimizer-output.js b/tools/test-js-optimizer-output.js
index 4c5b073c..aa494a05 100644
--- a/tools/test-js-optimizer-output.js
+++ b/tools/test-js-optimizer-output.js
@@ -145,4 +145,23 @@ function hoisting() {
     somethingElse();
   }
 }
+var FS = {
+  absolutePath: (function(relative, base) {
+    if (typeof relative !== "string") return null;
+    if (base === undefined) base = FS.currentPath;
+    if (relative && relative[0] == "/") base = "";
+    var full = base + "/" + relative;
+    var parts = full.split("/").reverse();
+    var absolute = [ "" ];
+    while (parts.length) {
+      var part = parts.pop();
+      if (part == "" || part == ".") {} else if (part == "..") {
+        if (absolute.length > 1) absolute.pop();
+      } else {
+        absolute.push(part);
+      }
+    }
+    return absolute.length == 1 ? "/" : absolute.join("/");
+  })
+};
 // EMSCRIPTEN_GENERATED_FUNCTIONS: ["abc", "xyz", "xyz2", "expr", "loopy", "bits", "maths", "hoisting"]
diff --git a/tools/test-js-optimizer.js b/tools/test-js-optimizer.js
index 8c2ad183..8d10d222 100644
--- a/tools/test-js-optimizer.js
+++ b/tools/test-js-optimizer.js
@@ -160,4 +160,25 @@ function hoisting() {
     somethingElse();
   }
 }
+var FS = {
+  absolutePath: function(relative, base) { // Don't touch this!
+    if (typeof relative !== 'string') return null;
+    if (base === undefined) base = FS.currentPath;
+    if (relative && relative[0] == '/') base = '';
+    var full = base + '/' + relative;
+    var parts = full.split('/').reverse();
+    var absolute = [''];
+    while (parts.length) {
+      var part = parts.pop();
+      if (part == '' || part == '.') {
+        // Nothing.
+      } else if (part == '..') {
+        if (absolute.length > 1) absolute.pop();
+      } else {
+        absolute.push(part);
+      }
+    }
+    return absolute.length == 1 ? '/' : absolute.join('/');
+  }
+}
 // EMSCRIPTEN_GENERATED_FUNCTIONS: ["abc", "xyz", "xyz2", "expr", "loopy", "bits", "maths", "hoisting"]