Merge pull request #95 from jterrace/master

Parallelized test suite

2 files changed, 193 insertions, 184 deletions

diff --git a/AUTHORS b/AUTHORS
index 2053ea31..f241a71c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -5,3 +5,4 @@ under the licensing terms detailed in LICENSE.
 * Tim Dawborn <tim.dawborn@gmail.com>
 * Max Shawabkeh <max99x@gmail.com>
 * Sigmund Vik <sigmund_vik@yahoo.com>
+* Jeff Terrace <jterrace@gmail.com>
diff --git a/tests/runner.py b/tests/runner.py
index 1178c2bf..85c5b674 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -41,6 +41,12 @@ Settings.saveJS = 0
 # Core test runner class, shared between normal tests and benchmarks
 
 class RunnerCore(unittest.TestCase):
+  def setUp(self):
+    dirname = tempfile.mkdtemp(prefix="ems_" + self.__class__.__name__ + "_", dir=TEMP_DIR)
+    if not os.path.exists(dirname):
+      os.makedirs(dirname)
+    self.working_dir = dirname
+    
   def tearDown(self):
     if Settings.saveJS:
       for name in os.listdir(self.get_dir()):
@@ -48,15 +54,13 @@ class RunnerCore(unittest.TestCase):
           suff = '.'.join(name.split('.')[-2:])
           shutil.copy(os.path.join(self.get_dir(), name),
                       os.path.join(TEMP_DIR, self.id().replace('__main__.', '').replace('.test_', '.')+'.'+suff))
+    shutil.rmtree(self.get_dir())
 
   def skip(self, why):
     print >> sys.stderr, '<skipping: %s> ' % why,
 
   def get_dir(self):
-    dirname = TEMP_DIR + '/tmp' # tempfile.mkdtemp(dir=TEMP_DIR)
-    if not os.path.exists(dirname):
-      os.makedirs(dirname)
-    return dirname
+    return self.working_dir
 
   # Similar to LLVM::createStandardModulePasses()
   def pick_llvm_opts(self, optimization_level, handpicked=None):
@@ -107,7 +111,7 @@ class RunnerCore(unittest.TestCase):
     output = Popen([LLVM_LINK] + files + ['-o', target], stdout=PIPE, stderr=STDOUT).communicate()[0]
     assert output is None or 'Could not open input file' not in output, 'Linking error: ' + output
 
-  def prep_ll_test(self, filename, ll_file, force_recompile=False, build_ll_hook=None):
+  def prep_ll_run(self, filename, ll_file, force_recompile=False, build_ll_hook=None):
     if ll_file.endswith(('.bc', '.o')):
       if ll_file != filename + '.o':
         shutil.copy(ll_file, filename + '.o')
@@ -174,7 +178,7 @@ class RunnerCore(unittest.TestCase):
         raise Exception("Linkage error");
 
     # Finalize
-    self.prep_ll_test(filename, filename + '.o', build_ll_hook=build_ll_hook)
+    self.prep_ll_run(filename, filename + '.o', build_ll_hook=build_ll_hook)
 
     self.do_emscripten(filename, output_processor, extra_args=extra_emscripten_args)
 
@@ -254,7 +258,7 @@ if 'benchmark' not in str(sys.argv):
 
   class T(RunnerCore): # Short name, to make it more fun to use manually on the commandline
     ## Does a complete test - builds, runs, checks output, etc.
-    def do_test(self, src, expected_output=None, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]):
+    def do_run(self, src, expected_output=None, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]):
         #print 'Running test:', inspect.stack()[1][3].replace('test_', ''), '[%s,%s,%s]' % (COMPILER.split(os.sep)[-1], 'llvm-optimizations' if LLVM_OPTS else '', 'reloop&optimize' if RELOOP else '')
         if force_c or (main_file is not None and main_file[-2:]) == '.c':
           basename = 'src.c'
@@ -290,13 +294,13 @@ if 'benchmark' not in str(sys.argv):
         #shutil.rmtree(dirname) # TODO: leave no trace in memory. But for now nice for debugging
 
     # No building - just process an existing .ll file (or .bc, which we turn into .ll)
-    def do_ll_test(self, ll_file, expected_output=None, args=[], js_engines=None, output_nicerizer=None, post_build=None, force_recompile=False, build_ll_hook=None, extra_emscripten_args=[]):
+    def do_ll_run(self, ll_file, expected_output=None, args=[], js_engines=None, output_nicerizer=None, post_build=None, force_recompile=False, build_ll_hook=None, extra_emscripten_args=[]):
 
       filename = os.path.join(self.get_dir(), 'src.cpp')
 
-      self.prep_ll_test(filename, ll_file, force_recompile, build_ll_hook)
+      self.prep_ll_run(filename, ll_file, force_recompile, build_ll_hook)
       self.do_emscripten(filename, extra_args=extra_emscripten_args)
-      self.do_test(None,
+      self.do_run(None,
                    expected_output,
                    args,
                    no_build=True,
@@ -313,7 +317,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, 'hello, world!')
+        self.do_run(src, 'hello, world!')
 
     def test_intvars(self):
         src = '''
@@ -366,7 +370,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*5,23,10,19,121,1,37,1,0*\n0:-1,1:134217727,2:4194303,3:131071,4:4095,5:127,6:3,7:0,8:0*\n*56,09*\nfixed:320434\n*21*')
+        self.do_run(src, '*5,23,10,19,121,1,37,1,0*\n0:-1,1:134217727,2:4194303,3:131071,4:4095,5:127,6:3,7:0,8:0*\n*56,09*\nfixed:320434\n*21*')
 
     def test_sintvars(self):
         global CORRECT_SIGNS; CORRECT_SIGNS = 1 # Relevant to this test
@@ -397,7 +401,7 @@ if 'benchmark' not in str(sys.argv):
         '''
         output = '*32780,32522,258*\n*258,2*\n*32780,32999,-219*\n*65317,510*'
         global CORRECT_OVERFLOWS; CORRECT_OVERFLOWS = 0 # We should not need overflow correction to get this right
-        self.do_test(src, output, force_c=True)
+        self.do_run(src, output, force_c=True)
 
     def test_bigint(self):
         if USE_TYPED_ARRAYS != 0: return self.skip('Typed arrays truncate i64')
@@ -421,7 +425,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*245127260211081,579378795077769,808077213656969,16428841631881,791648372025088*\n*13.00,6.00,3.00,*3*')
+        self.do_run(src, '*245127260211081,579378795077769,808077213656969,16428841631881,791648372025088*\n*13.00,6.00,3.00,*3*')
 
     def test_unsigned(self):
         global CORRECT_SIGNS; CORRECT_SIGNS = 1 # We test for exactly this sort of thing here
@@ -462,7 +466,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src)#, '*4294967295,0,4294967219*\n*-1,1,-1,1*\n*-2,1,-2,1*\n*246,296*\n*1,0*')
+        self.do_run(src)#, '*4294967295,0,4294967219*\n*-1,1,-1,1*\n*-2,1,-2,1*\n*246,296*\n*1,0*')
 
         # Now let's see some code that should just work in USE_TYPED_ARRAYS == 2, but requires
         # corrections otherwise
@@ -516,7 +520,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*255*\n*65535*\n*-1*\n*-1*\n*-1*')
+        self.do_run(src, '*255*\n*65535*\n*-1*\n*-1*\n*-1*')
 
     def test_bitfields(self):
         global SAFE_HEAP; SAFE_HEAP = 0 # bitfields do loads on invalid areas, by design
@@ -543,7 +547,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*0,0,0,0,0,1,0,1,0,0,1,1,1,0,0,1,0,1,1,1,0,1,1,1,*')
+        self.do_run(src, '*0,0,0,0,0,1,0,1,0,0,1,1,1,0,0,1,0,1,1,1,0,1,1,1,*')
 
     def test_floatvars(self):
         src = '''
@@ -566,7 +570,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*1,10,10.5,1,1.2340,0.00*')
+        self.do_run(src, '*1,10,10.5,1,1.2340,0.00*')
 
     def test_math(self):
         src = '''
@@ -587,12 +591,12 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*3.14,-3.14,inf,-inf,0,0,0,1,0,1,1,0*')
+        self.do_run(src, '*3.14,-3.14,inf,-inf,0,0,0,1,0,1,1,0*')
 
     def test_math_hyperbolic(self):
         src = open(path_from_root('tests', 'hyperbolic', 'src.c'), 'r').read()
         expected = open(path_from_root('tests', 'hyperbolic', 'output.txt'), 'r').read()
-        self.do_test(src, expected)
+        self.do_run(src, expected)
 
     def test_getgep(self):
         # Generated code includes getelementptr (getelementptr, 0, 1), i.e., GEP as the first param to GEP
@@ -613,7 +617,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*1 2*')
+        self.do_run(src, '*1 2*')
 
     def test_if(self):
         src = '''
@@ -627,11 +631,11 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*yes*')
+        self.do_run(src, '*yes*')
 
         # Test for issue 39
         if not LLVM_OPTS:
-          self.do_ll_test(path_from_root('tests', 'issue_39.ll'), '*yes*')
+          self.do_ll_run(path_from_root('tests', 'issue_39.ll'), '*yes*')
 
     def test_if_else(self):
         src = '''
@@ -647,7 +651,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*no*')
+        self.do_run(src, '*no*')
 
     def test_loop(self):
         src = '''
@@ -661,7 +665,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*3600*')
+        self.do_run(src, '*3600*')
 
     def test_stack(self):
         src = '''
@@ -683,7 +687,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*0,1*')
+        self.do_run(src, '*0,1*')
 
     def test_strings(self):
         src = '''
@@ -718,7 +722,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '4:10,177,543,def\n4\nwowie\ntoo\n76\n5\n(null)\n/* a comment */\n// another\ntest\n', ['wowie', 'too', '74'])
+        self.do_run(src, '4:10,177,543,def\n4\nwowie\ntoo\n76\n5\n(null)\n/* a comment */\n// another\ntest\n', ['wowie', 'too', '74'])
 
     def test_errar(self):
         src = r'''
@@ -748,7 +752,7 @@ if 'benchmark' not in str(sys.argv):
           <34>
           <123>
           '''
-        self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected))
+        self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected))
 
     def test_mainenv(self):
         src = '''
@@ -759,7 +763,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*(nil)*')
+        self.do_run(src, '*(nil)*')
 
     def test_funcs(self):
         src = '''
@@ -774,7 +778,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*72,90*')
+        self.do_run(src, '*72,90*')
 
     def test_structs(self):
         src = '''
@@ -799,7 +803,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*10,6,101,7018,101,7018,101,7018*')
+        self.do_run(src, '*10,6,101,7018,101,7018,101,7018*')
 
     gen_struct_src = '''
           #include <stdio.h>
@@ -821,10 +825,10 @@ if 'benchmark' not in str(sys.argv):
     '''
 
     def test_mallocstruct(self):
-        self.do_test(self.gen_struct_src.replace('{{gen_struct}}', '(S*)malloc(sizeof(S))').replace('{{del_struct}}', 'free'), '*51,62*')
+        self.do_run(self.gen_struct_src.replace('{{gen_struct}}', '(S*)malloc(sizeof(S))').replace('{{del_struct}}', 'free'), '*51,62*')
 
     def test_newstruct(self):
-        self.do_test(self.gen_struct_src.replace('{{gen_struct}}', 'new S').replace('{{del_struct}}', 'delete'), '*51,62*')
+        self.do_run(self.gen_struct_src.replace('{{gen_struct}}', 'new S').replace('{{del_struct}}', 'delete'), '*51,62*')
 
     def test_addr_of_stacked(self):
         src = '''
@@ -841,7 +845,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*7*')
+        self.do_run(src, '*7*')
 
     def test_globals(self):
         src = '''
@@ -857,7 +861,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*25,51*')
+        self.do_run(src, '*25,51*')
 
     def test_linked_list(self):
         src = '''
@@ -902,7 +906,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*1410,0*')
+        self.do_run(src, '*1410,0*')
 
     def test_sup(self):
         src = '''
@@ -950,9 +954,9 @@ if 'benchmark' not in str(sys.argv):
           }
         '''
         if QUANTUM_SIZE == 1:
-          self.do_test(src, 'sizeofs:6,8\n*C___: 0,3,6,9<24*\n*Carr: 0,3,6,9<24*\n*C__w: 0,3,9,12<24*\n*Cp1_: 1,2,5,8<24*\n*Cp2_: 0,2,5,8<24*\n*Cint: 0,3,4,7<24*\n*C4__: 0,3,4,7<24*\n*C4_2: 0,3,5,8<20*\n*C__z: 0,3,5,8<28*')
+          self.do_run(src, 'sizeofs:6,8\n*C___: 0,3,6,9<24*\n*Carr: 0,3,6,9<24*\n*C__w: 0,3,9,12<24*\n*Cp1_: 1,2,5,8<24*\n*Cp2_: 0,2,5,8<24*\n*Cint: 0,3,4,7<24*\n*C4__: 0,3,4,7<24*\n*C4_2: 0,3,5,8<20*\n*C__z: 0,3,5,8<28*')
         else:
-          self.do_test(src, 'sizeofs:6,8\n*C___: 0,6,12,20<24*\n*Carr: 0,6,12,20<24*\n*C__w: 0,6,12,20<24*\n*Cp1_: 4,6,12,20<24*\n*Cp2_: 0,6,12,20<24*\n*Cint: 0,8,12,20<24*\n*C4__: 0,8,12,20<24*\n*C4_2: 0,6,10,16<20*\n*C__z: 0,8,16,24<28*')
+          self.do_run(src, 'sizeofs:6,8\n*C___: 0,6,12,20<24*\n*Carr: 0,6,12,20<24*\n*C__w: 0,6,12,20<24*\n*Cp1_: 4,6,12,20<24*\n*Cp2_: 0,6,12,20<24*\n*Cint: 0,8,12,20<24*\n*C4__: 0,8,12,20<24*\n*C4_2: 0,6,10,16<20*\n*C__z: 0,8,16,24<28*')
 
     def test_assert(self):
         src = '''
@@ -964,7 +968,7 @@ if 'benchmark' not in str(sys.argv):
             return 1;
           }
         '''
-        self.do_test(src, 'Assertion failed: 1 == false')
+        self.do_run(src, 'Assertion failed: 1 == false')
 
     def test_exceptions(self):
         src = '''
@@ -990,11 +994,11 @@ if 'benchmark' not in str(sys.argv):
             return 1;
           }
         '''
-        self.do_test(src, '*throw...caught!infunc...done!*')
+        self.do_run(src, '*throw...caught!infunc...done!*')
 
         global DISABLE_EXCEPTION_CATCHING
         DISABLE_EXCEPTION_CATCHING = 1
-        self.do_test(src, 'Compiled code throwing an exception')
+        self.do_run(src, 'Compiled code throwing an exception')
 
     def test_typed_exceptions(self):
         return self.skip('TODO: fix this for llvm 3.0')
@@ -1003,7 +1007,7 @@ if 'benchmark' not in str(sys.argv):
         global EXCEPTION_DEBUG; EXCEPTION_DEBUG = 0  # Messes up expected output.
         src = open(path_from_root('tests', 'exceptions', 'typed.cpp'), 'r').read()
         expected = open(path_from_root('tests', 'exceptions', 'output.txt'), 'r').read()
-        self.do_test(src, expected)
+        self.do_run(src, expected)
 
     def test_class(self):
         src = '''
@@ -1032,7 +1036,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*0*')
+        self.do_run(src, '*0*')
 
     def test_inherit(self):
         src = '''
@@ -1060,7 +1064,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*51,87,78,550,100,78,550*')
+        self.do_run(src, '*51,87,78,550,100,78,550*')
 
     def test_polymorph(self):
         src = '''
@@ -1097,7 +1101,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*11,74,32,1012*\n*11*\n*22*')
+        self.do_run(src, '*11,74,32,1012*\n*11*\n*22*')
 
     def test_funcptr(self):
         src = '''
@@ -1135,7 +1139,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*26,26,90,90,26,90*\n*1,0,0,1*\n*goodbye!*')
+        self.do_run(src, '*26,26,90,90,26,90*\n*1,0,0,1*\n*goodbye!*')
 
     def test_mathfuncptr(self):
         src = '''
@@ -1150,7 +1154,7 @@ if 'benchmark' not in str(sys.argv):
            return 0;
           }
           '''
-        self.do_test(src, 'fn2(-5) = 5, fn(10) = 3.16')
+        self.do_run(src, 'fn2(-5) = 5, fn(10) = 3.16')
 
     def test_emptyclass(self):
         src = '''
@@ -1168,7 +1172,7 @@ if 'benchmark' not in str(sys.argv):
           return 0;
         }
         '''
-        self.do_test(src, '*zzcheezzz*')
+        self.do_run(src, '*zzcheezzz*')
 
     def test_alloca(self):
       src = '''
@@ -1181,7 +1185,7 @@ if 'benchmark' not in str(sys.argv):
           return 0;
         }
       '''
-      self.do_test(src, 'z:1*', force_c=True)
+      self.do_run(src, 'z:1*', force_c=True)
 
     def test_array2(self):
         src = '''
@@ -1199,7 +1203,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '0:-1.00,-0.33 1:0.33,-1.00 2:-0.33,1.00 3:1.00,0.33')
+        self.do_run(src, '0:-1.00,-0.33 1:0.33,-1.00 2:-0.33,1.00 3:1.00,0.33')
 
     def test_array2b(self):
         src = '''
@@ -1218,7 +1222,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*6,6\n7,95*')
+        self.do_run(src, '*6,6\n7,95*')
 
 
     def test_constglobalstructs(self):
@@ -1252,7 +1256,7 @@ if 'benchmark' not in str(sys.argv):
              return 0;
           }
           '''
-        self.do_test(src, '*97,15,3,10*')
+        self.do_run(src, '*97,15,3,10*')
 
     def test_conststructs(self):
         src = '''
@@ -1276,7 +1280,7 @@ if 'benchmark' not in str(sys.argv):
              return 0;
           }
           '''
-        self.do_test(src, '*70,97,15,3,3029,90*')
+        self.do_run(src, '*70,97,15,3,3029,90*')
 
     def test_mod_globalstruct(self):
         src = '''
@@ -1298,7 +1302,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_test(src, '*4096,4096,8192,69632*')
+        self.do_run(src, '*4096,4096,8192,69632*')
 
     def test_pystruct(self):
         src = '''
@@ -1363,9 +1367,9 @@ if 'benchmark' not in str(sys.argv):
         '''
         if QUANTUM_SIZE == 1:
           # Compressed memory. Note that sizeof() does give the fat sizes, however!
-          self.do_test(src, '*0,0,0,1,2,3,4,5*\n*1,0,0*\n*0*\n0:1,1\n1:1,1\n2:1,1\n*12,20,5*')
+          self.do_run(src, '*0,0,0,1,2,3,4,5*\n*1,0,0*\n*0*\n0:1,1\n1:1,1\n2:1,1\n*12,20,5*')
         else:
-          self.do_test(src, '*0,0,0,4,8,12,16,20*\n*1,0,0*\n*0*\n0:1,1\n1:1,1\n2:1,1\n*12,20,20*')
+          self.do_run(src, '*0,0,0,4,8,12,16,20*\n*1,0,0*\n*0*\n0:1,1\n1:1,1\n2:1,1\n*12,20,20*')
 
     def test_ptrtoint(self):
         src = '''
@@ -1387,7 +1391,7 @@ if 'benchmark' not in str(sys.argv):
         runner = self
         def check_warnings(output):
             runner.assertEquals(filter(lambda line: 'Warning' in line, output.split('\n')).__len__(), 4)
-        self.do_test(src, '*5*', output_processor=check_warnings)
+        self.do_run(src, '*5*', output_processor=check_warnings)
 
     def test_sizeof(self):
         # Has invalid writes between printouts
@@ -1422,7 +1426,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*2,2,5,8,8***8,8,5,8,8***7,2,6,990,7,2*', [], lambda x: x.replace('\n', '*'))
+        self.do_run(src, '*2,2,5,8,8***8,8,5,8,8***7,2,6,990,7,2*', [], lambda x: x.replace('\n', '*'))
 
     def test_emscripten_api(self):
         src = '''
@@ -1440,7 +1444,7 @@ if 'benchmark' not in str(sys.argv):
           src = open(filename, 'r').read()
           assert '// hello from the source' in src
 
-        self.do_test(src, 'hello world!', post_build=check)
+        self.do_run(src, 'hello world!', post_build=check)
 
     def test_ssr(self): # struct self-ref
         src = '''
@@ -1468,9 +1472,9 @@ if 'benchmark' not in str(sys.argv):
           }
           '''
         if QUANTUM_SIZE == 1:
-          self.do_test(src, '''*4*\n0:22016,0,8,12\n1:22018,1,12,8\n''')
+          self.do_run(src, '''*4*\n0:22016,0,8,12\n1:22018,1,12,8\n''')
         else:
-          self.do_test(src, '''*16*\n0:22016,0,32,48\n1:22018,1,48,32\n''')
+          self.do_run(src, '''*16*\n0:22016,0,32,48\n1:22018,1,48,32\n''')
 
     def test_tinyfuncstr(self):
         src = '''
@@ -1486,7 +1490,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*nameA,nameB*')
+        self.do_run(src, '*nameA,nameB*')
 
     def test_llvmswitch(self):
         src = '''
@@ -1511,7 +1515,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*96,97,98,101,101*')
+        self.do_run(src, '*96,97,98,101,101*')
 
     def test_indirectbr(self):
         src = '''
@@ -1535,7 +1539,7 @@ if 'benchmark' not in str(sys.argv):
             goto *addr;
           }
           '''
-        self.do_test(src, 'good\nbad')
+        self.do_run(src, 'good\nbad')
 
     def test_pack(self):
         src = '''
@@ -1567,9 +1571,9 @@ if 'benchmark' not in str(sys.argv):
           }
           '''
         if QUANTUM_SIZE == 1:
-          self.do_test(src, '*4,2,3*\n*6,2,3*')
+          self.do_run(src, '*4,2,3*\n*6,2,3*')
         else:
-          self.do_test(src, '*4,3,4*\n*6,4,6*')
+          self.do_run(src, '*4,3,4*\n*6,4,6*')
 
     def test_varargs(self):
         if QUANTUM_SIZE == 1: return self.skip('FIXME: Add support for this')
@@ -1636,7 +1640,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*cheez: 0+24*\n*cheez: 0+24*\n*albeit*\n*albeit*\nQ85*\nmaxxi:21*\nmaxxD:22.10*\n')
+        self.do_run(src, '*cheez: 0+24*\n*cheez: 0+24*\n*albeit*\n*albeit*\nQ85*\nmaxxi:21*\nmaxxD:22.10*\n')
 
     def test_stdlibs(self):
         if USE_TYPED_ARRAYS == 2:
@@ -1691,13 +1695,13 @@ if 'benchmark' not in str(sys.argv):
           }
           '''
 
-        self.do_test(src, '*1,2,3,5,5,6*\n*stdin==0:0*\n*%*\n*5*\n*66.0*\n*10*\n*0*\n*-10*\n*18*\n*10*\n*0*\n*4294967286*\n*cleaned*')
+        self.do_run(src, '*1,2,3,5,5,6*\n*stdin==0:0*\n*%*\n*5*\n*66.0*\n*10*\n*0*\n*-10*\n*18*\n*10*\n*0*\n*4294967286*\n*cleaned*')
 
     def test_time(self):
       if USE_TYPED_ARRAYS == 2: return self.skip('Typed arrays = 2 truncate i64s')
       src = open(path_from_root('tests', 'time', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'time', 'output.txt'), 'r').read()
-      self.do_test(src, expected,
+      self.do_run(src, expected,
                    extra_emscripten_args=['-H', 'libc/time.h'])
                    #extra_emscripten_args=['-H', 'libc/fcntl.h,libc/sys/unistd.h,poll.h,libc/math.h,libc/langinfo.h,libc/time.h'])
 
@@ -1744,7 +1748,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*staticccz*\n*1.00,2.00,3.00*')
+        self.do_run(src, '*staticccz*\n*1.00,2.00,3.00*')
 
     def test_copyop(self):
         # clang generated code is vulnerable to this, as it uses
@@ -1787,7 +1791,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src, '*0.00,0.00,0.00*\n*0,77,0*\n*0,77,0*\n*0,77,0*')
+        self.do_run(src, '*0.00,0.00,0.00*\n*0,77,0*\n*0,77,0*\n*0,77,0*')
 
     def test_memcpy(self):
         src = '''
@@ -1819,7 +1823,7 @@ if 'benchmark' not in str(sys.argv):
             return 0;
           }
           '''
-        self.do_test(src)
+        self.do_run(src)
 
     def test_nestedstructs(self):
         src = '''
@@ -1894,10 +1898,10 @@ if 'benchmark' not in str(sys.argv):
           '''
         if QUANTUM_SIZE == 1:
           # Compressed memory. Note that sizeof() does give the fat sizes, however!
-          self.do_test(src, '*16,0,1,2,2,3|20,0,1,1,2,3,3,4|24,0,5,0,1,1,2,3,3,4*\n*0,0,0,1,2,62,63,64,72*\n*2*')
+          self.do_run(src, '*16,0,1,2,2,3|20,0,1,1,2,3,3,4|24,0,5,0,1,1,2,3,3,4*\n*0,0,0,1,2,62,63,64,72*\n*2*')
         else:
           # Bloated memory; same layout as C/C++
-          self.do_test(src, '*16,0,4,8,8,12|20,0,4,4,8,12,12,16|24,0,20,0,4,4,8,12,12,16*\n*0,0,0,1,2,64,68,69,72*\n*2*')
+          self.do_run(src, '*16,0,4,8,8,12|20,0,4,4,8,12,12,16|24,0,20,0,4,4,8,12,12,16*\n*0,0,0,1,2,64,68,69,72*\n*2*')
 
     def test_dlfcn_basic(self):
       global BUILD_AS_SHARED_LIB
@@ -1944,7 +1948,7 @@ if 'benchmark' not in str(sys.argv):
           '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
         )
         open(filename, 'w').write(src)
-      self.do_test(src, 'Constructing main object.\nConstructing lib object.\n',
+      self.do_run(src, 'Constructing main object.\nConstructing lib object.\n',
                    post_build=add_pre_run_and_checks)
 
     def test_dlfcn_qsort(self):
@@ -2029,7 +2033,7 @@ if 'benchmark' not in str(sys.argv):
           '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
         )
         open(filename, 'w').write(src)
-      self.do_test(src, 'Sort with main comparison: 5 4 3 2 1 *Sort with lib comparison: 1 2 3 4 5 *',
+      self.do_run(src, 'Sort with main comparison: 5 4 3 2 1 *Sort with lib comparison: 1 2 3 4 5 *',
                    output_nicerizer=lambda x: x.replace('\n', '*'),
                    post_build=add_pre_run_and_checks)
 
@@ -2129,7 +2133,7 @@ if 'benchmark' not in str(sys.argv):
           '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
         )
         open(filename, 'w').write(src)
-      self.do_test(src, 'In func: 13*First calling main_fptr from lib.*Second calling lib_fptr from main.*parent_func called from child*parent_func called from child*Var: 42*',
+      self.do_run(src, 'In func: 13*First calling main_fptr from lib.*Second calling lib_fptr from main.*parent_func called from child*parent_func called from child*Var: 42*',
                    output_nicerizer=lambda x: x.replace('\n', '*'),
                    post_build=add_pre_run_and_checks)
 
@@ -2176,7 +2180,7 @@ if 'benchmark' not in str(sys.argv):
           '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
         )
         open(filename, 'w').write(src)
-      self.do_test(src, 'Parent global: 123.*Parent global: 456.*',
+      self.do_run(src, 'Parent global: 123.*Parent global: 456.*',
                    output_nicerizer=lambda x: x.replace('\n', '*'),
                    post_build=add_pre_run_and_checks,
                    extra_emscripten_args=['-H', 'libc/fcntl.h,libc/sys/unistd.h,poll.h,libc/math.h,libc/time.h,libc/langinfo.h'])
@@ -2233,7 +2237,7 @@ if 'benchmark' not in str(sys.argv):
           '''FS.createLazyFile('/', 'liblib.so', 'liblib.so', true, false);'''
         )
         open(filename, 'w').write(src)
-      self.do_test(src, '100*200*13*42*',
+      self.do_run(src, '100*200*13*42*',
                    output_nicerizer=lambda x: x.replace('\n', '*'),
                    post_build=add_pre_run_and_checks)
 
@@ -2277,7 +2281,7 @@ if 'benchmark' not in str(sys.argv):
         1406932606, 3554416254
         12345, 12345
         '''
-      self.do_test(src, re.sub(r'(^|\n)\s+', r'\1', expected))
+      self.do_run(src, re.sub(r'(^|\n)\s+', r'\1', expected))
 
     def test_strtod(self):
       if USE_TYPED_ARRAYS == 2: return self.skip('Typed arrays = 2 truncate doubles')
@@ -2332,19 +2336,19 @@ if 'benchmark' not in str(sys.argv):
         1.234e+57
         10
         '''
-      self.do_test(src, re.sub(r'\n\s+', '\n', expected))
+      self.do_run(src, re.sub(r'\n\s+', '\n', expected))
 
     def test_parseInt(self):
       if USE_TYPED_ARRAYS != 0: return self.skip('Typed arrays truncate i64')
       src = open(path_from_root('tests', 'parseInt', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'parseInt', 'output.txt'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_printf(self):
       if USE_TYPED_ARRAYS != 0: return self.skip('Typed arrays truncate i64')
       src = open(path_from_root('tests', 'printf', 'test.c'), 'r').read()
       expected = open(path_from_root('tests', 'printf', 'output.txt'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_printf_types(self):
       src = r'''
@@ -2363,7 +2367,7 @@ if 'benchmark' not in str(sys.argv):
           return 0;
         }
         '''
-      self.do_test(src, '1,2,3,4,5.5,6.6\n')
+      self.do_run(src, '1,2,3,4,5.5,6.6\n')
 
     def test_vprintf(self):
       src = r'''
@@ -2388,12 +2392,12 @@ if 'benchmark' not in str(sys.argv):
         Call with 1 variable argument.
         Call with 2 variable arguments.
         '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected))
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected))
 
     def test_langinfo(self):
       src = open(path_from_root('tests', 'langinfo', 'test.c'), 'r').read()
       expected = open(path_from_root('tests', 'langinfo', 'output.txt'), 'r').read()
-      self.do_test(src, expected, extra_emscripten_args=['-H', 'libc/langinfo.h'])
+      self.do_run(src, expected, extra_emscripten_args=['-H', 'libc/langinfo.h'])
 
     def test_files(self):
       global CORRECT_SIGNS; CORRECT_SIGNS = 1 # Just so our output is what we expect. Can flip them both.
@@ -2418,7 +2422,7 @@ if 'benchmark' not in str(sys.argv):
       other.close()
 
       src = open(path_from_root('tests', 'files.cpp'), 'r').read()
-      self.do_test(src, 'size: 7\ndata: 100,-56,50,25,10,77,123\ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n',
+      self.do_run(src, 'size: 7\ndata: 100,-56,50,25,10,77,123\ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\n',
                    post_build=post, extra_emscripten_args=['-H', 'libc/fcntl.h'])
 
     def test_folders(self):
@@ -2497,7 +2501,7 @@ if 'benchmark' not in str(sys.argv):
         --E: 20, D: 0
         --E: 9
       '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run)
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run)
 
     def test_stat(self):
       def add_pre_run(filename):
@@ -2514,7 +2518,7 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'stat', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'stat', 'output.txt'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
+      self.do_run(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
 
     def test_fcntl(self):
       def add_pre_run(filename):
@@ -2525,7 +2529,7 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'fcntl', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'fcntl', 'output.txt'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
+      self.do_run(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
 
     def test_fcntl_open(self):
       def add_pre_run(filename):
@@ -2540,7 +2544,7 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'fcntl-open', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'fcntl-open', 'output.txt'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
+      self.do_run(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
 
     def test_fcntl_misc(self):
       def add_pre_run(filename):
@@ -2551,7 +2555,7 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'fcntl-misc', 'src.c'), 'r').read()
       expected = open(path_from_root('tests', 'fcntl-misc', 'output.txt'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
+      self.do_run(src, expected, post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h'])
 
     def test_poll(self):
       def add_pre_run(filename):
@@ -2602,7 +2606,7 @@ if 'benchmark' not in str(sys.argv):
         multi[3].revents: 1
         multi[4].revents: 1
         '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h,poll.h'])
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run, extra_emscripten_args=['-H', 'libc/fcntl.h,poll.h'])
 
     def test_statvfs(self):
       src = r'''
@@ -2646,7 +2650,7 @@ if 'benchmark' not in str(sys.argv):
         f_flag: 2
         f_namemax: 255
         '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected))
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected))
 
     def test_libgen(self):
       src = r'''
@@ -2701,7 +2705,7 @@ if 'benchmark' not in str(sys.argv):
         (empty) -> . : .
         (null) -> . : .
       '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected))
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected))
 
     def test_utime(self):
       def add_pre_run_and_checks(filename):
@@ -2745,7 +2749,7 @@ if 'benchmark' not in str(sys.argv):
         first changed: true
         second changed: false
       '''
-      self.do_test(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run_and_checks)
+      self.do_run(src, re.sub('(^|\n)\s+', '\\1', expected), post_build=add_pre_run_and_checks)
 
     def test_fs_base(self):
       global INCLUDE_FULL_LIBRARY; INCLUDE_FULL_LIBRARY = 1
@@ -2757,7 +2761,7 @@ if 'benchmark' not in str(sys.argv):
           open(filename, 'w').write(src)
         src = 'int main() {return 0;}\n'
         expected = open(path_from_root('tests', 'filesystem', 'output.txt'), 'r').read()
-        self.do_test(src, expected, post_build=addJS, extra_emscripten_args=['-H', 'libc/fcntl.h,libc/sys/unistd.h,poll.h,libc/math.h,libc/langinfo.h,libc/time.h'])
+        self.do_run(src, expected, post_build=addJS, extra_emscripten_args=['-H', 'libc/fcntl.h,libc/sys/unistd.h,poll.h,libc/math.h,libc/langinfo.h,libc/time.h'])
       finally:
         INCLUDE_FULL_LIBRARY = 0
 
@@ -2770,7 +2774,7 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'unistd', 'access.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'access.out'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run)
+      self.do_run(src, expected, post_build=add_pre_run)
 
     def test_unistd_curdir(self):
       def add_pre_run(filename):
@@ -2781,17 +2785,17 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'unistd', 'curdir.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'curdir.out'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run)
+      self.do_run(src, expected, post_build=add_pre_run)
 
     def test_unistd_close(self):
       src = open(path_from_root('tests', 'unistd', 'close.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'close.out'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_unistd_confstr(self):
       src = open(path_from_root('tests', 'unistd', 'confstr.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'confstr.out'), 'r').read()
-      self.do_test(src, expected, extra_emscripten_args=['-H', 'libc/unistd.h'])
+      self.do_run(src, expected, extra_emscripten_args=['-H', 'libc/unistd.h'])
 
     def test_unistd_ttyname(self):
       def add_pre_run(filename):
@@ -2802,17 +2806,17 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'unistd', 'ttyname.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'ttyname.out'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run)
+      self.do_run(src, expected, post_build=add_pre_run)
 
     def test_unistd_dup(self):
       src = open(path_from_root('tests', 'unistd', 'dup.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'dup.out'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_unistd_pathconf(self):
       src = open(path_from_root('tests', 'unistd', 'pathconf.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'pathconf.out'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_unistd_truncate(self):
       def add_pre_run(filename):
@@ -2823,12 +2827,12 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'unistd', 'truncate.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'truncate.out'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run)
+      self.do_run(src, expected, post_build=add_pre_run)
 
     def test_unistd_swab(self):
       src = open(path_from_root('tests', 'unistd', 'swab.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'swab.out'), 'r').read()
-      self.do_test(src, expected)
+      self.do_run(src, expected)
 
     def test_unistd_isatty(self):
       def add_pre_run(filename):
@@ -2839,17 +2843,17 @@ if 'benchmark' not in str(sys.argv):
         open(filename, 'w').write(src)
       src = open(path_from_root('tests', 'unistd', 'isatty.c'), 'r').read()
       expected = open(path_from_root('tests', 'unistd', 'isatty.out'), 'r').read()
-      self.do_test(src, expected, post_build=add_pre_run)
+      self.do_run(src, expected, post_build=add_pre_run)
 
     def test_unistd_sysconf(self):
       src = open(path_from_root('tests', 'unistd', 'sysconf.c'), 'r').read()
       expected = open(path_from_