Make the test runner handle broken lli's correctly

lli on platforms such as Linux64 or Mac is broken, so it's best
not to rely on it for running our tests on those platforms.

1 files changed, 42 insertions, 28 deletions

diff --git a/tests/runner.py b/tests/runner.py
index 126071a0..5da40a89 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -14,7 +14,7 @@ will use 4 processes. To install nose do something like
 '''
 
 from subprocess import Popen, PIPE, STDOUT
-import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, tempfile, re, difflib, webbrowser
+import os, unittest, tempfile, shutil, time, inspect, sys, math, glob, tempfile, re, difflib, webbrowser, hashlib
 
 # Setup
 
@@ -37,6 +37,7 @@ except:
 class RunnerCore(unittest.TestCase):
   save_dir = os.environ.get('EM_SAVE_DIR')
   save_JS = 0
+  lli_available = False
   stderr_redirect = STDOUT # This avoids cluttering the test runner output, which is stderr too, with compiler warnings etc.
                            # Change this to None to get stderr reporting, for debugging purposes
 
@@ -51,6 +52,7 @@ class RunnerCore(unittest.TestCase):
       os.makedirs(dirname)
     self.working_dir = dirname
     os.chdir(dirname)
+    self.check_lli_available()
     
   def tearDown(self):
     if self.save_JS:
@@ -195,6 +197,14 @@ process(sys.argv[1])
     assert 'strict warning:' not in ret, 'We should pass all strict mode checks: ' + ret
     return ret
 
+  def check_lli_available(self):
+    target = path_from_root('tests', 'hello_world.ll')
+    p = Popen([LLVM_INTERPRETER, target], stdout=PIPE, stderr=self.stderr_redirect)
+    p.communicate()
+    self.lli_available = p.returncode == 0
+    if not self.lli_available:
+      self.skip("lli does not work on this platform...")
+
   def run_llvm_interpreter(self, args):
     return Popen([EXEC_LLVM] + args, stdout=PIPE, stderr=self.stderr_redirect).communicate()[0]
 
@@ -266,7 +276,7 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
 
   class T(RunnerCore): # Short name, to make it more fun to use manually on the commandline
     ## Does a complete test - builds, runs, checks output, etc.
-    def do_run(self, src, expected_output=None, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]):
+    def do_run(self, src, expected_output, args=[], output_nicerizer=None, output_processor=None, no_build=False, main_file=None, additional_files=[], js_engines=None, post_build=None, basename='src.cpp', libraries=[], includes=[], force_c=False, build_ll_hook=None, extra_emscripten_args=[]):
         if force_c or (main_file is not None and main_file[-2:]) == '.c':
           basename = 'src.c'
           Building.COMPILER = to_cc(Building.COMPILER)
@@ -277,11 +287,6 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
           self.build(src, dirname, filename, main_file=main_file, additional_files=additional_files, libraries=libraries, includes=includes,
                      build_ll_hook=build_ll_hook, extra_emscripten_args=extra_emscripten_args, post_build=post_build)
 
-        # If not provided with expected output, then generate it right now, using lli
-        if expected_output is None:
-          expected_output = self.run_llvm_interpreter([filename + '.o'])
-          print '[autogenerated expected output: %20s]' % (expected_output[0:30].replace('\n', '|')+'...')
-
         # Run in both JavaScript engines, if optimizing - significant differences there (typed arrays)
         if js_engines is None:
           js_engines = JS_ENGINES
@@ -656,7 +661,7 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
             return 0;
           }
         '''
-        self.do_run(src)#, '*4294967295,0,4294967219*\n*-1,1,-1,1*\n*-2,1,-2,1*\n*246,296*\n*1,0*')
+        self.do_run(src, '*4294967295,0,4294967219*\n*-1,1,-1,1*\n*-2,1,-2,1*\n*246,296*\n*1,0*')
 
         # Now let's see some code that should just work in USE_TYPED_ARRAYS == 2, but requires
         # corrections otherwise
@@ -858,7 +863,7 @@ if 'benchmark' not in str(sys.argv) and 'sanity' not in str(sys.argv):
           }
         '''
 
-        self.do_run(src)
+        self.do_run(src, '*1800*')
 
         generated = open('src.cpp.o.js', 'r').read()
         assert '__label__ ==' not in generated, 'We should hoist into the loop'
@@ -2352,7 +2357,10 @@ def process(filename):
             return 0;
           }
           '''
-        self.do_run(src)
+        def check(result):
+          return hashlib.sha1(result).hexdigest()
+        self.do_run(src, '6c9cdfe937383b79e52ca7a2cce83a21d9f5422c',
+                    output_nicerizer = check)
 
     def test_memmove(self):
       src = '''
@@ -2365,7 +2373,7 @@ def process(filename):
           return 0;
         }
       '''
-      self.do_run(src)
+      self.do_run(src, 'memmove can be very very useful')
 
     def test_bsearch(self):
       if Settings.QUANTUM_SIZE == 1: return self.skip('Test cannot work with q1')
@@ -3141,21 +3149,21 @@ at function.:blag
         #include <stdlib.h>
 
         int main () {
-          printf("%d\n", atoi(""));
-          printf("%d\n", atoi("a"));
-          printf("%d\n", atoi(" b"));
-          printf("%d\n", atoi(" c "));
-          printf("%d\n", atoi("6"));
-          printf("%d\n", atoi(" 5"));
-          printf("%d\n", atoi("4 "));
-          printf("%d\n", atoi("3 6"));
-          printf("%d\n", atoi(" 3 7"));
-          printf("%d\n", atoi("9 d"));
+          printf("%d*", atoi(""));
+          printf("%d*", atoi("a"));
+          printf("%d*", atoi(" b"));
+          printf("%d*", atoi(" c "));
+          printf("%d*", atoi("6"));
+          printf("%d*", atoi(" 5"));
+          printf("%d*", atoi("4 "));
+          printf("%d*", atoi("3 6"));
+          printf("%d*", atoi(" 3 7"));
+          printf("%d*", atoi("9 d"));
           printf("%d\n", atoi(" 8 e"));
           return 0;
         }
         '''
-      self.do_run(src)
+      self.do_run(src, '0*0*0*0*6*5*4*3*3*9*8')
 
     def test_sscanf(self):
       src = r'''
@@ -3181,7 +3189,8 @@ at function.:blag
           return 0;
         }
         '''
-      self.do_run(src)
+      self.do_run(src, 'en-us : 2*en-r : 99*en : 3*1.234567, 0.000000',
+                  output_nicerizer = lambda x: x.replace('\n', '*'))
 
       # Part 2: doubles (issue 148)
       if Settings.USE_TYPED_ARRAYS == 2:
@@ -3232,7 +3241,11 @@ Pass: 123456.789063 123456.789063
 Pass: 0.000012 0.000012
 Pass: 0.000012 0.000012''')
           else:
-            self.do_run(src)
+            self.do_run(src, '''Pass: 1.234568 1.234568
+Pass: 123456.789000 123456.789000
+Pass: 123456.789000 123456.789000
+Pass: 0.000012 0.000012
+Pass: 0.000012 0.000012''')
 
     def test_langinfo(self):
       src = open(path_from_root('tests', 'langinfo', 'test.c'), 'r').read()
@@ -4445,7 +4458,7 @@ def process(filename):
       self.do_autodebug(filename)
 
       # Compare to each other, and to expected output
-      self.do_ll_run(path_from_root('tests', filename+'.o.ll.ll'))
+      self.do_ll_run(path_from_root('tests', filename+'.o.ll.ll'), 'AD:')
       assert open('stdout').read().startswith('AD:-1'), 'We must note when we enter functions'
 
       # Test using build_ll_hook
@@ -4464,7 +4477,6 @@ def process(filename):
             return 0;
           }
         '''
-      self.do_run(src, build_ll_hook=self.do_autodebug)
       self.do_run(src, 'AD:', build_ll_hook=self.do_autodebug)
 
     def test_profiling(self):
@@ -5363,7 +5375,8 @@ Options that are modified or new in %s include:
           output = Popen([compiler, path_from_root('tests', 'hello_world' + suffix)] + args, stdout=PIPE, stderr=PIPE).communicate()
           assert len(output[0]) == 0, output[0]
           assert os.path.exists(target), 'Expected %s to exist since args are %s : %s' % (target, str(args), '\n'.join(output))
-          self.assertContained('hello, world!', self.run_llvm_interpreter([target]))
+          if self.lli_available:
+            self.assertContained('hello, world!', self.run_llvm_interpreter([target]))
 
         # emcc src.ll ==> generates .js
         clear()
@@ -5509,7 +5522,8 @@ Options that are modified or new in %s include:
           assert not os.path.exists(target)
           output = Popen([compiler, 'twopart_main.o', 'twopart_side.o', '-o', 'combined.bc'] + args, stdout=PIPE, stderr=PIPE).communicate()
           assert os.path.exists('combined.bc'), '\n'.join(output)
-          self.assertContained('side got: hello from main, over', self.run_llvm_interpreter(['combined.bc']))
+          if self.lli_available:
+            self.assertContained('side got: hello from main, over', self.run_llvm_interpreter(['combined.bc']))
 
         # --js-transform <transform>
         clear()