improve benchmark infrastructure in preparation for zlib benchmark

3 files changed, 98 insertions, 12 deletions

diff --git a/tests/runner.py b/tests/runner.py
index fe7b708c..f81684a4 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -288,8 +288,9 @@ process(sys.argv[1])
     assert 'strict warning:' not in ret, 'We should pass all strict mode checks: ' + ret
     return ret
 
-  def build_native(self, filename):
-    process = Popen([CLANG, '-O2', '-fno-math-errno', filename, '-o', filename+'.native'], stdout=PIPE);
+  def build_native(self, filename, args=[]):
+    compiler = CLANG if filename.endswith('cpp') else CLANG_CC
+    process = Popen([compiler, '-O2', '-fno-math-errno', filename, '-o', filename+'.native'] + args, stdout=PIPE, stderr=self.stderr_redirect)
     output = process.communicate()
     if process.returncode is not 0:
       print >> sys.stderr, "Building native executable with command '%s' failed with a return code %d!" % (' '.join([CLANG, '-O2', filename, '-o', filename+'.native']), process.returncode)
@@ -301,6 +302,7 @@ process(sys.argv[1])
     if process.returncode is not 0:
       print >> sys.stderr, "Running native executable with command '%s' failed with a return code %d!" % (' '.join([filename+'.native'] + args), process.returncode)
       print "Output: " + output[0]
+    return output[0]
 
   def assertIdentical(self, values, y):
     if type(values) not in [list, tuple]: values = [values]
@@ -339,7 +341,7 @@ process(sys.argv[1])
       os.makedirs(ret)
     return ret
 
-  def get_library(self, name, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=True, env_init={}, cache_name_extra=''):
+  def get_library(self, name, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=True, env_init={}, cache_name_extra='', native=False):
     build_dir = self.get_build_dir()
     output_dir = self.get_dir()
 
@@ -359,7 +361,7 @@ process(sys.argv[1])
     print >> sys.stderr, '<building and saving %s into cache> ' % cache_name,
 
     return Building.build_library(name, build_dir, output_dir, generated_libs, configure, configure_args, make, make_args, self.library_cache, cache_name,
-                                  copy_project=True, env_init=env_init)
+                                  copy_project=True, env_init=env_init, native=native)
 
   def clear(self, in_curr=False):
     for name in os.listdir(self.get_dir()):
@@ -10681,9 +10683,9 @@ elif 'benchmark' in str(sys.argv):
       print '   JavaScript: mean: %.3f (+-%.3f) secs  median: %.3f  range: %.3f-%.3f  (noise: %3.3f%%)  (%d runs)' % (mean, std, median, min(times), max(times), 100*std/mean, TEST_REPS)
       print '   Native    : mean: %.3f (+-%.3f) secs  median: %.3f  range: %.3f-%.3f  (noise: %3.3f%%)  JS is %.2f X slower' % (mean_native, std_native, median_native, min(native_times), max(native_times), 100*std_native/mean_native, final)
 
-    def do_benchmark(self, name, src, args=[], expected_output='FAIL', emcc_args=[]):
+    def do_benchmark(self, name, src, args=[], expected_output='FAIL', emcc_args=[], native_args=[], force_c=False):
       dirname = self.get_dir()
-      filename = os.path.join(dirname, name + '.cpp')
+      filename = os.path.join(dirname, name + '.c' + ('' if force_c else 'pp'))
       f = open(filename, 'w')
       f.write(src)
       f.close()
@@ -10713,12 +10715,15 @@ elif 'benchmark' in str(sys.argv):
           self.assertContained(expected_output, js_output)
 
       # Run natively
-      self.build_native(filename)
+      self.build_native(filename, native_args)
       global total_native_times
       native_times = []
       for i in range(TEST_REPS):
         start = time.time()
-        self.run_native(filename, args)
+        native_output = self.run_native(filename, args)
+        if i == 0:
+          # Sanity check on output
+          self.assertContained(expected_output, native_output)
         curr = time.time()-start
         native_times.append(curr)
         total_native_times[tests_done] += curr
@@ -10910,6 +10915,18 @@ elif 'benchmark' in str(sys.argv):
       src = open(path_from_root('system', 'lib', 'dlmalloc.c'), 'r').read() + '\n\n\n' + open(path_from_root('tests', 'dlmalloc_test.c'), 'r').read()
       self.do_benchmark('dlmalloc', src, ['400', '400'], '*400,0*', emcc_args=['-g', '-s', 'CORRECT_SIGNS=2', '-s', 'CORRECT_SIGNS_LINES=[4820, 4195, 4250, 4203, 4209, 4239, 4231]'])
 
+    def zzztest_zlib(self):
+      src = open(path_from_root('tests', 'zlib', 'benchmark.c'), 'r').read()
+      emcc_args = self.get_library('zlib', os.path.join('libz.a'), make_args=['libz.a']) + \
+                   ['-I' + path_from_root('tests', 'zlib')]
+      native_args = self.get_library('zlib_native', os.path.join('libz.a'), make_args=['libz.a'], native=True) + \
+                     ['-I' + path_from_root('tests', 'zlib')]
+      self.do_benchmark('zlib', src, ['100000', '100'], '''sum: 18710
+sizes: 100000,25906
+ok.
+''',
+                        force_c=True, emcc_args=emcc_args, native_args=native_args)
+
 elif 'sanity' in str(sys.argv):
 
   # Run some sanity checks on the test runner and emcc.
diff --git a/tests/zlib/benchmark.c b/tests/zlib/benchmark.c
new file mode 100644
index 00000000..f8ab6205
--- /dev/null
+++ b/tests/zlib/benchmark.c
@@ -0,0 +1,63 @@
+#include "zlib.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+
+
+// don't inline, to be friendly to js engine osr
+void __attribute__ ((noinline)) doit(char *buffer, int size, int i) {
+  static char *buffer2 = NULL;
+  static char *buffer3 = NULL;
+
+  int maxCompressedSize = compressBound(size);
+
+  if (!buffer2) buffer2 = (char*)malloc(maxCompressedSize);
+  if (!buffer3) buffer3 = (char*)malloc(size);
+
+  int compressedSize = maxCompressedSize;
+  compress(buffer2, &compressedSize, buffer, size);
+  if (i == 0) printf("sizes: %d,%d\n", size, compressedSize);
+
+  int decompressedSize = size;
+  uncompress(buffer3, &decompressedSize, buffer2, compressedSize);
+  assert(decompressedSize == size);
+  if (i == 0) assert(strcmp(buffer, buffer3) == 0);
+}
+
+int main(int argc, char **argv) {
+  int size = atoi(argv[1]);
+  int iters = atoi(argv[2]);
+  char *buffer = malloc(size);
+
+  int i = 0;
+  int run = 0;
+  char runChar = 17;
+  int sum = 0;
+  while (i < size) {
+    if (run > 0) {
+      run--;
+    } else {
+      if ((i & 7) == 0) {
+        runChar = i & 7;
+        run = i & 31;
+      } else {
+        runChar = (i*i) % 6714;
+      }
+    }
+    buffer[i] = runChar;
+    sum += buffer[i];
+    if (argc == 100) printf("%d: %d\n", i, buffer[i]); // confuse llvm optimizer, work around possible bug, this is not speed-relevant anyhow
+    i++;
+  }
+  printf("sum: %d\n", sum);
+
+  for (i = 0; i < iters; i++) {
+    doit(buffer, size, i);
+  }
+
+  printf("ok.\n");
+
+  return 0;
+}
+
diff --git a/tools/shared.py b/tools/shared.py
index f94bb263..ec135edc 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -588,8 +588,14 @@ class Building:
   COMPILER_TEST_OPTS = [] # For use of the test runner
 
   @staticmethod
-  def get_building_env():
+  def get_building_env(native=False):
     env = os.environ.copy()
+    if native:
+      env['CC'] = CLANG_CC
+      env['CXX'] = CLANG_CPP
+      env['LD'] = CLANG
+      env['CFLAGS'] = '-O2'
+      return env
     env['CC'] = EMCC if not WINDOWS else 'python %r' % EMCC
     env['CXX'] = EMXX if not WINDOWS else 'python %r' % EMXX
     env['AR'] = EMAR if not WINDOWS else 'python %r' % EMAR
@@ -667,14 +673,14 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
       raise
 
   @staticmethod
-  def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None):
+  def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None, native=False):
     ''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in
         memory since the test directory is destroyed and recreated for each test. Note that we cache separately
         for different compilers).
         This cache is just during the test runner. There is a different concept of caching as well, see |Cache|. '''
 
     if type(generated_libs) is not list: generated_libs = [generated_libs]
-    if source_dir is None: source_dir = path_from_root('tests', name)
+    if source_dir is None: source_dir = path_from_root('tests', name.replace('_native', ''))
 
     temp_dir = build_dir
     if copy_project:
@@ -695,7 +701,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
     #    os.unlink(lib) # make sure compilation completed successfully
     #  except:
     #    pass
-    env = Building.get_building_env()
+    env = Building.get_building_env(native)
     for k, v in env_init.iteritems():
       env[k] = v
     if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |link| call)