SAFE_HEAP now validates the load-store consistency assumption, plus minor related fixes

9 files changed, 143 insertions, 33 deletions

diff --git a/src/compiler.js b/src/compiler.js
index 7626bde5..395c7acd 100644
--- a/src/compiler.js
+++ b/src/compiler.js
@@ -12,12 +12,10 @@ load('settings.js');
 
 load('utility.js');
 load('enzymatic.js');
-load('library.js');
 load('parseTools.js');
 load('intertyper.js');
 load('analyzer.js');
 load('jsifier.js');
-load('runtime.js');
 
 //===============================
 // Main
@@ -28,6 +26,9 @@ var settings = JSON.parse(readline());
 for (setting in settings) {
   this[setting] = settings[setting];
 }
+var CONSTANTS = { 'QUANTUM_SIZE': QUANTUM_SIZE };
+
+load('runtime.js');
 
 // Sanity of settings
 
@@ -43,5 +44,6 @@ do {
 } while(true);
 
 // Do it
+eval(preprocess(read('library.js'), CONSTANTS));
 print(JSify(analyzer(intertyper(lines))));
 
diff --git a/src/intertyper.js b/src/intertyper.js
index e98012f9..db1a58f5 100644
--- a/src/intertyper.js
+++ b/src/intertyper.js
@@ -335,11 +335,17 @@ function intertyper(data, parseFunctions, baseLineNum) {
         var ident = item.tokens[0].text;
         while (item.tokens[2].text in set('private', 'constant', 'appending', 'global', 'weak_odr', 'internal', 'linkonce', 'linkonce_odr', 'weak', 'hidden'))
           item.tokens.splice(2, 1);
+        var external = false;
+        if (item.tokens[2].text === 'external') {
+          external = true;
+          item.tokens.splice(2, 1);
+        }
         var ret = {
           __result__: true,
           intertype: 'globalVariable',
           ident: ident,
           type: item.tokens[2].text,
+          external: external,
           lineNum: item.lineNum,
         };
         if (ident == '@llvm.global_ctors') {
@@ -699,6 +705,9 @@ function intertyper(data, parseFunctions, baseLineNum) {
   // external function stub
   substrate.addZyme('External', {
     processItem: function(item) {
+      if (item.tokens[1].text == 'noalias') {
+        item.tokens.splice(1, 1);
+      }
       return [{
         __result__: true,
         intertype: 'functionStub',
diff --git a/src/jsifier.js b/src/jsifier.js
index fb813e30..4720798c 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -76,7 +76,12 @@ function JSify(data, functionsOnly, givenTypes, givenFunctions) {
       return '{ ' + ret.join(', ') + ' }';
     }
 
-    return makeGetSlab(ptr, type) + '[' + calcFastOffset(ptr, pos, noNeedFirst) + ']';
+    var offset = calcFastOffset(ptr, pos, noNeedFirst);
+    if (SAFE_HEAP) {
+      return 'SAFE_HEAP_LOAD(' + offset + ', "' + safeQuote(type) + '")';
+    } else {
+      return makeGetSlab(ptr, type) + '[' + offset + ']';
+    }
   }
 
   function indexizeFunctions(value) { // TODO: Also check for other functions (externals, library, etc.)
@@ -99,15 +104,13 @@ function JSify(data, functionsOnly, givenTypes, givenFunctions) {
     value = indexizeFunctions(value);
     var offset = calcFastOffset(ptr, pos, noNeedFirst);
     if (SAFE_HEAP) {
-      return 'SAFE_HEAP_STORE(' + offset + ', ' + value + ')';
+      return 'SAFE_HEAP_STORE(' + offset + ', ' + value + ', "' + safeQuote(type) + '")';
     } else {
       return makeGetSlab(ptr, type) + '[' + offset + '] = ' + value;
     }
   }
 
   function makeEmptyStruct(type) {
-    dprint('types', '??makeemptystruct?? ' + dump(type));
-    // XXX hardcoded ptr impl
     var ret = [];
     var typeData = TYPES[type];
     assertTrue(typeData);
@@ -218,13 +221,16 @@ function JSify(data, functionsOnly, givenTypes, givenFunctions) {
         item.JS = '\n__globalConstructor__ = function() {\n' +
                     item.ctors.map(function(ctor) { return '  ' + toNiceIdent(ctor) + '();' }).join('\n') +
                   '\n}\n';
-      } else if (item.type == 'external') {
-        item.JS = 'var ' + item.ident + ' = ' + '0; /* external value? */';
       } else {
         item.JS = 'var ' + item.ident + ';';
         return [item, {
           intertype: 'GlobalVariable',
-          JS: 'globalFuncs.push(function() { return ' + item.ident + ' = ' + parseConst(item.value, item.type) + ' });',
+          JS: 'globalFuncs.push(function() { return ' + item.ident + ' = ' + (
+            item.external ?
+              makePointer(JSON.stringify(makeEmptyStruct(item.type)), null, 'ALLOC_STATIC', item.type) + ' /* external value? */'
+              :
+              parseConst(item.value, item.type) 
+            ) + ' });',
           __result__: true,
         }];
       }
@@ -868,10 +874,11 @@ function JSify(data, functionsOnly, givenTypes, givenFunctions) {
   function makeFunctionCall(ident, params, funcData) {
     // Special cases
     if (ident == '_llvm_va_start') {
+      // varargs
       var args = 'Array.prototype.slice.call(arguments, __numArgs__)';
       var data = 'Pointer_make([' + args + '.length].concat(' + args + '), 0)';
       if (SAFE_HEAP) {
-        return 'SAFE_HEAP_STORE(' + params[0].ident + ', ' + data + ', 0)';
+        return 'SAFE_HEAP_STORE(' + params[0].ident + ', ' + data + ', null)';
       } else {
         return 'IHEAP[' + params[0].ident + '] = ' + data;
       }
@@ -927,8 +934,7 @@ function JSify(data, functionsOnly, givenTypes, givenFunctions) {
 
     if (functionsOnly) return ret;
 
-    var params = { 'QUANTUM_SIZE': QUANTUM_SIZE };
-    var body = preprocess(read('preamble.js').replace('{{RUNTIME}}', getRuntime()) + ret + read('postamble.js'), params);
+    var body = preprocess(read('preamble.js').replace('{{RUNTIME}}', getRuntime()) + ret + read('postamble.js'), CONSTANTS);
     function reverse_(x) {
       if (LLVM_STYLE === 'old') {
         return x.reverse();
diff --git a/src/library.js b/src/library.js
index a052c44b..c1312e1f 100644
--- a/src/library.js
+++ b/src/library.js
@@ -117,7 +117,11 @@ var Library = {
   strcpy: function(pdest, psrc) {
     var i = 0;
     do {
+#if SAFE_HEAP
+      SAFE_HEAP_STORE(pdest+i, IHEAP[psrc+i], null);
+#else
       IHEAP[pdest+i] = IHEAP[psrc+i];
+#endif
       i ++;
     } while (IHEAP[psrc+i-1] != 0);
   },
@@ -125,7 +129,11 @@ var Library = {
   strncpy: function(pdest, psrc, num) {
     var padding = false;
     for (var i = 0; i < num; i++) {
+#if SAFE_HEAP
+      SAFE_HEAP_STORE(pdest+i, padding ? 0 : IHEAP[psrc+i], null);
+#else
       IHEAP[pdest+i] = padding ? 0 : IHEAP[psrc+i];
+#endif
       padding = padding || IHEAP[psrc+i] == 0;
     }
   },
diff --git a/src/parseTools.js b/src/parseTools.js
index d2e3655f..37e0a881 100644
--- a/src/parseTools.js
+++ b/src/parseTools.js
@@ -387,6 +387,10 @@ function parseNumerical(value, type) {
   } else if (value == 'null') {
     // NULL *is* 0, in C/C++. No JS null! (null == 0 is false, etc.)
     value = '0';
+  } else if (value === 'true') {
+    return '1';
+  } else if (value === 'false') {
+    return '0';
   }
   if (isNumber(value)) {
     return eval(value).toString(); // will change e.g. 5.000000e+01 to 50
diff --git a/src/preamble.js b/src/preamble.js
index 1623d387..f02ada7a 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -16,13 +16,47 @@ var __ATEXIT__ = [];
 #if SAFE_HEAP
 // Semi-manual memory corruption debugging
 var HEAP_WATCHED = {};
-function SAFE_HEAP_STORE(dest, value) {
+var HEAP_HISTORY = {};
+function SAFE_HEAP_CLEAR(dest) {
+  HEAP_HISTORY[dest] = [];
+}
+function SAFE_HEAP_ACCESS(dest, type, store) {
+  if (type && type[type.length-1] == '*') type = 'i32'; // pointers are ints, for our purposes here
+  // Note that this will pass even with unions: You can store X, load X, then store Y and load Y.
+  // You cannot, however, do the nonportable act of store X and load Y!
+  if (store) {
+    HEAP_HISTORY[dest] = [{ type: type, /*stack: new Error().stack */ }]; // |stack| is useful for debugging
+  } else {
+    if (!HEAP[dest] && HEAP[dest] !== 0) {
+      print('Warning: Reading an invalid value at ' + dest + ' :: ' + new Error().stack + '\n');
+    }
+    var history = HEAP_HISTORY[dest];
+    assert((history && history[0]) /* || HEAP[dest] === 0 */, "Loading from where there was no store! " + dest + ',' + HEAP[dest] + ',' + type + ', \n\n' + new Error().stack + '\n');
+    if (history[0].type && history[0].type !== type) {
+      print('Load-store consistency assumption failure! ' + dest);
+      print('\n');
+      print(history.map(function(item) {
+        return item.type + ' :: ' + JSON.stringify(item.stack);
+      }).join('\n\n'));
+      print('\n');
+      print('LOAD: ' + type + ', ' + new Error().stack);
+      print('\n');
+      assert(0, 'Load-store consistency assumption failure!');
+    }
+  }
+}
+function SAFE_HEAP_STORE(dest, value, type) {
+  SAFE_HEAP_ACCESS(dest, type, true);
   if (dest in HEAP_WATCHED) {
     print((new Error()).stack);
     throw "Bad store!" + dest;
   }
   HEAP[dest] = value;
 }
+function SAFE_HEAP_LOAD(dest, type) {
+  SAFE_HEAP_ACCESS(dest, type);
+  return HEAP[dest];
+}
 function __Z16PROTECT_HEAPADDRPv(dest) {
   HEAP_WATCHED[dest] = true;
 }
@@ -120,7 +154,7 @@ function Pointer_make(slab, pos, allocator) {
       curr = Runtime.getFunctionIndex(curr);
     }
 #if SAFE_HEAP
-    SAFE_HEAP_STORE(ret + i, curr);
+    SAFE_HEAP_STORE(ret + i, curr, null);
 #else
 #if USE_TYPED_ARRAYS
     // TODO: Check - also in non-typedarray case - for functions, and if so add |.__index__|
@@ -303,7 +337,7 @@ function _atoi(s) {
 function _llvm_memcpy_i32(dest, src, num, idunno) {
   for (var i = 0; i < num; i++) {
 #if SAFE_HEAP
-    SAFE_HEAP_STORE(dest + i, HEAP[src + i]);
+    SAFE_HEAP_STORE(dest + i, HEAP[src + i], null);
 #else
     HEAP[dest + i] = HEAP[src + i];
 #if USE_TYPED_ARRAYS
diff --git a/src/runtime.js b/src/runtime.js
index daa9086d..2ad54295 100644
--- a/src/runtime.js
+++ b/src/runtime.js
@@ -6,6 +6,9 @@ RuntimeGenerator = {
   alloc: function(size, type) {
     var ret = type + 'TOP';
 //    ret += '; for (var i = 0; i < ' + size + '; i++) HEAP[' + type + 'TOP+i] = 0'; // No need for typed arrays - per the spec, initialized to 0 anyhow
+    if (SAFE_HEAP) {
+      ret += '; for (var j = 0; j < ' + size + '; j++) SAFE_HEAP_CLEAR(' + type + 'TOP+j);';
+    }
     if (GUARD_MEMORY) {
       ret += '; assert(' + size + ' > 0)';
     }
@@ -26,11 +29,14 @@ RuntimeGenerator = {
   },
 
   stackEnter: function(initial) {
-    if (initial === 0) return ''; // XXX Note that we don't even push the stack! This is faster, but
-                                  // means that we don't clear stack allocations done in this function
-                                  // until the parent unwinds its stack. So potentially if we are in
-                                  // a loop, we can use a lot of memory.
+    if (!GUARD_MEMORY && initial === 0) return ''; // XXX Note that we don't even push the stack! This is faster, but
+                                                   // means that we don't clear stack allocations done in this function
+                                                   // until the parent unwinds its stack. So potentially if we are in
+                                                   // a loop, we can use a lot of memory.
     var ret = 'var __stackBase__  = STACKTOP; STACKTOP += ' + initial;
+    if (SAFE_HEAP) {
+      ret += '; for (var i = __stackBase__; i < STACKTOP; i++) SAFE_HEAP_STORE(i, 0, null);';
+    }
     if (GUARD_MEMORY) {
       ret += '; assert(STACKTOP < STACK_MAX)';
     }
@@ -39,7 +45,11 @@ RuntimeGenerator = {
 
   stackExit: function(initial) {
     if (initial === 0) return ''; // XXX See comment in stackEnter
-    return 'STACKTOP = __stackBase__';
+    var ret = '';
+    if (SAFE_HEAP) {
+      ret += 'for (var i = __stackBase__; i < STACKTOP; i++) SAFE_HEAP_CLEAR(i);';
+    }
+    return ret += 'STACKTOP = __stackBase__';
   },
 
   // An allocation that cannot be free'd
diff --git a/src/utility.js b/src/utility.js
index 3085fc61..fd8a1bce 100644
--- a/src/utility.js
+++ b/src/utility.js
@@ -1,5 +1,10 @@
 // General JS utilities
 
+function safeQuote(x) {
+  return x.replace(/"/g, '\\"')
+          .replace(/'/g, "\\'");
+}
+
 function dump(item) {
   var CHUNK = 500;
   function lineify(text) {
diff --git a/tests/runner.py b/tests/runner.py
index 0a8c50dc..c01f1ceb 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -73,7 +73,7 @@ class RunnerCore(unittest.TestCase):
   def do_emscripten(self, filename, output_processor=None):
     # Run Emscripten
     exported_settings = {}
-    for setting in ['QUANTUM_SIZE', 'RELOOP', 'OPTIMIZE', 'GUARD_MEMORY', 'USE_TYPED_ARRAYS']:
+    for setting in ['QUANTUM_SIZE', 'RELOOP', 'OPTIMIZE', 'GUARD_MEMORY', 'USE_TYPED_ARRAYS', 'SAFE_HEAP']:
       exported_settings[setting] = eval(setting)
     out = open(filename + '.o.js', 'w') if not OUTPUT_TO_SCREEN else None
     timeout_run(Popen([EMSCRIPTEN, filename + '.o.ll', COMPILER_ENGINE[0], str(exported_settings).replace("'", '"')], stdout=out, stderr=STDOUT), TIMEOUT, 'Compiling')
@@ -84,17 +84,15 @@ class RunnerCore(unittest.TestCase):
 
   def run_generated_code(self, engine, filename, args=[], check_timeout=True):
     return timeout_run(Popen(engine + [filename] + ([] if engine == SPIDERMONKEY_ENGINE else ['--']) + args,
-                       stdout=PIPE, stderr=STDOUT), 30 if check_timeout else None, 'Execution')
+                       stdout=PIPE, stderr=STDOUT), 120 if check_timeout else None, 'Execution')
 
   def assertContained(self, value, string):
     if value not in string:
-      print "Expected to find '%s' in '%s'" % (value, string)
-      self.assertTrue(value in string)
+      raise Exception("Expected to find '%s' in '%s'" % (value, string))
 
   def assertNotContained(self, value, string):
     if value in string:
-      print "Expected to NOT find '%s' in '%s'" % (value, string)
-      self.assertTrue(value not in string)
+      raise Exception("Expected to NOT find '%s' in '%s'" % (value, string))
 
 if 'benchmark' not in sys.argv:
   class T(RunnerCore): # Short name, to make it more fun to use manually on the commandline
@@ -122,9 +120,12 @@ if 'benchmark' not in sys.argv:
         #shutil.rmtree(dirname) # TODO: leave no trace in memory. But for now nice for debugging
 
     # No building - just process an existing .ll file
-    def do_ll_test(self, ll_file, output, args=[]):
+    def do_ll_test(self, ll_file, output, args=[], f_opt_ll_file=None, js_engines=[V8_ENGINE]):
       if COMPILER != LLVM_GCC: return # We use existing .ll, so which compiler is unimportant
-      if F_OPTS: return # We use existing .ll, so frontend stuff is unimportant
+      if F_OPTS:
+        return # TODO: enable the lines below
+        #if f_opt_ll_file is None: return # We use existing .ll, so frontend stuff is unimportant, unless we are given an optimized .ll
+        #ll_file = f_opt_ll_file
 
       filename = os.path.join(self.get_dir(), 'src.cpp')
       shutil.copy(ll_file, filename + '.o.ll')
@@ -133,7 +134,7 @@ if 'benchmark' not in sys.argv:
                    output,
                    args,
                    no_build=True,
-                   js_engines=[V8_ENGINE]) # mozilla bug XXX
+                   js_engines=js_engines)
 
     def test_hello_world(self):
         src = '''
@@ -1015,12 +1016,16 @@ if 'benchmark' not in sys.argv:
     ### 'Big' tests
 
     def test_fannkuch(self):
+        global SAFE_HEAP; SAFE_HEAP = 0 # Too slow for that
+
         results = [ (1,0), (2,1), (3,2), (4,4), (5,7), (6,10), (7, 16), (8,22) ]
         for i, j in results:
           src = open(path_from_root(['tests', 'fannkuch.cpp']), 'r').read()
           self.do_test(src, 'Pfannkuchen(%d) = %d.' % (i,j), [str(i)], no_build=i>1)
 
     def test_raytrace(self):
+        global SAFE_HEAP; SAFE_HEAP = 0 # Too slow for that
+
         src = open(path_from_root(['tests', 'raytrace.cpp']), 'r').read()
         output = open(path_from_root(['tests', 'raytrace.ppm']), 'r').read()
         self.do_test(src, output, ['3', '16'])
@@ -1042,6 +1047,8 @@ if 'benchmark' not in sys.argv:
       #              used, see Mozilla bug 593659.
       assert COMPILER_ENGINE != SPIDERMONKEY_ENGINE
 
+      global SAFE_HEAP; SAFE_HEAP = 0 # Has some actual loads of unwritten-to places, in the C++ code...
+
       self.do_test(path_from_root(['tests', 'cubescript']), '*\nTemp is 33\n9\n5\nhello, everyone\n*', main_file='command.cpp')
 
     def test_gcc_unmangler(self):
@@ -1049,10 +1056,15 @@ if 'benchmark' not in sys.argv:
       self.do_test(path_from_root(['third_party']), '*d_demangle(char const*, int, unsigned int*)*', args=['_ZL10d_demanglePKciPj'], main_file='gcc_demangler.c')
 
     def test_bullet(self):
+      global SAFE_HEAP; SAFE_HEAP = 0 # Too slow for that
       self.do_ll_test(path_from_root(['tests', 'bullet', 'bulletTest.ll']), open(path_from_root(['tests', 'bullet', 'output.txt']), 'r').read())
 
     def test_lua(self):
-      self.do_ll_test(path_from_root(['tests', 'lua', 'lua.ll']), 'hello lua world!', args=['-e', '''print("hello lua world!")'''])
+      self.do_ll_test(path_from_root(['tests', 'lua', 'lua.ll']),
+                      'hello lua world!',
+                      args=['-e', '''print("hello lua world!")'''],
+                      f_opt_ll_file=path_from_root(['tests', 'lua', 'lua.Os.ll']),
+                      js_engines=[SPIDERMONKEY_ENGINE])
 
     ### Test cases in separate files
 
@@ -1095,15 +1107,35 @@ if 'benchmark' not in sys.argv:
           open(filename, 'w').write(src)
         self.do_test(src, '*166*\n*ok*', post_build=post)
 
+    ### Tests for tools
+
+    def test_safe_heap(self):
+      if not SAFE_HEAP: return
+      if F_OPTS: return
+      src = '''
+        #include<stdio.h>
+        int main() {
+          int *x = new int;
+          *x = 20;
+          float *y = (float*)x;
+          printf("%f\\n", *y);
+          return 0;
+        }
+      '''
+      try:
+        self.do_test(src, '*nothingatall*')
+      except Exception, e:
+        assert 'Assertion failed: Load-store consistency assumption failure!' in str(e), str(e)
+
   # Generate tests for all our compilers
   def make_test(compiler, f_opts, embetter):
     class TT(T):
       def setUp(self):
-        global COMPILER, QUANTUM_SIZE, RELOOP, OPTIMIZE, GUARD_MEMORY, USE_TYPED_ARRAYS, F_OPTS
+        global COMPILER, QUANTUM_SIZE, RELOOP, OPTIMIZE, GUARD_MEMORY, USE_TYPED_ARRAYS, F_OPTS, SAFE_HEAP
         COMPILER = compiler['path']
         QUANTUM_SIZE = compiler['quantum_size']
         RELOOP = OPTIMIZE = USE_TYPED_ARRAYS = embetter
-        GUARD_MEMORY = 1-embetter
+        GUARD_MEMORY = SAFE_HEAP = 1-embetter
         F_OPTS = f_opts
     return TT
   for embetter in [0,1]:
@@ -1125,7 +1157,7 @@ else:
 
   QUANTUM_SIZE = 4
   RELOOP = OPTIMIZE = USE_TYPED_ARRAYS = 1
-  GUARD_MEMORY = 0
+  GUARD_MEMORY = SAFE_HEAP = 0
   F_OPTS = 1
 
   TEST_REPS = 10