refactor compiler to allow future batching of types and globals. move postsets from run() to the toplevel to boost performance and simplify compilation. fix various bugs that were noticed during this

13 files changed, 121 insertions, 63 deletions

diff --git a/src/analyzer.js b/src/analyzer.js
index 2dce7579..1b9b4378 100644
--- a/src/analyzer.js
+++ b/src/analyzer.js
@@ -29,7 +29,7 @@ function analyzer(data) {
   substrate.addActor('Gatherer', {
     processItem: function(item) {
       // Single-liners
-      ['globalVariable', 'functionStub', 'unparsedFunction', 'alias'].forEach(function(intertype) {
+      ['globalVariable', 'functionStub', 'unparsedFunction', 'unparsedGlobals', 'unparsedTypes', 'alias'].forEach(function(intertype) {
         var temp = splitter(item.items, function(item) { return item.intertype == intertype });
         item.items = temp.leftIn;
         item[intertype + 's'] = temp.splitOut;
diff --git a/src/intertyper.js b/src/intertyper.js
index a21a6427..45199091 100644
--- a/src/intertyper.js
+++ b/src/intertyper.js
@@ -7,6 +7,9 @@ function tokenize(text) {
 }
 
 function intertyper(data, sidePass, baseLineNum) {
+  var mainPass = !sidePass;
+  dprint('framework', 'Big picture: Starting intertyper, main pass=' + mainPass);
+
   baseLineNum = baseLineNum || 0;
 
   // Substrate
@@ -42,16 +45,47 @@ function intertyper(data, sidePass, baseLineNum) {
       var currFunctionLines;
       var currFunctionLineNum;
       var unparsedBundles = [];
+      var unparsedTypes, unparsedGlobals;
+      if (mainPass) {
+        unparsedTypes = {
+          intertype: 'unparsedTypes',
+          lines: []
+        };
+        unparsedBundles.push(unparsedTypes);
+        unparsedGlobals = {
+          intertype: 'unparsedGlobals',
+          lines: []
+        };
+        unparsedBundles.push(unparsedGlobals);
+      }
       for (var i = 0; i < lines.length; i++) {
         var line = lines[i];
         lines[i] = null; // lines may be very very large. Allow GCing to occur in the loop by releasing refs here
 
-        if (!sidePass && /^define .*/.test(line)) {
+        if (mainPass && (line[0] == '%' || line[0] == '@')) {
+          // If this isn't a type, it's a global variable, make a note of the information now, we will need it later
+          var testType = /[@%\w\d\.\" ]+ = type .*/.exec(line);
+          if (!testType) {
+            var global = /([@%\w\d\.\" ]+) = .*/.exec(line);
+            var globalIdent = toNiceIdent(global[1]);
+            var testAlias = /[@%\w\d\.\" ]+ = alias .*/.exec(line);
+            Variables.globals[globalIdent] = {
+              name: globalIdent,
+              alias: !!testAlias,
+              impl: VAR_EMULATED
+            };
+            unparsedGlobals.lines.push(line);
+          } else {
+            unparsedTypes.lines.push(line);
+          }
+          continue;
+        }
+        if (mainPass && /^define .*/.test(line)) {
           inFunction = true;
           currFunctionLines = [];
           currFunctionLineNum = i + 1;
         }
-        if (!inFunction || sidePass) {
+        if (!inFunction || !mainPass) {
           if (inContinual || new RegExp(/^\ +to.*/g).test(line)
                           || new RegExp(/^\ +catch .*/g).test(line)
                           || new RegExp(/^\ +filter .*/g).test(line)
@@ -74,9 +108,9 @@ function intertyper(data, sidePass, baseLineNum) {
         } else {
           currFunctionLines.push(line);
         }
-        if (!sidePass && /^}.*/.test(line)) {
+        if (mainPass && /^}.*/.test(line)) {
           inFunction = false;
-          if (!sidePass) {
+          if (mainPass) {
             var func = funcHeader.processItem(tokenizer.processItem({ lineText: currFunctionLines[0], lineNum: currFunctionLineNum }, true))[0];
             unparsedBundles.push({
               intertype: 'unparsedFunction',
@@ -93,7 +127,7 @@ function intertyper(data, sidePass, baseLineNum) {
       }
       // We need lines beginning with ';' inside functions, because older LLVM versions generated labels that way. But when not
       // parsing functions, we can ignore all such lines and save some time that way.
-      this.forwardItems(ret.filter(function(item) { return item.lineText && (item.lineText[0] != ';' || sidePass); }), 'Tokenizer');
+      this.forwardItems(ret.filter(function(item) { return item.lineText && (item.lineText[0] != ';' || !mainPass); }), 'Tokenizer');
       return unparsedBundles;
     }
   });
diff --git a/src/jsifier.js b/src/jsifier.js
index 0d9f2ebd..f7fc09b8 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -2,7 +2,7 @@
 // before this stage, which just does the final conversion to JavaScript.
 
 // Main function
-function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
+function JSify(data, functionsOnly, givenFunctions) {
   var mainPass = !functionsOnly;
 
   if (mainPass) {
@@ -33,7 +33,26 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     print(pre);
     print('Runtime.QUANTUM_SIZE = ' + QUANTUM_SIZE);
 
-    // Add additional necessary items for the main pass
+    Functions.implementedFunctions = set(data.unparsedFunctions.map(function(func) { return func.ident }));
+  }
+
+  // Does simple 'macro' substitution, using Django-like syntax,
+  // {{{ code }}} will be replaced with |eval(code)|.
+  function processMacros(text) {
+    return text.replace(/{{{[^}]+}}}/g, function(str) {
+      str = str.substr(3, str.length-6);
+      return eval(str).toString();
+    });
+  }
+
+  substrate = new Substrate('JSifyer');
+
+  if (mainPass) {
+    // Handle unparsed types TODO: Batch them
+    JSify(analyzer(intertyper(data.unparsedTypess[0].lines, true)), true, Functions);
+
+    // Add additional necessary items for the main pass. We can now do this since types are parsed (types can be used through
+    // generateStructInfo in library.js)
     LibraryManager.load();
     var libFuncsToInclude;
     if (INCLUDE_FULL_LIBRARY) {
@@ -53,23 +72,9 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
         ident: '_' + ident
       });
     });
-
-    Functions.implementedFunctions = set(data.unparsedFunctions.map(function(func) { return func.ident }));
-  }
-
-  // Does simple 'macro' substitution, using Django-like syntax,
-  // {{{ code }}} will be replaced with |eval(code)|.
-  function processMacros(text) {
-    return text.replace(/{{{[^}]+}}}/g, function(str) {
-      str = str.substr(3, str.length-6);
-      return eval(str).toString();
-    });
   }
 
-  substrate = new Substrate('JSifyer');
-
-  var GLOBAL_VARIABLES = !mainPass ? givenGlobalVariables : data.globalVariables;
-  Variables.globals = GLOBAL_VARIABLES;
+  // Functions
 
   Functions.currFunctions = !mainPass ? givenFunctions.currFunctions : {};
   Functions.currExternalFunctions = !mainPass ? givenFunctions.currExternalFunctions : {};
@@ -101,7 +106,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     data.unparsedFunctions[i] = null;
     dprint('unparsedFunctions', '====================\n// Processing |' + func.ident + '|, ' + i + '/' + data.unparsedFunctions.length);
     if (DEBUG_MEMORY) MemoryDebugger.tick('pre-func ' + func.ident);
-    JSify(analyzer(intertyper(func.lines, true, func.lineNum-1)), true, Functions, GLOBAL_VARIABLES);
+    JSify(analyzer(intertyper(func.lines, true, func.lineNum-1)), true, Functions);
     if (DEBUG_MEMORY) MemoryDebugger.tick('func ' + func.ident);
   }
   func = null; // Do not hold on to anything from inside that loop (JS function scoping..)
@@ -225,7 +230,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   substrate.addActor('GlobalVariable', {
     processItem: function(item) {
       item.intertype = 'GlobalVariableStub';
-      delete item.lines; // Save some memory
+      item.lines = null; // Save some memory
       var ret = [item];
       if (item.ident == '_llvm_global_ctors') {
         item.JS = '\n__globalConstructor__ = function() {\n' +
@@ -585,7 +590,14 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   });
 
   function getVarData(funcData, ident) {
-    return funcData.variables[ident] || GLOBAL_VARIABLES[ident] || null;
+    var local = funcData.variables[ident];
+    if (local) return local;
+    var global = Variables.globals[ident];
+    // FIXME: Currently, if something is an alias, we assume it is not a simple variable, so no need for
+    //        FUNCTION_TABLE when calling it (which we do need for a normal simple global variable). In
+    //        theory though an alias could be simple, we should probably check the type if this ever becomes a problem.
+    if (global && global.alias) global = null;
+    return global || null;
   }
 
   function getVarImpl(funcData, ident) {
@@ -1057,7 +1069,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     //
 
     if (!mainPass) {
-      var generated = itemsDict.function;
+      var generated = itemsDict.function.concat(itemsDict.type).concat(itemsDict.GlobalVariableStub).concat(itemsDict.GlobalVariable).concat(itemsDict.GlobalVariablePostSet);
       Functions.allIdents = Functions.allIdents.concat(itemsDict.function.map(function(func) {
         return func.ident;
       }).filter(function(func) {
@@ -1070,7 +1082,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     // This is the main pass. Print out the generated code that we have here, together with the
     // rest of the output that we started to print out earlier (see comment on the
     // "Final shape that will be created").
-    var generated = itemsDict.type.concat(itemsDict.GlobalVariableStub).concat(itemsDict.functionStub);
+    var generated = itemsDict.functionStub.concat(itemsDict.GlobalVariablePostSet);
     generated.forEach(function(item) { print(indentify(item.JS || '', 2)); });
     if (RUNTIME_TYPE_INFO) {
       Types.cleanForRuntime();
@@ -1080,8 +1092,10 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     var postFile = BUILD_AS_SHARED_LIB ? 'postamble_sharedlib.js' : 'postamble.js';
     var postParts = processMacros(preprocess(read(postFile), CONSTANTS)).split('{{GLOBAL_VARS}}');
     print(postParts[0]);
-    itemsDict.GlobalVariable.forEach(function(item) { print(indentify(item.JS, 4)); });
-    itemsDict.GlobalVariablePostSet.forEach(function(item) { print(indentify(item.JS, 4)); });
+
+    // Print out global variables and postsets TODO: batching
+    JSify(analyzer(intertyper(data.unparsedGlobalss[0].lines, true)), true, Functions);
+
     print(Functions.generateIndexing()); // done last, as it may rely on aliases set in postsets
     print(postParts[1]);
     print(shellParts[1]);
@@ -1094,15 +1108,17 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   // Data
 
   if (mainPass) {
-    substrate.addItems(values(Types.types).filter(function(type) { return type.lineNum != '?' }), 'Type');
-    substrate.addItems(values(data.globalVariables), 'GlobalVariable');
     substrate.addItems(data.functionStubs, 'FunctionStub');
-    substrate.addItems(data.aliass, 'Alias');
     assert(data.functions.length == 0);
   } else {
+    substrate.addItems(values(Types.types).filter(function(type) { return type.lineNum != '?' && type.name_ }), 'Type');
+    substrate.addItems(values(data.globalVariables), 'GlobalVariable');
+    substrate.addItems(data.aliass, 'Alias');
     substrate.addItems(data.functions, 'FunctionSplitter');
   }
 
   finalCombiner(substrate.solve());
+
+  dprint('framework', 'Big picture: Finishing JSifier, main pass=' + mainPass);
 }
 
diff --git a/src/library.js b/src/library.js
index c180b7c4..2a67ee88 100644
--- a/src/library.js
+++ b/src/library.js
@@ -388,9 +388,6 @@ LibraryManager.library = {
       __impure_ptr = allocate([ allocate(
         {{{ QUANTUM_SIZE === 4 ? '[0, 0, 0, 0, _stdin, 0, 0, 0, _stdout, 0, 0, 0, _stderr, 0, 0, 0]' : '[0, _stdin, _stdout, _stderr]' }}},
         'void*', ALLOC_STATIC) ], 'void*', ALLOC_STATIC);
-
-      // Once initialized, permissions start having effect.
-      FS.ignorePermissions = false;
     },
 
     quit: function() {
@@ -5481,11 +5478,6 @@ LibraryManager.library = {
     eval(Pointer_stringify(ptr));
   },
 
-  EMSCRIPTEN_COMMENT__inline: function(param) {
-    param = stripCorrections(param);
-    return '// ' + Variables.globals[param].value.text.replace('\\00', '') + '   ';
-  },
-
   $Profiling: {
     max_: 0,
     times: null,
diff --git a/src/modules.js b/src/modules.js
index fec73f34..39baabba 100644
--- a/src/modules.js
+++ b/src/modules.js
@@ -191,7 +191,7 @@ var Debugging = {
 };
 
 var Variables = {
-  globals: null
+  globals: {}
 };
 
 var Types = {
diff --git a/src/parseTools.js b/src/parseTools.js
index cedfeaad..f2ef3374 100644
--- a/src/parseTools.js
+++ b/src/parseTools.js
@@ -337,7 +337,7 @@ function parseParamTokens(params) {
           value: null,
           ident: toNiceIdent('%') + anonymousIndex
         });
-        Types.needAnalysis[ret.type] = 0;
+        Types.needAnalysis[ret[ret.length-1].type] = 0;
         anonymousIndex ++;
       }
     } else if (segment[1].text in PARSABLE_LLVM_FUNCTIONS) {
@@ -358,9 +358,7 @@ function parseParamTokens(params) {
         value: segment[1],
         ident: toNiceIdent(parseNumerical(segment[1].text))
       });
-      Types.needAnalysis[ret.type] = 0;
-      //          } else {
-      //            throw "what is this params token? " + JSON.stringify(segment);
+      Types.needAnalysis[removeAllPointing(ret[ret.length-1].type)] = 0;
     }
     ret[ret.length-1].byVal = byVal;
   }
diff --git a/src/postamble.js b/src/postamble.js
index f1d48ead..c8fd44bd 100644
--- a/src/postamble.js
+++ b/src/postamble.js
@@ -20,11 +20,11 @@ Module.callMain = function callMain(args) {
   return _main(argc, argv, 0);
 }
 
+{{GLOBAL_VARS}}
+
 function run(args) {
   args = args || Module['arguments'];
 
-{{GLOBAL_VARS}}
-
   __globalConstructor__();
 
   var ret = null;
@@ -38,6 +38,11 @@ Module['run'] = run;
 
 // {{PRE_RUN_ADDITIONS}}
 
+// In a hackish way, we disable permissions until now, so setup code works, but enable them for runtime so compile code works with permissions
+try {
+  FS.ignorePermissions = false;
+} catch(e){}
+
 #if INVOKE_RUN
 #else
 Module['noInitialRun'] = true;
diff --git a/src/postamble_sharedlib.js b/src/postamble_sharedlib.js
index 1fd78f23..c7a03597 100644
--- a/src/postamble_sharedlib.js
+++ b/src/postamble_sharedlib.js
@@ -1,8 +1,9 @@
 
 // === Auto-generated postamble setup entry stuff ===
 
+{{GLOBAL_VARS}}
+
 function run(args) {
-  {{GLOBAL_VARS}}
   __globalConstructor__();
 }
 Module['run'] = run;
diff --git a/system/include/emscripten.h b/system/include/emscripten.h
index 0ce31e9f..482f7ef3 100644
--- a/system/include/emscripten.h
+++ b/system/include/emscripten.h
@@ -20,8 +20,10 @@ extern void emscripten_run_script(const char *script);
 /*
  * This macro-looking function will cause Emscripten to
  * generate a comment in the generated code.
+ * XXX This is deprecated for now, because it requires us to
+ *     hold all global vars in memory. We need a better solution.
  */
-extern void EMSCRIPTEN_COMMENT(const char *text);
+//extern void EMSCRIPTEN_COMMENT(const char *text);
 
 /*
  * Profiling tools.
diff --git a/tests/cases/gepoverflow.txt b/tests/cases/gepoverflow.txt
index 6d4d90bc..01514709 100644
--- a/tests/cases/gepoverflow.txt
+++ b/tests/cases/gepoverflow.txt
@@ -1,2 +1,2 @@
-*1052186,1052756*
+*1052222,1052792*
 *-514,56*
diff --git a/tests/runner.py b/tests/runner.py
index 3fe70ea9..b372ee51 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -1580,7 +1580,7 @@ if 'benchmark' not in str(sys.argv):
           #include "emscripten.h"
 
           int main() {
-            EMSCRIPTEN_COMMENT("hello from the source");
+            // EMSCRIPTEN_COMMENT("hello from the source");
             emscripten_run_script("print('hello world' + '!')");
             return 0;
           }
@@ -1588,7 +1588,7 @@ if 'benchmark' not in str(sys.argv):
 
         def check(filename):
           src = open(filename, 'r').read()
-          assert '// hello from the source' in src
+          # TODO: restore this (see comment in emscripten.h) assert '// hello from the source' in src
 
         self.do_run(src, 'hello world!', post_build=check)
 
@@ -2712,7 +2712,7 @@ if 'benchmark' not in str(sys.argv):
     def test_files(self):
       Settings.CORRECT_SIGNS = 1 # Just so our output is what we expect. Can flip them both.
       def post(filename):
-        src = open(filename, 'r').read().replace(
+        src = open(filename, 'r').read().replace('FS.init();', '').replace( # Disable normal initialization, replace with ours
           '// {{PRE_RUN_ADDITIONS}}',
           '''
             FS.createDataFile('/', 'somefile.binary', [100, 200, 50, 25, 10, 77, 123], true, false);  // 200 becomes -56, since signed chars are used in memory
@@ -3067,7 +3067,7 @@ if 'benchmark' not in str(sys.argv):
       Settings.INCLUDE_FULL_LIBRARY = 1
       try:
         def addJS(filename):
-          src = open(filename, 'r').read().replace(
+          src = open(filename, 'r').read().replace('FS.init();', '').replace( # Disable normal initialization, replace with ours
             '// {{PRE_RUN_ADDITIONS}}',
             open(path_from_root('tests', 'filesystem', 'src.js'), 'r').read())
           open(filename, 'w').write(src)
@@ -4546,6 +4546,7 @@ class %s(T):
     llvm_opts = %d # 1 is yes, 2 is yes and unsafe
     embetter = %d
     quantum_size = %d
+    # TODO: Move much of these to a init() function in shared.py, and reuse that
     Settings.USE_TYPED_ARRAYS = %d
     Settings.INVOKE_RUN = 1
     Settings.RELOOP = Settings.MICRO_OPTS = embetter
@@ -4564,11 +4565,15 @@ class %s(T):
     Settings.RUNTIME_TYPE_INFO = 0
     Settings.DISABLE_EXCEPTION_CATCHING = 0
     Settings.PROFILE = 0
+    Settings.INCLUDE_FULL_LIBRARY = 0
+    Settings.BUILD_AS_SHARED_LIB = 0
     Settings.TOTAL_MEMORY = Settings.FAST_MEMORY = None
     Settings.EMULATE_UNALIGNED_ACCESSES = Settings.USE_TYPED_ARRAYS == 2 and Building.LLVM_OPTS == 2
     if Settings.USE_TYPED_ARRAYS == 2:
       Settings.I64_MODE = 1
       Settings.SAFE_HEAP = 1 # only checks for alignment problems, which is very important with unsafe opts
+    else:
+      Settings.I64_MODE = 0
 
     if Settings.QUANTUM_SIZE == 1 or Settings.USE_TYPED_ARRAYS == 2:
       Settings.RELOOP = 0 # XXX Would be better to use this, but it isn't really what we test in these cases, and is very slow
diff --git a/tests/stat/output.txt b/tests/stat/output.txt
index 1e6ae74e..b5d66fa5 100644
--- a/tests/stat/output.txt
+++ b/tests/stat/output.txt
@@ -2,7 +2,7 @@
 ret: 0
 errno: 0
 st_dev: 1
-st_ino: 2
+st_ino: 8
 st_mode: 040777
 st_nlink: 1
 st_rdev: 0
@@ -24,7 +24,7 @@ S_ISSOCK: 0
 ret: 0
 errno: 0
 st_dev: 1
-st_ino: 3
+st_ino: 9
 st_mode: 0100777
 st_nlink: 1
 st_rdev: 0
@@ -45,11 +45,11 @@ S_ISSOCK: 0
 --stat DEVICE--
 ret: 0
 errno: 0
-st_dev: 5
-st_ino: 5
+st_dev: 11
+st_ino: 11
 st_mode: 020777
 st_nlink: 1
-st_rdev: 5
+st_rdev: 11
 st_size: 0
 st_atime: 1200000000
 st_mtime: 1200000000
@@ -68,7 +68,7 @@ S_ISSOCK: 0
 ret: 0
 errno: 0
 st_dev: 1
-st_ino: 3
+st_ino: 9
 st_mode: 0100777
 st_nlink: 1
 st_rdev: 0
@@ -90,7 +90,7 @@ S_ISSOCK: 0
 ret: 0
 errno: 0
 st_dev: 1
-st_ino: 4
+st_ino: 10
 st_mode: 0120777
 st_nlink: 1
 st_rdev: 0
@@ -112,7 +112,7 @@ S_ISSOCK: 0
 ret: 0
 errno: 0
 st_dev: 1
-st_ino: 3
+st_ino: 9
 st_mode: 0100777
 st_nlink: 1
 st_rdev: 0
diff --git a/tests/stat/src.c b/tests/stat/src.c
index 3b3b8421..bdf27ca2 100644
--- a/tests/stat/src.c
+++ b/tests/stat/src.c
@@ -1,3 +1,8 @@
+/*
+Note: Hardcoded st_ino values etc. may change with minor changes to the library impl.
+      In such an event, we will need to update output.txt here.
+*/
+
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>