refactor compiler to allow future batching of types and globals. move postsets from run() to the toplevel to boost performance and simplify compilation. fix various bugs that were noticed during this

8 files changed, 95 insertions, 49 deletions

diff --git a/src/analyzer.js b/src/analyzer.js
index 2dce7579..1b9b4378 100644
--- a/src/analyzer.js
+++ b/src/analyzer.js
@@ -29,7 +29,7 @@ function analyzer(data) {
   substrate.addActor('Gatherer', {
     processItem: function(item) {
       // Single-liners
-      ['globalVariable', 'functionStub', 'unparsedFunction', 'alias'].forEach(function(intertype) {
+      ['globalVariable', 'functionStub', 'unparsedFunction', 'unparsedGlobals', 'unparsedTypes', 'alias'].forEach(function(intertype) {
         var temp = splitter(item.items, function(item) { return item.intertype == intertype });
         item.items = temp.leftIn;
         item[intertype + 's'] = temp.splitOut;
diff --git a/src/intertyper.js b/src/intertyper.js
index a21a6427..45199091 100644
--- a/src/intertyper.js
+++ b/src/intertyper.js
@@ -7,6 +7,9 @@ function tokenize(text) {
 }
 
 function intertyper(data, sidePass, baseLineNum) {
+  var mainPass = !sidePass;
+  dprint('framework', 'Big picture: Starting intertyper, main pass=' + mainPass);
+
   baseLineNum = baseLineNum || 0;
 
   // Substrate
@@ -42,16 +45,47 @@ function intertyper(data, sidePass, baseLineNum) {
       var currFunctionLines;
       var currFunctionLineNum;
       var unparsedBundles = [];
+      var unparsedTypes, unparsedGlobals;
+      if (mainPass) {
+        unparsedTypes = {
+          intertype: 'unparsedTypes',
+          lines: []
+        };
+        unparsedBundles.push(unparsedTypes);
+        unparsedGlobals = {
+          intertype: 'unparsedGlobals',
+          lines: []
+        };
+        unparsedBundles.push(unparsedGlobals);
+      }
       for (var i = 0; i < lines.length; i++) {
         var line = lines[i];
         lines[i] = null; // lines may be very very large. Allow GCing to occur in the loop by releasing refs here
 
-        if (!sidePass && /^define .*/.test(line)) {
+        if (mainPass && (line[0] == '%' || line[0] == '@')) {
+          // If this isn't a type, it's a global variable, make a note of the information now, we will need it later
+          var testType = /[@%\w\d\.\" ]+ = type .*/.exec(line);
+          if (!testType) {
+            var global = /([@%\w\d\.\" ]+) = .*/.exec(line);
+            var globalIdent = toNiceIdent(global[1]);
+            var testAlias = /[@%\w\d\.\" ]+ = alias .*/.exec(line);
+            Variables.globals[globalIdent] = {
+              name: globalIdent,
+              alias: !!testAlias,
+              impl: VAR_EMULATED
+            };
+            unparsedGlobals.lines.push(line);
+          } else {
+            unparsedTypes.lines.push(line);
+          }
+          continue;
+        }
+        if (mainPass && /^define .*/.test(line)) {
           inFunction = true;
           currFunctionLines = [];
           currFunctionLineNum = i + 1;
         }
-        if (!inFunction || sidePass) {
+        if (!inFunction || !mainPass) {
           if (inContinual || new RegExp(/^\ +to.*/g).test(line)
                           || new RegExp(/^\ +catch .*/g).test(line)
                           || new RegExp(/^\ +filter .*/g).test(line)
@@ -74,9 +108,9 @@ function intertyper(data, sidePass, baseLineNum) {
         } else {
           currFunctionLines.push(line);
         }
-        if (!sidePass && /^}.*/.test(line)) {
+        if (mainPass && /^}.*/.test(line)) {
           inFunction = false;
-          if (!sidePass) {
+          if (mainPass) {
             var func = funcHeader.processItem(tokenizer.processItem({ lineText: currFunctionLines[0], lineNum: currFunctionLineNum }, true))[0];
             unparsedBundles.push({
               intertype: 'unparsedFunction',
@@ -93,7 +127,7 @@ function intertyper(data, sidePass, baseLineNum) {
       }
       // We need lines beginning with ';' inside functions, because older LLVM versions generated labels that way. But when not
       // parsing functions, we can ignore all such lines and save some time that way.
-      this.forwardItems(ret.filter(function(item) { return item.lineText && (item.lineText[0] != ';' || sidePass); }), 'Tokenizer');
+      this.forwardItems(ret.filter(function(item) { return item.lineText && (item.lineText[0] != ';' || !mainPass); }), 'Tokenizer');
       return unparsedBundles;
     }
   });
diff --git a/src/jsifier.js b/src/jsifier.js
index 0d9f2ebd..f7fc09b8 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -2,7 +2,7 @@
 // before this stage, which just does the final conversion to JavaScript.
 
 // Main function
-function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
+function JSify(data, functionsOnly, givenFunctions) {
   var mainPass = !functionsOnly;
 
   if (mainPass) {
@@ -33,7 +33,26 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     print(pre);
     print('Runtime.QUANTUM_SIZE = ' + QUANTUM_SIZE);
 
-    // Add additional necessary items for the main pass
+    Functions.implementedFunctions = set(data.unparsedFunctions.map(function(func) { return func.ident }));
+  }
+
+  // Does simple 'macro' substitution, using Django-like syntax,
+  // {{{ code }}} will be replaced with |eval(code)|.
+  function processMacros(text) {
+    return text.replace(/{{{[^}]+}}}/g, function(str) {
+      str = str.substr(3, str.length-6);
+      return eval(str).toString();
+    });
+  }
+
+  substrate = new Substrate('JSifyer');
+
+  if (mainPass) {
+    // Handle unparsed types TODO: Batch them
+    JSify(analyzer(intertyper(data.unparsedTypess[0].lines, true)), true, Functions);
+
+    // Add additional necessary items for the main pass. We can now do this since types are parsed (types can be used through
+    // generateStructInfo in library.js)
     LibraryManager.load();
     var libFuncsToInclude;
     if (INCLUDE_FULL_LIBRARY) {
@@ -53,23 +72,9 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
         ident: '_' + ident
       });
     });
-
-    Functions.implementedFunctions = set(data.unparsedFunctions.map(function(func) { return func.ident }));
-  }
-
-  // Does simple 'macro' substitution, using Django-like syntax,
-  // {{{ code }}} will be replaced with |eval(code)|.
-  function processMacros(text) {
-    return text.replace(/{{{[^}]+}}}/g, function(str) {
-      str = str.substr(3, str.length-6);
-      return eval(str).toString();
-    });
   }
 
-  substrate = new Substrate('JSifyer');
-
-  var GLOBAL_VARIABLES = !mainPass ? givenGlobalVariables : data.globalVariables;
-  Variables.globals = GLOBAL_VARIABLES;
+  // Functions
 
   Functions.currFunctions = !mainPass ? givenFunctions.currFunctions : {};
   Functions.currExternalFunctions = !mainPass ? givenFunctions.currExternalFunctions : {};
@@ -101,7 +106,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     data.unparsedFunctions[i] = null;
     dprint('unparsedFunctions', '====================\n// Processing |' + func.ident + '|, ' + i + '/' + data.unparsedFunctions.length);
     if (DEBUG_MEMORY) MemoryDebugger.tick('pre-func ' + func.ident);
-    JSify(analyzer(intertyper(func.lines, true, func.lineNum-1)), true, Functions, GLOBAL_VARIABLES);
+    JSify(analyzer(intertyper(func.lines, true, func.lineNum-1)), true, Functions);
     if (DEBUG_MEMORY) MemoryDebugger.tick('func ' + func.ident);
   }
   func = null; // Do not hold on to anything from inside that loop (JS function scoping..)
@@ -225,7 +230,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   substrate.addActor('GlobalVariable', {
     processItem: function(item) {
       item.intertype = 'GlobalVariableStub';
-      delete item.lines; // Save some memory
+      item.lines = null; // Save some memory
       var ret = [item];
       if (item.ident == '_llvm_global_ctors') {
         item.JS = '\n__globalConstructor__ = function() {\n' +
@@ -585,7 +590,14 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   });
 
   function getVarData(funcData, ident) {
-    return funcData.variables[ident] || GLOBAL_VARIABLES[ident] || null;
+    var local = funcData.variables[ident];
+    if (local) return local;
+    var global = Variables.globals[ident];
+    // FIXME: Currently, if something is an alias, we assume it is not a simple variable, so no need for
+    //        FUNCTION_TABLE when calling it (which we do need for a normal simple global variable). In
+    //        theory though an alias could be simple, we should probably check the type if this ever becomes a problem.
+    if (global && global.alias) global = null;
+    return global || null;
   }
 
   function getVarImpl(funcData, ident) {
@@ -1057,7 +1069,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     //
 
     if (!mainPass) {
-      var generated = itemsDict.function;
+      var generated = itemsDict.function.concat(itemsDict.type).concat(itemsDict.GlobalVariableStub).concat(itemsDict.GlobalVariable).concat(itemsDict.GlobalVariablePostSet);
       Functions.allIdents = Functions.allIdents.concat(itemsDict.function.map(function(func) {
         return func.ident;
       }).filter(function(func) {
@@ -1070,7 +1082,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     // This is the main pass. Print out the generated code that we have here, together with the
     // rest of the output that we started to print out earlier (see comment on the
     // "Final shape that will be created").
-    var generated = itemsDict.type.concat(itemsDict.GlobalVariableStub).concat(itemsDict.functionStub);
+    var generated = itemsDict.functionStub.concat(itemsDict.GlobalVariablePostSet);
     generated.forEach(function(item) { print(indentify(item.JS || '', 2)); });
     if (RUNTIME_TYPE_INFO) {
       Types.cleanForRuntime();
@@ -1080,8 +1092,10 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     var postFile = BUILD_AS_SHARED_LIB ? 'postamble_sharedlib.js' : 'postamble.js';
     var postParts = processMacros(preprocess(read(postFile), CONSTANTS)).split('{{GLOBAL_VARS}}');
     print(postParts[0]);
-    itemsDict.GlobalVariable.forEach(function(item) { print(indentify(item.JS, 4)); });
-    itemsDict.GlobalVariablePostSet.forEach(function(item) { print(indentify(item.JS, 4)); });
+
+    // Print out global variables and postsets TODO: batching
+    JSify(analyzer(intertyper(data.unparsedGlobalss[0].lines, true)), true, Functions);
+
     print(Functions.generateIndexing()); // done last, as it may rely on aliases set in postsets
     print(postParts[1]);
     print(shellParts[1]);
@@ -1094,15 +1108,17 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
   // Data
 
   if (mainPass) {
-    substrate.addItems(values(Types.types).filter(function(type) { return type.lineNum != '?' }), 'Type');
-    substrate.addItems(values(data.globalVariables), 'GlobalVariable');
     substrate.addItems(data.functionStubs, 'FunctionStub');
-    substrate.addItems(data.aliass, 'Alias');
     assert(data.functions.length == 0);
   } else {
+    substrate.addItems(values(Types.types).filter(function(type) { return type.lineNum != '?' && type.name_ }), 'Type');
+    substrate.addItems(values(data.globalVariables), 'GlobalVariable');
+    substrate.addItems(data.aliass, 'Alias');
     substrate.addItems(data.functions, 'FunctionSplitter');
   }
 
   finalCombiner(substrate.solve());
+
+  dprint('framework', 'Big picture: Finishing JSifier, main pass=' + mainPass);
 }
 
diff --git a/src/library.js b/src/library.js
index c180b7c4..2a67ee88 100644
--- a/src/library.js
+++ b/src/library.js
@@ -388,9 +388,6 @@ LibraryManager.library = {
       __impure_ptr = allocate([ allocate(
         {{{ QUANTUM_SIZE === 4 ? '[0, 0, 0, 0, _stdin, 0, 0, 0, _stdout, 0, 0, 0, _stderr, 0, 0, 0]' : '[0, _stdin, _stdout, _stderr]' }}},
         'void*', ALLOC_STATIC) ], 'void*', ALLOC_STATIC);
-
-      // Once initialized, permissions start having effect.
-      FS.ignorePermissions = false;
     },
 
     quit: function() {
@@ -5481,11 +5478,6 @@ LibraryManager.library = {
     eval(Pointer_stringify(ptr));
   },
 
-  EMSCRIPTEN_COMMENT__inline: function(param) {
-    param = stripCorrections(param);
-    return '// ' + Variables.globals[param].value.text.replace('\\00', '') + '   ';
-  },
-
   $Profiling: {
     max_: 0,
     times: null,
diff --git a/src/modules.js b/src/modules.js
index fec73f34..39baabba 100644
--- a/src/modules.js
+++ b/src/modules.js
@@ -191,7 +191,7 @@ var Debugging = {
 };
 
 var Variables = {
-  globals: null
+  globals: {}
 };
 
 var Types = {
diff --git a/src/parseTools.js b/src/parseTools.js
index cedfeaad..f2ef3374 100644
--- a/src/parseTools.js
+++ b/src/parseTools.js
@@ -337,7 +337,7 @@ function parseParamTokens(params) {
           value: null,
           ident: toNiceIdent('%') + anonymousIndex
         });
-        Types.needAnalysis[ret.type] = 0;
+        Types.needAnalysis[ret[ret.length-1].type] = 0;
         anonymousIndex ++;
       }
     } else if (segment[1].text in PARSABLE_LLVM_FUNCTIONS) {
@@ -358,9 +358,7 @@ function parseParamTokens(params) {
         value: segment[1],
         ident: toNiceIdent(parseNumerical(segment[1].text))
       });
-      Types.needAnalysis[ret.type] = 0;
-      //          } else {
-      //            throw "what is this params token? " + JSON.stringify(segment);
+      Types.needAnalysis[removeAllPointing(ret[ret.length-1].type)] = 0;
     }
     ret[ret.length-1].byVal = byVal;
   }
diff --git a/src/postamble.js b/src/postamble.js
index f1d48ead..c8fd44bd 100644
--- a/src/postamble.js
+++ b/src/postamble.js
@@ -20,11 +20,11 @@ Module.callMain = function callMain(args) {
   return _main(argc, argv, 0);
 }
 
+{{GLOBAL_VARS}}
+
 function run(args) {
   args = args || Module['arguments'];
 
-{{GLOBAL_VARS}}
-
   __globalConstructor__();
 
   var ret = null;
@@ -38,6 +38,11 @@ Module['run'] = run;
 
 // {{PRE_RUN_ADDITIONS}}
 
+// In a hackish way, we disable permissions until now, so setup code works, but enable them for runtime so compile code works with permissions
+try {
+  FS.ignorePermissions = false;
+} catch(e){}
+
 #if INVOKE_RUN
 #else
 Module['noInitialRun'] = true;
diff --git a/src/postamble_sharedlib.js b/src/postamble_sharedlib.js
index 1fd78f23..c7a03597 100644
--- a/src/postamble_sharedlib.js
+++ b/src/postamble_sharedlib.js
@@ -1,8 +1,9 @@
 
 // === Auto-generated postamble setup entry stuff ===
 
+{{GLOBAL_VARS}}
+
 function run(args) {
-  {{GLOBAL_VARS}}
   __globalConstructor__();
 }
 Module['run'] = run;