parse each function in a separate pass, to save RAM

5 files changed, 80 insertions, 32 deletions

diff --git a/src/analyzer.js b/src/analyzer.js
index 4582f0b1..1b46a076 100644
--- a/src/analyzer.js
+++ b/src/analyzer.js
@@ -11,7 +11,7 @@ function cleanFunc(func) {
   });
 }
 
-function analyzer(data) {
+function analyzer(data, givenTypes) {
   substrate = new Substrate('Analyzer');
 
   // Sorter
@@ -26,7 +26,11 @@ function analyzer(data) {
   substrate.addZyme('Gatherer', {
     processItem: function(item) {
       // Single-liners
-      ['globalVariable', 'functionStub', 'type'].forEach(function(intertype) {
+      ['globalVariable', 'functionStub', 'type', 'unparsedFunction'].forEach(function(intertype) {
+        if (intertype === 'type' && givenTypes) {
+          item.types = givenTypes;
+          return;
+        }
         var temp = splitter(item.items, function(item) { return item.intertype == intertype });
         item[intertype + 's'] = temp.splitOut;
         item.items = temp.leftIn;
@@ -54,7 +58,7 @@ function analyzer(data) {
         } else if (subItem.intertype == 'label') {
           item.functions.slice(-1)[0].labels.push(subItem);
           subItem.lines = [];
-        } else if (item.functions.slice(-1)[0].endLineNum === null) {
+        } else if (item.functions.length > 0 && item.functions.slice(-1)[0].endLineNum === null) {
           // Internal line
           item.functions.slice(-1)[0].lines.push(subItem);
           item.functions.slice(-1)[0].labels.slice(-1)[0].lines.push(subItem);
@@ -121,9 +125,11 @@ function analyzer(data) {
   substrate.addZyme('Typevestigator', {
     processItem: function(data) {
       // Convert types list to dict
-      var old = data.types;
-      data.types = {};
-      old.forEach(function(type) { data.types[type.name_] = type });
+      if (data.types.length !== undefined) {
+        var old = data.types;
+        data.types = {};
+        old.forEach(function(type) { data.types[type.name_] = type });
+      }
 
       // Find additional types
       walkJSON(data, function(item) {
diff --git a/src/enzymatic.js b/src/enzymatic.js
index e6af4ab5..b0fd4619 100644
--- a/src/enzymatic.js
+++ b/src/enzymatic.js
@@ -56,7 +56,7 @@ Substrate.prototype = {
   },
 
   solve: function() {
-    print("// Solving " + this.name_ + "...");
+    dprint('enzymatic', "// Solving " + this.name_ + "...");
 
     var startTime = Date.now();
     var midTime = startTime;
@@ -64,12 +64,12 @@ Substrate.prototype = {
     function midComment(force) {
       var curr = Date.now();
       if (curr - midTime > 1000 || force) {
-        print('// Working on ' + that.name_ + ', so far ' + ((curr-startTime)/1000).toString().substr(0,10) + ' seconds.');
+        dprint('enzymatic', '// Working on ' + that.name_ + ', so far ' + ((curr-startTime)/1000).toString().substr(0,10) + ' seconds.');
         midTime = curr;
       }
     }
     function finalComment() {
-      print('// Completed ' + that.name_ + ' in ' + ((Date.now() - startTime)/1000).toString().substr(0,10) + ' seconds.');
+      dprint('enzymatic', '// Completed ' + that.name_ + ' in ' + ((Date.now() - startTime)/1000).toString().substr(0,10) + ' seconds.');
     }
 
     var finalResult = null;
@@ -89,10 +89,10 @@ Substrate.prototype = {
         var outputs;
         var currResultCount = that.results.length;
         try {
-          dprint('Processing using ' + zyme.name_ + ': ' + inputs.length);
+          dprint('enzymatic', 'Processing using ' + zyme.name_ + ': ' + inputs.length);
           zyme.items = []; // More may be added in process(); we'll get to them next time
           outputs = zyme.process(inputs);
-          dprint('New results: ' + (outputs.length + that.results.length - currResultCount) + ' out of ' + (that.results.length + outputs.length));
+          dprint('enzymatic', 'New results: ' + (outputs.length + that.results.length - currResultCount) + ' out of ' + (that.results.length + outputs.length));
         } catch (e) {
           //print("Exception, current selected are: " + inputs.map(dump).join('\n\n'));
           print("Stack: " + new Error().stack);
diff --git a/src/intertyper.js b/src/intertyper.js
index f500a18f..c0708daa 100644
--- a/src/intertyper.js
+++ b/src/intertyper.js
@@ -1,15 +1,23 @@
 // llvm => internal intermediate representation
 
-function intertyper(data) {
+var LLVM_STYLE = null;
+
+//! @param parseFunctions We parse functions only on later passes, since we do not
+//!                       want to parse all of them at once, and have all their
+//!                       lines and data in memory at the same time.
+function intertyper(data, parseFunctions) {
+  //parseFunctions = true; // Uncomment to do all parsing in a single big RAM-heavy pass
+
   // Substrate
 
-  LLVM_STYLE = (data.indexOf('<label>') == -1 && data.indexOf('entry:') != -1) ? 'old' : 'new'; // new = clang on 2.8, old = llvm-gcc anywhere or clang on 2.7
-  dprint('LLVM_STYLE: ' + LLVM_STYLE);
+  if (LLVM_STYLE === null) {
+    // new = clang on 2.8, old = llvm-gcc anywhere or clang on 2.7
+    LLVM_STYLE = (data.indexOf('<label>') == -1 && data.indexOf('entry:') != -1) ? 'old' : 'new';
+    dprint('LLVM_STYLE: ' + LLVM_STYLE);
+  }
 
   substrate = new Substrate('Intertyper');
 
-  var IGNORE_FUNCTIONS = 0; // Debugging
-
   // Line splitter.
   substrate.addZyme('LineSplitter', {
     processItem: function(item) {
@@ -17,12 +25,17 @@ function intertyper(data) {
       var ret = [];
       var inContinual = false;
       var inFunction = false;
+      var currFunctionLines;
+      var currFunctionLineNum;
+      var unparsedFunctions = [];
       for (var i = 0; i < lines.length; i++) {
         var line = lines[i];
-        if (/^define .*/.test(line)) {
+        if (!parseFunctions && /^define .*/.test(line)) {
           inFunction = true;
+          currFunctionLines = [];
+          currFunctionLineNum = i + 1;
         }
-        if (!(IGNORE_FUNCTIONS && inFunction)) {
+        if (!inFunction || parseFunctions) {
           if (inContinual || new RegExp(/^\ +to.*/g).test(line)) {
             // to after invoke
             ret.slice(-1)[0].lineText += line;
@@ -39,12 +52,26 @@ function intertyper(data) {
               inContinual = true;
             }
           }
+        } else {
+          currFunctionLines.push(line);
         }
-        if (/^}.*/.test(line)) {
+        if (!parseFunctions && /^}.*/.test(line)) {
           inFunction = false;
+          if (!parseFunctions) {
+            unparsedFunctions.push({
+              __result__: true,
+              intertype: 'unparsedFunction',
+              // We need this early, to know all function idents
+              ident: toNiceIdent(funcHeader.processItem(tokenizer.processItem({ lineText: currFunctionLines[0] }, true))[0].ident),
+              lineNum: currFunctionLineNum,
+              lines: currFunctionLines,
+            });
+            currFunctionLines = [];
+          }
         }
       }
       this.forwardItems(ret.filter(function(item) { return item.lineText; }), 'Tokenizer');
+      return unparsedFunctions;
     },
   });
 
@@ -332,7 +359,7 @@ function intertyper(data) {
     },
   });
   // function header
-  substrate.addZyme('FuncHeader', {
+  funcHeader = substrate.addZyme('FuncHeader', {
     processItem: function(item) {
       item.tokens = item.tokens.filter(function(token) {
         return ['noalias', 'available_externally', 'weak', 'internal', 'signext', 'zeroext', 'nounwind', 'define', 'linkonce_odr', 'inlinehint', '{', 'fastcc'].indexOf(token.text) == -1;
diff --git a/src/jsifier.js b/src/jsifier.js
index 5cf3d5d1..a86cc2a4 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -1,14 +1,21 @@
 // Convert analyzed data to javascript
 
-function JSify(data) {
+function JSify(data, functionsOnly, givenTypes, givenFunctions) {
   substrate = new Substrate('JSifyer');
 
-  var TYPES = data.types;
-  var FUNCTIONS = {};
-  data.functions.forEach(function(func) {
+  var TYPES = functionsOnly ? givenTypes : data.types;
+
+  var FUNCTIONS = functionsOnly ? givenFunctions : {};
+
+  // Now that analysis has completed, we can get around to handling unparsedFunctions
+  (functionsOnly ? data.functions : data.unparsedFunctions.concat(data.functions)).forEach(function(func) {
     FUNCTIONS[func.ident] = func;
   });
 
+  data.unparsedFunctions.forEach(function(func) {
+    func.JS = JSify(analyzer(intertyper(func.lines, true), TYPES), true, TYPES, FUNCTIONS);
+  });
+
   // type
   substrate.addZyme('Type', {
     processItem: function(item) {
@@ -710,11 +717,11 @@ function JSify(data) {
       case 'zext': case 'fpext': case 'trunc': case 'sext': case 'fptrunc': return ident1;
       case 'select': return ident1 + ' ? ' + ident2 + ' : ' + ident3;
       case 'ptrtoint': {
-        if (type != 'i8*') print('// XXX Warning: Risky ptrtoint operation on line ' + lineNum);
+        //if (type != 'i8*') print('// XXX Warning: Risky ptrtoint operation on line ' + lineNum);
         return ident1;
       }
       case 'inttoptr': {
-        print('// XXX Warning: inttoptr operation on line ' + lineNum);
+        //print('// XXX Warning: inttoptr operation on line ' + lineNum);
         return ident1;
       }
       default: throw 'Unknown mathcmp op: ' + item.op
@@ -895,14 +902,21 @@ function JSify(data) {
   // Final combiner
 
   function finalCombiner(items) {
-    var ret = items.filter(function(item) { return item.intertype == 'type' });
-    ret = ret.concat(items.filter(function(item) { return item.intertype == 'GlobalVariableStub' }));
-    ret.push('\n');
-    ret = ret.concat(items.filter(function(item) { return item.intertype == 'functionStub' }));
-    ret.push('\n');
+    var ret = [];
+    if (!functionsOnly) {
+      ret = ret.concat(items.filter(function(item) { return item.intertype == 'type' }));
+      ret = ret.concat(items.filter(function(item) { return item.intertype == 'GlobalVariableStub' }));
+      ret.push('\n');
+      ret = ret.concat(items.filter(function(item) { return item.intertype == 'functionStub' }));
+      ret.push('\n');
+    }
     ret = ret.concat(items.filter(function(item) { return item.intertype == 'function' }));
+    ret = ret.concat(data.unparsedFunctions);
+
     ret = ret.map(function(item) { return item.JS }).join('\n');
 
+    if (functionsOnly) return ret;
+
     var params = { 'QUANTUM_SIZE': QUANTUM_SIZE };
     var body = preprocess(read('preamble.js').replace('{{RUNTIME}}', getRuntime()) + ret + read('postamble.js'), params);
     function reverse_(x) {
diff --git a/src/settings.js b/src/settings.js
index 717bd334..7462af7f 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -34,8 +34,9 @@ EXCEPTION_DEBUG = 1; // Print out exceptions in emscriptened code
 EXECUTION_TIMEOUT = -1; // Throw an exception after X seconds - useful to debug infinite loops
 
 // Compiler debugging options
-DEBUG_TAGS_SHOWING = ['enzymatic'];
+DEBUG_TAGS_SHOWING = [];
   // Some useful items:
+  //    enzymatic
   //    gconst
   //    types
   //    relooping