10 files changed, 128 insertions, 41 deletions
diff --git a/emcc b/emcc
index bc76b01d..1d285cbb 100755
--- a/emcc
+++ b/emcc
@@ -470,6 +470,11 @@ Options that are modified or new in %s include:
                            libraries, and after any link-time optimizations
                            (if any).
 
+  --memory-init-file <on>  If on, we generate a separate memory initialization
+                           file. This is more efficient than storing the
+                           memory initialization data embedded inside
+                           JavaScript as text. (default is on)
+
 The target file, if specified (-o <target>), defines what will
 be generated:
 
@@ -716,6 +721,7 @@ try:
   bind = False
   jcache = False
   save_bc = False
+  memory_init_file = False # XXX TODO True
 
   if use_cxx:
     default_cxx_std = '-std=c++03' # Enforce a consistent C++ standard when compiling .cpp files, if user does not specify one on the cmdline.
@@ -849,6 +855,11 @@ try:
       save_bc = newargs[i+1]
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i] == '--memory-init-file':
+      check_bad_eq(newargs[i])
+      memory_init_file = int(newargs[i+1])
+      newargs[i] = ''
+      newargs[i+1] = ''
     elif newargs[i].startswith(('-I/', '-L/')):
       if not absolute_warning_shown:
         print >> sys.stderr, 'emcc: warning: -I or -L of an absolute path encountered. If this is to a local system header/library, it may cause problems (local system files make sense for compiling natively on your system, but not necessarily to JavaScript)' # Of course an absolute path to a non-system-specific library or header is fine, and you can ignore this warning. The danger are system headers that are e.g. x86 specific and nonportable. The emscripten bundled headers are modified to be portable, local system ones are generally not
@@ -1429,6 +1440,32 @@ try:
   src = re.sub(r'\n+[ \n]*\n+', '\n', src)
   open(final, 'w').write(src)
 
+  if memory_init_file:
+    memfile = target + '.mem' 
+    seen_memory_init = False
+    def repl(m):
+      seen_memory_init = True
+      # handle chunking of the memory initializer
+      s = re.sub('[\[\]\n\(\)\. ]', '', m.groups(0)[0])
+      s = s.replace('concat', ',')
+      if s[-1] == ',': s = s[:-1]
+      open(memfile, 'wb').write(''.join(map(lambda x: chr(int(x or '0')), s.split(','))))
+      if DEBUG:
+        # Copy into temp dir as well, so can be run there too
+        temp_memfile = os.path.join(shared.EMSCRIPTEN_TEMP_DIR, os.path.basename(memfile))
+        if os.path.abspath(memfile) != os.path.abspath(memfile):
+          shutil.copyfile(memfile, temp_memfile)
+      return 'loadMemoryInitializer("%s");' % os.path.basename(memfile)
+    src = re.sub('/\* memory initializer \*/ allocate\(([\d,\.concat\(\)\[\]\\n ]+)"i8", ALLOC_NONE, TOTAL_STACK\)', repl, src, count=1)
+    open(final + '.mem.js', 'w').write(src)
+    final += '.mem.js'
+    if DEBUG:
+      if seen_memory_init:
+        save_intermediate('meminit')
+        print >> sys.stderr, 'emcc: wrote memory initialization to %s' % memfile
+      else:
+        print >> sys.stderr, 'emcc: did not see memory initialization'
+
   # If we were asked to also generate HTML, do that
   if final_suffix == 'html':
     if DEBUG: print >> sys.stderr, 'emcc: generating HTML'
diff --git a/src/jsifier.js b/src/jsifier.js
index ce089334..50030268 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -1524,6 +1524,8 @@ function JSify(data, functionsOnly, givenFunctions) {
           });
           // write out the singleton big memory initialization value
           print('/* memory initializer */ ' + makePointer(memoryInitialization, null, 'ALLOC_NONE', 'i8', 'TOTAL_STACK', true)); // we assert on TOTAL_STACK == GLOBAL_BASE
+        } else {
+          print('/* memory initializer */'); // test purposes
         }
       }
 
diff --git a/src/library_browser.js b/src/library_browser.js
index 099516a8..c1add740 100644
--- a/src/library_browser.js
+++ b/src/library_browser.js
@@ -379,7 +379,7 @@ mergeInto(LibraryManager.library, {
       xhr.open('GET', url, true);
       xhr.responseType = 'arraybuffer';
       xhr.onload = function() {
-        if (xhr.status == 200) {
+        if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
           onload(xhr.response);
         } else {
           onerror();
diff --git a/src/parseTools.js b/src/parseTools.js
index 20049094..7dafbebe 100644
--- a/src/parseTools.js
+++ b/src/parseTools.js
@@ -1560,7 +1560,7 @@ function makePointer(slab, pos, allocator, type, ptr, finalMemoryInitialization)
   }
 
   // JS engines sometimes say array initializers are too large. Work around that by chunking and calling concat to combine at runtime
-  var chunkSize = 10240;
+  var chunkSize = JS_CHUNK_SIZE;
   function chunkify(array) {
     // break very large slabs into parts
     var ret = '';
diff --git a/src/postamble.js b/src/postamble.js
index dd4f4f37..4b3e831d 100644
--- a/src/postamble.js
+++ b/src/postamble.js
@@ -2,6 +2,12 @@
 // === Auto-generated postamble setup entry stuff ===
 
 Module.callMain = function callMain(args) {
+  assert(runDependencies == 0, 'cannot call main when async dependencies remain! (listen on __ATMAIN__)');
+
+  args = args || [];
+
+  ensureInitRuntime();
+
   var argc = args.length+1;
   function pad() {
     for (var i = 0; i < {{{ QUANTUM_SIZE }}}-1; i++) {
@@ -70,6 +76,8 @@ function run(args) {
   }
 
   function doRun() {
+    ensureInitRuntime();
+
     var ret = 0;
     calledRun = true;
     if (Module['_main']) {
@@ -112,8 +120,6 @@ if (Module['preInit']) {
   }
 }
 
-initRuntime();
-
 #if INVOKE_RUN
 var shouldRunNow = true;
 #else
diff --git a/src/preamble.js b/src/preamble.js
index 2cff440c..68510e3a 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -718,7 +718,11 @@ var __ATINIT__ = []; // functions called during startup
 var __ATMAIN__ = []; // functions called when main() is to be run
 var __ATEXIT__ = []; // functions called during shutdown
 
-function initRuntime() {
+var runtimeInitialized = false;
+
+function ensureInitRuntime() {
+  if (runtimeInitialized) return;
+  runtimeInitialized = true;
   callRuntimeCallbacks(__ATINIT__);
 }
 function preMain() {
@@ -805,7 +809,7 @@ Math.imul = function(a, b) {
 // the dependencies are met.
 var runDependencies = 0;
 var runDependencyTracking = {};
-var calledRun = false;
+var calledInit = false, calledRun = false;
 var runDependencyWatcher = null;
 function addRunDependency(id) {
   runDependencies++;
@@ -861,6 +865,12 @@ Module['removeRunDependency'] = removeRunDependency;
 Module["preloadedImages"] = {}; // maps url to image data
 Module["preloadedAudios"] = {}; // maps url to audio data
 
+function addPreRun(func) {
+  if (!Module['preRun']) Module['preRun'] = [];
+  else if (typeof Module['preRun'] == 'function') Module['preRun'] = [Module['preRun']];
+  Module['preRun'].push(func);
+}
+
 #if PGO
 var PGOMonitor = {
   called: {},
@@ -874,9 +884,32 @@ var PGOMonitor = {
   }
 };
 __ATEXIT__.push({ func: function() { PGOMonitor.dump() } });
-if (!Module.preRun) Module.preRun = [];
-Module.preRun.push(function() { addRunDependency('pgo') });
+addPreRun(function() { addRunDependency('pgo') });
+#endif
+
+function loadMemoryInitializer(filename) {
+  function applyData(data) {
+#if USE_TYPED_ARRAYS == 2
+    HEAPU8.set(data, TOTAL_STACK);
+#else
+    allocate(data, 'i8', ALLOC_NONE, TOTAL_STACK);
 #endif
+  }
+
+  if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
+    // synchronous
+    applyData(Module['readBinary'](filename));
+  } else {
+    // asynchronous
+    addPreRun(function() {
+      Browser.asyncLoad(filename, function(data) {
+        applyData(data);
+      }, function(data) {
+        throw 'could not load memory initializer ' + filename;
+      });
+    });
+  }
+}
 
 // === Body ===
 
diff --git a/src/settings.js b/src/settings.js
index 6b054443..f9b47228 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -357,6 +357,8 @@ var NECESSARY_BLOCKADDRS = []; // List of (function, block) for all block addres
 
 var EMIT_GENERATED_FUNCTIONS = 0; // whether to emit the list of generated functions, needed for external JS optimization passes
 
+var JS_CHUNK_SIZE = 10240; // Used as a maximum size before breaking up expressions and lines into smaller pieces
+
 // Compiler debugging options
 var DEBUG_TAGS_SHOWING = [];
   // Some useful items:
diff --git a/src/shell.js b/src/shell.js
index 8c37bf5b..c8f3644a 100644
--- a/src/shell.js
+++ b/src/shell.js
@@ -24,17 +24,20 @@ if (ENVIRONMENT_IS_NODE) {
   var nodeFS = require('fs');
   var nodePath = require('path');
 
-  Module['read'] = function(filename) {
+  Module['read'] = function(filename, binary) {
     filename = nodePath['normalize'](filename);
-    var ret = nodeFS['readFileSync'](filename).toString();
+    var ret = nodeFS['readFileSync'](filename);
     // The path is absolute if the normalized version is the same as the resolved.
     if (!ret && filename != nodePath['resolve'](filename)) {
       filename = path.join(__dirname, '..', 'src', filename);
-      ret = nodeFS['readFileSync'](filename).toString();
+      ret = nodeFS['readFileSync'](filename);
     }
+    if (ret && !binary) ret = ret.toString();
     return ret;
   };
 
+  Module['readBinary'] = function(filename) { return Module['read'](filename, true) };
+
   Module['load'] = function(f) {
     globalEval(read(f));
   };
@@ -48,12 +51,10 @@ if (ENVIRONMENT_IS_SHELL) {
   Module['print'] = print;
   if (typeof printErr != 'undefined') Module['printErr'] = printErr; // not present in v8 or older sm
 
-  // Polyfill over SpiderMonkey/V8 differences
-  if (typeof read != 'undefined') {
-    Module['read'] = read;
-  } else {
-    Module['read'] = function(f) { snarf(f) };
-  }
+  Module['read'] = read;
+  Module['readBinary'] = function(f) {
+    return read(f, 'binary');
+  };
 
   if (!Module['arguments']) {
     if (typeof scriptArgs != 'undefined') {
diff --git a/tests/hello_world_gles_shell.html b/tests/hello_world_gles_shell.html
index 343ddb64..4abee90c 100644
--- a/tests/hello_world_gles_shell.html
+++ b/tests/hello_world_gles_shell.html
@@ -46,7 +46,7 @@
           reportResult(firstImage != secondImage);
         }, 0);
       }
-      addEventListener("load", doTest, false);
+      Module.postRun = doTest;
 
       // The compiled code
       {{{ SCRIPT_CODE }}}
diff --git a/tests/runner.py b/tests/runner.py
index 62a85085..4afdaa99 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -266,6 +266,13 @@ process(sys.argv[1])
     if output_processor is not None:
       output_processor(open(filename + '.o.js').read())
 
+    if 0:# XXX TODO self.emcc_args is not None:
+      if '--memory-init-file' in self.emcc_args:
+        memory_init_file = int(self.emcc_args[self.emcc_args.index('--memory-init-file')+1])
+      else:
+        memory_init_file = 1
+      assert ('/* memory initializer */' in open(filename + '.o.js').read()) == (not memory_init_file)
+
   def run_generated_code(self, engine, filename, args=[], check_timeout=True, output_nicerizer=None):
     stdout = os.path.join(self.get_dir(), 'stdout') # use files, as PIPE can get too full and hang us
     stderr = os.path.join(self.get_dir(), 'stderr')
@@ -8911,11 +8918,11 @@ TT = %s
   exec('o1 = make_run("o1", compiler=CLANG, emcc_args=["-O1", "-s", "SAFE_HEAP=1"])')
 
   # Make one run with -O2, but without closure (we enable closure in specific tests, otherwise on everything it is too slow)
-  exec('o2 = make_run("o2", compiler=CLANG, emcc_args=["-O2"])')
+  exec('o2 = make_run("o2", compiler=CLANG, emcc_args=["-O2", "-s", "JS_CHUNK_SIZE=1024"])')
 
   # asm.js
   exec('asm2 = make_run("asm2", compiler=CLANG, emcc_args=["-O2", "-s", "ASM_JS=1"])')
-  exec('asm2g = make_run("asm2g", compiler=CLANG, emcc_args=["-O2", "-s", "ASM_JS=1", "-g", "-s", "ASSERTIONS=1"])')
+  exec('asm2g = make_run("asm2g", compiler=CLANG, emcc_args=["-O2", "-s", "ASM_JS=1", "-g", "-s", "ASSERTIONS=1", "--memory-init-file", "0"])')
 
   # Make custom runs with various options
   for compiler, quantum, embetter, typed_arrays, llvm_opts in [
@@ -9018,20 +9025,23 @@ Options that are modified or new in %s include:
 
         # emcc [..] -o [path] ==> should work with absolute paths
         try:
-          os.mkdir('a_dir')
-          os.chdir('a_dir')
-          os.mkdir('b_dir')
           for path in [os.path.abspath(os.path.join('..', 'file1.js')), os.path.join('b_dir', 'file2.js')]:
+            print path
             self.clear(in_curr=True)
+            os.chdir(self.get_dir())
+            if not os.path.exists('a_dir'): os.mkdir('a_dir')
+            os.chdir('a_dir')
+            if not os.path.exists('b_dir'): os.mkdir('b_dir')
             output = Popen([PYTHON, compiler, path_from_root('tests', 'hello_world.ll'), '-o', path], stdout=PIPE, stderr=PIPE).communicate()
+            print output
             assert os.path.exists(path), path + ' does not exist; ' + '\n'.join(output)
-            self.assertContained('hello, world!', run_js(path))
+            last = os.getcwd()
+            os.chdir(os.path.dirname(path))
+            self.assertContained('hello, world!', run_js(os.path.basename(path)))
+            os.chdir(last)
         finally:
           os.chdir(self.get_dir())
-          try:
-            shutil.rmtree('a_dir')
-          except:
-            pass
+        self.clear()
 
         # dlmalloc. dlmalloc is special in that it is the only part of libc that is (1) hard to write well, and
         # very speed-sensitive. So we do not implement it in JS in library.js, instead we compile it from source
@@ -9975,7 +9985,8 @@ f.close()
 
         if no_initial_run:
           # Calling main later should still work, filesystem etc. must be set up.
-          src = open(os.path.join(self.get_dir(), 'a.out.js')).read() + '\n_main();\n';
+          print 'call main later'
+          src = open(os.path.join(self.get_dir(), 'a.out.js')).read() + '\nModule.callMain();\n';
           open(os.path.join(self.get_dir(), 'a.out.js'), 'w').write(src)
           assert 'hello from main' in run_js(os.path.join(self.get_dir(), 'a.out.js')), 'main should print when called manually'
 
@@ -10094,12 +10105,14 @@ f.close()
 
     def test_chunking(self):
       if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
+      if os.environ.get('EMCC_CORES'): return self.skip('cannot run if cores are altered')
       if multiprocessing.cpu_count() < 2: return self.skip('need multiple cores')
       try:
         os.environ['EMCC_DEBUG'] = '1'
+        os.environ['EMCC_CORES'] = '2'
         for asm, linkable, chunks, js_chunks in [
-            (0, 0, 3, 2), (0, 1, 4, 4),
-            (1, 0, 3, 2), (1, 1, 4, 4)
+            (0, 0, 3, 2), (0, 1, 3, 4),
+            (1, 0, 3, 2), (1, 1, 3, 4)
           ]:
           print asm, linkable, chunks, js_chunks
           output, err = Popen([PYTHON, EMCC, path_from_root('tests', 'hello_libcxx.cpp'), '-O1', '-s', 'LINKABLE=%d' % linkable, '-s', 'ASM_JS=%d' % asm, '-s', 'UNRESOLVED_AS_DEAD=1'] + (['-O2'] if asm else []), stdout=PIPE, stderr=PIPE).communicate()
@@ -10113,6 +10126,7 @@ f.close()
           assert ok, err
       finally:
         del os.environ['EMCC_DEBUG']
+        del os.environ['EMCC_CORES']
 
     def test_debuginfo(self):
       if os.environ.get('EMCC_DEBUG'): return self.skip('cannot run in debug mode')
@@ -10331,11 +10345,11 @@ seeked= file.
           assert (' with -O3 since EMCC_OPTIMIZE_NORMALLY defined' in err) == optimize_normally
 
           for last in ['both.o', 'both2.o']:
-            out, err = Popen([PYTHON, EMCC, self.in_dir('both.o'), '-O2', '-o', last + '.js'], stdout=PIPE, stderr=PIPE).communicate()
+            out, err = Popen([PYTHON, EMCC, self.in_dir('both.o'), '-O2', '-o', last + '.js', '--memory-init-file', '0'], stdout=PIPE, stderr=PIPE).communicate()
             assert ("emcc: LLVM opts: ['-O3']" not in err) == optimize_normally
             assert ' with -O3 since EMCC_OPTIMIZE_NORMALLY defined' not in err
             output = run_js(last + '.js')
-            assert 'yello' in output, 'code works'
+            assert 'yello' in output, 'code works ' + err
           assert open('both.o.js').read() == open('both2.o.js').read()
 
         finally:
@@ -11409,14 +11423,6 @@ elif 'browser' in str(sys.argv):
           self.run_browser('something.html', 'You should see animating gears.', '/report_gl_result?true')
           assert ('var GLEmulation' in open(self.in_dir('something.html')).read()) == emulation, "emulation code should be added when asked for"
 
-    def test_glgears_bad(self):
-      # Make sure that OpenGL ES is not available if typed arrays are not used
-      Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world_gles.c'), '-o', 'something.html',
-                                           '-DHAVE_BUILTIN_SINCOS', '-s', 'GL_TESTING=1',
-                                           '-s', 'USE_TYPED_ARRAYS=0',
-                                           '--shell-file', path_from_root('tests', 'hello_world_gles_shell.html')]).communicate()
-      self.run_browser('something.html', 'You should not see animating gears.', '/report_gl_result?false')
-
     def test_glgears_deriv(self):
       self.reftest(path_from_root('tests', 'gears.png'))
       Popen([PYTHON, EMCC, path_from_root('tests', 'hello_world_gles_deriv.c'), '-o', 'something.html', '-s', 'GL_TESTING=1',