Merge branch 'incoming'

84 files changed, 4537 insertions, 1978 deletions

diff --git a/.gitignore b/.gitignore
index 92043241..747394e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@
 *.bc
 src/relooper*.js
 
+node_modules/
+
 # Ignore generated files 
 src/relooper.js
 src/relooper.js.raw.js
diff --git a/emcc b/emcc
index a07ce4f1..958ed7cd 100755
--- a/emcc
+++ b/emcc
@@ -663,6 +663,12 @@ if '-M' in sys.argv or '-MM' in sys.argv:
   logging.debug('just dependencies: ' + ' '.join(cmd))
   exit(subprocess.call(cmd))
 
+if '-E' in sys.argv:
+  # Just run the preprocessor
+  cmd = [CC] + sys.argv[1:]
+  logging.debug('just preprocssor ' + ' '.join(cmd))
+  exit(subprocess.call(cmd))
+
 # Check if a target is specified
 target = None
 for i in range(len(sys.argv)-1):
@@ -1070,9 +1076,6 @@ try:
       shared.Settings.CORRECT_OVERFLOWS = 1
     assert not shared.Settings.PGO, 'cannot run PGO in ASM_JS mode'
 
-    if shared.Settings.ASSERTIONS and shared.Settings.ALIASING_FUNCTION_POINTERS:
-      logging.warning('ALIASING_FUNCTION_POINTERS is on, function pointer comparisons may be invalid across types')
-
   if shared.Settings.CORRECT_SIGNS >= 2 or shared.Settings.CORRECT_OVERFLOWS >= 2 or shared.Settings.CORRECT_ROUNDINGS >= 2:
     debug_level = 4 # must keep debug info to do line-by-line operations
 
@@ -1107,13 +1110,24 @@ try:
     shared.Settings.LINKABLE = 1 # TODO: add FORCE_DCE option for the brave people that do want to dce here and in side modules
     debug_level = max(debug_level, 2)
 
-  if shared.Settings.DLOPEN_SUPPORT:
-    shared.Settings.LINKABLE = 1
+  if shared.Settings.ASSERTIONS and shared.Settings.ALIASING_FUNCTION_POINTERS:
+    logging.warning('ALIASING_FUNCTION_POINTERS is on, function pointer comparisons may be invalid across types')
 
   if shared.Settings.STB_IMAGE and final_suffix in JS_CONTAINING_SUFFIXES:
     input_files.append(shared.path_from_root('third_party', 'stb_image.c'))
     shared.Settings.EXPORTED_FUNCTIONS += ['_stbi_load', '_stbi_load_from_memory', '_stbi_image_free']
 
+  if type(shared.Settings.EXPORTED_FUNCTIONS) in (list, tuple):
+    # always need malloc and free to be kept alive and exported, for internal use and other modules
+    for required_export in ['_malloc', '_free']:
+      if required_export not in shared.Settings.EXPORTED_FUNCTIONS:
+        shared.Settings.EXPORTED_FUNCTIONS.append(required_export)
+  else:
+    logging.debug('using response file for EXPORTED_FUNCTIONS, make sure it includes _malloc and _free')
+
+  if shared.Settings.ASM_JS and shared.Settings.DLOPEN_SUPPORT:
+    assert shared.Settings.DISABLE_EXCEPTION_CATCHING, 'no exceptions support with dlopen in asm yet'
+
   ## Compile source code to bitcode
 
   logging.debug('compiling to bitcode')
diff --git a/emlibtool b/emlibtool
deleted file mode 100755
index 1eb18edc..00000000
--- a/emlibtool
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/usr/bin/env python2
-
-'''
-This is a helper script. See emcc.
-'''
-
-import os, sys
-from tools import shared
-
-raise Exception('TODO: emlibtool')
-
diff --git a/emlibtool.bat b/emlibtool.bat
deleted file mode 100644
index 4ea705be..00000000
--- a/emlibtool.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-@echo off
-python "%~dp0\emlibtool" %*
\ No newline at end of file
diff --git a/emscripten.py b/emscripten.py
index c5e235d8..257527fe 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -444,6 +444,11 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
        forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'):
       basic_vars += ['cttz_i8', 'ctlz_i8']
 
+    if settings.get('DLOPEN_SUPPORT'):
+      for sig in last_forwarded_json['Functions']['tables'].iterkeys():
+        basic_vars.append('F_BASE_%s' % sig)
+        asm_setup += '  var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n'
+
     asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)]
     # function tables
     def asm_coerce(value, sig):
@@ -475,8 +480,12 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
 
 ''' % (sig, i, args, arg_coercions, jsret))
       from tools import shared
+      shared.Settings.copy(settings)
       asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n'
       basic_funcs.append('invoke_%s' % sig)
+      if settings.get('DLOPEN_SUPPORT'):
+        asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n'
+        basic_funcs.append('extCall_%s' % sig)
 
     # calculate exports
     exported_implemented_functions = list(exported_implemented_functions)
@@ -606,23 +615,36 @@ function setTempRet%d(value) {
 // EMSCRIPTEN_END_ASM
 (%s, %s, buffer);
 %s;
+''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n  '), exports, the_global, sending, receiving)]
+
+    if not settings.get('SIDE_MODULE'):
+      funcs_js.append('''
 Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) };
 Runtime.stackSave = function() { return asm['stackSave']() };
 Runtime.stackRestore = function(top) { asm['stackRestore'](top) };
-''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n  '), exports, the_global, sending, receiving)]
+''')
 
     # Set function table masks
-    def function_table_maskize(js):
-      masks = {}
-      default = None
-      for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
-        masks[sig] = str(table.count(','))
-        default = sig
+    masks = {}
+    max_mask = 0
+    for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
+      mask = table.count(',')
+      masks[sig] = str(mask)
+      max_mask = max(mask, max_mask)
+    def function_table_maskize(js, masks):
       def fix(m):
         sig = m.groups(0)[0]
         return masks[sig]
       return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]]
-    funcs_js = map(function_table_maskize, funcs_js)
+    funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js)
+
+    if settings.get('DLOPEN_SUPPORT'):
+      funcs_js.append('''
+  asm.maxFunctionIndex = %(max_mask)d;
+  DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module);
+  Module.SYMBOL_TABLE = SYMBOL_TABLE;
+''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) })
+
   else:
     function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()])
     outfile.write(function_tables_defs)
@@ -637,13 +659,18 @@ Runtime.stackRestore = function(top) { asm['stackRestore'](top) };
     symbol_table = {}
     for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems():
        if forwarded_json['Variables']['globals'][k]['named']:
-         symbol_table[k] = v + forwarded_json['Runtime']['GLOBAL_BASE']
+         symbol_table[k] = str(v + forwarded_json['Runtime']['GLOBAL_BASE'])
     for raw in last_forwarded_json['Functions']['tables'].itervalues():
       if raw == '': continue
       table = map(string.strip, raw[raw.find('[')+1:raw.find(']')].split(","))
-      symbol_table.update(map(lambda x: (x[1], x[0]),
-        filter(lambda x: x[1] != '0', enumerate(table))))
-    outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table))
+      for i in range(len(table)):
+        value = table[i]
+        if value != '0':
+          if settings.get('SIDE_MODULE'):
+            symbol_table[value] = 'FUNCTION_TABLE_OFFSET+' + str(i)
+          else:
+            symbol_table[value] = str(i)
+    outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', ''))
 
   for funcs_js_item in funcs_js: # do this loop carefully to save memory
     funcs_js_item = indexize(funcs_js_item)
@@ -663,69 +690,6 @@ def main(args, compiler_engine, cache, jcache, relooper, temp_files, DEBUG, DEBU
     name, value = setting.strip().split('=', 1)
     settings[name] = json.loads(value)
 
-  # Add header defines to settings
-  defines = {}
-  include_root = path_from_root('system', 'include')
-  headers = args.headers[0].split(',') if len(args.headers) > 0 else []
-  seen_headers = set()
-  while len(headers) > 0:
-    header = headers.pop(0)
-    if not os.path.isabs(header):
-      header = os.path.join(include_root, header)
-    seen_headers.add(header)
-    for line in open(header, 'r'):
-      line = line.replace('\t', ' ')
-      m = re.match('^ *# *define +(?P<name>[-\w_.]+) +\(?(?P<value>[-\w_.|]+)\)?.*', line)
-      if not m:
-        # Catch enum defines of a very limited sort
-        m = re.match('^ +(?P<name>[A-Z_\d]+) += +(?P<value>\d+).*', line)
-      if m:
-        if m.group('name') != m.group('value'):
-          defines[m.group('name')] = m.group('value')
-        #else:
-        #  print 'Warning: %s #defined to itself' % m.group('name') # XXX this can happen if we are set to be equal to an enum (with the same name)
-      m = re.match('^ *# *include *["<](?P<name>[\w_.-/]+)[">].*', line)
-      if m:
-        # Find this file
-        found = False
-        for w in [w for w in os.walk(include_root)]:
-          for f in w[2]:
-            curr = os.path.join(w[0], f)
-            if curr.endswith(m.group('name')) and curr not in seen_headers:
-              headers.append(curr)
-              found = True
-              break
-          if found: break
-        #assert found, 'Could not find header: ' + m.group('name')
-  if len(defines) > 0:
-    def lookup(value):
-      try:
-        while not unicode(value).isnumeric():
-          value = defines[value]
-        return value
-      except:
-        pass
-      try: # 0x300 etc.
-        value = eval(value)
-        return value
-      except:
-        pass
-      try: # CONST1|CONST2
-        parts = map(lookup, value.split('|'))
-        value = reduce(lambda a, b: a|b, map(eval, parts))
-        return value
-      except:
-        pass
-      return None
-    for key, value in defines.items():
-      value = lookup(value)
-      if value is not None:
-        defines[key] = str(value)
-      else:
-        del defines[key]
-    #print >> sys.stderr, 'new defs:', str(defines).replace(',', ',\n  '), '\n\n'
-    settings.setdefault('C_DEFINES', {}).update(defines)
-
   # libraries
   libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else []
 
diff --git a/package.json b/package.json
new file mode 100644
index 00000000..a1447c9f
--- /dev/null
+++ b/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "emscripten",
+  "version": "1.0.0",
+  "dependencies": {
+    "ws": "~0.4.28"
+  }
+}
diff --git a/src/intertyper.js b/src/intertyper.js
index 31e97bd0..ddb93d71 100644
--- a/src/intertyper.js
+++ b/src/intertyper.js
@@ -707,16 +707,24 @@ function intertyper(data, sidePass, baseLineNums) {
     var tokensLeft = item.tokens.slice(2);
     item.ident = eatLLVMIdent(tokensLeft);
     if (item.ident == 'asm') {
+      if (ASM_JS) {
+        warnOnce('inline JS in asm.js mode can cause the code to no longer fall in the asm.js subset of JavaScript');
+      }
+      assert(TARGET_LE32, 'inline js is only supported in le32');
       // Inline assembly is just JavaScript that we paste into the code
       item.intertype = 'value';
       if (tokensLeft[0].text == 'sideeffect') tokensLeft.splice(0, 1);
       item.ident = tokensLeft[0].text.substr(1, tokensLeft[0].text.length-2) || ';'; // use ; for empty inline assembly
       var i = 0;
+      var params = [], args = [];
       splitTokenList(tokensLeft[3].item.tokens).map(function(element) {
         var ident = toNiceIdent(element[1].text);
         var type = element[0].text;
-        item.ident = item.ident.replace(new RegExp('\\$' + i++, 'g'), ident);
+        params.push('$' + (i++));
+        args.push(ident);
       });
+      if (item.assignTo) item.ident = 'return ' + item.ident;
+      item.ident = '(function(' + params + ') { ' + item.ident + ' })(' + args + ');';
       return { forward: null, ret: [item], item: item };
     } 
     if (item.ident.substr(-2) == '()') {
diff --git a/src/jsifier.js b/src/jsifier.js
index 7273f54c..38f3bd5e 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -338,6 +338,7 @@ function JSify(data, functionsOnly, givenFunctions) {
           // External variables in shared libraries should not be declared as
           // they would shadow similarly-named globals in the parent, so do nothing here.
           if (BUILD_AS_SHARED_LIB) return ret;
+          if (SIDE_MODULE) return [];
           // Library items need us to emit something, but everything else requires nothing.
           if (!LibraryManager.library[item.ident.slice(1)]) return ret;
         }
@@ -1142,8 +1143,8 @@ function JSify(data, functionsOnly, givenFunctions) {
     });
     var range = maxx - minn;
     var useIfs = (item.switchLabels.length+1) < 6 || range > 10*1024 || (range/item.switchLabels.length) > 1024; // heuristics
-    if (VERBOSE && useIfs && item.switchLabels.length > 2) {
-      warn('not optimizing llvm switch into js switch because ' + [range, range/item.switchLabels.length]);
+    if (VERBOSE && useIfs && item.switchLabels.length >= 6) {
+      warn('not optimizing llvm switch into js switch because range of values is ' + range + ', density is ' + range/item.switchLabels.length);
     }
 
     var phiSets = calcPhiSets(item);
@@ -1405,7 +1406,10 @@ function JSify(data, functionsOnly, givenFunctions) {
     // We cannot compile assembly. See comment in intertyper.js:'Call'
     assert(ident != 'asm', 'Inline assembly cannot be compiled to JavaScript!');
 
+    var extCall = false;
+
     if (ASM_JS && funcData.setjmpTable) forceByPointer = true; // in asm.js mode, we must do an invoke for each call
+    if (ASM_JS && DLOPEN_SUPPORT && !invoke && !funcData.setjmpTable) extCall = true; // go out, to be able to access other modules TODO: optimize
 
     ident = Variables.resolveAliasToIdent(ident);
     var shortident = ident.slice(1);
@@ -1466,7 +1470,7 @@ function JSify(data, functionsOnly, givenFunctions) {
 
     args = args.map(function(arg, i) { return indexizeFunctions(arg, argsTypes[i]) });
     if (ASM_JS) {
-      if (shortident in Functions.libraryFunctions || simpleIdent in Functions.libraryFunctions || byPointerForced || invoke || funcData.setjmpTable) {
+      if (shortident in Functions.libraryFunctions || simpleIdent in Functions.libraryFuncti