Merge pull request #32 from max99x/master

Basic dynamic loading support

12 files changed, 407 insertions, 11 deletions

diff --git a/src/analyzer.js b/src/analyzer.js
index 33238e8a..bae05ec8 100644
--- a/src/analyzer.js
+++ b/src/analyzer.js
@@ -1127,7 +1127,13 @@ function analyzer(data) {
   var ret = substrate.solve();
 
   // Add additional necessary items
-  ['memset', 'malloc', 'free'].forEach(function(ident) {
+  if (INCLUDE_FULL_LIBRARY) {
+    assert(!BUILD_AS_SHARED_LIB, 'Cannot have both INCLUDE_FULL_LIBRARY and BUILD_AS_SHARED_LIB set.')
+    var libFuncsToInclude = keys(Library);
+  } else {
+    var libFuncsToInclude = ['memset', 'malloc', 'free'];
+  }
+  libFuncsToInclude.forEach(function(ident) {
     ret.functionStubs.push({
       intertype: 'functionStub',
       ident: '_' + ident
diff --git a/src/compiler.js b/src/compiler.js
index e5e8a755..4fe68e88 100644
--- a/src/compiler.js
+++ b/src/compiler.js
@@ -42,6 +42,7 @@ if (SAFE_HEAP >= 2) {
 }
 
 EXPORTED_FUNCTIONS = set(EXPORTED_FUNCTIONS);
+EXPORTED_GLOBALS = set(EXPORTED_GLOBALS);
 
 // Settings sanity checks
 
diff --git a/src/jsifier.js b/src/jsifier.js
index d3a197b6..a0c6ec72 100644
--- a/src/jsifier.js
+++ b/src/jsifier.js
@@ -178,9 +178,13 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
           }
           constant = makePointer(constant, null, 'ALLOC_STATIC', item.type);
 
+          var js = item.ident + '=' + constant + ';';
+          if (item.ident in EXPORTED_GLOBALS) {
+            js += '\nModule["' + item.ident + '"] = ' + item.ident + ';';
+          }
           return ret.concat({
             intertype: 'GlobalVariable',
-            JS: item.ident + '=' + constant + ';',
+            JS: js,
           });
         }
       }
@@ -196,7 +200,10 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     processItem: function(item) {
       var ret = [item];
       var shortident = item.ident.substr(1);
-      if (shortident in Library) {
+      if (BUILD_AS_SHARED_LIB) {
+        // Shared libraries reuse the runtime of their parents.
+        item.JS = '';
+      } else if (Library.hasOwnProperty(shortident)) {
         function addFromLibrary(ident) {
           if (ident in addedLibraryItems) return '';
           // Don't replace implemented functions with library ones (which can happen when we add dependencies).
@@ -764,14 +771,17 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) {
     //        postamble
     //          global_vars
 
-    var shellParts = read('shell.js').split('{{BODY}}');
+    var shellFile = BUILD_AS_SHARED_LIB ? 'shell_sharedlib.js' : 'shell.js';
+    var shellParts = read(shellFile).split('{{BODY}}');
     print(shellParts[0]);
-      var pre = processMacros(preprocess(read('preamble.js').replace('{{RUNTIME}}', getRuntime()), CONSTANTS));
+      var preFile = BUILD_AS_SHARED_LIB ? 'preamble_sharedlib.js' : 'preamble.js';
+      var pre = processMacros(preprocess(read(preFile).replace('{{RUNTIME}}', getRuntime()), CONSTANTS));
       print(pre);
       generated.forEach(function(item) { print(indentify(item.JS || '', 2)); });
       print(Functions.generateIndexing());
 
-      var postParts = processMacros(preprocess(read('postamble.js'), CONSTANTS)).split('{{GLOBAL_VARS}}');
+      var postFile = BUILD_AS_SHARED_LIB ? 'postamble_sharedlib.js' : 'postamble.js';
+      var postParts = processMacros(preprocess(read(postFile), CONSTANTS)).split('{{GLOBAL_VARS}}');
       print(postParts[0]);
         itemsDict.GlobalVariable.forEach(function(item) { print(indentify(item.JS, 4)); });
         itemsDict.GlobalVariablePostSet.forEach(function(item) { print(indentify(item.JS, 4)); });
diff --git a/src/library.js b/src/library.js
index 42e2cc5c..083bf29c 100644
--- a/src/library.js
+++ b/src/library.js
@@ -277,7 +277,7 @@ var Library = {
         }
       } else {
         Module.stdin = function stdin(prompt) {
-          return window.prompt(prompt);
+          return window.prompt(prompt) || '';
         };
       }
 
@@ -1194,6 +1194,126 @@ var Library = {
     return Math.pow(2, x);
   },
 
+  
+  // ==========================================================================
+  // dlfcn.h
+  // ==========================================================================
+
+  // Data for dlfcn.h.
+  $DLFCN_DATA: {
+    error: null,
+    isError: false,
+    loadedLibs: {}, // handle -> [refcount, name, lib_object]
+    loadedLibNames: {}, // name -> handle
+  },
+  // void* dlopen(const char* filename, int flag);
+  dlopen__deps: ['$DLFCN_DATA'],
+  dlopen: function(filename, flag) {
+    // TODO: Add support for LD_LIBRARY_PATH.
+    filename = Pointer_stringify(filename);
+    filename += '.js';
+
+    if (DLFCN_DATA.loadedLibNames[filename]) {
+      // Already loaded; increment ref count and return.
+      var handle = DLFCN_DATA.loadedLibNames[filename];
+      DLFCN_DATA.loadedLibs[handle][0]++;
+      return handle;
+    }
+
+    try {
+      var lib_data = read(filename);
+    } catch (e) {
+      DLFCN_DATA.isError = true;
+      return 0;
+    }
+
+    try {
+      var lib_module = eval(lib_data)(FUNCTION_TABLE.length);
+    } catch (e) {
+      DLFCN_DATA.isError = true;
+      return 0;
+    }
+
+    // Not all browsers support Object.keys().
+    var handle = 1;
+    for (var key in DLFCN_DATA.loadedLibs) {
+      if (DLFCN_DATA.loadedLibs.hasOwnProperty(key)) handle++;
+    }
+
+    DLFCN_DATA.loadedLibs[handle] = [1, filename, lib_module];
+    DLFCN_DATA.loadedLibNames[filename] = handle;
+
+    // We don't care about RTLD_NOW and RTLD_LAZY.
+    if (flag & 256) { // RTLD_GLOBAL
+      for (var ident in lib_module) {
+        if (lib_module.hasOwnProperty(ident)) {
+          // TODO: Check if we need to unmangle here.
+          Module[ident] = lib_module[ident];
+        }
+      }
+    }
+
+    return handle;
+  },
+  // int dlclose(void* handle);
+  dlclose__deps: ['$DLFCN_DATA'],
+  dlclose: function(handle) {
+    if (!DLFCN_DATA.loadedLibs[handle]) {
+      DLFCN_DATA.isError = true;
+      return 1;
+    } else {
+      var lib_record = DLFCN_DATA.loadedLibs[handle];
+      if (lib_record[0]-- == 0) {
+        delete DLFCN_DATA.loadedLibNames[lib_record[1]];
+        delete DLFCN_DATA.loadedLibs[handle];
+      }
+      return 0;
+    }
+  },
+  // void* dlsym(void* handle, const char* symbol);
+  dlsym__deps: ['$DLFCN_DATA'],
+  dlsym: function(handle, symbol) {
+    symbol = Pointer_stringify(symbol);
+    // TODO: Properly mangle.
+    symbol = '_' + symbol;
+
+    if (!DLFCN_DATA.loadedLibs[handle]) {
+      DLFCN_DATA.isError = true;
+      return 0;
+    } else {
+      var lib_module = DLFCN_DATA.loadedLibs[handle][2];
+      if (!lib_module[symbol]) {
+        DLFCN_DATA.isError = true;
+        return 0;
+      } else {
+        var result = lib_module[symbol];
+        if (typeof result == 'function') {
+          // TODO: Cache functions rather than appending on every lookup.
+          FUNCTION_TABLE.push(result);
+          FUNCTION_TABLE.push(0);
+          result = FUNCTION_TABLE.length - 2;
+        }
+        return result;
+      }
+    }
+  },
+  // char* dlerror(void);
+  dlerror__deps: ['$DLFCN_DATA'],
+  dlerror: function() {
+    if (DLFCN_DATA.isError) {
+      return 0;
+    } else {
+      // TODO: Return non-generic error messages.
+      if (DLFCN_DATA.error === null) {
+        var msg = 'An error occurred while loading dynamic library.';
+        var arr = Module.intArrayFromString(msg)
+        DLFCN_DATA.error = Pointer_make(arr, 0, 2, 'i8');
+      }
+      DLFCN_DATA.isError = false;
+      return DLFCN_DATA.error;
+    }
+  },
+
   // ==========================================================================
   // unistd.h
   // ==========================================================================
diff --git a/src/modules.js b/src/modules.js
index fd18fc72..91758609 100644
--- a/src/modules.js
+++ b/src/modules.js
@@ -140,7 +140,13 @@ var Functions = {
 
   // Generate code for function indexing
   generateIndexing: function() {
-    return 'var FUNCTION_TABLE = [' + this.indexedFunctions.toString().replace('"', '') + '];';
+    var indices = this.indexedFunctions.toString().replace('"', '');
+    if (BUILD_AS_SHARED_LIB) {
+      // Shared libraries reuse the parent's function table.
+      return 'FUNCTION_TABLE = FUNCTION_TABLE.concat([' + indices + ']);';
+    } else {
+      return 'var FUNCTION_TABLE = [' + indices + '];';
+    }
   }
 };
 
diff --git a/src/parseTools.js b/src/parseTools.js
index e520f919..39537ab2 100644
--- a/src/parseTools.js
+++ b/src/parseTools.js
@@ -694,7 +694,11 @@ function makeGetValue(ptr, pos, type, noNeedFirst, unsigned) {
 
 function indexizeFunctions(value) {
   if (value in Functions.currFunctions) {
-    return Functions.getIndex(value);
+    if (BUILD_AS_SHARED_LIB) {
+      return '(FUNCTION_TABLE_OFFSET + ' + Functions.getIndex(value) + ')';
+    } else {
+      return Functions.getIndex(value);
+    }
   }
   if (value && value[0] && value[0] == '_') {
     var rootIdent = LibraryManager.getRootIdent(value.slice(1));
diff --git a/src/postamble_sharedlib.js b/src/postamble_sharedlib.js
new file mode 100644
index 00000000..1fd78f23
--- /dev/null
+++ b/src/postamble_sharedlib.js
@@ -0,0 +1,15 @@
+
+// === Auto-generated postamble setup entry stuff ===
+
+function run(args) {
+  {{GLOBAL_VARS}}
+  __globalConstructor__();
+}
+Module['run'] = run;
+
+// {{PRE_RUN_ADDITIONS}}
+
+run();
+
+// {{POST_RUN_ADDITIONS}}
+
diff --git a/src/preamble.js b/src/preamble.js
index eff0b8e9..408ef6aa 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -598,7 +598,7 @@ if (!this['read']) {
     xhr.send(null);
     if (xhr.status != 200 && xhr.status != 0) throw 'failed to open: ' + url;
     return xhr.responseText;
-  }
+  };
 }
 
 function readBinary(filename) {
diff --git a/src/preamble_sharedlib.js b/src/preamble_sharedlib.js
new file mode 100644
index 00000000..4b91c44c
--- /dev/null
+++ b/src/preamble_sharedlib.js
@@ -0,0 +1,11 @@
+// === Auto-generated preamble library stuff ===
+
+//========================================
+// Runtime essentials
+//========================================
+
+var __globalConstructor__ = function globalConstructor() {
+}
+
+// === Body ===
+
diff --git a/src/settings.js b/src/settings.js
index f2dd9065..437d61b5 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -101,8 +101,20 @@ AUTO_OPTIMIZE = 0; // When run with the CHECK_* options, will not fail on errors
 EXPORTED_FUNCTIONS = ['_main']; // Functions that are explicitly exported, so they are guaranteed to
                                 // be accessible outside of the generated code.
 
+EXPORTED_GLOBALS = []; // Global non-function variables that are explicitly
+                       // exported, so they are guaranteed to be
+                       // accessible outside of the generated code.
+
+INCLUDE_FULL_LIBRARY = 0; // Whether to include the whole library rather than just the
+                          // functions used by the generated code. This is needed when
+                          // dynamically loading modules that make use of runtime
+                          // library functions that are not used in the main module.
+
 SHOW_LABELS = 0; // Show labels in the generated code
 
+BUILD_AS_SHARED_LIB = 0; // Whether to build the code as a shared library, which
+                         // must be loaded dynamically using dlopen().
+
 // Compiler debugging options
 DEBUG_TAGS_SHOWING = [];
   // Some useful items:
diff --git a/src/shell_sharedlib.js b/src/shell_sharedlib.js
new file mode 100644
index 00000000..cbdac74b
--- /dev/null
+++ b/src/shell_sharedlib.js
@@ -0,0 +1,14 @@
+"use strict";
+
+// Capture the output of this into a variable, if you want
+(function(FUNCTION_TABLE_OFFSET) {
+  var Module = {};
+  var args = [];
+  Module.arguments = [];
+
+  {{BODY}}
+
+  // {{MODULE_ADDITIONS}}
+
+  return Module;
+});
diff --git a/tests/runner.py b/tests/runner.py
index 4215bfea..bba28e5a 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -239,7 +239,7 @@ class RunnerCore(unittest.TestCase):
   def do_emscripten(self, filename, output_processor=None):
     # Run Emscripten
     exported_settings = {}
-    for setting in ['QUANTUM_SIZE', 'RELOOP', 'OPTIMIZE', 'ASSERTIONS', 'USE_TYPED_ARRAYS', 'SAFE_HEAP', 'CHECK_OVERFLOWS', 'CORRECT_OVERFLOWS', 'CORRECT_SIGNS', 'CHECK_SIGNS', 'CORRECT_OVERFLOWS_LINES', 'CORRECT_SIGNS_LINES', 'CORRECT_ROUNDINGS', 'CORRECT_ROUNDINGS_LINES', 'INVOKE_RUN', 'SAFE_HEAP_LINES', 'INIT_STACK', 'AUTO_OPTIMIZE', 'EXPORTED_FUNCTIONS']:
+    for setting in ['QUANTUM_SIZE', 'RELOOP', 'OPTIMIZE', 'ASSERTIONS', 'USE_TYPED_ARRAYS', 'SAFE_HEAP', 'CHECK_OVERFLOWS', 'CORRECT_OVERFLOWS', 'CORRECT_SIGNS', 'CHECK_SIGNS', 'CORRECT_OVERFLOWS_LINES', 'CORRECT_SIGNS_LINES', 'CORRECT_ROUNDINGS', 'CORRECT_ROUNDINGS_LINES', 'INVOKE_RUN', 'SAFE_HEAP_LINES', 'INIT_STACK', 'AUTO_OPTIMIZE', 'EXPORTED_FUNCTIONS', 'EXPORTED_GLOBALS', 'BUILD_AS_SHARED_LIB', 'INCLUDE_FULL_LIBRARY']:
       try:
         value = eval(setting)
         exported_settings[setting] = value
@@ -1779,6 +1779,203 @@ if 'benchmark' not in sys.argv:
           # Bloated memory; same layout as C/C++
           self.do_test(src, '*16,0,4,8,8,12|20,0,4,4,8,12,12,16|24,0,20,0,4,4,8,12,12,16*\n*0,0,0,1,2,64,68,69,72*\n*2*')
 
+    def test_dlfcn_basic(self):
+      global BUILD_AS_SHARED_LIB
+      lib_src = '''
+        #include <cstdio>
+
+        class Foo {
+        public:
+          Foo() {
+            printf("Constructing lib object.\\n");
+          }
+        };
+
+        Foo global;
+        '''
+      dirname = self.get_dir()
+      filename = os.path.join(dirname, 'liblib.cpp')
+      BUILD_AS_SHARED_LIB = 1
+      self.build(lib_src, dirname, filename)
+      shutil.move(filename + '.o.js', os.path.join(dirname, 'liblib.so.js'))
+
+      src = '''
+        #include <cstdio>
+        #include <dlfcn.h>
+
+        class Bar {
+        public:
+          Bar() {
+            printf("Constructing main object.\\n");
+          }
+        };
+
+        Bar global;
+
+        int main() {
+          dlopen("liblib.so", RTLD_NOW);
+          return 0;
+        }
+        '''
+      BUILD_AS_SHARED_LIB = 0
+      self.do_test(src, 'Constructing main object.\nConstructing lib object.\n')
+
+    def test_dlfcn_qsort(self):
+      global BUILD_AS_SHARED_LIB, EXPORTED_FUNCTIONS
+      lib_src = '''
+        int lib_cmp(const void* left, const void* right) {
+          const int* a = (const int*) left;
+          const int* b = (const int*) right;
+          if(*a > *b) return 1;
+          else if(*a == *b) return  0;
+          else return -1;
+        }
+
+        typedef int (*CMP_TYPE)(const void*, const void*);
+
+        CMP_TYPE get_cmp() {
+          return lib_cmp;
+        }
+        '''
+      dirname = self.get_dir()
+      filename = os.path.join(dirname, 'liblib.cpp')
+      BUILD_AS_SHARED_LIB = 1
+      EXPORTED_FUNCTIONS = ['__Z7get_cmpv']
+      self.build(lib_src, dirname, filename)
+      shutil.move(filename + '.o.js', os.path.join(dirname, 'liblib.so.js'))
+
+      src = '''
+        #include <stdio.h>
+        #include <stdlib.h>
+        #include <dlfcn.h>
+
+        typedef int (*CMP_TYPE)(const void*, const void*);
+
+        int main_cmp(const void* left, const void* right) {
+          const int* a = (const int*) left;
+          const int* b = (const int*) right;
+          if(*a < *b) return 1;
+          else if(*a == *b) return  0;
+          else return -1;
+        }
+
+        int main() {
+          void* lib_handle;
+          CMP_TYPE (*getter_ptr)();
+          CMP_TYPE lib_cmp_ptr;
+          int arr[5] = {4, 2, 5, 1, 3};
+
+          lib_handle = dlopen("liblib.so", RTLD_NOW);
+          if (lib_handle == NULL) {
+            printf("Could not load lib.\\n");
+            return 1;
+          }
+          getter_ptr = (CMP_TYPE (*)()) dlsym(lib_handle, "_Z7get_cmpv");
+          if (getter_ptr == NULL) {
+            printf("Could not find func.\\n");
+            return 1;
+          }
+          lib_cmp_ptr = getter_ptr();
+
+          qsort((void*)arr, 5, sizeof(int), main_cmp);
+          printf("Sort with main comparison: ");
+          for (int i = 0; i < 5; i++) {
+            printf("%d ", arr[i]);
+          }
+          printf("\\n");
+
+          qsort((void*)arr, 5, sizeof(int), lib_cmp_ptr);
+          printf("Sort with lib comparison: ");
+          for (int i = 0; i < 5; i++) {
+            printf("%d ", arr[i]);
+          }
+          printf("\\n");
+
+          return 0;
+        }
+        '''
+      BUILD_AS_SHARED_LIB = 0
+      EXPORTED_FUNCTIONS = ['_main']
+      self.do_test(src, 'Sort with main comparison: 5 4 3 2 1 *Sort with lib comparison: 1 2 3 4 5 *',
+                   output_nicerizer=lambda x: x.replace('\n', '*'))
+
+    def test_dlfcn_data_and_fptr(self):
+      global BUILD_AS_SHARED_LIB, EXPORTED_FUNCTIONS, EXPORTED_GLOBALS
+      lib_src = '''
+        #include <stdio.h>
+
+        int global = 42;
+
+        void lib_fptr() {
+          printf("Second calling lib_fptr from main.\\n");
+        }
+
+        void (*func(int x, void(*fptr)()))() {
+          printf("In func: %d\\n", x);
+          fptr();
+          return lib_fptr;
+        }
+        '''
+      dirname = self.get_dir()
+      filename = os.path.join(dirname, 'liblib.cpp')
+      BUILD_AS_SHARED_LIB = 1
+      EXPORTED_FUNCTIONS = ['__Z4funciPFvvE']
+      EXPORTED_GLOBALS = ['_global']
+      self.build(lib_src, dirname, filename)
+      shutil.move(filename + '.o.js', os.path.join(dirname, 'liblib.so.js'))
+
+      src = '''
+        #include <stdio.h>
+        #include <dlfcn.h>
+
+        typedef void (*FUNCTYPE(int, void(*)()))();
+
+        FUNCTYPE func;
+
+        void main_fptr() {
+          printf("First calling main_fptr from lib.\\n");
+        }
+
+        int main() {
+          void* lib_handle;
+          FUNCTYPE* func_fptr;
+
+          // Test basic lib loading.
+          lib_handle = dlopen("liblib.so", RTLD_NOW);
+          if (lib_handle == NULL) {
+            printf("Could not load lib.\\n");
+            return 1;
+          }
+
+          // Test looked up function.
+          func_fptr = (FUNCTYPE*) dlsym(lib_handle, "_Z4funciPFvvE");
+          if (func_fptr == NULL) {
+            printf("Could not find func.\\n");
+            return 1;
+          }
+
+          // Test passing function pointers across module bounds.
+          void (*fptr)() = func_fptr(13, main_fptr);
+          fptr();
+
+          // Test global data.
+          int* global = (int*) dlsym(lib_handle, "global");
+          if (global == NULL) {
+            printf("Could not find global.\\n");
+            return 1;
+          }
+
+          printf("Var: %d\\n", *global);
+
+          return 0;
+        }
+        '''
+      BUILD_AS_SHARED_LIB = 0
+      EXPORTED_FUNCTIONS = ['_main']
+      EXPORTED_GLOBALS = []
+      self.do_test(src, 'In func: 13*First calling main_fptr from lib.*Second calling lib_fptr from main.*Var: 42*',
+                   output_nicerizer=lambda x: x.replace('\n', '*'))
+
     def test_files(self):
       global CORRECT_SIGNS; CORRECT_SIGNS = 1 # Just so our output is what we expect. Can flip them both.
       def post(filename):