completely automatic malloc inclusion in emcc

6 files changed, 141 insertions, 28 deletions

diff --git a/emcc b/emcc
index a1db0e36..88730539 100755
--- a/emcc
+++ b/emcc
@@ -186,6 +186,8 @@ def unsuffixed(name):
 def unsuffixed_basename(name):
   return os.path.basename(unsuffixed(name))
 
+LLVM_INTERNAL_OPT_LEVEL = 2
+
 # ---------------- End configs -------------
 
 if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
@@ -272,6 +274,7 @@ try:
   settings_changes = []
   for i in range(len(newargs)):
     if newargs[i] == '-s':
+      assert '=' in newargs[i+1], 'Incorrect syntax for -s (use -s OPT=VAL): ' + newargs[i+1]
       settings_changes.append(newargs[i+1])
       newargs[i] = newargs[i+1] = ''
     elif newargs[i].startswith('--typed-arrays'):
@@ -324,6 +327,11 @@ try:
     shared.Settings.DOUBLE_MODE = 0
     print >> sys.stderr, 'Warning: Applying some potentially unsafe optimizations! (Use -O2 if this fails.)'
 
+  # Apply -s settings in newargs here (after optimization levels, so they can override them)
+  for change in settings_changes:
+    key, value = change.split('=')
+    exec('shared.Settings.' + key + ' = ' + value)
+
   ## Compile source code to bitcode
 
   if DEBUG: print >> sys.stderr, 'emcc: compiling to bitcode'
@@ -343,7 +351,7 @@ try:
   if llvm_opt_level > 0:
     if DEBUG: print >> sys.stderr, 'emcc: LLVM opts'
     for input_file in input_files:
-      shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), 2, safe=llvm_opt_level < 2)
+      shared.Building.llvm_opt(in_temp(unsuffixed_basename(input_file) + '.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2)
 
   # If we were just asked to generate bitcode, stop there
   if final_suffix not in ['js', 'html']:
@@ -354,7 +362,7 @@ try:
       if len(input_files) == 1:
         shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), specified_target)
       else:
-        assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files'
+        assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv)
         # We have a specified target (-o <target>), which is not JavaScript or HTML, and
         # we have multiple files: Link them. TODO: Pass complex linker args along
         shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), specified_target)
@@ -365,18 +373,46 @@ try:
 
   if DEBUG: print >> sys.stderr, 'emcc: generating JavaScript'
 
+  extra_files_to_link = []
+
+  # Check if we need to include dlmalloc. Note that we assume a single symbol is enough to know if we have/do not have dlmalloc. If you
+  # include just a few symbols but want the rest, this will not work.
+  need_dlmalloc = False
+  has_dlmalloc = False
+  for input_file in input_files:
+    symbols = shared.Building.llvm_nm(in_temp(unsuffixed_basename(input_file) + '.o'))
+    for malloc_def in ['malloc', 'free', 'calloc', 'memalign', 'realloc', 'valloc', 'pvalloc', 'mallinfo', 'mallopt', 'malloc_trim', 'malloc_stats', 'malloc_usable_size', 'malloc_footprint', 'malloc_max_footprint', 'independent_calloc', 'independent_comalloc']:
+      if malloc_def in symbols.undefs:
+        need_dlmalloc = True
+      if malloc_def in symbols.defs:
+        has_dlmalloc = True
+  if need_dlmalloc and not has_dlmalloc:
+    # We need to build and link dlmalloc in
+    if DEBUG: print >> sys.stderr, 'emcc: including dlmalloc'
+    Popen([shared.EMCC, shared.path_from_root('src', 'dlmalloc.c'), '-g', '-o', in_temp('dlmalloc.o')]).communicate()
+    if llvm_opt_level > 0:
+      shared.Building.llvm_opt(in_temp('dlmalloc.o'), LLVM_INTERNAL_OPT_LEVEL, safe=llvm_opt_level < 2)
+    extra_files_to_link.append(in_temp('dlmalloc.o'))
+
+    # dlmalloc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
+    try:
+      if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
+    except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
+      shared.Settings.CORRECT_SIGNS = 2
+    if shared.Settings.CORRECT_SIGNS == 2:
+      shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
+    # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
+    # so all is well anyhow too.
+
   # First, combine the bitcode files if there are several
-  if len(input_files) > 1:
-    shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files), in_temp(target_basename + '.bc'))
+  if len(input_files) + len(extra_files_to_link) > 1:
+    shared.Building.link(map(lambda input_file: in_temp(unsuffixed_basename(input_file) + '.o'), input_files) + extra_files_to_link,
+                         in_temp(target_basename + '.bc'))
+    # TODO: LLVM link-time opts?
   else:
     shutil.move(in_temp(unsuffixed_basename(input_files[0]) + '.o'), in_temp(target_basename + '.bc'))
 
-  # Apply -s settings in newargs here (after -Ox, so they can override it)
-
-  for change in settings_changes:
-    key, value = change.split('=')
-    exec('shared.Settings.' + key + ' = ' + value)
-
+  # Emscripten
   if opt_level >= 2:
     print >> sys.stderr, 'Warning: The relooper optimization can be very slow.'
 
diff --git a/tests/hello_malloc.cpp b/tests/hello_malloc.cpp
new file mode 100644
index 00000000..d3774f70
--- /dev/null
+++ b/tests/hello_malloc.cpp
@@ -0,0 +1,18 @@
+#include<stdio.h>
+#include<stdlib.h>
+#include<assert.h>
+
+int main() {
+  // Check that a real malloc is used by allocating, freeing, then seeing that we did actually free by a new allocation going to the original place
+  int N = 100;
+  void *allocs[N];
+  for (int i = 0; i < N; i++)
+    allocs[i] = malloc((i+1)*1024);
+  for (int i = 0; i < N; i++)
+    free(allocs[i]);
+  void *another = malloc(1024);
+  assert(another == allocs[0]);
+  printf("hello, world!\n");
+  return 1;
+}
+
diff --git a/tests/hello_world_loop.cpp b/tests/hello_world_loop.cpp
index b9361834..46ec9f23 100644
--- a/tests/hello_world_loop.cpp
+++ b/tests/hello_world_loop.cpp
@@ -10,7 +10,7 @@ extern "C" {
 
 int main() {
   char *original = (char*)"h e l l o ,   w o r l d ! ";
-  char *copy = (char*)malloc(strlen(original));
+  char copy[strlen(original)];
   for (int i = 0; i < strlen(original); i += 2) {
     copy[i/2] = original[i];
   }
diff --git a/tests/hello_world_loop_malloc.cpp b/tests/hello_world_loop_malloc.cpp
new file mode 100644
index 00000000..b9361834
--- /dev/null
+++ b/tests/hello_world_loop_malloc.cpp
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<string.h>
+#include<stdlib.h>
+
+extern "C" {
+  void dump(char *s) {
+    printf("%s\n", s);
+  }
+}
+
+int main() {
+  char *original = (char*)"h e l l o ,   w o r l d ! ";
+  char *copy = (char*)malloc(strlen(original));
+  for (int i = 0; i < strlen(original); i += 2) {
+    copy[i/2] = original[i];
+  }
+  copy[strlen(copy)+1] = (int)&original; // force original to be on the stack
+  dump(copy);
+  return 1;
+}
+
diff --git a/tests/runner.py b/tests/runner.py
index d5489dd6..9224cb4b 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -3560,6 +3560,17 @@ at function.:blag
       self.do_run(src, '*1,0*', ['200', '1'], extra_emscripten_args=['-m'])
       self.do_run(src, '*400,0*', ['400', '400'], extra_emscripten_args=['-m'], no_build=True)
 
+      if self.use_defaults: # TODO: do this in other passes too, passing their opts into emcc
+        # emcc should build in dlmalloc automatically, and do all the sign correction etc. for it
+
+        try_delete(os.path.join(self.get_dir(), 'src.cpp.o.js'))
+        # XXX find out why we fail without TOTAL_MEMORY here. that should not happen!
+        output = Popen([EMCC, '-g', '-s', 'TOTAL_MEMORY=104857600', path_from_root('tests', 'dlmalloc_test.c'),
+                        '-o', os.path.join(self.get_dir(), 'src.cpp.o.js')], stdout=PIPE, stderr=PIPE).communicate()
+
+        self.do_run('x', '*1,0*', ['200', '1'], no_build=True)
+        self.do_run('x', '*400,0*', ['400', '400'], no_build=True)
+
     def test_libcxx(self):
       self.do_run(path_from_root('tests', 'libcxx'),
                    'june -> 30\nPrevious (in alphabetical order) is july\nNext (in alphabetical order) is march',
@@ -4928,26 +4939,36 @@ Options that are modified or new in %s include:
           assert os.path.exists(target), 'Expected %s to exist since args are %s : %s' % (target, str(args), '\n'.join(output))
           self.assertContained('hello, world!', self.run_llvm_interpreter([target]))
 
+        # dlmalloc. dlmalloc is special in that it is the only part of libc that is (1) hard to write well, and
+        # very speed-sensitive. So we do not implement it in JS in library.js, instead we compile it from source
+        for source, has_malloc in [('hello_world' + suffix, False), ('hello_malloc.cpp', True)]:
+          clear()
+          output = Popen([compiler, path_from_root('tests', source)], stdout=PIPE, stderr=PIPE).communicate()
+          assert os.path.exists('a.out.js'), '\n'.join(output)
+          self.assertContained('hello, world!', run_js('a.out.js'))
+          generated = open('a.out.js').read()
+          assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not'
+
         # Optimization: emcc src.cpp -o something.js [-Ox]. -O0 is the same as not specifying any optimization setting
-        for params, opt_level, bc_params, closure in [ # bc params are used after compiling to bitcode
-          (['-o', 'something.js'],                          0, None, 0),
-          (['-o', 'something.js', '-O0'],                   0, None, 0),
-          (['-o', 'something.js', '-O1'],                   1, None, 0),
-          (['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1),
-          (['-o', 'something.js', '-O2'],                   2, None, 1),
-          (['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0),
-          (['-o', 'something.js', '-O3'],                   3, None, 1),
-          (['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0),
+        for params, opt_level, bc_params, closure, has_malloc in [ # bc params are used after compiling to bitcode
+          (['-o', 'something.js'],                          0, None, 0, 1),
+          (['-o', 'something.js', '-O0'],                   0, None, 0, 0),
+          (['-o', 'something.js', '-O1'],                   1, None, 0, 0),
+          (['-o', 'something.js', '-O1', '--closure', '1'], 1, None, 1, 0),
+          (['-o', 'something.js', '-O2'],                   2, None, 1, 1),
+          (['-o', 'something.js', '-O2', '--closure', '0'], 2, None, 0, 0),
+          (['-o', 'something.js', '-O3'],                   3, None, 1, 1),
+          (['-o', 'something.js', '-O3', '--closure', '0'], 3, None, 0, 0),
           # and, test compiling to bitcode first
-          (['-o', 'something.bc'], 0, [],      0),
-          (['-o', 'something.bc'], 0, ['-O0'], 0),
-          (['-o', 'something.bc'], 1, ['-O1'], 0),
-          (['-o', 'something.bc'], 2, ['-O2'], 1),
-          (['-o', 'something.bc'], 3, ['-O3'], 1),
+          (['-o', 'something.bc'], 0, [],      0, 0),
+          (['-o', 'something.bc'], 0, ['-O0'], 0, 0),
+          (['-o', 'something.bc'], 1, ['-O1'], 0, 0),
+          (['-o', 'something.bc'], 2, ['-O2'], 1, 0),
+          (['-o', 'something.bc'], 3, ['-O3'], 1, 0),
         ]:
           #print params, opt_level, bc_params, closure
           clear()
-          output = Popen([compiler, path_from_root('tests', 'hello_world_loop.cpp')] + params,
+          output = Popen([compiler, path_from_root('tests', 'hello_world_loop' + ('_malloc' if has_malloc else '') + '.cpp')] + params,
                          stdout=PIPE, stderr=PIPE).communicate()
           assert len(output[0]) == 0, output[0]
           if bc_params is not None:
@@ -4975,6 +4996,7 @@ Options that are modified or new in %s include:
             assert 'var $i;' in generated, 'micro opts should always be on'
             if opt_level >= 1: assert 'HEAP8[HEAP32[' in generated, 'eliminator should create compound expressions, and fewer one-time vars'
             assert ('_puts(' in generated) == (opt_level >= 1), 'with opt >= 1, llvm opts are run and they should optimize printf to puts'
+            assert ('function _malloc(bytes) {' in generated) == (not has_malloc), 'If malloc is needed, it should be there, if not not'
 
         # emcc -s RELOOP=1 src.cpp ==> should pass -s to emscripten.py. --typed-arrays is a convenient alias for -s USE_TYPED_ARRAYS
         for params, test, text in [
@@ -5036,7 +5058,6 @@ Options that are modified or new in %s include:
           self.assertContained('side got: hello from main, over', self.run_llvm_interpreter(['combined.bc']))
 
       # TODO: compile .ll inputs to emcc into .bc
-      # TODO: dlmalloc in emcc (just pass the arg to emscripten.py)
       # TODO: test normal project linking, static and dynamic: get_library should not need to be told what to link!
       # TODO: when ready, switch tools/shared building to use emcc over emmaken
       # TODO: when this is done, more test runner to test these (i.e., test all -Ox thoroughly)
diff --git a/tools/shared.py b/tools/shared.py
index eef1dd68..7bd31729 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1,4 +1,4 @@
-import shutil, time, os, json, tempfile
+import shutil, time, os, sys, json, tempfile
 from subprocess import Popen, PIPE, STDOUT
 
 __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -18,6 +18,7 @@ LLVM_LD=os.path.join(LLVM_ROOT, 'llvm-ld')
 LLVM_OPT=os.path.expanduser(os.path.join(LLVM_ROOT, 'opt'))
 LLVM_AS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-as'))
 LLVM_DIS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-dis'))
+LLVM_NM=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-nm'))
 LLVM_DIS_OPTS = ['-show-annotations'] # For LLVM 2.8+. For 2.7, you may need to do just    []
 LLVM_INTERPRETER=os.path.expanduser(os.path.join(LLVM_ROOT, 'lli'))
 LLVM_COMPILER=os.path.expanduser(os.path.join(LLVM_ROOT, 'llc'))
@@ -342,6 +343,22 @@ class Building:
     output = Popen([LLVM_AS, filename + '.o.ll', '-o=' + filename + '.o'], stdout=PIPE).communicate()[0]
     assert os.path.exists(filename + '.o'), 'Could not create bc file: ' + output
 
+  @staticmethod
+  def llvm_nm(filename):
+    # LLVM binary ==> list of symbols
+    output = Popen([LLVM_NM, filename], stdout=PIPE).communicate()[0]
+    class ret:
+      defs = []
+      undefs = []
+    for line in output.split('\n'):
+      if len(line) == 0: continue
+      status, symbol = filter(lambda seg: len(seg) > 0, line.split(' '))
+      if status == 'U':
+        ret.undefs.append(symbol)
+      else:
+        ret.defs.append(symbol)
+    return ret
+
   @staticmethod # TODO: make this use emcc instead of emmaken
   def emmaken(filename, args=[], stdout=None, stderr=None, env=None):
     try: