emcc --remove-duplicates option

3 files changed, 57 insertions, 5 deletions

diff --git a/emcc b/emcc
index 9b1090ba..56b59d60 100755
--- a/emcc
+++ b/emcc
@@ -273,6 +273,11 @@ Options that are modified or new in %s include:
                            -v to Clang, and also enable EMCC_DEBUG
                            to details emcc's operations
 
+  --remove-duplicates      If set, will remove duplicate symbols when
+                           linking. This can be useful because
+                           llvm-link's behavior is not as permissive
+                           as ld is.
+
 The target file, if specified (-o <target>), defines what will
 be generated:
 
@@ -418,6 +423,7 @@ try:
   ignore_dynamic_linking = False
   shell_path = shared.path_from_root('src', 'shell.html')
   js_libraries = []
+  remove_duplicates = False
 
   def check_bad_eq(arg):
     assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)'
@@ -504,6 +510,9 @@ try:
       js_libraries.append(newargs[i+1])
       newargs[i] = ''
       newargs[i+1] = ''
+    elif newargs[i] == '--remove-duplicates':
+      remove_duplicates = True
+      newargs[i] = ''
   newargs = [ arg for arg in newargs if arg is not '' ]
 
   if llvm_opts is None: llvm_opts = LLVM_OPT_LEVEL[opt_level]
@@ -687,7 +696,7 @@ try:
         # We have a specified target (-o <target>), which is not JavaScript or HTML, and
         # we have multiple files: Link them
         if DEBUG: print >> sys.stderr, 'emcc: link: ' + str(temp_files)
-        shared.Building.link(temp_files, specified_target)
+        shared.Building.link(temp_files, specified_target, remove_duplicates=remove_duplicates)
     exit(0)
 
   ## Continue on to create JavaScript
@@ -779,8 +788,7 @@ try:
      (not LEAVE_INPUTS_RAW and not (suffix(temp_files[0]) in BITCODE_SUFFIXES or suffix(temp_files[0]) in DYNAMICLIB_SUFFIXES) and shared.Building.is_ar(temp_files[0])):
     linker_inputs = temp_files + extra_files_to_link
     if DEBUG: print >> sys.stderr, 'emcc: linking: ', linker_inputs
-    shared.Building.link(linker_inputs,
-                         in_temp(target_basename + '.bc'))
+    shared.Building.link(linker_inputs, in_temp(target_basename + '.bc'), remove_duplicates=remove_duplicates)
     final = in_temp(target_basename + '.bc')
   else:
     if not LEAVE_INPUTS_RAW:
diff --git a/tests/runner.py b/tests/runner.py
index 24533e16..654b0a97 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -6863,6 +6863,30 @@ f.close()
       Popen(['python', EMCC, os.path.join(self.get_dir(), 'foo', 'main.o'), os.path.join(self.get_dir(), 'bar', 'main.o')]).communicate()
       self.assertContained('hello there', run_js(os.path.join(self.get_dir(), 'a.out.js')))
 
+    def test_remove_duplicates(self):
+      # can happen with .a files. we do a best-effort, removing dupes
+      open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write('''
+        #include<stdio.h>
+        void printey() { printf("bye bye\\n"); }
+        int main() {
+          printey();
+          return 0;
+        }
+      ''')
+      open(os.path.join(self.get_dir(), 'side.cpp'), 'w').write('''
+        #include<stdio.h>
+        void printey() { printf("bye bye\\n"); }
+      ''')
+
+      # without --remove-duplicates, we fail
+      err = Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp'), os.path.join(self.get_dir(), 'side.cpp')], stderr=PIPE).communicate()[1]
+      assert not os.path.exists('a.out.js')
+      assert 'multiply' in err
+
+      # with it, we succeed
+      err = Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp'), os.path.join(self.get_dir(), 'side.cpp'), '--remove-duplicates'], stderr=PIPE).communicate()[1]
+      self.assertContained('bye bye', run_js(os.path.join(self.get_dir(), 'a.out.js')))
+
     def test_embed_file(self):
       open(os.path.join(self.get_dir(), 'somefile.txt'), 'w').write('''hello from a file with lots of data and stuff in it thank you very much''')
       open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(r'''
diff --git a/tools/shared.py b/tools/shared.py
index f431318b..3db2fe69 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -114,6 +114,7 @@ LLVM_DIS=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-dis'))
 LLVM_NM=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-nm'))
 LLVM_INTERPRETER=os.path.expanduser(os.path.join(LLVM_ROOT, 'lli'))
 LLVM_COMPILER=os.path.expanduser(os.path.join(LLVM_ROOT, 'llc'))
+LLVM_EXTRACT=os.path.expanduser(os.path.join(LLVM_ROOT, 'llvm-extract'))
 COFFEESCRIPT = path_from_root('tools', 'eliminator', 'node_modules', 'coffee-script', 'bin', 'coffee')
 
 EMSCRIPTEN = path_from_root('emscripten.py')
@@ -530,7 +531,7 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
     return generated_libs
 
   @staticmethod
-  def link(files, target):
+  def link(files, target, remove_duplicates=False):
     actual_files = []
     unresolved_symbols = set() # necessary for .a linking, see below
     resolved_symbols = set()
@@ -579,8 +580,27 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)''' % { 'winfix': '' if not WINDOWS e
         finally:
           os.chdir(cwd)
     try_delete(target)
+
+    if remove_duplicates:
+      # Remove duplicate symbols. This is a workaround for how we compile .a files, we try to
+      # emulate ld behavior which is permissive TODO: cache llvm-nm results
+      seen_symbols = set()
+      print >> sys.stderr, actual_files
+      for actual in actual_files:
+        symbols = Building.llvm_nm(actual)
+        dupes = seen_symbols.intersection(symbols.defs)
+        if len(dupes) > 0:
+          print >> sys.stderr, 'emcc: warning: removing duplicates in', actual
+          for dupe in dupes:
+            print >> sys.stderr, 'emcc: warning: removing duplicate', dupe
+            Popen([LLVM_EXTRACT, actual, '-delete', '-glob=' + dupe, '-o', actual], stderr=PIPE).communicate()
+            Popen([LLVM_EXTRACT, actual, '-delete', '-func=' + dupe, '-o', actual], stderr=PIPE).communicate()
+          Popen([LLVM_EXTRACT, actual, '-delete', '-glob=.str', '-o', actual], stderr=PIPE).communicate() # garbage that appears here
+        seen_symbols = seen_symbols.union(symbols.defs)
+
+    # Finish link
     output = Popen([LLVM_LINK] + actual_files + ['-o', target], stdout=PIPE).communicate()[0]
-    assert os.path.exists(target) and (output is None or 'Could not open input file' not in output), 'Linking error: ' + output
+    assert os.path.exists(target) and (output is None or 'Could not open input file' not in output), 'Linking error: ' + output + '\nemcc: If you get duplicate symbol errors, try --remove-duplicates'
     if temp_dir:
       try_delete(temp_dir)