refactor temp files handling code, and first passing compilation test for emcc

6 files changed, 197 insertions, 128 deletions

diff --git a/emcc b/emcc
index 6be979bc..d1b9155e 100755
--- a/emcc
+++ b/emcc
@@ -92,12 +92,11 @@ emcc can be influenced by a few environment variables:
 
 '''
 
-import sys
-import os
-import subprocess
+import os, sys, shutil
+from subprocess import Popen, PIPE, STDOUT
 from tools import shared
 
-DEBUG = 0
+DEBUG = 1
 
 ################### XXX
 print >> sys.stderr, '\n***This is a WORK IN PROGRESS***'
@@ -164,116 +163,140 @@ if CONFIGURE_CONFIG or CMAKE_CONFIG:
   if DEBUG: print >> sys.stderr, 'emcc.py, just configuring: ', cmd
   exit(os.execvp(compiler, cmd))
 
-try:
-  #f=open('/dev/shm/tmp/waka.txt', 'a')
-  #f.write('Args: ' + ' '.join(sys.argv) + '\nCMake? ' + str(CMAKE_CONFIG) + '\n')
-  #f.close()
+if os.environ.get('EMMAKEN_COMPILER'):
+  CXX = os.environ['EMMAKEN_COMPILER']
+else:
+  CXX = shared.CLANG
 
-  if os.environ.get('EMMAKEN_COMPILER'):
-    CXX = os.environ['EMMAKEN_COMPILER']
-  else:
-    CXX = CLANG
-
-  CC = to_cc(CXX)
+CC = shared.to_cc(CXX)
 
-  # If we got here from a redirection through emmakenxx.py, then force a C++ compiler here
-  if os.environ.get('EMMAKEN_CXX'):
-    CC = CXX
+# If we got here from a redirection through emmakenxx.py, then force a C++ compiler here
+if os.environ.get('EMMAKEN_CXX'):
+  CC = CXX
 
-  CC_ADDITIONAL_ARGS = COMPILER_OPTS # + ['-g']?
-  ALLOWED_LINK_ARGS = ['-f', '-help', '-o', '-print-after', '-print-after-all', '-print-before',
-                       '-print-before-all', '-time-passes', '-v', '-verify-dom-info', '-version' ]
-  TWO_PART_DISALLOWED_LINK_ARGS = ['-L'] # Ignore thingsl like |-L .|
+CC_ADDITIONAL_ARGS = shared.COMPILER_OPTS # + ['-g']?
+ALLOWED_LINK_ARGS = ['-f', '-help', '-o', '-print-after', '-print-after-all', '-print-before',
+                     '-print-before-all', '-time-passes', '-v', '-verify-dom-info', '-version' ]
+TWO_PART_DISALLOWED_LINK_ARGS = ['-L'] # Ignore thingsl like |-L .|
 
-  EMMAKEN_CFLAGS = os.environ.get('EMMAKEN_CFLAGS')
-  if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += EMMAKEN_CFLAGS.split(' ')
+EMMAKEN_CFLAGS = os.environ.get('EMMAKEN_CFLAGS')
+if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += EMMAKEN_CFLAGS.split(' ')
 
-  # ----------------  End configs -------------
+# ----------------  End configs -------------
 
-  if len(sys.argv) == 2 and 'conftest' not in ' '.join(sys.argv): # Avoid messing with configure, see below too
-    # ranlib
-    sys.exit(0)
-  if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
-    # noop ar
-    sys.exit(0)
+if len(sys.argv) == 1 or sys.argv[1] in ['x', 't']:
+  # noop ar
+  if DEBUG: print >> sys.stderr, 'emcc.py, just ar'
+  sys.exit(0)
 
-  use_cxx = True
-  use_linker = True
-  header = False # pre-compiled headers. We fake that by just copying the file
+use_cxx = True
+use_linker = True
+header = False # pre-compiled headers. We fake that by just copying the file
 
-  opts = []
-  files = []
-  for i in range(1, len(sys.argv)):
+opts = []
+files = []
+for i in range(1, len(sys.argv)):
+  arg = sys.argv[i]
+  if arg.startswith('-'):
+    opts.append(arg)
+  else:
+    files.append(arg)
+    if arg.endswith('.c'):
+      use_cxx = False
+    if arg.endswith(('.c', '.cc', '.cpp', '.dT')):
+      use_linker = False
+    if arg.endswith('.h') and sys.argv[i-1] != '-include':
+      header = True
+      use_linker = False
+
+if '--version' in opts:
+  use_linker = False
+
+use_compiler = not use_linker and not header
+
+if set(sys.argv[1]).issubset(set('-cruqs')): # ar
+  sys.argv = sys.argv[:1] + sys.argv[3:] + ['-o='+sys.argv[2]]
+  assert use_linker, 'Linker should be used in this case'
+
+# Check if a target is specified
+target = None
+for i in range(len(sys.argv)-1):
+  if sys.argv[i] == '-o':
+    target = sys.argv[i+1]
+    sys.argv = sys.argv[:i] + sys.argv[i+2:]
+    break
+
+if use_linker:
+  call = LLVM_LD
+  newargs = ['-disable-opt']
+  i = 0
+  while i < len(sys.argv)-1:
+    i += 1
     arg = sys.argv[i]
     if arg.startswith('-'):
-      opts.append(arg)
-    else:
-      files.append(arg)
-      if arg.endswith('.c'):
-        use_cxx = False
-      if arg.endswith(('.c', '.cc', '.cpp', '.dT')):
-        use_linker = False
-      if arg.endswith('.h') and sys.argv[i-1] != '-include':
-        header = True
-        use_linker = False
-
-  if '--version' in opts:
-    use_linker = False
-
-  if set(sys.argv[1]).issubset(set('-cruqs')): # ar
-    sys.argv = sys.argv[:1] + sys.argv[3:] + ['-o='+sys.argv[2]]
-    assert use_linker, 'Linker should be used in this case'
-
-  if use_linker:
-    call = LLVM_LD
-    newargs = ['-disable-opt']
-    found_o = False
-    i = 0
-    while i < len(sys.argv)-1:
-      i += 1
-      arg = sys.argv[i]
-      if found_o:
-        newargs.append('-o=%s' % arg)
-        found_o = False
-        continue
-      if arg.startswith('-'):
-        if arg == '-o':
-          found_o = True
-          continue
-        prefix = arg.split('=')[0]
-        if prefix in ALLOWED_LINK_ARGS:
-          newargs.append(arg)
-        if arg in TWO_PART_DISALLOWED_LINK_ARGS:
-          i += 1
-      elif arg.endswith('.so'):
-        continue # .so's do not exist yet, in many cases
-      else:
-        # not option, so just append
+      prefix = arg.split('=')[0]
+      if prefix in ALLOWED_LINK_ARGS:
         newargs.append(arg)
-  elif not header:
-    call = CXX if use_cxx else CC
-    newargs = sys.argv[1:]
-    for i in range(len(newargs)):
-      if newargs[i].startswith('-O'):
-        if DEBUG: print >> sys.stderr, 'emcc.py: WARNING: Optimization flags (-Ox) are ignored in emcc. Tell emscripten.py to do that, or run LLVM opt.'
-        newargs[i] = ''
-    newargs = [ arg for arg in newargs if arg is not '' ] + CC_ADDITIONAL_ARGS
-    newargs.append('-emit-llvm')
-    if not use_linker:
-      newargs.append('-c') 
-  else:
-    if DEBUG: print >> sys.stderr, 'Just copy.'
-    shutil.copy(sys.argv[-1], sys.argv[-2])
-    exit(0)
+      if arg in TWO_PART_DISALLOWED_LINK_ARGS:
+        i += 1
+    elif arg.endswith('.so'):
+      continue # .so's do not exist yet, in many cases
+    else:
+      # not option, so just append
+      newargs.append(arg)
+  if target:
+    newargs.append('-o=' + target)
+
+  if DEBUG: print >> sys.stderr, "Running:", call, ' '.join(newargs)
+  Popen([call] + newargs).communicate()
+  exit(0)
+
+elif use_compiler:
+  call = CXX if use_cxx else CC
+  newargs = sys.argv[1:]
+  for i in range(len(newargs)):
+    if newargs[i].startswith('-O'):
+      if DEBUG: print >> sys.stderr, 'emcc.py: WARNING: Optimization flags (-Ox) are ignored in emcc. Tell emscripten.py to do that, or run LLVM opt.'
+      newargs[i] = ''
+  newargs = [ arg for arg in newargs if arg is not '' ] + CC_ADDITIONAL_ARGS
 
-  #f=open('/dev/shm/tmp/waka.txt', 'a')
-  #f.write('Calling: ' + ' '.join(newargs) + '\n\n')
-  #f.close()
+  if target is None:
+    # No explicit -o specified, so do the most natural thing, compile to .js
+    target = 'a.out.js'
+
+  target_basename = '.'.join(target.split('.')[:-1])
+  final_suffix = target.split('.')[-1]
+
+  # First, generate LLVM bitcode TODO: handle |emcc a.cpp b.cpp -c| which generate *two* bitcode files
+  newargs = newargs + ['-emit-llvm', '-c', '-o', target_basename + '.bc']
 
   if DEBUG: print >> sys.stderr, "Running:", call, ' '.join(newargs)
+  Popen([call] + newargs).communicate()
+
+  # If we were just asked to generate bitcode, stop there
+  if final_suffix in ['o', 'bc']:
+    if final_suffix == 'o':
+      shutil.move(target_basename + '.bc', target_basename + '.o')
+    exit(0)
+
+  # Continue on to create JavaScript
+  temp_files = shared.TempFiles()
+  temp_files.note(target_basename + '.bc')
+  try:
+    shared.Building.emscripten(target_basename + '.bc', append_ext=False)
+    shutil.move(target_basename + '.bc.o.js', target_basename + '.js')
+
+    ## TODO: If we were asked to also generate HTML, do that
+    #if final_suffix == 'html':
+  finally:
+    temp_files.clean()
+
+  exit(0)
+
+else: # header or such
+  if DEBUG: print >> sys.stderr, 'Just copy.'
+  shutil.copy(sys.argv[-1], sys.argv[-2])
+  exit(0)
+
 
-  os.execvp(call, [call] + newargs)
-except Exception, e:
-  print 'Error in emcc.py. (Is the config file ~/.emscripten set up properly?) Error:', e
-  raise
 
diff --git a/emscripten.py b/emscripten.py
index 4b3740d1..0b08b8f1 100755
--- a/emscripten.py
+++ b/emscripten.py
@@ -20,14 +20,9 @@ import os
 import subprocess
 import re
 import sys
-import tempfile
 from tools import shared
 
 
-# Temporary files that should be deleted once the program is finished.
-TEMP_FILES_TO_CLEAN = []
-
-
 __rootpath__ = os.path.abspath(os.path.dirname(__file__))
 def path_from_root(*pathelems):
   """Returns the absolute path for which the given path elements are
@@ -35,14 +30,7 @@ def path_from_root(*pathelems):
   """
   return os.path.join(__rootpath__, *pathelems)
 
-
-def get_temp_file(suffix):
-  """Returns a named temp file  with the given prefix."""
-  named_file = tempfile.NamedTemporaryFile(
-      dir=shared.TEMP_DIR, suffix=suffix, delete=False)
-  TEMP_FILES_TO_CLEAN.append(named_file.name)
-  return named_file
-
+temp_files = shared.TempFiles()
 
 def assemble(filepath):
   """Converts human-readable LLVM assembly to binary LLVM bitcode.
@@ -56,7 +44,7 @@ def assemble(filepath):
   """
   if not filepath.endswith('.bc'):
     command = [shared.LLVM_AS, '-o=-', filepath]
-    with get_temp_file('.bc') as out: ret = subprocess.call(command, stdout=out)
+    with temp_files.get('.bc') as out: ret = subprocess.call(command, stdout=out)
     if ret != 0: raise RuntimeError('Could not assemble %s.' % filepath)
     filepath = out.name
   return filepath
@@ -74,7 +62,7 @@ def disassemble(filepath):
   """
   if not filepath.endswith('.ll'):
     command = [shared.LLVM_DIS, '-o=-', filepath] + shared.LLVM_DIS_OPTS
-    with get_temp_file('.ll') as out: ret = subprocess.call(command, stdout=out)
+    with temp_files.get('.ll') as out: ret = subprocess.call(command, stdout=out)
     if ret != 0: raise RuntimeError('Could not disassemble %s.' % filepath)
     filepath = out.name
   return filepath
@@ -92,7 +80,7 @@ def optimize(filepath):
   shared.Building.LLVM_OPTS = 1
   shared.Settings.QUANTUM_SIZE = 1 # just so it isn't 4, and we assume we can do things that fail on q1
   command = [shared.LLVM_OPT, '-o=-', filepath] + shared.Building.pick_llvm_opts(3, True)
-  with get_temp_file('.bc') as out: ret = subprocess.call(command, stdout=out)
+  with temp_files.get('.bc') as out: ret = subprocess.call(command, stdout=out)
   if ret != 0: raise RuntimeError('Could not optimize %s.' % filepath)
   return out.name
 
@@ -107,7 +95,7 @@ def link(*objects):
     The path to the linked file.
   """
   command = [shared.LLVM_LINK] + list(objects)
-  with get_temp_file('.bc') as out: ret = subprocess.call(command, stdout=out)
+  with temp_files.get('.bc') as out: ret = subprocess.call(command, stdout=out)
   if ret != 0: raise RuntimeError('Could not link %s.' % objects)
   return out.name
 
@@ -121,7 +109,7 @@ def compile_malloc():
   src = path_from_root('src', 'dlmalloc.c')
   includes = '-I' + path_from_root('src', 'include')
   command = [shared.CLANG, '-c', '-g', '-emit-llvm'] + shared.COMPILER_OPTS + ['-o-', includes, src]
-  with get_temp_file('.bc') as out: ret = subprocess.call(command, stdout=out)
+  with temp_files.get('.bc') as out: ret = subprocess.call(command, stdout=out)
   if ret != 0: raise RuntimeError('Could not compile dlmalloc.')
   return out.name
 
@@ -147,7 +135,7 @@ def emscript(infile, settings, outfile):
       defined in src/settings.js.
     outfile: The file where the output is written.
   """
-  settings_file = get_temp_file('.txt').name # Save settings to a file to work around v8 issue 1579
+  settings_file = temp_files.get('.txt').name # Save settings to a file to work around v8 issue 1579
   s = open(settings_file, 'w')
   s.write(settings)
   s.close()
@@ -287,8 +275,5 @@ if __name__ == '__main__':
   if isinstance(keywords.outfile, basestring):
     keywords.outfile = open(keywords.outfile, 'w')
 
-  try:
-    main(keywords)
-  finally:
-    for filename in TEMP_FILES_TO_CLEAN:
-      os.unlink(filename)
+  temp_files.run_and_clean(lambda: main(keywords))
+
diff --git a/tests/hello_world.c b/tests/hello_world.c
new file mode 100644
index 00000000..cad52a2c
--- /dev/null
+++ b/tests/hello_world.c
@@ -0,0 +1,7 @@
+#include<stdio.h>
+
+int main() {
+  printf("hello, world!\n");
+  return 1;
+}
+
diff --git a/tests/hello_world.cpp b/tests/hello_world.cpp
new file mode 100644
index 00000000..441d892b
--- /dev/null
+++ b/tests/hello_world.cpp
@@ -0,0 +1,9 @@
+#include<stdio.h>
+
+class Test {}; // This will fail in C mode
+
+int main() {
+  printf("hello, world!\n");
+  return 1;
+}
+
diff --git a/tests/runner.py b/tests/runner.py
index 8a4315f7..0d54fd34 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -4871,6 +4871,7 @@ TT = %s
     def test_emcc(self):
       for compiler in [EMCC, EMXX]:
         shortcompiler = os.path.basename(compiler)
+        suffix = '.c' if compiler == EMCC else '.cpp'
 
         # --version
         output = Popen([compiler, '--version'], stdout=PIPE, stderr=PIPE).communicate(input)
@@ -4914,23 +4915,34 @@ JavaScript in the final linking stage of building.
 
 ''' % (shortcompiler, shortcompiler, shortcompiler), output[0], output[1])
 
+        # emcc src.cpp   or   emcc src.cpp -o src.js ==> should give a .js file
+        try_delete('a.out.js')
+        output = Popen([compiler, path_from_root('tests', 'hello_world' + suffix)], stdout=PIPE, stderr=PIPE).communicate(input)
+        assert len(output[0]) == 0, output[0]
+        #assert len(output[1]) == 0, output[1] # we have some debug warnings there now, FIXME
+        assert os.path.exists('a.out.js'), output # should be created in the current directory just like gcc, with a name similar to a.out
+        self.assertContained('hello, world!', run_js(None, 'a.out.js'))
+
       # TODO: make sure all of these match gcc
       # TODO: when this is done, more test runner to test these (i.e., test all -Ox thoroughly)
-      # emcc src.cpp   or   emcc src.cpp -o src.js ==> should give a .js file
       # emcc src.cpp -c    and   emcc src.cpp -o src.[o|bc] ==> should give a .bc file
-      # emcc src.cpp -o src.html ==> should embed the js in an html file for immediate running on the web. only tricky part is sdl
+      # emcc src.cpp -o src.html ==> should embed the js in an html file for immediate running on the web. only tricky part is sdl. TODO: update library_sdl
       # emcc -O0 src.cpp ==> same as without -O0: assertions, etc., and greatest chance of code working: i64 1, ta2, etc., micro-opts
       # emcc -O1 src.cpp ==> no assertions, plus eliminator, plus js optimizer
       # emcc -O2 src.cpp ==> plus reloop (warn about speed)
       # emcc -O3 src.cpp ==> no corrections, relax some other stuff like i64 1 into 0, etc., do closure: dangerous stuff, warn, suggest -O2!
       # emcc --typed-arrays=x .. ==> should use typed arrays. default should be 2
       # emcc --llvm-opts=x .. ==> pick level of LLVM optimizations (default is 0, to be safe?)
+      # emcc -s RELOOP=1 src.cpp ==> should pass -s to emscripten.py
       # When doing unsafe opts, can we run -Ox on the source, not just at the very end?
       # linking - TODO. in particular, test normal project linking, static and dynamic: get_library should not need to be told what to link!
+      #   emcc a.cpp b.cpp => one .js
+      #   emcc a.cpp b.cpp -c => two .o files
       #     annotate each .bc with emscripten info, like "compiled with -O2: do the O2 opts when going to final .js"
       #     warn if linking files with different annotations etc.
       #     use llvm metadata, example: !0 = metadata !{i32 720913, i32 0, i32 4, metadata !"/dev/shm/tmp/src.cpp", metadata !"/dev/shm/tmp", metadata !"clang version 3.0 (tags/RELEASE_30/rc3)", i1 true, i1 false, metadata !"EMSCRIPTEN:O3", i32 0, metadata !1, metadata !1, metadata !3, metadata !1} ; [ DW_TAG_compile_unit ]
       # TODO: when ready, switch tools/shared building to use emcc over emmaken
+      # TODO: add shebang to generated .js files, using JS_ENGINES[0]? #!/usr/bin/python etc
 
     def test_eliminator(self):
       input = open(path_from_root('tools', 'eliminator', 'eliminator-test.js')).read()
diff --git a/tools/shared.py b/tools/shared.py
index 5c5da098..b517c9c7 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1,4 +1,4 @@
-import shutil, time, os, json
+import shutil, time, os, json, tempfile
 from subprocess import Popen, PIPE, STDOUT
 
 __rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -80,6 +80,38 @@ if USE_EMSDK:
 if 'gcparam' not in str(SPIDERMONKEY_ENGINE):
   SPIDERMONKEY_ENGINE += ['-e', "gcparam('maxBytes', 1024*1024*1024);"] # Our very large files need lots of gc heap
 
+# Temp file utilities
+
+def try_delete(filename):
+  try:
+    os.unlink(filename)
+  except:
+    pass
+
+class TempFiles:
+  def __init__(self):
+    self.to_clean = []
+
+  def note(self, filename):
+    self.to_clean.append(filename)
+
+  def get(self, suffix):
+    """Returns a named temp file  with the given prefix."""
+    named_file = tempfile.NamedTemporaryFile(dir=TEMP_DIR, suffix=suffix, delete=False)
+    self.note(named_file.name)
+    return named_file
+
+  def clean(self):
+    for filename in self.to_clean:
+      try_delete(filename)
+    self.to_clean = []
+
+  def run_and_clean(self, func):
+    try:
+      func()
+    finally:
+      self.clean()
+
 # Utilities
 
 def timeout_run(proc, timeout, note):
@@ -93,6 +125,7 @@ def timeout_run(proc, timeout, note):
   return proc.communicate()[0]
 
 def run_js(engine, filename, args=[], check_timeout=False, stdout=PIPE, stderr=None, cwd=None):
+  if engine is None: engine = JS_ENGINES[0]
   if type(engine) is not list: engine = [engine]
   return timeout_run(Popen(engine + [filename] + (['--'] if 'd8' in engine[0] else []) + args,
                      stdout=stdout, stderr=stderr, cwd=cwd), 15*60 if check_timeout else None, 'Execution')