Merge pull request #1336 from int3/csmith

Some tweaks to make CSmith easier to setup / debug.

3 files changed, 56 insertions, 39 deletions

diff --git a/tests/fuzz/creduce_tester.py b/tests/fuzz/creduce_tester.py
index c3460e9d..d5618c2e 100755
--- a/tests/fuzz/creduce_tester.py
+++ b/tests/fuzz/creduce_tester.py
@@ -1,53 +1,53 @@
 #!/usr/bin/python
 
 '''
-Runs csmith, a C fuzzer, and looks for bugs
+Usage: creduce ./creduce_tester.py newfail1.c
 '''
 
-import os, sys, difflib
+import os, sys
 from subprocess import Popen, PIPE, STDOUT
 
 sys.path += [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools')]
-import shared
+import shared, jsrun
+
+# creduce will only pass the filename of the C file as the first arg, so other
+# configuration options will have to be hardcoded.
+CSMITH_CFLAGS = ['-I', os.path.join(os.environ['CSMITH_PATH'], 'runtime')]
+ENGINE = shared.JS_ENGINES[0]
+EMCC_ARGS = ['-O2', '-s', 'ASM_JS=1', '-s', 'PRECISE_I64_MATH=1', '-s',
+  'PRECISE_I32_MUL=1']
 
 filename = sys.argv[1]
+obj_filename = os.path.splitext(filename)[0]
+js_filename = obj_filename + '.js'
 print 'testing file', filename
 
-print '2) Compile natively'
-shared.try_delete(filename)
-shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename] + CSMITH_CFLAGS, stderr=PIPE)
-assert os.path.exists(filename)
-print '3) Run natively'
 try:
-  correct = shared.timeout_run(Popen([filename], stdout=PIPE, stderr=PIPE), 3)
+  print '2) Compile natively'
+  shared.check_execute([shared.CLANG_CC, '-O2', filename, '-o', obj_filename] + CSMITH_CFLAGS)
+  print '3) Run natively'
+  correct = jsrun.timeout_run(Popen([obj_filename], stdout=PIPE, stderr=PIPE), 3)
 except Exception, e:
   print 'Failed or infinite looping in native, skipping', e
-  notes['invalid'] += 1
-  os.exit(0) # boring 
+  sys.exit(1) # boring
 
 print '4) Compile JS-ly and compare'
 
 def try_js(args):
-  shared.try_delete(filename + '.js')
-  shared.execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', '-s', 'PRECISE_I64_MATH=1', '-s', 'PRECISE_I32_MUL=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args, stderr=PIPE)
-  assert os.path.exists(filename + '.js')
-  js = shared.run_js(filename + '.js', stderr=PIPE, engine=engine1)
-  assert correct == js, ''.join([a.rstrip()+'\n' for a in difflib.unified_diff(correct.split('\n'), js.split('\n'), fromfile='expected', tofile='actual')])
+  shared.check_execute([shared.EMCC] + EMCC_ARGS + CSMITH_CFLAGS + args +
+    [filename, '-o', js_filename])
+  js = shared.run_js(js_filename, stderr=PIPE, engine=ENGINE)
+  assert correct == js
 
 # Try normally, then try unaligned because csmith does generate nonportable code that requires x86 alignment
-ok = False
-normal = True
-for args, note in [([], None), (['-s', 'UNALIGNED_MEMORY=1'], 'unaligned')]:
+# If you are sure that alignment is not the cause, disable it for a faster reduction
+for args in [[]]:
   try:
     try_js(args)
-    ok = True
-    if note:
-      notes[note] += 1
     break
   except Exception, e:
-    print e
-    normal = False
-if not ok: sys.exit(1)
-
-sys.exit(0) # boring
+    pass
+else:
+  sys.exit(0)
 
+sys.exit(1) # boring
diff --git a/tests/fuzz/csmith_driver.py b/tests/fuzz/csmith_driver.py
index b60e67f7..c987a3be 100755
--- a/tests/fuzz/csmith_driver.py
+++ b/tests/fuzz/csmith_driver.py
@@ -1,11 +1,14 @@
 #!/usr/bin/python
 
 '''
-Runs csmith, a C fuzzer, and looks for bugs
+Runs csmith, a C fuzzer, and looks for bugs.
+
+CSMITH_PATH should be set to something like /usr/local/include/csmith
 '''
 
 import os, sys, difflib, shutil
-from subprocess import Popen, PIPE, STDOUT
+from distutils.spawn import find_executable
+from subprocess import check_call, Popen, PIPE, STDOUT, CalledProcessError
 
 sys.path += [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools')]
 import shared
@@ -15,8 +18,11 @@ engine2 = eval('shared.' + sys.argv[2]) if len(sys.argv) > 2 else None
 
 print 'testing js engines', engine1, engine2
 
-CSMITH = os.path.expanduser('~/Dev/csmith/src/csmith')
-CSMITH_CFLAGS = ['-I' + os.path.expanduser('~/Dev/csmith/runtime/')]
+CSMITH = os.environ.get('CSMITH') or find_executable('csmith')
+assert CSMITH, 'Could not find CSmith on your PATH. Please set the environment variable CSMITH.'
+CSMITH_PATH = os.environ.get('CSMITH_PATH')
+assert CSMITH_PATH, 'Please set the environment variable CSMITH_PATH.'
+CSMITH_CFLAGS = ['-I', os.path.join(CSMITH_PATH, 'runtime')]
 
 filename = os.path.join(shared.CANONICAL_TEMP_DIR, 'fuzzcode')
 
@@ -31,7 +37,7 @@ fails = 0
 while 1:
   print 'Tried %d, notes: %s' % (tried, notes)
   print '1) Generate C'
-  shared.execute([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# +
+  check_call([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# +
                  #['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'],
                  stdout=open(filename + '.c', 'w'))
   #shutil.copyfile(filename + '.c', 'testcase%d.c' % tried)
@@ -41,11 +47,11 @@ while 1:
 
   print '2) Compile natively'
   shared.try_delete(filename)
-  shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS, stderr=PIPE) #  + shared.EMSDK_OPTS
-  shared.execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS, stderr=PIPE)
-  shared.execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stdout=PIPE, stderr=PIPE)
+  shared.check_execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS) #  + shared.EMSDK_OPTS
+  shared.check_execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS)
+  shared.check_execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'])
   shutil.move(filename + '.bc.run', filename + '2')
-  shared.execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS, stderr=PIPE)
+  shared.check_execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS)
   print '3) Run natively'
   try:
     correct1 = shared.jsrun.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3)
@@ -65,7 +71,7 @@ while 1:
   def try_js(args):
     shared.try_delete(filename + '.js')
     print '(compile)'
-    shared.execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args, stderr=PIPE)
+    shared.check_execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args)
     assert os.path.exists(filename + '.js')
     print '(run)'
     js = shared.run_js(filename + '.js', stderr=PIPE, engine=engine1, check_timeout=True)
@@ -91,7 +97,7 @@ while 1:
     print "EMSCRIPTEN BUG"
     notes['embug'] += 1
     fails += 1
-    shutil.copyfile('fuzzcode.c', 'newfail%d.c' % fails)
+    shutil.copyfile(filename + '.c', 'newfail%d.c' % fails)
     continue
   #if not ok:
   #  try: # finally, try with safe heap. if that is triggered, this is nonportable code almost certainly
@@ -118,7 +124,7 @@ while 1:
       print "ODIN VALIDATION BUG"
       notes['embug'] += 1
       fails += 1
-      shutil.copyfile('fuzzcode.c', 'newfail%d.c' % fails)
+      shutil.copyfile(filename + '.c', 'newfail%d.c' % fails)
       continue
 
     js2 = js2.replace('\nwarning: Successfully compiled asm.js code\n', '')
diff --git a/tools/shared.py b/tools/shared.py
index da6d3e57..776001cd 100644
--- a/tools/shared.py
+++ b/tools/shared.py
@@ -1379,6 +1379,17 @@ def execute(cmd, *args, **kw):
     logging.error('Invoking Process failed: <<< ' + cmd + ' >>>')
     raise
 
+def check_execute(cmd, *args, **kw):
+  # TODO: use in more places. execute doesn't actually check that return values
+  # are nonzero
+  try:
+    kw['stderr'] = STDOUT
+    subprocess.check_output(cmd, *args, **kw)
+    logging.debug("Successfuly executed %s" % " ".join(cmd))
+  except subprocess.CalledProcessError as e:
+    logging.error("'%s' failed with output:\n%s" % (" ".join(e.cmd), e.output))
+    raise
+
 def suffix(name):
   parts = name.split('.')
   if len(parts) > 1: