diff options
| -rwxr-xr-x | tests/fuzz/creduce_tester.py | 54 | ||||
| -rwxr-xr-x | tests/fuzz/csmith_driver.py | 30 | ||||
| -rw-r--r-- | tools/shared.py | 11 |
3 files changed, 56 insertions, 39 deletions
diff --git a/tests/fuzz/creduce_tester.py b/tests/fuzz/creduce_tester.py index c3460e9d..d5618c2e 100755 --- a/tests/fuzz/creduce_tester.py +++ b/tests/fuzz/creduce_tester.py @@ -1,53 +1,53 @@ #!/usr/bin/python ''' -Runs csmith, a C fuzzer, and looks for bugs +Usage: creduce ./creduce_tester.py newfail1.c ''' -import os, sys, difflib +import os, sys from subprocess import Popen, PIPE, STDOUT sys.path += [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools')] -import shared +import shared, jsrun + +# creduce will only pass the filename of the C file as the first arg, so other +# configuration options will have to be hardcoded. +CSMITH_CFLAGS = ['-I', os.path.join(os.environ['CSMITH_PATH'], 'runtime')] +ENGINE = shared.JS_ENGINES[0] +EMCC_ARGS = ['-O2', '-s', 'ASM_JS=1', '-s', 'PRECISE_I64_MATH=1', '-s', + 'PRECISE_I32_MUL=1'] filename = sys.argv[1] +obj_filename = os.path.splitext(filename)[0] +js_filename = obj_filename + '.js' print 'testing file', filename -print '2) Compile natively' -shared.try_delete(filename) -shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename] + CSMITH_CFLAGS, stderr=PIPE) -assert os.path.exists(filename) -print '3) Run natively' try: - correct = shared.timeout_run(Popen([filename], stdout=PIPE, stderr=PIPE), 3) + print '2) Compile natively' + shared.check_execute([shared.CLANG_CC, '-O2', filename, '-o', obj_filename] + CSMITH_CFLAGS) + print '3) Run natively' + correct = jsrun.timeout_run(Popen([obj_filename], stdout=PIPE, stderr=PIPE), 3) except Exception, e: print 'Failed or infinite looping in native, skipping', e - notes['invalid'] += 1 - os.exit(0) # boring + sys.exit(1) # boring print '4) Compile JS-ly and compare' def try_js(args): - shared.try_delete(filename + '.js') - shared.execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', '-s', 'PRECISE_I64_MATH=1', '-s', 'PRECISE_I32_MUL=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args, stderr=PIPE) - assert os.path.exists(filename + '.js') - js = shared.run_js(filename + '.js', stderr=PIPE, engine=engine1) - assert correct == js, ''.join([a.rstrip()+'\n' for a in difflib.unified_diff(correct.split('\n'), js.split('\n'), fromfile='expected', tofile='actual')]) + shared.check_execute([shared.EMCC] + EMCC_ARGS + CSMITH_CFLAGS + args + + [filename, '-o', js_filename]) + js = shared.run_js(js_filename, stderr=PIPE, engine=ENGINE) + assert correct == js # Try normally, then try unaligned because csmith does generate nonportable code that requires x86 alignment -ok = False -normal = True -for args, note in [([], None), (['-s', 'UNALIGNED_MEMORY=1'], 'unaligned')]: +# If you are sure that alignment is not the cause, disable it for a faster reduction +for args in [[]]: try: try_js(args) - ok = True - if note: - notes[note] += 1 break except Exception, e: - print e - normal = False -if not ok: sys.exit(1) - -sys.exit(0) # boring + pass +else: + sys.exit(0) +sys.exit(1) # boring diff --git a/tests/fuzz/csmith_driver.py b/tests/fuzz/csmith_driver.py index b60e67f7..c987a3be 100755 --- a/tests/fuzz/csmith_driver.py +++ b/tests/fuzz/csmith_driver.py @@ -1,11 +1,14 @@ #!/usr/bin/python ''' -Runs csmith, a C fuzzer, and looks for bugs +Runs csmith, a C fuzzer, and looks for bugs. + +CSMITH_PATH should be set to something like /usr/local/include/csmith ''' import os, sys, difflib, shutil -from subprocess import Popen, PIPE, STDOUT +from distutils.spawn import find_executable +from subprocess import check_call, Popen, PIPE, STDOUT, CalledProcessError sys.path += [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools')] import shared @@ -15,8 +18,11 @@ engine2 = eval('shared.' + sys.argv[2]) if len(sys.argv) > 2 else None print 'testing js engines', engine1, engine2 -CSMITH = os.path.expanduser('~/Dev/csmith/src/csmith') -CSMITH_CFLAGS = ['-I' + os.path.expanduser('~/Dev/csmith/runtime/')] +CSMITH = os.environ.get('CSMITH') or find_executable('csmith') +assert CSMITH, 'Could not find CSmith on your PATH. Please set the environment variable CSMITH.' +CSMITH_PATH = os.environ.get('CSMITH_PATH') +assert CSMITH_PATH, 'Please set the environment variable CSMITH_PATH.' +CSMITH_CFLAGS = ['-I', os.path.join(CSMITH_PATH, 'runtime')] filename = os.path.join(shared.CANONICAL_TEMP_DIR, 'fuzzcode') @@ -31,7 +37,7 @@ fails = 0 while 1: print 'Tried %d, notes: %s' % (tried, notes) print '1) Generate C' - shared.execute([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# + + check_call([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# + #['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'], stdout=open(filename + '.c', 'w')) #shutil.copyfile(filename + '.c', 'testcase%d.c' % tried) @@ -41,11 +47,11 @@ while 1: print '2) Compile natively' shared.try_delete(filename) - shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS, stderr=PIPE) # + shared.EMSDK_OPTS - shared.execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS, stderr=PIPE) - shared.execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stdout=PIPE, stderr=PIPE) + shared.check_execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS) # + shared.EMSDK_OPTS + shared.check_execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS) + shared.check_execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc']) shutil.move(filename + '.bc.run', filename + '2') - shared.execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS, stderr=PIPE) + shared.check_execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS) print '3) Run natively' try: correct1 = shared.jsrun.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3) @@ -65,7 +71,7 @@ while 1: def try_js(args): shared.try_delete(filename + '.js') print '(compile)' - shared.execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args, stderr=PIPE) + shared.check_execute([shared.EMCC, '-O2', '-s', 'ASM_JS=1', filename + '.c', '-o', filename + '.js'] + CSMITH_CFLAGS + args) assert os.path.exists(filename + '.js') print '(run)' js = shared.run_js(filename + '.js', stderr=PIPE, engine=engine1, check_timeout=True) @@ -91,7 +97,7 @@ while 1: print "EMSCRIPTEN BUG" notes['embug'] += 1 fails += 1 - shutil.copyfile('fuzzcode.c', 'newfail%d.c' % fails) + shutil.copyfile(filename + '.c', 'newfail%d.c' % fails) continue #if not ok: # try: # finally, try with safe heap. if that is triggered, this is nonportable code almost certainly @@ -118,7 +124,7 @@ while 1: print "ODIN VALIDATION BUG" notes['embug'] += 1 fails += 1 - shutil.copyfile('fuzzcode.c', 'newfail%d.c' % fails) + shutil.copyfile(filename + '.c', 'newfail%d.c' % fails) continue js2 = js2.replace('\nwarning: Successfully compiled asm.js code\n', '') diff --git a/tools/shared.py b/tools/shared.py index da6d3e57..776001cd 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1379,6 +1379,17 @@ def execute(cmd, *args, **kw): logging.error('Invoking Process failed: <<< ' + cmd + ' >>>') raise +def check_execute(cmd, *args, **kw): + # TODO: use in more places. execute doesn't actually check that return values + # are nonzero + try: + kw['stderr'] = STDOUT + subprocess.check_output(cmd, *args, **kw) + logging.debug("Successfuly executed %s" % " ".join(cmd)) + except subprocess.CalledProcessError as e: + logging.error("'%s' failed with output:\n%s" % (" ".join(e.cmd), e.output)) + raise + def suffix(name): parts = name.split('.') if len(parts) > 1: |