diff options
| -rw-r--r-- | test/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | test/Makefile | 1 | ||||
| -rw-r--r-- | test/lit.cfg | 12 | ||||
| -rwxr-xr-x | utils/test/MultiTestRunner.py | 223 | ||||
| -rwxr-xr-x | utils/test/TestRunner.py | 102 | ||||
| -rw-r--r-- | utils/test/TestingConfig.py | 21 | ||||
| -rw-r--r-- | utils/test/Util.py | 70 |
7 files changed, 243 insertions, 188 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 794e80b0b2..58e73682f6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,6 +33,7 @@ if(PYTHONINTERP_FOUND) add_custom_target(clang-test-${testdir} ${PYTHON_EXECUTABLE} ${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py + "--root=${LLVM_SOURCE_DIR}/tools/clang/test" "--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}" "--path=${LLVM_SOURCE_DIR}/test/Scripts" -s ${CLANG_TEST_EXTRA_ARGS} @@ -48,6 +49,7 @@ if(PYTHONINTERP_FOUND) add_custom_target(clang-test ${PYTHON_EXECUTABLE} ${LLVM_SOURCE_DIR}/tools/clang/utils/test/MultiTestRunner.py + "--root=${LLVM_SOURCE_DIR}/tools/clang/test" "--path=${LLVM_TOOLS_PATH}/${CMAKE_CFG_INTDIR}" "--path=${LLVM_SOURCE_DIR}/test/Scripts" -s ${CLANG_TEST_EXTRA_ARGS} diff --git a/test/Makefile b/test/Makefile index 97c9d27404..13cbfe6b3c 100644 --- a/test/Makefile +++ b/test/Makefile @@ -21,6 +21,7 @@ endif all:: @ echo '--- Running clang tests for $(TARGET_TRIPLE) ---' @ $(PROJ_SRC_DIR)/../utils/test/MultiTestRunner.py \ + --root $(PROJ_SRC_DIR) \ --path $(ToolDir) \ --path $(LLVM_SRC_ROOT)/test/Scripts \ $(TESTARGS) $(TESTDIRS) $(VGARG) diff --git a/test/lit.cfg b/test/lit.cfg new file mode 100644 index 0000000000..89ca044f70 --- /dev/null +++ b/test/lit.cfg @@ -0,0 +1,12 @@ +# -*- Python -*- + +# Configuration file for the 'lit' test runner. + +# suffixes: A list of file extensions to treat as test files. +suffixes = ['.c', '.cpp'] + +# environment: The base environment to use when running test commands. +# +# The 'PATH' and 'SYSTEMROOT' variables will be set automatically from the lit +# command line variables. +environment = {} diff --git a/utils/test/MultiTestRunner.py b/utils/test/MultiTestRunner.py index aaceb7f657..9980e651a3 100755 --- a/utils/test/MultiTestRunner.py +++ b/utils/test/MultiTestRunner.py @@ -15,33 +15,40 @@ TODO in a separate category). """ -# TOD import os, sys, re, random, time import threading +from Queue import Queue + import ProgressBar import TestRunner +import Util + +from TestingConfig import TestingConfig from TestRunner import TestStatus -from Queue import Queue -kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll']) +kConfigName = 'lit.cfg' -def getTests(inputs): +def getTests(cfg, inputs): for path in inputs: if not os.path.exists(path): - print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,) + Util.warning('Invalid test %r' % path) continue if not os.path.isdir(path): yield path + foundOne = False for dirpath,dirnames,filenames in os.walk(path): # FIXME: This doesn't belong here if 'Output' in dirnames: dirnames.remove('Output') for f in filenames: base,ext = os.path.splitext(f) - if ext in kTestFileExtensions: + if ext in cfg.suffixes: yield os.path.join(dirpath,f) + foundOne = True + if not foundOne: + Util.warning('No tests in input directory %r' % path) class TestingProgressDisplay: def __init__(self, opts, numTests, progressBar=None): @@ -109,7 +116,8 @@ class TestResult: return self.code in (TestStatus.Fail,TestStatus.XPass) class TestProvider: - def __init__(self, opts, tests, display): + def __init__(self, config, opts, tests, display): + self.config = config self.opts = opts self.tests = tests self.index = 0 @@ -156,14 +164,12 @@ class Tester(threading.Thread): elapsed = None try: opts = self.provider.opts - if opts.debugDoNotTest: - code = None - else: - startTime = time.time() - code, output = TestRunner.runOneTest(path, base, - opts.clang, opts.clangcc, - opts.useValgrind) - elapsed = time.time() - startTime + startTime = time.time() + code, output = TestRunner.runOneTest(self.provider.config, + path, base, + opts.clang, opts.clangcc, + opts.useValgrind) + elapsed = time.time() - startTime except KeyboardInterrupt: # This is a sad hack. Unfortunately subprocess goes # bonkers with ctrl-c and we start forking merrily. @@ -172,102 +178,127 @@ class Tester(threading.Thread): self.provider.setResult(index, TestResult(path, code, output, elapsed)) -def detectCPUs(): - """ - Detects the number of CPUs on a system. Cribbed from pp. - """ - # Linux, Unix and MacOS: - if hasattr(os, "sysconf"): - if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): - # Linux & Unix: - ncpus = os.sysconf("SC_NPROCESSORS_ONLN") - if isinstance(ncpus, int) and ncpus > 0: - return ncpus - else: # OSX: - return int(os.popen2("sysctl -n hw.ncpu")[1].read()) - # Windows: - if os.environ.has_key("NUMBER_OF_PROCESSORS"): - ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]); - if ncpus > 0: - return ncpus - return 1 # Default +def findConfigPath(root): + prev = None + while root != prev: + cfg = os.path.join(root, kConfigName) + if os.path.exists(cfg): + return cfg + + prev,root = root,os.path.dirname(root) + + raise ValueError,"Unable to find config file %r" % kConfigName def main(): global options - from optparse import OptionParser - parser = OptionParser("usage: %prog [options] {inputs}") - parser.add_option("-j", "--threads", dest="numThreads", - help="Number of testing threads", - type=int, action="store", - default=detectCPUs()) - parser.add_option("", "--clang", dest="clang", - help="Program to use as \"clang\"", - action="store", default=None) - parser.add_option("", "--clang-cc", dest="clangcc", - help="Program to use as \"clang-cc\"", - action="store", default=None) - parser.add_option("", "--vg", dest="useValgrind", - help="Run tests under valgrind", - action="store_true", default=False) - parser.add_option("-v", "--verbose", dest="showOutput", - help="Show all test output", - action="store_true", default=False) - parser.add_option("-q", "--quiet", dest="quiet", - help="Suppress no error output", - action="store_true", default=False) - parser.add_option("-s", "--succinct", dest="succinct", - help="Reduce amount of output", - action="store_true", default=False) - parser.add_option("", "--max-tests", dest="maxTests", - help="Maximum number of tests to run", - action="store", type=int, default=None) - parser.add_option("", "--max-time", dest="maxTime", - help="Maximum time to spend testing (in seconds)", - action="store", type=float, default=None) - parser.add_option("", "--shuffle", dest="shuffle", - help="Run tests in random order", - action="store_true", default=False) - parser.add_option("", "--seed", dest="seed", - help="Seed for random number generator (default: random)", + from optparse import OptionParser, OptionGroup + parser = OptionParser("usage: %prog [options] {file-or-path}") + + parser.add_option("", "--root", dest="root", + help="Path to root test directory", action="store", default=None) - parser.add_option("", "--no-progress-bar", dest="useProgressBar", - help="Do not use curses based progress bar", - action="store_false", default=True) - parser.add_option("", "--debug-do-not-test", dest="debugDoNotTest", - help="DEBUG: Skip running actual test script", - action="store_true", default=False) - parser.add_option("", "--time-tests", dest="timeTests", - help="Track elapsed wall time for each test", - action="store_true", default=False) - parser.add_option("", "--path", dest="path", - help="Additional paths to add to testing environment", - action="append", type=str, default=[]) + parser.add_option("", "--config", dest="config", + help="Testing configuration file [default='%default']", + action="store", default=kConfigName) + + group = OptionGroup(parser, "Output Format") + # FIXME: I find these names very confusing, although I like the + # functionality. + group.add_option("-q", "--quiet", dest="quiet", + help="Suppress no error output", + action="store_true", default=False) + group.add_option("-s", "--succinct", dest="succinct", + help="Reduce amount of output", + action="store_true", default=False) + group.add_option("-v", "--verbose", dest="showOutput", + help="Show all test output", + action="store_true", default=False) + group.add_option("", "--no-progress-bar", dest="useProgressBar", + help="Do not use curses based progress bar", + action="store_false", default=True) + parser.add_option_group(group) + + group = OptionGroup(parser, "Test Execution") + group.add_option("-j", "--threads", dest="numThreads", + help="Number of testing threads", + type=int, action="store", + default=None) + group.add_option("", "--clang", dest="clang", + help="Program to use as \"clang\"", + action="store", default=None) + group.add_option("", "--clang-cc", dest="clangcc", + help="Program to use as \"clang-cc\"", + action="store", default=None) + group.add_option("", "--path", dest="path", + help="Additional paths to add to testing environment", + action="append", type=str, default=[]) + group.add_option("", "--vg", dest="useValgrind", + help="Run tests under valgrind", + action="store_true", default=False) + group.add_option("", "--time-tests", dest="timeTests", + help="Track elapsed wall time for each test", + action="store_true", default=False) + parser.add_option_group(group) + + group = OptionGroup(parser, "Test Selection") + group.add_option("", "--max-tests", dest="maxTests", + help="Maximum number of tests to run", + action="store", type=int, default=None) + group.add_option("", "--max-time", dest="maxTime", + help="Maximum time to spend testing (in seconds)", + action="store", type=float, default=None) + group.add_option("", "--shuffle", dest="shuffle", + help="Run tests in random order", + action="store_true", default=False) + parser.add_option_group(group) (opts, args) = parser.parse_args() - + if not args: parser.error('No inputs specified') - # FIXME: Move into configuration object. - TestRunner.kChildEnv["PATH"] = os.pathsep.join(opts.path + - [TestRunner.kChildEnv['PATH']]) + if opts.numThreads is None: + opts.numThreads = Util.detectCPUs() + + inputs = args + + # Resolve root if not given, either infer it from the config file if given, + # otherwise from the inputs. + if not opts.root: + if opts.config: + opts.root = os.path.dirname(opts.config) + else: + opts.root = os.path.commonprefix(inputs) + + # Find the config file, if not specified. + if not opts.config: + try: + opts.config = findConfigPath(opts.root) + except ValueError,e: + parser.error(e.args[0]) + + cfg = TestingConfig.frompath(opts.config) + + # Update the configuration based on the command line arguments. + for name in ('PATH','SYSTEMROOT'): + if name in cfg.environment: + parser.error("'%s' should not be set in configuration!" % name) + + cfg.root = opts.root + cfg.environment['PATH'] = os.pathsep.join(opts.path + + [os.environ.get('PATH','')]) + cfg.environment['SYSTEMROOT'] = os.environ.get('SYSTEMROOT','') if opts.clang is None: - opts.clang = TestRunner.inferClang() + opts.clang = TestRunner.inferClang(cfg) if opts.clangcc is None: - opts.clangcc = TestRunner.inferClangCC(opts.clang) + opts.clangcc = TestRunner.inferClangCC(cfg, opts.clang) # FIXME: It could be worth loading these in parallel with testing. - allTests = list(getTests(args)) + allTests = list(getTests(cfg, args)) allTests.sort() tests = allTests - if opts.seed is not None: - try: - seed = int(opts.seed) - except: - parser.error('--seed argument should be an integer') - random.seed(seed) if opts.shuffle: random.shuffle(tests) if opts.maxTests is not None: @@ -292,7 +323,7 @@ def main(): print header display = TestingProgressDisplay(opts, len(tests), progressBar) - provider = TestProvider(opts, tests, display) + provider = TestProvider(cfg, opts, tests, display) testers = [Tester(provider) for i in range(opts.numThreads)] startTime = time.time() @@ -309,7 +340,7 @@ def main(): if not opts.quiet: print 'Testing Time: %.2fs'%(time.time() - startTime) - # List test results organized organized by kind. + # List test results organized by kind. byCode = {} for t in provider.results: if t: diff --git a/utils/test/TestRunner.py b/utils/test/TestRunner.py index d3ffef2b75..fb3d172fb6 100755 --- a/utils/test/TestRunner.py +++ b/utils/test/TestRunner.py @@ -23,10 +23,7 @@ import signal import subprocess import sys -# Increase determinism by explicitly choosing the environment. -kChildEnv = {} -for var in ('PATH', 'SYSTEMROOT'): - kChildEnv[var] = os.environ.get(var, '') +import Util kSystemName = platform.system() @@ -42,22 +39,7 @@ class TestStatus: def getName(code): return TestStatus.kNames[code] -def mkdir_p(path): - if not path: - pass - elif os.path.exists(path): - pass - else: - parent = os.path.dirname(path) - if parent != path: - mkdir_p(parent) - try: - os.mkdir(path) - except OSError,e: - if e.errno != errno.EEXIST: - raise - -def executeScript(script, commands, cwd, useValgrind): +def executeScript(cfg, script, commands, cwd, useValgrind): # Write script file f = open(script,'w') if kSystemName == 'Windows': @@ -82,7 +64,7 @@ def executeScript(script, commands, cwd, useValgrind): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=kChildEnv) + env=cfg.environment) out,err = p.communicate() exitCode = p.wait() @@ -93,14 +75,14 @@ def executeScript(script, commands, cwd, useValgrind): return out, err, exitCode import StringIO -def runOneTest(testPath, tmpBase, clang, clangcc, useValgrind): +def runOneTest(cfg, testPath, tmpBase, clang, clangcc, useValgrind): # Make paths absolute. tmpBase = os.path.abspath(tmpBase) testPath = os.path.abspath(testPath) # Create the output directory if it does not already exist. - mkdir_p(os.path.dirname(tmpBase)) + Util.mkdir_p(os.path.dirname(tmpBase)) script = tmpBase + '.script' if kSystemName == 'Windows': script += '.bat' @@ -154,7 +136,7 @@ def runOneTest(testPath, tmpBase, clang, clangcc, useValgrind): # Strip off '&&' scriptLines[i] = ln[:-2] - out, err, exitCode = executeScript(script, scriptLines, + out, err, exitCode = executeScript(cfg, script, scriptLines, os.path.dirname(testPath), useValgrind) if xfailLines: @@ -188,31 +170,7 @@ def capture(args): out,_ = p.communicate() return out -def which(command): - # FIXME: Take configuration object. - - # Check for absolute match first. - if os.path.exists(command): - return command - - # Would be nice if Python had a lib function for this. - paths = kChildEnv['PATH'] - if not paths: - paths = os.defpath - - # Get suffixes to search. - pathext = os.environ.get('PATHEXT', '').split(os.pathsep) - - # Search the paths... - for path in paths.split(os.pathsep): - for ext in pathext: - p = os.path.join(path, command + ext) - if os.path.exists(p): - return p - - return None - -def inferClang(): +def inferClang(cfg): # Determine which clang to use. clang = os.getenv('CLANG') @@ -222,7 +180,7 @@ def inferClang(): return clang # Otherwise look in the path. - clang = which('clang') + clang = Util.which('clang', cfg.environment['PATH']) if not clang: print >>sys.stderr, "error: couldn't find 'clang' program, try setting CLANG in your environment" @@ -230,7 +188,7 @@ def inferClang(): return clang -def inferClangCC(clang): +def inferClangCC(cfg, clang): clangcc = os.getenv('CLANGCC') # If the user set clang in the environment, definitely use that and don't @@ -244,7 +202,7 @@ def inferClangCC(clang): clangccName = clang[:-4] + '-cc.exe' else: clangccName = clang + '-cc' - clangcc = which(clangccName) + clangcc = Util.which(clangccName, cfg.environment['PATH']) if not clangcc: # Otherwise ask clang. res = capture([clang, '-print-prog-name=clang-cc']) @@ -267,43 +225,3 @@ def getTestOutputBase(dir, testpath): return os.path.join(dir, os.path.basename(os.path.dirname(testpath)), os.path.basename(testpath)) - -def main(): - global options - from optparse import OptionParser - parser = OptionParser("usage: %prog [options] {tests}") - parser.add_option("", "--clang", dest="clang", - help="Program to use as \"clang\"", - action="store", default=None) - parser.add_option("", "--clang-cc", dest="clangcc", - help="Program to use as \"clang-cc\"", - action="store", default=None) - parser.add_option("", "--vg", dest="useValgrind", - help="Run tests under valgrind", - action="store_true", default=False) - (opts, args) = parser.parse_args() - - if not args: - parser.error('No tests specified') - - if opts.clang is None: - opts.clang = inferClang() - if opts.clangcc is None: - opts.clangcc = inferClangCC(opts.clang) - - for path in args: - base = getTestOutputBase('Output', path) + '.out' - - status,output = runOneTest(path, base, opts.clang, opts.clangcc, - opts.useValgrind) - print '%s: %s' % (TestStatus.getName(status).upper(), path) - if status == TestStatus.Fail or status == TestStatus.XPass: - print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20) - sys.stdout.write(output) - print "*" * 20 - sys.exit(1) - - sys.exit(0) - -if __name__=='__main__': - main() diff --git a/utils/test/TestingConfig.py b/utils/test/TestingConfig.py new file mode 100644 index 0000000000..6b33589da5 --- /dev/null +++ b/utils/test/TestingConfig.py @@ -0,0 +1,21 @@ +class TestingConfig: + """" + TestingConfig - Information on a how to run a group of tests. + """ + + @staticmethod + def frompath(path): + data = {} + f = open(path) + exec f in {},data + + return TestingConfig(suffixes = data.get('suffixes', []), + environment = data.get('environment', {})) + + def __init__(self, suffixes, environment): + self.root = None + self.suffixes = set(suffixes) + self.environment = dict(environment) + + + diff --git a/utils/test/Util.py b/utils/test/Util.py new file mode 100644 index 0000000000..bacdab8a58 --- /dev/null +++ b/utils/test/Util.py @@ -0,0 +1,70 @@ +import os, sys + +def warning(msg): + print >>sys.stderr, '%s: warning: %s' % (sys.argv[0], msg) + +def detectCPUs(): + """ + Detects the number of CPUs on a system. Cribbed from pp. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + else: # OSX: + return int(os.popen2("sysctl -n hw.ncpu")[1].read()) + # Windows: + if os.environ.has_key("NUMBER_OF_PROCESSORS"): + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]); + if ncpus > 0: + return ncpus + return 1 # Default + +def mkdir_p(path): + """mkdir_p(path) - Make the "path" directory, if it does not exist; this + will also make directories for any missing parent directories.""" + + if not path or os.path.exists(path): + return + + parent = os.path.dirname(path) + if parent != path: + mkdir_p(parent) + + try: + os.mkdir(path) + except OSError,e: + # Ignore EEXIST, which may occur during a race condition. + if e.errno != errno.EEXIST: + raise + +def which(command, paths = None): + """which(command, [paths]) - Look up the given command in the paths string (or + the PATH environment variable, if unspecified).""" + + if paths is None: + paths = os.environ.get('PATH','') + + # Check for absolute match first. + if os.path.exists(command): + return command + + # Would be nice if Python had a lib function for this. + if not paths: + paths = os.defpath + + # Get suffixes to search. + pathext = os.environ.get('PATHEXT', '').split(os.pathsep) + + # Search the paths... + for path in paths.split(os.pathsep): + for ext in pathext: + p = os.path.join(path, command + ext) + if os.path.exists(p): + return p + + return None + |