Fix PR 8199. This patch prepends the build tool dir to LLVM programs

being tested.  This ensures that we test the tools just built and not
some random tools that might happen to be in the user's PATH.  This
makes LLVM testing much more stable and predictable.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@122341 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 50 insertions, 2 deletions

diff --git a/docs/TestingGuide.html b/docs/TestingGuide.html
index 1150ac8f96..4d14fe4a8c 100644
--- a/docs/TestingGuide.html
+++ b/docs/TestingGuide.html
@@ -376,6 +376,11 @@ clang/test directory. </p>
   shell. Consequently the syntax differs from normal shell script syntax in a 
   few ways.  You can specify as many RUN lines as needed.</p>
 
+  <p>lit performs substitution on each RUN line to replace LLVM tool
+  names with the full paths to the executable built for each tool (in
+  $(LLVM_OBJ_ROOT)/$(BuildMode)/bin).  This ensures that lit does not
+  invoke any stray LLVM tools in the user's path during testing.</p>
+
   <p>Each RUN line is executed on its own, distinct from other lines unless
   its last character is <tt>\</tt>. This continuation character causes the RUN
   line to be concatenated with the next one. In this way you can build up long
diff --git a/test/lit.cfg b/test/lit.cfg
index 98f72092e3..7df27029bb 100644
--- a/test/lit.cfg
+++ b/test/lit.cfg
@@ -4,6 +4,7 @@
 
 import os
 import sys
+import re
 
 # name: The name of this test suite.
 config.name = 'LLVM'
@@ -148,6 +149,44 @@ for sub in ['llvmgcc', 'llvmgxx', 'emitir', 'compile_cxx', 'compile_c',
     else:
         config.substitutions.append(('%' + sub, site_exp[sub]))
 
+# For each occurrence of an llvm tool name as its own word, replace it
+# with the full path to the build directory holding that tool.  This
+# ensures that we are testing the tools just built and not some random
+# tools that might happen to be in the user's PATH.  Thus this list
+# includes every tool placed in $(LLVM_OBJ_ROOT)/$(BuildMode)/bin
+# (llvm_tools_dir in lit parlance).
+                # Don't match 'bugpoint-'.
+for pattern in [r"\bbugpoint\b(?!-)",   r"\bclang\b",
+                r"\bedis\b",            r"\bgold\b",
+                r"\bllc\b",             r"\blli\b",
+                r"\bllvm-ar\b",         r"\bllvm-as\b",
+                r"\bllvm-bcanalyzer\b", r"\bllvm-config\b",
+                r"\bllvm-diff\b",       r"\bllvm-dis\b",
+                r"\bllvm-extract\b",    r"\bllvm-ld\b",
+                r"\bllvm-link\b",       r"\bllvm-mc\b",
+                r"\bllvm-nm\b",         r"\bllvm-prof\b",
+                r"\bllvm-ranlib\b",     r"\bllvm-shlib\b",
+                r"\bllvm-stub\b",       r"\bllvm2cpp\b",
+                # Don't match '-llvmc'.
+                r"(?<!-)\bllvmc\b",     r"\blto\b",
+                                        # Don't match '.opt', '-opt'
+                                        # or '^opt'.
+                r"\bmacho-dump\b",      r"(?<!\.|-|\^)\bopt\b",
+                r"\btblgen\b",          r"\bFileCheck\b",
+                r"\bFileUpdate\b",      r"\bc-index-test\b",
+                r"\bfpcmp\b",           r"\bllvm-PerfectShuffle\b",
+                # Handle these specially as they are strings searched
+                # for during testing.
+                r"\| \bcount\b",         r"\| \bnot\b"]:
+    # Extract the tool name from the pattern.  This relies on the tool
+    # name being surrounded by \b word match operators.  If the
+    # pattern starts with "| ", include it in the string to be
+    # substituted.
+    substitution = re.sub(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
+                          r"\2" + llvm_tools_dir + "/" + r"\4",
+                          pattern)
+    config.substitutions.append((pattern, substitution))
+
 excludes = []
 
 # Provide target_triple for use in XFAIL and XTARGET.
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index a2f97cdb5e..bf6eed8691 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -8,6 +8,8 @@ import Util
 import platform
 import tempfile
 
+import re
+
 class InternalShellError(Exception):
     def __init__(self, command, message):
         self.command = command
@@ -444,11 +446,13 @@ def parseIntegratedTestScript(test, normalize_slashes=False):
             if ln[ln.index('END.'):].strip() == 'END.':
                 break
 
-    # Apply substitutions to the script.
+    # Apply substitutions to the script.  Allow full regular
+    # expression syntax.  Replace each matching occurrence of regular
+    # expression pattern a with substitution b in line ln.
     def processLine(ln):
         # Apply substitutions
         for a,b in substitutions:
-            ln = ln.replace(a,b)
+            ln = re.sub(a, b, ln)
 
         # Strip the trailing newline and any extra whitespace.
         return ln.strip()