[msan] Fix handling of equality comparison of pointer vectors.

Also improve test coveration of the handling of relational comparisons.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@172539 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 79 insertions, 6 deletions

diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 76da970682..db0de4d797 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -574,7 +574,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
       return IT;
     if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
-      uint32_t EltSize = MS.TD->getTypeStoreSizeInBits(VT->getElementType());
+      uint32_t EltSize = MS.TD->getTypeSizeInBits(VT->getElementType());
       return VectorType::get(IntegerType::get(*MS.C, EltSize),
                              VT->getNumElements());
     }
@@ -586,7 +586,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
       return Res;
     }
-    uint32_t TypeSize = MS.TD->getTypeStoreSizeInBits(OrigTy);
+    uint32_t TypeSize = MS.TD->getTypeSizeInBits(OrigTy);
     return IntegerType::get(*MS.C, TypeSize);
   }
 
@@ -1127,10 +1127,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *B = I.getOperand(1);
     Value *Sa = getShadow(A);
     Value *Sb = getShadow(B);
-    if (A->getType()->isPointerTy())
-      A = IRB.CreatePointerCast(A, MS.IntptrTy);
-    if (B->getType()->isPointerTy())
-      B = IRB.CreatePointerCast(B, MS.IntptrTy);
+
+    // Get rid of pointers and vectors of pointers.
+    // For ints (and vectors of ints), types of A and Sa match,
+    // and this is a no-op.
+    A = IRB.CreatePointerCast(A, Sa->getType());
+    B = IRB.CreatePointerCast(B, Sb->getType());
+
     // A == B  <==>  (C = A^B) == 0
     // A != B  <==>  (C = A^B) != 0
     // Sc = Sa | Sb
diff --git a/test/Instrumentation/MemorySanitizer/msan_basic.ll b/test/Instrumentation/MemorySanitizer/msan_basic.ll
index cd90f8836a..d62ebfe535 100644
--- a/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -362,6 +362,76 @@ define zeroext i1 @ICmpSLE(i32 %x) nounwind uwtable readnone {
 ; CHECK: ret i1
 
 
+; Check that we propagate shadow for x<0, x>=0, etc (i.e. sign bit tests)
+; of the vector arguments.
+
+define <2 x i1> @ICmpSLT_vector(<2 x i32*> %x) nounwind uwtable readnone {
+  %1 = icmp slt <2 x i32*> %x, zeroinitializer
+  ret <2 x i1> %1
+}
+
+; CHECK: @ICmpSLT_vector
+; CHECK: icmp slt <2 x i64>
+; CHECK-NOT: call void @__msan_warning
+; CHECK: icmp slt <2 x i32*>
+; CHECK-NOT: call void @__msan_warning
+; CHECK: ret <2 x i1>
+
+
+; Check that we propagate shadow for x == y comparison.
+; This is a bit complex. See the comment in handleEqualityComparison.
+
+define i1 @ICmpEQ(i32 %x, i32 %y) nounwind uwtable readnone {
+  %1 = icmp eq i32 %x, %y
+  ret i1 %1
+}
+
+; CHECK: @ICmpEQ
+; CHECK: xor i32 %x, %y
+; CHECK: or i32
+; CHECK: xor i32
+; CHECK: and i32
+; CHECK: icmp eq i32
+; CHECK: icmp ne i32
+; CHECK: and i1
+; CHECK: icmp eq i32 %x, %y
+; CHECK: ret i1
+
+define <2 x i1> @ICmpEQ_vector(<2 x i32> %x, <2 x i32> %y) nounwind uwtable readnone {
+  %1 = icmp eq <2 x i32> %x, %y
+  ret <2 x i1> %1
+}
+
+; CHECK: @ICmpEQ_vector
+; CHECK: xor <2 x i32> %x, %y
+; CHECK: or <2 x i32>
+; CHECK: xor <2 x i32>
+; CHECK: and <2 x i32>
+; CHECK: icmp eq <2 x i32>
+; CHECK: icmp ne <2 x i32>
+; CHECK: and <2 x i1>
+; CHECK: icmp eq <2 x i32> %x, %y
+; CHECK: ret <2 x i1>
+
+define <2 x i1> @ICmpEQ_pointer_vector(<2 x i32*> %x, <2 x i32*> %y) nounwind uwtable readnone {
+  %1 = icmp eq <2 x i32*> %x, %y
+  ret <2 x i1> %1
+}
+
+; CHECK: @ICmpEQ_pointer_vector
+; CHECK: ptrtoint <2 x i32*> %x to <2 x i64>
+; CHECK: ptrtoint <2 x i32*> %y to <2 x i64>
+; CHECK: xor <2 x i64>
+; CHECK: or <2 x i64>
+; CHECK: xor <2 x i64>
+; CHECK: and <2 x i64>
+; CHECK: icmp eq <2 x i64>
+; CHECK: icmp ne <2 x i64>
+; CHECK: and <2 x i1>
+; CHECK: icmp eq <2 x i32*> %x, %y
+; CHECK: ret <2 x i1>
+
+
 ; Check that loads of shadow have the same aligment as the original loads.
 ; Check that loads of origin have the aligment of max(4, original alignment).