3 files changed, 28 insertions, 11 deletions
diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a9ec243bc0..024dd337ac 100644
--- a/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -83,7 +83,7 @@ struct SLPVectorizer : public FunctionPass {
 
       // Use the bollom up slp vectorizer to construct chains that start with
       // he store instructions.
-      BoUpSLP R(BB, SE, DL, TTI, AA);
+      BoUpSLP R(BB, SE, DL, TTI, AA, LI->getLoopFor(BB));
 
       // Vectorize trees that end at reductions.
       BBChanged |= vectorizeReductions(BB, R);
diff --git a/lib/Transforms/Vectorize/VecUtils.cpp b/lib/Transforms/Vectorize/VecUtils.cpp
index c08bed7170..244af394b1 100644
--- a/lib/Transforms/Vectorize/VecUtils.cpp
+++ b/lib/Transforms/Vectorize/VecUtils.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/Verifier.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
@@ -44,8 +45,8 @@ static const unsigned RecursionMaxDepth = 6;
 namespace llvm {
 
 BoUpSLP::BoUpSLP(BasicBlock *Bb, ScalarEvolution *S, DataLayout *Dl,
-                             TargetTransformInfo *Tti, AliasAnalysis *Aa) :
-                             BB(Bb), SE(S), DL(Dl), TTI(Tti), AA(Aa) {
+                 TargetTransformInfo *Tti, AliasAnalysis *Aa, Loop *Lp) :
+  BB(Bb), SE(S), DL(Dl), TTI(Tti), AA(Aa), L(Lp)  {
   numberInstructions();
 }
 
@@ -121,7 +122,7 @@ bool BoUpSLP::vectorizeStoreChain(ValueList &Chain, int CostThreshold) {
     if (Cost < CostThreshold) {
       DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
       vectorizeTree(Operands, VF);
-      i += VF;
+      i += VF - 1;
       Changed = true;
     }
   }
@@ -381,13 +382,15 @@ int BoUpSLP::getTreeCost_rec(ValueList &VL, unsigned Depth) {
   // Check if all of the operands are constants.
   bool AllConst = true;
   bool AllSameScalar = true;
+  bool MustScalarizeFlag = false;
   for (unsigned i = 0, e = VL.size(); i < e; ++i) {
     AllConst &= isa<Constant>(VL[i]);
     AllSameScalar &= (VL[0] == VL[i]);
     // Must have a single use.
     Instruction *I = dyn_cast<Instruction>(VL[i]);
-    // This instruction is outside the basic block or if it is a known hazard.
-    if (MustScalarize.count(VL[i]) || (I && I->getParent() != BB))
+    MustScalarizeFlag |= MustScalarize.count(VL[i]);
+    // This instruction is outside the basic block.
+    if (I && I->getParent() != BB)
       return getScalarizationCost(VecTy);
   }
 
@@ -395,11 +398,23 @@ int BoUpSLP::getTreeCost_rec(ValueList &VL, unsigned Depth) {
   if (AllConst) return 0;
 
   // If all of the operands are identical we can broadcast them.
-  if (AllSameScalar)
+  Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
+  if (AllSameScalar) {
+    // If we are in a loop, and this is not an instruction (e.g. constant or
+    // argument) or the instruction is defined outside the loop then assume
+    // that the cost is zero.
+    if (L && (!VL0 || !L->contains(VL0)))
+      return 0;
+
+    // We need to broadcast the scalar.
     return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
+  }
+
+  // If this is not a constant, or a scalar from outside the loop then we
+  // need to scalarize it.
+  if (MustScalarizeFlag)
+    return getScalarizationCost(VecTy);
 
-  // Scalarize unknown structures.
-  Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
   if (!VL0) return getScalarizationCost(VecTy);
   assert(VL0->getParent() == BB && "Wrong BB");
 
diff --git a/lib/Transforms/Vectorize/VecUtils.h b/lib/Transforms/Vectorize/VecUtils.h
index 3f6866407a..c756bd3968 100644
--- a/lib/Transforms/Vectorize/VecUtils.h
+++ b/lib/Transforms/Vectorize/VecUtils.h
@@ -27,6 +27,7 @@ class BasicBlock; class Instruction; class Type;
 class VectorType; class StoreInst; class Value;
 class ScalarEvolution; class DataLayout;
 class TargetTransformInfo; class AliasAnalysis;
+class Loop;
 
 /// Bottom Up SLP vectorization utility class.
 struct BoUpSLP  {
@@ -37,7 +38,7 @@ struct BoUpSLP  {
 
   // \brief C'tor.
   BoUpSLP(BasicBlock *Bb, ScalarEvolution *Se, DataLayout *Dl,
-         TargetTransformInfo *Tti, AliasAnalysis *Aa);
+         TargetTransformInfo *Tti, AliasAnalysis *Aa, Loop *Lp);
 
   /// \brief Take the pointer operand from the Load/Store instruction.
   /// \returns NULL if this is not a valid Load/Store instruction.
@@ -112,7 +113,7 @@ private:
 
   /// \returns a vector from a collection of scalars in \p VL.
   Value *Scalarize(ValueList &VL, VectorType *Ty);
-  
+
 private:
   /// Maps instructions to numbers and back.
   SmallDenseMap<Value*, int> InstrIdx;
@@ -155,6 +156,7 @@ private:
   DataLayout *DL;
   TargetTransformInfo *TTI;
   AliasAnalysis *AA;
+  Loop *L;
 };
 
 } // end of namespace