diff options
25 files changed, 2157 insertions, 1610 deletions
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index a350d5fd93..9ae92f388e 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -353,13 +353,6 @@ public: SDValue getConvertRndSat(MVT VT, DebugLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code); - - /// getVectorShuffle - Return an ISD::VECTOR_SHUFFLE node. The number of - /// elements in VT, which must be a vector type, must match the number of - /// mask elements NumElts. A negative integer mask element is treated as - /// undefined. - SDValue getVectorShuffle(MVT VT, DebugLoc dl, SDValue N1, SDValue N2, - const int *MaskElts); /// getZeroExtendInReg - Return the expression required to zero extend the Op /// value assuming it was the smaller SrcTy value. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index a850b3fcb5..aaa39674d7 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1703,32 +1703,6 @@ public: } }; -class ShuffleVectorSDNode : public SDNode { - SDUse Ops[2]; - int *Mask; -protected: - friend class SelectionDAG; - ShuffleVectorSDNode(MVT VT, DebugLoc dl, SDValue N1, SDValue N2, int *M) - : SDNode(ISD::VECTOR_SHUFFLE, dl, getSDVTList(VT)), Mask(M) { - InitOperands(Ops, N1, N2); - } -public: - - const int * getMask() const { return Mask; } - - bool isSplat() { return isSplatMask(Mask, getValueType(0)); } - int getSplatIndex() { - assert(isSplat() && "Cannot get splat index for non-splat!"); - return Mask[0]; - } - static bool isSplatMask(const int *Mask, MVT VT); - - static bool classof(const ShuffleVectorSDNode *) { return true; } - static bool classof(const SDNode *N) { - return N->getOpcode() == ISD::VECTOR_SHUFFLE; - } -}; - class ConstantSDNode : public SDNode { const ConstantInt *Value; friend class SelectionDAG; @@ -2110,7 +2084,7 @@ public: return N->getOpcode() == ISD::CONDCODE; } }; - + /// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the /// future and most targets don't support it. class CvtRndSatSDNode : public SDNode { diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index a5f064dcde..477505e2f7 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -328,7 +328,7 @@ public: /// support *some* VECTOR_SHUFFLE operations, those with specific masks. /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values /// are assumed to be legal. - virtual bool isShuffleMaskLegal(const int *Mask, MVT VT) const { + virtual bool isShuffleMaskLegal(SDValue Mask, MVT VT) const { return true; } @@ -336,7 +336,9 @@ public: /// used by Targets can use this to indicate if there is a suitable /// VECTOR_SHUFFLE that can be used to replace a VAND with a constant /// pool entry. - virtual bool isVectorClearMaskLegal(const int *Mask, MVT VT) const { + virtual bool isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, + MVT EVT, + SelectionDAG &DAG) const { return false; } diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td index 2cd29676db..7f39bb2f83 100644 --- a/include/llvm/Target/TargetSelectionDAG.td +++ b/include/llvm/Target/TargetSelectionDAG.td @@ -51,6 +51,15 @@ class SDTCisOpSmallerThanOp<int SmallOp, int BigOp> : SDTypeConstraint<SmallOp>{ int BigOperandNum = BigOp; } +/// SDTCisIntVectorOfSameSize - This indicates that ThisOp and OtherOp are +/// vector types, and that ThisOp is the result of +/// MVT::getIntVectorWithNumElements with the number of elements +/// that ThisOp has. +class SDTCisIntVectorOfSameSize<int ThisOp, int OtherOp> + : SDTypeConstraint<ThisOp> { + int OtherOpNum = OtherOp; +} + /// SDTCisEltOfVec - This indicates that ThisOp is a scalar type of the same /// type as the element type of OtherOp, which is a vector type. class SDTCisEltOfVec<int ThisOp, int OtherOp> @@ -166,8 +175,8 @@ def SDTIStore : SDTypeProfile<1, 3, [ // indexed store SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3> ]>; -def SDTVecShuffle : SDTypeProfile<1, 2, [ - SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2> +def SDTVecShuffle : SDTypeProfile<1, 3, [ + SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisIntVectorOfSameSize<3, 0> ]>; def SDTVecExtract : SDTypeProfile<1, 2, [ // vector extract SDTCisEltOfVec<0, 1>, SDTCisPtrTy<2> diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 40807d8909..72b3e3627e 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -5098,21 +5098,7 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), InVec.getValueType(), &Ops[0], Ops.size()); } - // If the invec is an UNDEF and if EltNo is a constant, create a new - // BUILD_VECTOR with undef elements and the inserted element. - if (!LegalOperations && InVec.getOpcode() == ISD::UNDEF && - isa<ConstantSDNode>(EltNo)) { - MVT VT = InVec.getValueType(); - MVT EVT = VT.getVectorElementType(); - unsigned NElts = VT.getVectorNumElements(); - SmallVector<SDValue, 8> Ops(NElts, DAG.getUNDEF(EVT)); - unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); - if (Elt < Ops.size()) - Ops[Elt] = InVal; - return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - InVec.getValueType(), &Ops[0], Ops.size()); - } return SDValue(); } @@ -5174,8 +5160,9 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { // to examine the mask. if (BCNumEltsChanged) return SDValue(); - int Idx = cast<ShuffleVectorSDNode>(InVec)->getMask()[Elt]; - int NumElems = InVec.getValueType().getVectorNumElements(); + unsigned Idx = cast<ConstantSDNode>(InVec.getOperand(2). + getOperand(Elt))->getZExtValue(); + unsigned NumElems = InVec.getOperand(2).getNumOperands(); InVec = (Idx < NumElems) ? InVec.getOperand(0) : InVec.getOperand(1); if (InVec.getOpcode() == ISD::BIT_CONVERT) InVec = InVec.getOperand(0); @@ -5222,6 +5209,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { unsigned NumInScalars = N->getNumOperands(); MVT VT = N->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); MVT EltType = VT.getVectorElementType(); // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT @@ -5264,36 +5252,56 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { } // If everything is good, we can make a shuffle operation. + MVT IndexVT = MVT::i32; if (VecIn1.getNode()) { - SmallVector<int, 8> Mask; + SmallVector<SDValue, 8> BuildVecIndices; for (unsigned i = 0; i != NumInScalars; ++i) { if (N->getOperand(i).getOpcode() == ISD::UNDEF) { - Mask.push_back(-1); + BuildVecIndices.push_back(DAG.getUNDEF(IndexVT)); continue; } - // If extracting from the first vector, just use the index directly. SDValue Extract = N->getOperand(i); + + // If extracting from the first vector, just use the index directly. SDValue ExtVal = Extract.getOperand(1); if (Extract.getOperand(0) == VecIn1) { - Mask.push_back(cast<ConstantSDNode>(ExtVal)->getZExtValue()); + if (ExtVal.getValueType() == IndexVT) + BuildVecIndices.push_back(ExtVal); + else { + unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue(); + BuildVecIndices.push_back(DAG.getConstant(Idx, IndexVT)); + } continue; } // Otherwise, use InIdx + VecSize unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue(); - Mask.push_back(Idx+NumInScalars); + BuildVecIndices.push_back(DAG.getConstant(Idx+NumInScalars, IndexVT)); } // Add count and size info. - if (!TLI.isTypeLegal(VT) && LegalTypes) + MVT BuildVecVT = MVT::getVectorVT(IndexVT, NumElts); + if (!TLI.isTypeLegal(BuildVecVT) && LegalTypes) return SDValue(); // Return the new VECTOR_SHUFFLE node. - SDValue Ops[2]; + SDValue Ops[5]; Ops[0] = VecIn1; - Ops[1] = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); - return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]); + if (VecIn2.getNode()) { + Ops[1] = VecIn2; + } else { + // Use an undef build_vector as input for the second operand. + std::vector<SDValue> UnOps(NumInScalars, + DAG.getUNDEF(EltType)); + Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, + &UnOps[0], UnOps.size()); + AddToWorkList(Ops[1].getNode()); + } + + Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), BuildVecVT, + &BuildVecIndices[0], BuildVecIndices.size()); + return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), VT, Ops, 3); } return SDValue(); @@ -5313,10 +5321,8 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { } SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { - return SDValue(); - - MVT VT = N->getValueType(0); - unsigned NumElts = VT.getVectorNumElements(); + SDValue ShufMask = N->getOperand(2); + unsigned NumElts = ShufMask.getNumOperands(); SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -5324,13 +5330,60 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { assert(N0.getValueType().getVectorNumElements() == NumElts && "Vector shuffle must be normalized in DAG"); - // FIXME: implement canonicalizations from DAG.getVectorShuffle() + // If the shuffle mask is an identity operation on the LHS, return the LHS. + bool isIdentity = true; + for (unsigned i = 0; i != NumElts; ++i) { + if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && + cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() != i) { + isIdentity = false; + break; + } + } + if (isIdentity) return N->getOperand(0); + + // If the shuffle mask is an identity operation on the RHS, return the RHS. + isIdentity = true; + for (unsigned i = 0; i != NumElts; ++i) { + if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && + cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() != + i+NumElts) { + isIdentity = false; + break; + } + } + if (isIdentity) return N->getOperand(1); + + // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not + // needed at all. + bool isUnary = true; + bool isSplat = true; + int VecNum = -1; + unsigned BaseIdx = 0; + for (unsigned i = 0; i != NumElts; ++i) + if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) { + unsigned Idx=cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue(); + int V = (Idx < NumElts) ? 0 : 1; + if (VecNum == -1) { + VecNum = V; + BaseIdx = Idx; + } else { + if (BaseIdx != Idx) + isSplat = false; + if (VecNum != V) { + isUnary = false; + break; + } + } + } + + // Normalize unary shuffle so the RHS is undef. + if (isUnary && VecNum == 1) + std::swap(N0, N1); // If it is a splat, check if the argument vector is a build_vector with // all scalar elements the same. - if (cast<ShuffleVectorSDNode>(N)->isSplat()) { + if (isSplat) { SDNode *V = N0.getNode(); - // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to @@ -5344,7 +5397,6 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { if (V->getOpcode() == ISD::BUILD_VECTOR) { unsigned NumElems = V->getNumOperands(); - unsigned BaseIdx = cast<ShuffleVectorSDNode>(N)->getSplatIndex(); if (NumElems > BaseIdx) { SDValue Base; bool AllSame = true; @@ -5369,6 +5421,38 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { } } } + + // If it is a unary or the LHS and the RHS are the same node, turn the RHS + // into an undef. + if (isUnary || N0 == N1) { + // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the + // first operand. + SmallVector<SDValue, 8> MappedOps; + + for (unsigned i = 0; i != NumElts; ++i) { + if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF || + cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() < + NumElts) { + MappedOps.push_back(ShufMask.getOperand(i)); + } else { + unsigned NewIdx = + cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() - + NumElts; + MappedOps.push_back(DAG.getConstant(NewIdx, + ShufMask.getOperand(i).getValueType())); + } + } + + ShufMask = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), + ShufMask.getValueType(), + &MappedOps[0], MappedOps.size()); + AddToWorkList(ShufMask.getNode()); + return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), + N->getValueType(0), N0, + DAG.getUNDEF(N->getValueType(0)), + ShufMask); + } + return SDValue(); } @@ -5377,42 +5461,52 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> /// vector_shuffle V, Zero, <0, 4, 2, 4> SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { - MVT VT = N->getValueType(0); - DebugLoc dl = N->getDebugLoc(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); if (N->getOpcode() == ISD::AND) { if (RHS.getOpcode() == ISD::BIT_CONVERT) RHS = RHS.getOperand(0); if (RHS.getOpcode() == ISD::BUILD_VECTOR) { - SmallVector<int, 8> Indices; - unsigned NumElts = RHS.getNumOperands(); + std::vector<SDValue> IdxOps; + unsigned NumOps = RHS.getNumOperands(); + unsigned NumElts = NumOps; for (unsigned i = 0; i != NumElts; ++i) { SDValue Elt = RHS.getOperand(i); if (!isa<ConstantSDNode>(Elt)) return SDValue(); else if (cast<ConstantSDNode>(Elt)->isAllOnesValue()) - Indices.push_back(i); + IdxOps.push_back(DAG.getIntPtrConstant(i)); else if (cast<ConstantSDNode>(Elt)->isNullValue()) - Indices.push_back(NumElts); + IdxOps.push_back(DAG.getIntPtrConstant(NumElts)); else return SDValue(); } // Let's see if the target supports this vector_shuffle. - MVT RVT = RHS.getValueType(); - if (!TLI.isVectorClearMaskLegal(&Indices[0], RVT)) + if (!TLI.isVectorClearMaskLegal(IdxOps, TLI.getPointerTy(), DAG)) return SDValue(); // Return the new VECTOR_SHUFFLE node. - MVT EVT = RVT.getVectorElementType(); - SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(), - DAG.getConstant(0, EVT)); - SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - RVT, &ZeroOps[0], ZeroOps.size()); - LHS = DAG.getNode(ISD::BIT_CONVERT, dl, RVT, LHS); - SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]); - return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Shuf); + MVT EVT = RHS.getValueType().getVectorElementType(); + MVT VT = MVT::getVectorVT(EVT, NumElts); + MVT MaskVT = MVT::getVectorVT(TLI.getPointerTy(), NumElts); + std::vector<SDValue> Ops; + LHS = DAG.getNode(ISD::BIT_CONVERT, LHS.getDebugLoc(), VT, LHS); + Ops.push_back(LHS); + AddToWorkList(LHS.getNode()); + std::vector<SDValue> ZeroOps(NumElts, DAG.getConstant(0, EVT)); + Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), + VT, &ZeroOps[0], ZeroOps.size())); + Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), + MaskVT, &IdxOps[0], IdxOps.size())); + SDValue Result = DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), + VT, &Ops[0], Ops.size()); + + if (VT != N->getValueType(0)) + Result = DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), + N->getValueType(0), Result); + + return Result; } } diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index fed2b02ea5..5ea1ce3430 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -267,10 +267,16 @@ private: bool isVolatile, SDValue ValOp, unsigned StWidth, DebugLoc dl); - /// promoteShuffle - Promote a shuffle mask of a vector VT to perform the - /// same shuffle on a vector of NVT. Must not create an illegal shuffle mask. - SDValue promoteShuffle(MVT NVT, MVT VT, DebugLoc dl, SDValue N1, SDValue N2, - const int *Mask) const; + /// isShuffleLegal - Return non-null if a vector shuffle is legal with the + /// specified mask and type. Targets can specify exactly which masks they + /// support and the code generator is tasked with not creating illegal masks. + /// + /// Note that this will also return true for shuffles that are promoted to a + /// different type. + /// + /// If this is a legal shuffle, this method returns the (possibly promoted) + /// build_vector Mask. If it's not a legal shuffle, it returns null. + SDNode *isShuffleLegal(MVT VT, SDValue Mask) const; bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, SmallPtrSet<SDNode*, 32> &NodesLeadingTo); @@ -313,35 +319,50 @@ private: }; } -/// promoteShuffle - Promote a shuffle mask of a vector VT to perform the -/// same shuffle on a vector of NVT. Must not create an illegal shuffle mask. -/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> -SDValue SelectionDAGLegalize::promoteShuffle(MVT NVT, MVT VT, DebugLoc dl, - SDValue N1, SDValue N2, - const int *Mask) const { - MVT EltVT = NVT.getVectorElementType(); - int NumMaskElts = VT.getVectorNumElements(); - int NumDestElts = NVT.getVectorNumElements(); - unsigned NumEltsGrowth = NumDestElts / NumMaskElts; - - assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); - - if (NumEltsGrowth == 1) - return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); - - SmallVector<int, 8> NewMask; - for (int i = 0; i != NumMaskElts; ++i) { - int Idx = Mask[i]; - for (unsigned j = 0; j != NumEltsGrowth; ++j) { - if (Idx < 0) - NewMask.push_back(-1); - else - NewMask.push_back(Idx * NumEltsGrowth + j); +/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the +/// specified mask and type. Targets can specify exactly which masks they +/// support and the code generator is tasked with not creating illegal masks. +/// +/// Note that this will also return true for shuffles that are promoted to a +/// different type. +SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const { + switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { + default: return 0; + case TargetLowering::Legal: + case TargetLowering::Custom: + break; + case TargetLowering::Promote: { + // If this is promoted to a different type, convert the shuffle mask and + // ask if it is legal in the promoted type! + MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT); + MVT EltVT = NVT.getVectorElementType(); + + // If we changed # elements, change the shuffle mask. + unsigned NumEltsGrowth = + NVT.getVectorNumElements() / VT.getVectorNumElements(); + assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); + if (NumEltsGrowth > 1) { + // Renumber the elements. + SmallVector<SDValue, 8> Ops; + for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) { + SDValue InOp = Mask.getOperand(i); + for (unsigned j = 0; j != NumEltsGrowth; ++j) { + if (InOp.getOpcode() == ISD::UNDEF) + Ops.push_back(DAG.getUNDEF(EltVT)); + else { + unsigned InEltNo = cast<ConstantSDNode>(InOp)->getZExtValue(); + Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT)); + } + } + } + Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getDebugLoc(), + NVT, &Ops[0], Ops.size()); } + VT = NVT; + break; } - assert((int)NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); - assert(TLI.isShuffleMaskLegal(&Mask[0], NVT) && "Shuffle not legal?"); - return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); + } + return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0; } SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, @@ -1631,15 +1652,25 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1.getValueType(), Tmp2); unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); + MVT ShufMaskVT = + MVT::getIntVectorWithNumElements(NumElts); + MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType(); + // We generate a shuffle of InVec and ScVec, so the shuffle mask // should be 0,1,2,3,4,5... with the appropriate element replaced with // elt 0 of the RHS. - SmallVector<int, 8> ShufOps; - for (unsigned i = 0; i != NumElts; ++i) - ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); - - Result = DAG.getVectorShuffle(Tmp1.getValueType(), dl, Tmp1, ScVec, - &ShufOps[0]); + SmallVector<SDValue, 8> ShufOps; + for (unsigned i = 0; i != NumElts; ++i) { + if (i != InsertPos->getZExtValue()) + ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT)); + else + ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT)); + } + SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, dl, ShufMaskVT, + &ShufOps[0], ShufOps.size()); + + Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, Tmp1.getValueType(), + Tmp1, ScVec, ShufMask); Result = LegalizeOp(Result); break; } @@ -1677,14 +1708,13 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::VECTOR_SHUFFLE: Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors, Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask. - Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); + Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); // Allow targets to custom lower the SHUFFLEs they support. switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, Result.getValueType())){ default: assert(0 && "Unknown operation action!"); case TargetLowering::Legal: - assert(TLI.isShuffleMaskLegal(cast<ShuffleVectorSDNode>(Node)->getMask(), - Result.getValueType()) && + assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) && "vector shuffle should not be created if not legal!"); break; case TargetLowering::Custom: @@ -1698,21 +1728,23 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { MVT VT = Node->getValueType(0); MVT EltVT = VT.getVectorElementType(); MVT PtrVT = TLI.getPointerTy(); - const int *Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); - int NumElems = VT.getVectorNumElements(); + SDValue Mask = Node->getOperand(2); + unsigned NumElems = Mask.getNumOperands(); SmallVector<SDValue, 8> Ops; - for (int i = 0; i != NumElems; ++i) { - if (Mask[i] < 0) { + for (unsigned i = 0; i != NumElems; ++i) { + SDValue Arg = Mask.getOperand(i); + if (Arg.getOpcode() == ISD::UNDEF) { Ops.push_back(DAG.getUNDEF(EltVT)); - continue; + } else { + assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); + unsigned Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); + if (Idx < NumElems) + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, + DAG.getConstant(Idx, PtrVT))); + else + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, + DAG.getConstant(Idx - NumElems, PtrVT))); } - int Idx = Mask[i]; - if (Idx < NumElems) - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, - DAG.getConstant(Idx, PtrVT))); - else - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, - DAG.getConstant(Idx - NumElems, PtrVT))); } Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); break; @@ -1727,8 +1759,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); // Convert the shuffle mask to the right # elements. - Result = promoteShuffle(NVT, OVT, dl, Tmp1, Tmp2, - cast<ShuffleVectorSDNode>(Node)->getMask()); + Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0); + assert(Tmp3.getNode() && "Shuffle not legal?"); + Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, NVT, Tmp1, Tmp2, Tmp3); Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); break; } @@ -5457,7 +5490,6 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // FIXME: it would be far nicer to change this into map<SDValue,uint64_t> // and use a bitmask instead of a list of elements. - // FIXME: this doesn't treat <0, u, 0, u> for example, as a splat. std::map<SDValue, std::vector<unsigned> > Values; Values[SplatValue].push_back(0); bool isConstant = true; @@ -5514,17 +5546,21 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { if (SplatValue.getNode()) { // Splat of one value? // Build the shuffle constant vector: <0, 0, 0, 0> - SmallVector<int, 8> ZeroVec(NumElems, 0); + MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); + SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); + std::vector<SDValue> ZeroVec(NumElems, Zero); + SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, + &ZeroVec[0], ZeroVec.size()); // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. - if (TLI.isShuffleMaskLegal(&ZeroVec[0], Node->getValueType(0))) { + if (isShuffleLegal(VT, SplatMask)) { // Get the splatted value into the low element of a vector register. SDValue LowValVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, SplatValue); // Return shuffle(LowValVec, undef, <0,0,0,0>) - return DAG.getVectorShuffle(VT, dl, LowValVec, DAG.getUNDEF(VT), - &ZeroVec[0]); + return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, LowValVec, + DAG.getUNDEF(VT), SplatMask); } } @@ -5546,25 +5582,35 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { std::swap(Val1, Val2); // Build the shuffle constant vector: e.g. <0, 4, 0, 4> - SmallVector<int, 8> ShuffleMask(NumElems, -1); + MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); + MVT MaskEltVT = MaskVT.getVectorElementType(); + std::vector<SDValue> MaskVec(NumElems); // Set elements of the shuffle mask for Val1. std::vector<unsigned> &Val1Elts = Values[Val1]; for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i) - ShuffleMask[Val1Elts[i]] = 0; + MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT); // Set elements of the shuffle mask for Val2. std::vector<unsigned> &Val2Elts = Values[Val2]; for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i) if (Val2.getOpcode() != ISD::UNDEF) - ShuffleMask[Val2Elts[i]] = NumElems; + MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT); + else + MaskVec[Val2Elts[i]] = DAG.getUNDEF(MaskEltVT); + + SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, + &MaskVec[0], MaskVec.size()); // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR, VT) && - TLI.isShuffleMaskLegal(&ShuffleMask[0], VT)) { + isShuffleLegal(VT, ShuffleMask)) { Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val1); |