diff options
25 files changed, 1610 insertions, 2157 deletions
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 9ae92f388e..a350d5fd93 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -353,6 +353,13 @@ public: SDValue getConvertRndSat(MVT VT, DebugLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code); + + /// getVectorShuffle - Return an ISD::VECTOR_SHUFFLE node. The number of + /// elements in VT, which must be a vector type, must match the number of + /// mask elements NumElts. A negative integer mask element is treated as + /// undefined. + SDValue getVectorShuffle(MVT VT, DebugLoc dl, SDValue N1, SDValue N2, + const int *MaskElts); /// getZeroExtendInReg - Return the expression required to zero extend the Op /// value assuming it was the smaller SrcTy value. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index aaa39674d7..a850b3fcb5 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1703,6 +1703,32 @@ public: } }; +class ShuffleVectorSDNode : public SDNode { + SDUse Ops[2]; + int *Mask; +protected: + friend class SelectionDAG; + ShuffleVectorSDNode(MVT VT, DebugLoc dl, SDValue N1, SDValue N2, int *M) + : SDNode(ISD::VECTOR_SHUFFLE, dl, getSDVTList(VT)), Mask(M) { + InitOperands(Ops, N1, N2); + } +public: + + const int * getMask() const { return Mask; } + + bool isSplat() { return isSplatMask(Mask, getValueType(0)); } + int getSplatIndex() { + assert(isSplat() && "Cannot get splat index for non-splat!"); + return Mask[0]; + } + static bool isSplatMask(const int *Mask, MVT VT); + + static bool classof(const ShuffleVectorSDNode *) { return true; } + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::VECTOR_SHUFFLE; + } +}; + class ConstantSDNode : public SDNode { const ConstantInt *Value; friend class SelectionDAG; @@ -2084,7 +2110,7 @@ public: return N->getOpcode() == ISD::CONDCODE; } }; - + /// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the /// future and most targets don't support it. class CvtRndSatSDNode : public SDNode { diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index 477505e2f7..a5f064dcde 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -328,7 +328,7 @@ public: /// support *some* VECTOR_SHUFFLE operations, those with specific masks. /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values /// are assumed to be legal. - virtual bool isShuffleMaskLegal(SDValue Mask, MVT VT) const { + virtual bool isShuffleMaskLegal(const int *Mask, MVT VT) const { return true; } @@ -336,9 +336,7 @@ public: /// used by Targets can use this to indicate if there is a suitable /// VECTOR_SHUFFLE that can be used to replace a VAND with a constant /// pool entry. - virtual bool isVectorClearMaskLegal(const std::vector<SDValue> &BVOps, - MVT EVT, - SelectionDAG &DAG) const { + virtual bool isVectorClearMaskLegal(const int *Mask, MVT VT) const { return false; } diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td index 7f39bb2f83..2cd29676db 100644 --- a/include/llvm/Target/TargetSelectionDAG.td +++ b/include/llvm/Target/TargetSelectionDAG.td @@ -51,15 +51,6 @@ class SDTCisOpSmallerThanOp<int SmallOp, int BigOp> : SDTypeConstraint<SmallOp>{ int BigOperandNum = BigOp; } -/// SDTCisIntVectorOfSameSize - This indicates that ThisOp and OtherOp are -/// vector types, and that ThisOp is the result of -/// MVT::getIntVectorWithNumElements with the number of elements -/// that ThisOp has. -class SDTCisIntVectorOfSameSize<int ThisOp, int OtherOp> - : SDTypeConstraint<ThisOp> { - int OtherOpNum = OtherOp; -} - /// SDTCisEltOfVec - This indicates that ThisOp is a scalar type of the same /// type as the element type of OtherOp, which is a vector type. class SDTCisEltOfVec<int ThisOp, int OtherOp> @@ -175,8 +166,8 @@ def SDTIStore : SDTypeProfile<1, 3, [ // indexed store SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3> ]>; -def SDTVecShuffle : SDTypeProfile<1, 3, [ - SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisIntVectorOfSameSize<3, 0> +def SDTVecShuffle : SDTypeProfile<1, 2, [ + SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2> ]>; def SDTVecExtract : SDTypeProfile<1, 2, [ // vector extract SDTCisEltOfVec<0, 1>, SDTCisPtrTy<2> diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 72b3e3627e..40807d8909 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -5098,7 +5098,21 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), InVec.getValueType(), &Ops[0], Ops.size()); } + // If the invec is an UNDEF and if EltNo is a constant, create a new + // BUILD_VECTOR with undef elements and the inserted element. + if (!LegalOperations && InVec.getOpcode() == ISD::UNDEF && + isa<ConstantSDNode>(EltNo)) { + MVT VT = InVec.getValueType(); + MVT EVT = VT.getVectorElementType(); + unsigned NElts = VT.getVectorNumElements(); + SmallVector<SDValue, 8> Ops(NElts, DAG.getUNDEF(EVT)); + unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); + if (Elt < Ops.size()) + Ops[Elt] = InVal; + return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), + InVec.getValueType(), &Ops[0], Ops.size()); + } return SDValue(); } @@ -5160,9 +5174,8 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { // to examine the mask. if (BCNumEltsChanged) return SDValue(); - unsigned Idx = cast<ConstantSDNode>(InVec.getOperand(2). - getOperand(Elt))->getZExtValue(); - unsigned NumElems = InVec.getOperand(2).getNumOperands(); + int Idx = cast<ShuffleVectorSDNode>(InVec)->getMask()[Elt]; + int NumElems = InVec.getValueType().getVectorNumElements(); InVec = (Idx < NumElems) ? InVec.getOperand(0) : InVec.getOperand(1); if (InVec.getOpcode() == ISD::BIT_CONVERT) InVec = InVec.getOperand(0); @@ -5209,7 +5222,6 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { unsigned NumInScalars = N->getNumOperands(); MVT VT = N->getValueType(0); - unsigned NumElts = VT.getVectorNumElements(); MVT EltType = VT.getVectorElementType(); // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT @@ -5252,56 +5264,36 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { } // If everything is good, we can make a shuffle operation. - MVT IndexVT = MVT::i32; if (VecIn1.getNode()) { - SmallVector<SDValue, 8> BuildVecIndices; + SmallVector<int, 8> Mask; for (unsigned i = 0; i != NumInScalars; ++i) { if (N->getOperand(i).getOpcode() == ISD::UNDEF) { - BuildVecIndices.push_back(DAG.getUNDEF(IndexVT)); + Mask.push_back(-1); continue; } - SDValue Extract = N->getOperand(i); - // If extracting from the first vector, just use the index directly. + SDValue Extract = N->getOperand(i); SDValue ExtVal = Extract.getOperand(1); if (Extract.getOperand(0) == VecIn1) { - if (ExtVal.getValueType() == IndexVT) - BuildVecIndices.push_back(ExtVal); - else { - unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue(); - BuildVecIndices.push_back(DAG.getConstant(Idx, IndexVT)); - } + Mask.push_back(cast<ConstantSDNode>(ExtVal)->getZExtValue()); continue; } // Otherwise, use InIdx + VecSize unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue(); - BuildVecIndices.push_back(DAG.getConstant(Idx+NumInScalars, IndexVT)); + Mask.push_back(Idx+NumInScalars); } // Add count and size info. - MVT BuildVecVT = MVT::getVectorVT(IndexVT, NumElts); - if (!TLI.isTypeLegal(BuildVecVT) && LegalTypes) + if (!TLI.isTypeLegal(VT) && LegalTypes) return SDValue(); // Return the new VECTOR_SHUFFLE node. - SDValue Ops[5]; + SDValue Ops[2]; Ops[0] = VecIn1; - if (VecIn2.getNode()) { - Ops[1] = VecIn2; - } else { - // Use an undef build_vector as input for the second operand. - std::vector<SDValue> UnOps(NumInScalars, - DAG.getUNDEF(EltType)); - Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), VT, - &UnOps[0], UnOps.size()); - AddToWorkList(Ops[1].getNode()); - } - - Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), BuildVecVT, - &BuildVecIndices[0], BuildVecIndices.size()); - return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), VT, Ops, 3); + Ops[1] = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); + return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]); } return SDValue(); @@ -5321,8 +5313,10 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { } SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { - SDValue ShufMask = N->getOperand(2); - unsigned NumElts = ShufMask.getNumOperands(); + return SDValue(); + + MVT VT = N->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -5330,60 +5324,13 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { assert(N0.getValueType().getVectorNumElements() == NumElts && "Vector shuffle must be normalized in DAG"); - // If the shuffle mask is an identity operation on the LHS, return the LHS. - bool isIdentity = true; - for (unsigned i = 0; i != NumElts; ++i) { - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() != i) { - isIdentity = false; - break; - } - } - if (isIdentity) return N->getOperand(0); - - // If the shuffle mask is an identity operation on the RHS, return the RHS. - isIdentity = true; - for (unsigned i = 0; i != NumElts; ++i) { - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() != - i+NumElts) { - isIdentity = false; - break; - } - } - if (isIdentity) return N->getOperand(1); - - // Check if the shuffle is a unary shuffle, i.e. one of the vectors is not - // needed at all. - bool isUnary = true; - bool isSplat = true; - int VecNum = -1; - unsigned BaseIdx = 0; - for (unsigned i = 0; i != NumElts; ++i) - if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) { - unsigned Idx=cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue(); - int V = (Idx < NumElts) ? 0 : 1; - if (VecNum == -1) { - VecNum = V; - BaseIdx = Idx; - } else { - if (BaseIdx != Idx) - isSplat = false; - if (VecNum != V) { - isUnary = false; - break; - } - } - } - - // Normalize unary shuffle so the RHS is undef. - if (isUnary && VecNum == 1) - std::swap(N0, N1); + // FIXME: implement canonicalizations from DAG.getVectorShuffle() // If it is a splat, check if the argument vector is a build_vector with // all scalar elements the same. - if (isSplat) { + if (cast<ShuffleVectorSDNode>(N)->isSplat()) { SDNode *V = N0.getNode(); + // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to @@ -5397,6 +5344,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { if (V->getOpcode() == ISD::BUILD_VECTOR) { unsigned NumElems = V->getNumOperands(); + unsigned BaseIdx = cast<ShuffleVectorSDNode>(N)->getSplatIndex(); if (NumElems > BaseIdx) { SDValue Base; bool AllSame = true; @@ -5421,38 +5369,6 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { } } } - - // If it is a unary or the LHS and the RHS are the same node, turn the RHS - // into an undef. - if (isUnary || N0 == N1) { - // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the - // first operand. - SmallVector<SDValue, 8> MappedOps; - - for (unsigned i = 0; i != NumElts; ++i) { - if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF || - cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() < - NumElts) { - MappedOps.push_back(ShufMask.getOperand(i)); - } else { - unsigned NewIdx = - cast<ConstantSDNode>(ShufMask.getOperand(i))->getZExtValue() - - NumElts; - MappedOps.push_back(DAG.getConstant(NewIdx, - ShufMask.getOperand(i).getValueType())); - } - } - - ShufMask = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - ShufMask.getValueType(), - &MappedOps[0], MappedOps.size()); - AddToWorkList(ShufMask.getNode()); - return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), - N->getValueType(0), N0, - DAG.getUNDEF(N->getValueType(0)), - ShufMask); - } - return SDValue(); } @@ -5461,52 +5377,42 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> /// vector_shuffle V, Zero, <0, 4, 2, 4> SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { + MVT VT = N->getValueType(0); + DebugLoc dl = N->getDebugLoc(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); if (N->getOpcode() == ISD::AND) { if (RHS.getOpcode() == ISD::BIT_CONVERT) RHS = RHS.getOperand(0); if (RHS.getOpcode() == ISD::BUILD_VECTOR) { - std::vector<SDValue> IdxOps; - unsigned NumOps = RHS.getNumOperands(); - unsigned NumElts = NumOps; + SmallVector<int, 8> Indices; + unsigned NumElts = RHS.getNumOperands(); for (unsigned i = 0; i != NumElts; ++i) { SDValue Elt = RHS.getOperand(i); if (!isa<ConstantSDNode>(Elt)) return SDValue(); else if (cast<ConstantSDNode>(Elt)->isAllOnesValue()) - IdxOps.push_back(DAG.getIntPtrConstant(i)); + Indices.push_back(i); else if (cast<ConstantSDNode>(Elt)->isNullValue()) - IdxOps.push_back(DAG.getIntPtrConstant(NumElts)); + Indices.push_back(NumElts); else return SDValue(); } // Let's see if the target supports this vector_shuffle. - if (!TLI.isVectorClearMaskLegal(IdxOps, TLI.getPointerTy(), DAG)) + MVT RVT = RHS.getValueType(); + if (!TLI.isVectorClearMaskLegal(&Indices[0], RVT)) return SDValue(); // Return the new VECTOR_SHUFFLE node. - MVT EVT = RHS.getValueType().getVectorElementType(); - MVT VT = MVT::getVectorVT(EVT, NumElts); - MVT MaskVT = MVT::getVectorVT(TLI.getPointerTy(), NumElts); - std::vector<SDValue> Ops; - LHS = DAG.getNode(ISD::BIT_CONVERT, LHS.getDebugLoc(), VT, LHS); - Ops.push_back(LHS); - AddToWorkList(LHS.getNode()); - std::vector<SDValue> ZeroOps(NumElts, DAG.getConstant(0, EVT)); - Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - VT, &ZeroOps[0], ZeroOps.size())); - Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), - MaskVT, &IdxOps[0], IdxOps.size())); - SDValue Result = DAG.getNode(ISD::VECTOR_SHUFFLE, N->getDebugLoc(), - VT, &Ops[0], Ops.size()); - - if (VT != N->getValueType(0)) - Result = DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), - N->getValueType(0), Result); - - return Result; + MVT EVT = RVT.getVectorElementType(); + SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(), + DAG.getConstant(0, EVT)); + SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), + RVT, &ZeroOps[0], ZeroOps.size()); + LHS = DAG.getNode(ISD::BIT_CONVERT, dl, RVT, LHS); + SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]); + return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Shuf); } } diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 5ea1ce3430..fed2b02ea5 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -267,16 +267,10 @@ private: bool isVolatile, SDValue ValOp, unsigned StWidth, DebugLoc dl); - /// isShuffleLegal - Return non-null if a vector shuffle is legal with the - /// specified mask and type. Targets can specify exactly which masks they - /// support and the code generator is tasked with not creating illegal masks. - /// - /// Note that this will also return true for shuffles that are promoted to a - /// different type. - /// - /// If this is a legal shuffle, this method returns the (possibly promoted) - /// build_vector Mask. If it's not a legal shuffle, it returns null. - SDNode *isShuffleLegal(MVT VT, SDValue Mask) const; + /// promoteShuffle - Promote a shuffle mask of a vector VT to perform the + /// same shuffle on a vector of NVT. Must not create an illegal shuffle mask. + SDValue promoteShuffle(MVT NVT, MVT VT, DebugLoc dl, SDValue N1, SDValue N2, + const int *Mask) const; bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, SmallPtrSet<SDNode*, 32> &NodesLeadingTo); @@ -319,50 +313,35 @@ private: }; } -/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the -/// specified mask and type. Targets can specify exactly which masks they -/// support and the code generator is tasked with not creating illegal masks. -/// -/// Note that this will also return true for shuffles that are promoted to a -/// different type. -SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const { - switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { - default: return 0; - case TargetLowering::Legal: - case TargetLowering::Custom: - break; - case TargetLowering::Promote: { - // If this is promoted to a different type, convert the shuffle mask and - // ask if it is legal in the promoted type! - MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT); - MVT EltVT = NVT.getVectorElementType(); - - // If we changed # elements, change the shuffle mask. - unsigned NumEltsGrowth = - NVT.getVectorNumElements() / VT.getVectorNumElements(); - assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); - if (NumEltsGrowth > 1) { - // Renumber the elements. - SmallVector<SDValue, 8> Ops; - for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) { - SDValue InOp = Mask.getOperand(i); - for (unsigned j = 0; j != NumEltsGrowth; ++j) { - if (InOp.getOpcode() == ISD::UNDEF) - Ops.push_back(DAG.getUNDEF(EltVT)); - else { - unsigned InEltNo = cast<ConstantSDNode>(InOp)->getZExtValue(); - Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT)); - } - } - } - Mask = DAG.getNode(ISD::BUILD_VECTOR, Mask.getDebugLoc(), - NVT, &Ops[0], Ops.size()); +/// promoteShuffle - Promote a shuffle mask of a vector VT to perform the +/// same shuffle on a vector of NVT. Must not create an illegal shuffle mask. +/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> +SDValue SelectionDAGLegalize::promoteShuffle(MVT NVT, MVT VT, DebugLoc dl, + SDValue N1, SDValue N2, + const int *Mask) const { + MVT EltVT = NVT.getVectorElementType(); + int NumMaskElts = VT.getVectorNumElements(); + int NumDestElts = NVT.getVectorNumElements(); + unsigned NumEltsGrowth = NumDestElts / NumMaskElts; + + assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); + + if (NumEltsGrowth == 1) + return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); + + SmallVector<int, 8> NewMask; + for (int i = 0; i != NumMaskElts; ++i) { + int Idx = Mask[i]; + for (unsigned j = 0; j != NumEltsGrowth; ++j) { + if (Idx < 0) + NewMask.push_back(-1); + else + NewMask.push_back(Idx * NumEltsGrowth + j); } - VT = NVT; - break; } - } - return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0; + assert((int)NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); + assert(TLI.isShuffleMaskLegal(&Mask[0], NVT) && "Shuffle not legal?"); + return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]); } SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, @@ -1652,25 +1631,15 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1.getValueType(), Tmp2); unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); - MVT ShufMaskVT = - MVT::getIntVectorWithNumElements(NumElts); - MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType(); - // We generate a shuffle of InVec and ScVec, so the shuffle mask // should be 0,1,2,3,4,5... with the appropriate element replaced with // elt 0 of the RHS. - SmallVector<SDValue, 8> ShufOps; - for (unsigned i = 0; i != NumElts; ++i) { - if (i != InsertPos->getZExtValue()) - ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT)); - else - ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT)); - } - SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, dl, ShufMaskVT, - &ShufOps[0], ShufOps.size()); - - Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, Tmp1.getValueType(), - Tmp1, ScVec, ShufMask); + SmallVector<int, 8> ShufOps; + for (unsigned i = 0; i != NumElts; ++i) + ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); + + Result = DAG.getVectorShuffle(Tmp1.getValueType(), dl, Tmp1, ScVec, + &ShufOps[0]); Result = LegalizeOp(Result); break; } @@ -1708,13 +1677,14 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::VECTOR_SHUFFLE: Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input vectors, Tmp2 = LegalizeOp(Node->getOperand(1)); // but not the shuffle mask. - Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); + Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); // Allow targets to custom lower the SHUFFLEs they support. switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, Result.getValueType())){ default: assert(0 && "Unknown operation action!"); case TargetLowering::Legal: - assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) && + assert(TLI.isShuffleMaskLegal(cast<ShuffleVectorSDNode>(Node)->getMask(), + Result.getValueType()) && "vector shuffle should not be created if not legal!"); break; case TargetLowering::Custom: @@ -1728,23 +1698,21 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { MVT VT = Node->getValueType(0); MVT EltVT = VT.getVectorElementType(); MVT PtrVT = TLI.getPointerTy(); - SDValue Mask = Node->getOperand(2); - unsigned NumElems = Mask.getNumOperands(); + const int *Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); + int NumElems = VT.getVectorNumElements(); SmallVector<SDValue, 8> Ops; - for (unsigned i = 0; i != NumElems; ++i) { - SDValue Arg = Mask.getOperand(i); - if (Arg.getOpcode() == ISD::UNDEF) { + for (int i = 0; i != NumElems; ++i) { + if (Mask[i] < 0) { Ops.push_back(DAG.getUNDEF(EltVT)); - } else { - assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!"); - unsigned Idx = cast<ConstantSDNode>(Arg)->getZExtValue(); - if (Idx < NumElems) - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, - DAG.getConstant(Idx, PtrVT))); - else - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, - DAG.getConstant(Idx - NumElems, PtrVT))); + continue; } + int Idx = Mask[i]; + if (Idx < NumElems) + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp1, + DAG.getConstant(Idx, PtrVT))); + else + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Tmp2, + DAG.getConstant(Idx - NumElems, PtrVT))); } Result = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], Ops.size()); break; @@ -1759,9 +1727,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Tmp2); // Convert the shuffle mask to the right # elements. - Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0); - assert(Tmp3.getNode() && "Shuffle not legal?"); - Result = DAG.getNode(ISD::VECTOR_SHUFFLE, dl, NVT, Tmp1, Tmp2, Tmp3); + Result = promoteShuffle(NVT, OVT, dl, Tmp1, Tmp2, + cast<ShuffleVectorSDNode>(Node)->getMask()); Result = DAG.getNode(ISD::BIT_CONVERT, dl, OVT, Result); break; } @@ -5490,6 +5457,7 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // FIXME: it would be far nicer to change this into map<SDValue,uint64_t> // and use a bitmask instead of a list of elements. + // FIXME: this doesn't treat <0, u, 0, u> for example, as a splat. std::map<SDValue, std::vector<unsigned> > Values; Values[SplatValue].push_back(0); bool isConstant = true; @@ -5546,21 +5514,17 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { if (SplatValue.getNode()) { // Splat of one value? // Build the shuffle constant vector: <0, 0, 0, 0> - MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); - SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); - std::vector<SDValue> ZeroVec(NumElems, Zero); - SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, - &ZeroVec[0], ZeroVec.size()); + SmallVector<int, 8> ZeroVec(NumElems, 0); // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. - if (isShuffleLegal(VT, SplatMask)) { + if (TLI.isShuffleMaskLegal(&ZeroVec[0], Node->getValueType(0))) { // Get the splatted value into the low element of a vector register. SDValue LowValVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, SplatValue); // Return shuffle(LowValVec, undef, <0,0,0,0>) - return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, LowValVec, - DAG.getUNDEF(VT), SplatMask); + return DAG.getVectorShuffle(VT, dl, LowValVec, DAG.getUNDEF(VT), + &ZeroVec[0]); } } @@ -5582,35 +5546,25 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { std::swap(Val1, Val2); // Build the shuffle constant vector: e.g. <0, 4, 0, 4> - MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); - MVT MaskEltVT = MaskVT.getVectorElementType(); - std::vector<SDValue> MaskVec(NumElems); + SmallVector<int, 8> ShuffleMask(NumElems, -1); // Set elements of the shuffle mask for Val1. std::vector<unsigned> &Val1Elts = Values[Val1]; for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i) - MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT); + ShuffleMask[Val1Elts[i]] = 0; // Set elements of the shuffle mask for Val2. std::vector<unsigned> &Val2Elts = Values[Val2]; for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i) if (Val2.getOpcode() != ISD::UNDEF) - MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT); - else - MaskVec[Val2Elts[i]] = DAG.getUNDEF(MaskEltVT); - - SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVT, - &MaskVec[0], MaskVec.size()); + ShuffleMask[Val2Elts[i]] = NumElems; // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR, VT) && - isShuffleLegal(VT, ShuffleMask)) { + TLI.isShuffleMaskLegal(&ShuffleMask[0], VT)) { Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val1); Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Val2); - SDValue Ops[] = { Val1, Val2, ShuffleMask }; - - // Return shuffle(LoValVec, HiValVec, <0,1,0,1>) - return DAG.getNode(ISD::VECTOR_SHUFFLE, dl, VT, Ops, 3); + return DAG.getVectorShuffle(VT, dl, Val1, Val2, &ShuffleMask[0]); } } @@ -8066,36 +8020,19 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { case ISD::VECTOR_SHUFFLE: { SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); - // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is - // used as permutation array. We build the vector here instead of widening - // because we don't want to legalize and have it turned to something else. - SDValue PermOp = Node->getOperand(2); - SDValueVector NewOps; - MVT PVT = PermOp.getValueType().getVectorElementType(); + const int *Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); + SmallVector<int, 8> NewMask; for (unsigned i = 0; i < NumElts; ++i) { - if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) { - NewOps.push_back(PermOp.getOperand(i)); - |