aboutsummaryrefslogtreecommitdiff
path: root/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen')
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeDAG.cpp70
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp8
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeTypes.cpp36
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeTypes.h106
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp1033
-rw-r--r--lib/CodeGen/SelectionDAG/TargetLowering.cpp28
6 files changed, 1238 insertions, 43 deletions
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 18ba9125d1..4a2af06966 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -491,9 +491,11 @@ void SelectionDAGLegalize::HandleOp(SDValue Op) {
MVT VT = Op.getValueType();
// If the type legalizer was run then we should never see any illegal result
// types here except for target constants (the type legalizer does not touch
- // those).
+ // those) or for build vector used as a mask for a vector shuffle.
+ // FIXME: We can removed the BUILD_VECTOR case when we fix PR2957.
assert((TypesNeedLegalizing || getTypeAction(VT) == Legal ||
- Op.getOpcode() == ISD::TargetConstant) &&
+ Op.getOpcode() == ISD::TargetConstant ||
+ Op.getOpcode() == ISD::BUILD_VECTOR) &&
"Illegal type introduced after type legalization?");
switch (getTypeAction(VT)) {
default: assert(0 && "Bad type action!");
@@ -3299,7 +3301,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
0);
break;
}
-
+
// Check to see if we have a libcall for this operator.
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
bool isSigned = false;
@@ -7646,7 +7648,10 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo,
case ISD::XOR:
case ISD::UREM:
case ISD::SREM:
- case ISD::FREM: {
+ case ISD::FREM:
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL: {
SDValue LL, LH, RL, RH;
SplitVectorOp(Node->getOperand(0), LL, LH);
SplitVectorOp(Node->getOperand(1), RL, RH);
@@ -8067,30 +8072,33 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
SDValue Tmp1 = Node->getOperand(0);
// Converts between two different types so we need to determine
// the correct widen type for the input operand.
- MVT TVT = Tmp1.getValueType();
- assert(TVT.isVector() && "can not widen non vector type");
- MVT TEVT = TVT.getVectorElementType();
- assert(WidenVT.getSizeInBits() % EVT.getSizeInBits() == 0 &&
- "can not widen bit bit convert that are not multiple of element type");
- MVT TWidenVT = MVT::getVectorVT(TEVT,
- WidenVT.getSizeInBits()/EVT.getSizeInBits());
- Tmp1 = WidenVectorOp(Tmp1, TWidenVT);
- assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits());
- Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+ MVT InVT = Tmp1.getValueType();
+ unsigned WidenSize = WidenVT.getSizeInBits();
+ if (InVT.isVector()) {
+ MVT InEltVT = InVT.getVectorElementType();
+ unsigned InEltSize = InEltVT.getSizeInBits();
+ assert(WidenSize % InEltSize == 0 &&
+ "can not widen bit convert that are not multiple of element type");
+ MVT NewInWidenVT = MVT::getVectorVT(InEltVT, WidenSize / InEltSize);
+ Tmp1 = WidenVectorOp(Tmp1, NewInWidenVT);
+ assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits());
+ Result = DAG.getNode(ISD::BIT_CONVERT, WidenVT, Tmp1);
+ } else {
+ // If the result size is a multiple of the input size, widen the input
+ // and then convert.
+ unsigned InSize = InVT.getSizeInBits();
+ assert(WidenSize % InSize == 0 &&
+ "can not widen bit convert that are not multiple of element type");
+ unsigned NewNumElts = WidenSize / InSize;
+ SmallVector<SDValue, 16> Ops(NewNumElts);
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ Ops[0] = Tmp1;
+ for (unsigned i = 1; i < NewNumElts; ++i)
+ Ops[i] = UndefVal;
- TargetLowering::LegalizeAction action =
- TLI.getOperationAction(Node->getOpcode(), WidenVT);
- switch (action) {
- default: assert(0 && "action not supported");
- case TargetLowering::Legal:
- break;
- case TargetLowering::Promote:
- // We defer the promotion to when we legalize the op
- break;
- case TargetLowering::Expand:
- // Expand the operation into a bunch of nasty scalar code.
- Result = LegalizeOp(UnrollVectorOp(Result));
- break;
+ MVT NewInVT = MVT::getVectorVT(InVT, NewNumElts);
+ Result = DAG.getNode(ISD::BUILD_VECTOR, NewInVT, &Ops[0], NewNumElts);
+ Result = DAG.getNode(ISD::BIT_CONVERT, WidenVT, Result);
}
break;
}
@@ -8098,7 +8106,8 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP:
case ISD::FP_TO_SINT:
- case ISD::FP_TO_UINT: {
+ case ISD::FP_TO_UINT:
+ case ISD::FP_ROUND: {
SDValue Tmp1 = Node->getOperand(0);
// Converts between two different types so we need to determine
// the correct widen type for the input operand.
@@ -8118,7 +8127,6 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::ANY_EXTEND:
- case ISD::FP_ROUND:
case ISD::SIGN_EXTEND_INREG:
case ISD::FABS:
case ISD::FNEG:
@@ -8129,7 +8137,7 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
case ISD::CTTZ:
case ISD::CTLZ: {
// Unary op widening
- SDValue Tmp1;
+ SDValue Tmp1;
Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
assert(Tmp1.getValueType() == WidenVT);
Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
@@ -8306,7 +8314,7 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts);
Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT);
SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT);
- Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2,
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2,
Node->getOperand(2));
break;
}
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 8abf1448db..5f23677880 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -217,7 +217,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) {
// Convert the element to an integer and promote it by hand.
return DAG.getNode(ISD::ANY_EXTEND, NOutVT,
BitConvertToInteger(GetScalarizedVector(InOp)));
- case SplitVector:
+ case SplitVector: {
// For example, i32 = BIT_CONVERT v2i16 on alpha. Convert the split
// pieces of the input into integers and reassemble in the final type.
SDValue Lo, Hi;
@@ -233,9 +233,13 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BIT_CONVERT(SDNode *N) {
JoinIntegers(Lo, Hi));
return DAG.getNode(ISD::BIT_CONVERT, NOutVT, InOp);
}
+ case WidenVector:
+ if (OutVT.bitsEq(NInVT))
+ // The input is widened to the same size. Convert to the widened value.
+ return DAG.getNode(ISD::BIT_CONVERT, OutVT, GetWidenedVector(InOp));
+ }
// Otherwise, lower the bit-convert to a store/load from the stack.
-
// Create the stack frame object. Make sure it is aligned for both
// the source and destination types.
SDValue FIPtr = DAG.CreateStackTemporary(InVT, OutVT);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 8454dda303..704c537bb4 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -108,6 +108,8 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
Mapped |= 32;
if (SplitVectors.find(Res) != SplitVectors.end())
Mapped |= 64;
+ if (WidenedVectors.find(Res) != WidenedVectors.end())
+ Mapped |= 128;
if (I->getNodeId() != Processed) {
if (Mapped != 0) {
@@ -115,7 +117,10 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
Failed = true;
}
} else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(I)) {
- if (Mapped > 1) {
+ // FIXME: Because of PR2957, the build vector can be placed on this
+ // list but if the associated vector shuffle is split, the build vector
+ // can also be split so we allow this to go through for now.
+ if (Mapped > 1 && Res.getOpcode() != ISD::BUILD_VECTOR) {
cerr << "Value with legal type was transformed!";
Failed = true;
}
@@ -144,6 +149,8 @@ void DAGTypeLegalizer::PerformExpensiveChecks() {
cerr << " ExpandedFloats";
if (Mapped & 64)
cerr << " SplitVectors";
+ if (Mapped & 128)
+ cerr << " WidenedVectors";
cerr << "\n";
abort();
}
@@ -241,6 +248,10 @@ bool DAGTypeLegalizer::run() {
SplitVectorResult(N, i);
Changed = true;
goto NodeDone;
+ case WidenVector:
+ WidenVectorResult(N, i);
+ Changed = true;
+ goto NodeDone;
}
}
@@ -255,6 +266,13 @@ ScanOperands:
if (IgnoreNodeResults(N->getOperand(i).getNode()))
continue;
+ if (N->getOpcode() == ISD::VECTOR_SHUFFLE && i == 2) {
+ // The shuffle mask doesn't need to be a legal vector type.
+ // FIXME: We can remove this once we fix PR2957.
+ SetIgnoredNodeResult(N->getOperand(2).getNode());
+ continue;
+ }
+
MVT OpVT = N->getOperand(i).getValueType();
switch (getTypeAction(OpVT)) {
default:
@@ -288,6 +306,10 @@ ScanOperands:
NeedsReanalyzing = SplitVectorOperand(N, i);
Changed = true;
break;
+ case WidenVector:
+ NeedsReanalyzing = WidenVectorOperand(N, i);
+ Changed = true;
+ break;
}
break;
}
@@ -791,6 +813,18 @@ void DAGTypeLegalizer::SetSplitVector(SDValue Op, SDValue Lo,
Entry.second = Hi;
}
+void DAGTypeLegalizer::SetWidenedVector(SDValue Op, SDValue Result) {
+ AnalyzeNewValue(Result);
+
+ SDValue &OpEntry = WidenedVectors[Op];
+ assert(OpEntry.getNode() == 0 && "Node is already promoted!");
+ OpEntry = Result;
+}
+
+// Set to ignore result
+void DAGTypeLegalizer::SetIgnoredNodeResult(SDNode* N) {
+ IgnoredNodesResultsSet.insert(N);
+}
//===----------------------------------------------------------------------===//
// Utilities.
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 9a42c5656e..2d4cde813f 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
@@ -63,7 +64,8 @@ private:
SoftenFloat, // Convert this float type to a same size integer type.
ExpandFloat, // Split this float type into two of half the size.
ScalarizeVector, // Replace this one-element vector with its element type.
- SplitVector // This vector type should be split into smaller vectors.
+ SplitVector, // This vector type should be split into smaller vectors.
+ WidenVector // This vector type should be widened into larger vectors.
};
/// ValueTypeActions - This is a bitvector that contains two bits for each
@@ -88,11 +90,8 @@ private:
// 2) For vectors, use a wider vector type (e.g. v3i32 -> v4i32).
if (!VT.isVector())
return PromoteInteger;
- else if (VT.getVectorNumElements() == 1)
- return ScalarizeVector;
else
- // TODO: move widen code to LegalizeTypes.
- return SplitVector;
+ return WidenVector;
case TargetLowering::Expand:
// Expand can mean
// 1) split scalar in half, 2) convert a float to an integer,
@@ -120,9 +119,13 @@ private:
/// IgnoreNodeResults - Pretend all of this node's results are legal.
bool IgnoreNodeResults(SDNode *N) const {
- return N->getOpcode() == ISD::TargetConstant;
+ return N->getOpcode() == ISD::TargetConstant ||
+ IgnoredNodesResultsSet.count(N);
}
+ /// IgnoredNode - Set of nodes whose result don't need to be legal.
+ DenseSet<SDNode*> IgnoredNodesResultsSet;
+
/// PromotedIntegers - For integer nodes that are below legal width, this map
/// indicates what promoted value to use.
DenseMap<SDValue, SDValue> PromotedIntegers;
@@ -147,6 +150,10 @@ private:
/// which operands are the expanded version of the input.
DenseMap<SDValue, std::pair<SDValue, SDValue> > SplitVectors;
+ /// WidenVectors - For vector nodes that need to be widened, indicates
+ /// the widen value to use.
+ DenseMap<SDValue, SDValue> WidenedVectors;
+
/// ReplacedValues - For values that have been replaced with another,
/// indicates the replacement value to use.
DenseMap<SDValue, SDValue> ReplacedValues;
@@ -201,6 +208,8 @@ private:
SDValue GetVectorElementPointer(SDValue VecPtr, MVT EltVT, SDValue Index);
+ void SetIgnoredNodeResult(SDNode* N);
+
//===--------------------------------------------------------------------===//
// Integer Promotion Support: LegalizeIntegerTypes.cpp
//===--------------------------------------------------------------------===//
@@ -563,6 +572,91 @@ private:
SDValue SplitVecOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
+ // Vector Widening Support: LegalizeVectorTypes.cpp
+ //===--------------------------------------------------------------------===//
+ SDValue GetWidenedVector(SDValue Op) {
+ SDValue &WidenedOp = WidenedVectors[Op];
+ RemapValue(WidenedOp);
+ assert(WidenedOp.getNode() && "Operand wasn't widened?");
+ return WidenedOp;
+ }
+ void SetWidenedVector(SDValue Op, SDValue Result);
+
+ // Widen Vector Result Promotion.
+ void WidenVectorResult(SDNode *N, unsigned ResNo);
+ SDValue WidenVecRes_BIT_CONVERT(SDNode* N);
+ SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
+ SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
+ SDValue WidenVecRes_CONVERT_RNDSAT(SDNode* N);
+ SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
+ SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
+ SDValue WidenVecRes_LOAD(SDNode* N);
+ SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
+ SDValue WidenVecRes_SELECT(SDNode* N);
+ SDValue WidenVecRes_SELECT_CC(SDNode* N);
+ SDValue WidenVecRes_UNDEF(SDNode *N);
+ SDValue WidenVecRes_VECTOR_SHUFFLE(SDNode *N);
+ SDValue WidenVecRes_VSETCC(SDNode* N);
+
+ SDValue WidenVecRes_Binary(SDNode *N);
+ SDValue WidenVecRes_Convert(SDNode *N);
+ SDValue WidenVecRes_Shift(SDNode *N);
+ SDValue WidenVecRes_Unary(SDNode *N);
+
+ // Widen Vector Operand.
+ bool WidenVectorOperand(SDNode *N, unsigned ResNo);
+ SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
+ SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
+ SDValue WidenVecOp_STORE(SDNode* N);
+
+ SDValue WidenVecOp_Convert(SDNode *N);
+
+ //===--------------------------------------------------------------------===//
+ // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
+ //===--------------------------------------------------------------------===//
+
+ /// Helper genWidenVectorLoads - Helper function to generate a set of
+ /// loads to load a vector with a resulting wider type. It takes
+ /// ExtType: Extension type
+ /// LdChain: list of chains for the load we have generated.
+ /// Chain: incoming chain for the ld vector.
+ /// BasePtr: base pointer to load from.
+ /// SV: memory disambiguation source value.
+ /// SVOffset: memory disambiugation offset.
+ /// Alignment: alignment of the memory.
+ /// isVolatile: volatile load.
+ /// LdWidth: width of memory that we want to load.
+ /// ResType: the wider result result type for the resulting vector.
+ SDValue GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain, SDValue Chain,
+ SDValue BasePtr, const Value *SV,
+ int SVOffset, unsigned Alignment,
+ bool isVolatile, unsigned LdWidth,
+ MVT ResType);
+
+ /// Helper genWidenVectorStores - Helper function to generate a set of
+ /// stores to store a widen vector into non widen memory
+ /// It takes
+ /// StChain: list of chains for the stores we have generated
+ /// Chain: incoming chain for the ld vector
+ /// BasePtr: base pointer to load from
+ /// SV: memory disambiguation source value
+ /// SVOffset: memory disambiugation offset
+ /// Alignment: alignment of the memory
+ /// isVolatile: volatile lod
+ /// ValOp: value to store
+ /// StWidth: width of memory that we want to store
+ void GenWidenVectorStores(SmallVector<SDValue, 16>& StChain, SDValue Chain,
+ SDValue BasePtr, const Value *SV,
+ int SVOffset, unsigned Alignment,
+ bool isVolatile, SDValue ValOp,
+ unsigned StWidth);
+
+ /// Modifies a vector input (widen or narrows) to a vector of NVT. The
+ /// input vector must have the same element type as NVT.
+ SDValue ModifyToType(SDValue InOp, MVT WidenVT);
+
+
+ //===--------------------------------------------------------------------===//
// Generic Splitting: LegalizeTypesGeneric.cpp
//===--------------------------------------------------------------------===//
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5f15fa506d..5c34db489b 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -371,7 +371,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
- case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break;
+ case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break;
case ISD::VECTOR_SHUFFLE: SplitVecRes_VECTOR_SHUFFLE(N, Lo, Hi); break;
case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break;
@@ -1062,3 +1062,1034 @@ SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE(SDNode *N, unsigned OpNo) {
assert(false && "Failed to find an appropriate mask type!");
return SDValue(N, 0);
}
+
+
+//===----------------------------------------------------------------------===//
+// Result Vector Widening
+//===----------------------------------------------------------------------===//
+
+void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
+ DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG);
+ cerr << "\n");
+ SDValue Res = SDValue();
+
+ switch (N->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ cerr << "WidenVectorResult #" << ResNo << ": ";
+ N->dump(&DAG); cerr << "\n";
+#endif
+ assert(0 && "Do not know how to widen the result of this operator!");
+ abort();
+
+ case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break;
+ case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
+ case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
+ case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break;
+ case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
+ case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
+ case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
+ case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
+ case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
+ case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
+ case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
+ case ISD::VECTOR_SHUFFLE: Res = WidenVecRes_VECTOR_SHUFFLE(N); break;
+ case ISD::VSETCC: Res = WidenVecRes_VSETCC(N); break;
+
+ case ISD::ADD:
+ case ISD::AND:
+ case ISD::BSWAP:
+ case ISD::FADD:
+ case ISD::FCOPYSIGN:
+ case ISD::FDIV:
+ case ISD::FMUL:
+ case ISD::FPOW:
+ case ISD::FPOWI:
+ case ISD::FREM:
+ case ISD::FSUB:
+ case ISD::MUL:
+ case ISD::MULHS:
+ case ISD::MULHU:
+ case ISD::OR:
+ case ISD::SDIV:
+ case ISD::SREM:
+ case ISD::UDIV:
+ case ISD::UREM:
+ case ISD::SUB:
+ case ISD::XOR: Res = WidenVecRes_Binary(N); break;
+
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL: Res = WidenVecRes_Shift(N); break;
+
+ case ISD::ANY_EXTEND:
+ case ISD::FP_ROUND:
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ case ISD::SIGN_EXTEND:
+ case ISD::SINT_TO_FP:
+ case ISD::TRUNCATE:
+ case ISD::ZERO_EXTEND:
+ case ISD::UINT_TO_FP: Res = WidenVecRes_Convert(N); break;
+
+ case ISD::CTLZ:
+ case ISD::CTPOP:
+ case ISD::CTTZ:
+ case ISD::FABS:
+ case ISD::FCOS:
+ case ISD::FNEG:
+ case ISD::FSIN:
+ case ISD::FSQRT: Res = WidenVecRes_Unary(N); break;
+ }
+
+ // If Res is null, the sub-method took care of registering the result.
+ if (Res.getNode())
+ SetWidenedVector(SDValue(N, ResNo), Res);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
+ // Binary op widening.
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ SDValue InOp1 = GetWidenedVector(N->getOperand(0));
+ SDValue InOp2 = GetWidenedVector(N->getOperand(1));
+ return DAG.getNode(N->getOpcode(), WidenVT, InOp1, InOp2);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
+ SDValue InOp = N->getOperand(0);
+
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ unsigned WidenNumElts = WidenVT.getVectorNumElements();
+
+ MVT InVT = InOp.getValueType();
+ MVT InEltVT = InVT.getVectorElementType();
+ MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
+
+ unsigned Opcode = N->getOpcode();
+ unsigned InVTNumElts = InVT.getVectorNumElements();
+
+ if (getTypeAction(InVT) == WidenVector) {
+ InOp = GetWidenedVector(N->getOperand(0));
+ InVT = InOp.getValueType();
+ InVTNumElts = InVT.getVectorNumElements();
+ if (InVTNumElts == WidenNumElts)
+ return DAG.getNode(Opcode, WidenVT, InOp);
+ }
+
+ if (TLI.isTypeLegal(InWidenVT)) {
+ // Because the result and the input are different vector types, widening
+ // the result could create a legal type but widening the input might make
+ // it an illegal type that might lead to repeatedly splitting the input
+ // and then widening it. To avoid this, we widen the input only if
+ // it results in a legal type.
+ if (WidenNumElts % InVTNumElts == 0) {
+ // Widen the input and call convert on the widened input vector.
+ unsigned NumConcat = WidenNumElts/InVTNumElts;
+ SmallVector<SDValue, 16> Ops(NumConcat);
+ Ops[0] = InOp;
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ for (unsigned i = 1; i != NumConcat; ++i)
+ Ops[i] = UndefVal;
+ return DAG.getNode(Opcode, WidenVT,
+ DAG.getNode(ISD::CONCAT_VECTORS, InWidenVT,
+ &Ops[0], NumConcat));
+ }
+
+ if (InVTNumElts % WidenNumElts == 0) {
+ // Extract the input and convert the shorten input vector.
+ return DAG.getNode(Opcode, WidenVT,
+ DAG.getNode(ISD::EXTRACT_SUBVECTOR, InWidenVT, InOp,
+ DAG.getIntPtrConstant(0)));
+ }
+ }
+
+ // Otherwise unroll into some nasty scalar code and rebuild the vector.
+ SmallVector<SDValue, 16> Ops(WidenNumElts);
+ MVT EltVT = WidenVT.getVectorElementType();
+ unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
+ unsigned i;
+ for (i=0; i < MinElts; ++i)
+ Ops[i] = DAG.getNode(Opcode, EltVT,
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, InEltVT, InOp,
+ DAG.getIntPtrConstant(i)));
+
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, EltVT);
+ for (; i < WidenNumElts; ++i)
+ Ops[i] = UndefVal;
+
+ return DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &Ops[0], WidenNumElts);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ SDValue InOp = GetWidenedVector(N->getOperand(0));
+ SDValue ShOp = N->getOperand(1);
+
+ MVT ShVT = ShOp.getValueType();
+ if (getTypeAction(ShVT) == WidenVector) {
+ ShOp = GetWidenedVector(ShOp);
+ ShVT = ShOp.getValueType();
+ }
+ MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(),
+ WidenVT.getVectorNumElements());
+ if (ShVT != ShWidenVT)
+ ShOp = ModifyToType(ShOp, ShWidenVT);
+
+ return DAG.getNode(N->getOpcode(), WidenVT, InOp, ShOp);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
+ // Unary op widening.
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ SDValue InOp = GetWidenedVector(N->getOperand(0));
+ return DAG.getNode(N->getOpcode(), WidenVT, InOp);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) {
+ SDValue InOp = N->getOperand(0);
+ MVT InVT = InOp.getValueType();
+ MVT VT = N->getValueType(0);
+ MVT WidenVT = TLI.getTypeToTransformTo(VT);
+
+ switch (getTypeAction(InVT)) {
+ default:
+ assert(false && "Unknown type action!");
+ break;
+ case Legal:
+ break;
+ case PromoteInteger:
+ // If the InOp is promoted to the same size, convert it. Otherwise,
+ // fall out of the switch and widen the promoted input.
+ InOp = GetPromotedInteger(InOp);
+ InVT = InOp.getValueType();
+ if (WidenVT.bitsEq(InVT))
+ return DAG.getNode(ISD::BIT_CONVERT, WidenVT, InOp);
+ break;
+ case SoftenFloat:
+ case ExpandInteger:
+ case ExpandFloat:
+ case ScalarizeVector:
+ case SplitVector:
+ break;
+ case WidenVector:
+ // If the InOp is widened to the same size, convert it. Otherwise, fall
+ // out of the switch and widen the widened input.
+ InOp = GetWidenedVector(InOp);
+ InVT = InOp.getValueType();
+ if (WidenVT.bitsEq(InVT))
+ // The input widens to the same size. Convert to the widen value.
+ return DAG.getNode(ISD::BIT_CONVERT, WidenVT, InOp);
+ break;
+ }
+
+ unsigned WidenSize = WidenVT.getSizeInBits();
+ unsigned InSize = InVT.getSizeInBits();
+ if (WidenSize % InSize == 0) {
+ // Determine new input vector type. The new input vector type will use
+ // the same element type (if its a vector) or use the input type as a
+ // vector. It is the same size as the type to widen to.
+ MVT NewInVT;
+ unsigned NewNumElts = WidenSize / InSize;
+ if (InVT.isVector()) {
+ MVT InEltVT = InVT.getVectorElementType();
+ NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits());
+ } else {
+ NewInVT = MVT::getVectorVT(InVT, NewNumElts);
+ }
+
+ if (TLI.isTypeLegal(NewInVT)) {
+ // Because the result and the input are different vector types, widening
+ // the result could create a legal type but widening the input might make
+ // it an illegal type that might lead to repeatedly splitting the input
+ // and then widening it. To avoid this, we widen the input only if
+ // it results in a legal type.
+ SmallVector<SDValue, 16> Ops(NewNumElts);
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ Ops[0] = InOp;
+ for (unsigned i = 1; i < NewNumElts; ++i)
+ Ops[i] = UndefVal;
+
+ SDValue NewVec;
+ if (InVT.isVector())
+ NewVec = DAG.getNode(ISD::CONCAT_VECTORS, NewInVT, &Ops[0], NewNumElts);
+ else
+ NewVec = DAG.getNode(ISD::BUILD_VECTOR, NewInVT, &Ops[0], NewNumElts);
+ return DAG.getNode(ISD::BIT_CONVERT, WidenVT, NewVec);
+ }
+ }
+
+ // This should occur rarely. Lower the bit-convert to a store/load
+ // from the stack. Create the stack frame object. Make sure it is aligned
+ // for both the source and destination types.
+ SDValue FIPtr = DAG.CreateStackTemporary(InVT, WidenVT);
+
+ // Emit a store to the stack slot.
+ SDValue Store = DAG.getStore(DAG.getEntryNode(), InOp, FIPtr, NULL, 0);
+
+ // Result is a load from the stack slot.
+ return DAG.getLoad(WidenVT, Store, FIPtr, NULL, 0);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
+ // Build a vector with undefined for the new nodes.
+ MVT VT = N->getValueType(0);
+ MVT EltVT = VT.getVectorElementType();
+ unsigned NumElts = VT.getVectorNumElements();
+
+ MVT WidenVT = TLI.getTypeToTransformTo(VT);
+ unsigned WidenNumElts = WidenVT.getVectorNumElements();
+
+ SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
+ NewOps.reserve(WidenNumElts);
+ for (unsigned i = NumElts; i < WidenNumElts; ++i)
+ NewOps.push_back(DAG.getNode(ISD::UNDEF, EltVT));
+
+ return DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size());
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
+ MVT InVT = N->getOperand(0).getValueType();
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ unsigned WidenNumElts = WidenVT.getVectorNumElements();
+ unsigned NumOperands = N->getNumOperands();
+
+ bool InputWidened = false; // Indicates we need to widen the input.
+ if (getTypeAction(InVT) != WidenVector) {
+ if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
+ // Add undef vectors to widen to correct length.
+ unsigned NumConcat = WidenVT.getVectorNumElements() /
+ InVT.getVectorNumElements();
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ SmallVector<SDValue, 16> Ops(NumConcat);
+ for (unsigned i=0; i < NumOperands; ++i)
+ Ops[i] = N->getOperand(i);
+ for (unsigned i = NumOperands; i != NumConcat; ++i)
+ Ops[i] = UndefVal;
+ return DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, &Ops[0], NumConcat);
+ }
+ } else {
+ InputWidened = true;
+ if (WidenVT == TLI.getTypeToTransformTo(InVT)) {
+ // The inputs and the result are widen to the same value.
+ unsigned i;
+ for (i=1; i < NumOperands; ++i)
+ if (N->getOperand(i).getOpcode() != ISD::UNDEF)
+ break;
+
+ if (i > NumOperands)
+ // Everything but the first operand is an UNDEF so just return the
+ // widened first operand.
+ return GetWidenedVector(N->getOperand(0));
+
+ if (NumOperands == 2) {
+ // Replace concat of two operands with a shuffle.
+ MVT PtrVT = TLI.getPointerTy();
+ SmallVector<SDValue, 16> MaskOps(WidenNumElts);
+ for (unsigned i=0; i < WidenNumElts/2; ++i) {
+ MaskOps[i] = DAG.getConstant(i, PtrVT);
+ MaskOps[i+WidenNumElts/2] = DAG.getConstant(i+WidenNumElts, PtrVT);
+ }
+ SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR,
+ MVT::getVectorVT(PtrVT, WidenNumElts),
+ &MaskOps[0], WidenNumElts);
+ return DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT,
+ GetWidenedVector(N->getOperand(0)),
+ GetWidenedVector(N->getOperand(1)), Mask);
+ }
+ }
+ }
+
+ // Fall back to use extracts and build vector.
+ MVT EltVT = WidenVT.getVectorElementType();
+ unsigned NumInElts = InVT.getVectorNumElements();
+ SmallVector<SDValue, 16> Ops(WidenNumElts);
+ unsigned Idx = 0;
+ for (unsigned i=0; i < NumOperands; ++i) {
+ SDValue InOp = N->getOperand(i);
+ if (InputWidened)
+ InOp = GetWidenedVector(InOp);
+ for (unsigned j=0; j < NumInElts; ++j)
+ Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, InOp,
+ DAG.getIntPtrConstant(j));
+ }
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, EltVT);
+ for (; Idx < WidenNumElts; ++Idx)
+ Ops[Idx] = UndefVal;
+ return DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &Ops[0], WidenNumElts);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
+ SDValue InOp = N->getOperand(0);
+ SDValue RndOp = N->getOperand(3);
+ SDValue SatOp = N->getOperand(4);
+
+ MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
+ unsigned WidenNumElts = WidenVT.getVectorNumElements();
+
+ MVT InVT = InOp.getValueType();
+ MVT InEltVT = InVT.getVectorElementType();
+ MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
+
+ SDValue DTyOp = DAG.getValueType(WidenVT);
+ SDValue STyOp = DAG.getValueType(InWidenVT);
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
+
+ unsigned InVTNumElts = InVT.getVectorNumElements();
+ if (getTypeAction(InVT) == WidenVector) {
+ InOp = GetWidenedVector(InOp);
+ InVT = InOp.getValueType();
+ InVTNumElts = InVT.getVectorNumElements();
+ if (InVTNumElts == WidenNumElts)
+ return DAG.getConvertRndSat(WidenVT, InOp, DTyOp, STyOp, RndOp,
+ SatOp, CvtCode);
+ }
+
+ if (TLI.isTypeLegal(InWidenVT)) {
+ // Because the result and the input are different vector types, widening
+ // the result could create a legal type but widening the input might make
+ // it an illegal type that might lead to repeatedly splitting the input
+ // and then widening it. To avoid this, we widen the input only if
+ // it results in a legal type.
+ if (WidenNumElts % InVTNumElts == 0) {
+ // Widen the input and call convert on the widened input vector.
+ unsigned NumConcat = WidenNumElts/InVTNumElts;
+ SmallVector<SDValue, 16> Ops(NumConcat);
+ Ops[0] = InOp;
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ Ops[i] = UndefVal;
+ }
+ InOp = DAG.getNode(ISD::CONCAT_VECTORS, InWidenVT, &Ops[0], NumConcat);
+ return DAG.getConvertRndSat(WidenVT, InOp, DTyOp, STyOp, RndOp,
+ SatOp, CvtCode);
+ }
+
+ if (InVTNumElts % WidenNumElts == 0) {
+ // Extract the input and convert the shorten input vector.
+ InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, InWidenVT, InOp,
+ DAG.getIntPtrConstant(0));
+ return DAG.getConvertRndSat(WidenVT, InOp, DTyOp, STyOp, RndOp,
+ SatOp, CvtCode);
+ }
+ }
+
+ // Otherwise unroll into some nasty scalar code and rebuild the vector.
+ SmallVector<SDValue, 16> Ops(WidenNumElts);
+ MVT EltVT = WidenVT.getVectorElementType();
+ DTyOp = DAG.getValueType(EltVT);
+ STyOp = DAG.getValueType(InEltVT);
+
+ unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
+ unsigned i;
+ for (i=0; i < MinElts; ++i) {
+ SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, InEltVT, InOp,
+ DAG.getIntPtrConstant(i));
+ Ops[i] = DAG.getConvertRndSat(WidenVT, ExtVal, DTyOp, STyOp, RndOp,
+ SatOp, CvtCode);
+ }
+
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, EltVT);
+ for (; i < WidenNumElts; ++i)
+ Ops[i] = UndefVal;
+
+ return DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &Ops[0], WidenNumElts);
+}
+
+SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
+ MVT VT = N->getValueType(0);
+ MVT WidenVT = TLI.getTypeToTransformTo(VT);
+ unsigned WidenNumElts = WidenVT.getVectorNumElements();
+ SDValue InOp = N->getOperand(0);
+ SDValue Idx = N->getOperand(1);
+
+ if (getTypeAction(InOp.getValueType()) == WidenVector)
+ InOp = GetW