diff options
31 files changed, 687 insertions, 750 deletions
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 2b8d678eb3..8ea14a5dfd 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -323,17 +323,20 @@ public: SDOperand getNode(unsigned Opcode, SDVTList VTs, const SDOperand *Ops, unsigned NumOps); - SDOperand getMemcpy(SDOperand Chain, SDOperand Dest, SDOperand Src, - SDOperand Size, SDOperand Align, - SDOperand AlwaysInline); - - SDOperand getMemmove(SDOperand Chain, SDOperand Dest, SDOperand Src, - SDOperand Size, SDOperand Align, - SDOperand AlwaysInline); - - SDOperand getMemset(SDOperand Chain, SDOperand Dest, SDOperand Src, - SDOperand Size, SDOperand Align, - SDOperand AlwaysInline); + SDOperand getMemcpy(SDOperand Chain, SDOperand Dst, SDOperand Src, + SDOperand Size, unsigned Align, + bool AlwaysInline, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff); + + SDOperand getMemmove(SDOperand Chain, SDOperand Dst, SDOperand Src, + SDOperand Size, unsigned Align, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff); + + SDOperand getMemset(SDOperand Chain, SDOperand Dst, SDOperand Src, + SDOperand Size, unsigned Align, + Value *DstSV, uint64_t DstOff); /// getSetCC - Helper function to make it easier to build SetCC's if you just /// have an ISD::CondCode instead of an SDOperand. diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 6b2b8572c3..deded1a36a 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -497,14 +497,6 @@ namespace ISD { // it returns an output chain. STACKRESTORE, - // MEMSET/MEMCPY/MEMMOVE - The first operand is the chain. The following - // correspond to the operands of the LLVM intrinsic functions and the last - // one is AlwaysInline. The only result is a token chain. The alignment - // argument is guaranteed to be a Constant node. - MEMSET, - MEMMOVE, - MEMCPY, - // CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of // a call sequence, and carry arbitrary information that target might want // to know. The first operand is a chain, the rest are specified by the diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h index 719f719846..16f9ed63b3 100644 --- a/include/llvm/Target/TargetLowering.h +++ b/include/llvm/Target/TargetLowering.h @@ -948,18 +948,61 @@ public: SDOperand Callee, ArgListTy &Args, SelectionDAG &DAG); - virtual SDOperand LowerMEMCPY(SDOperand Op, SelectionDAG &DAG); - virtual SDOperand LowerMEMCPYCall(SDOperand Chain, SDOperand Dest, - SDOperand Source, SDOperand Count, - SelectionDAG &DAG); - virtual SDOperand LowerMEMCPYInline(SDOperand Chain, SDOperand Dest, - SDOperand Source, unsigned Size, - unsigned Align, SelectionDAG &DAG) { - assert(0 && "Not Implemented"); - return SDOperand(); // this is here to silence compiler errors + /// EmitTargetCodeForMemcpy - Emit target-specific code that performs a + /// memcpy. This can be used by targets to provide code sequences for cases + /// that don't fit the target's parameters for simple loads/stores and can be + /// more efficient than using a library call. This function can return a null + /// SDOperand if the target declines to use inline code and a different + /// lowering strategy should be used. + /// + /// If AlwaysInline is true, the size is constant and the target should not + /// emit any calls and is strongly encouraged to attempt to emit inline code + /// even if it is beyond the usual threshold because this intrinsic is being + /// expanded in a place where calls are not feasible (e.g. within the prologue + /// for another call). If the target chooses to decline an AlwaysInline + /// request here, legalize will resort to using simple loads and stores. + virtual SDOperand + EmitTargetCodeForMemcpy(SelectionDAG &DAG, + SDOperand Chain, + SDOperand Op1, SDOperand Op2, + SDOperand Op3, unsigned Align, + bool AlwaysInline, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff) { + return SDOperand(); + } + + /// EmitTargetCodeForMemmove - Emit target-specific code that performs a + /// memmove. This can be used by targets to provide code sequences for cases + /// that don't fit the target's parameters for simple loads/stores and can be + /// more efficient than using a library call. This function can return a null + /// SDOperand if the target declines to use code and a different lowering + /// strategy should be used. + virtual SDOperand + EmitTargetCodeForMemmove(SelectionDAG &DAG, + SDOperand Chain, + SDOperand Op1, SDOperand Op2, + SDOperand Op3, unsigned Align, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff) { + return SDOperand(); + } + + /// EmitTargetCodeForMemset - Emit target-specific code that performs a + /// memset. This can be used by targets to provide code sequences for cases + /// that don't fit the target's parameters for simple stores and can be more + /// efficient than using a library call. This function can return a null + /// SDOperand if the target declines to use code and a different lowering + /// strategy should be used. + virtual SDOperand + EmitTargetCodeForMemset(SelectionDAG &DAG, + SDOperand Chain, + SDOperand Op1, SDOperand Op2, + SDOperand Op3, unsigned Align, + Value *DstSV, uint64_t DstOff) { + return SDOperand(); } - /// LowerOperation - This callback is invoked for operations that are /// unsupported by the target, which are registered to use 'custom' lowering, /// and whose defined values are all legal. diff --git a/include/llvm/Target/TargetSubtarget.h b/include/llvm/Target/TargetSubtarget.h index 1096b1635c..fde8f44669 100644 --- a/include/llvm/Target/TargetSubtarget.h +++ b/include/llvm/Target/TargetSubtarget.h @@ -28,9 +28,6 @@ class TargetSubtarget { protected: // Can only create subclasses... TargetSubtarget(); public: - /// getMaxInlineSizeThreshold - Returns the maximum memset / memcpy size - /// that still makes it profitable to inline the call. - virtual unsigned getMaxInlineSizeThreshold() const {return 0; } virtual ~TargetSubtarget(); }; diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 5cb13e3f44..2df363e5be 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -22,6 +22,7 @@ #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Target/TargetSubtarget.h" #include "llvm/CallingConv.h" #include "llvm/Constants.h" #include "llvm/DerivedTypes.h" @@ -2842,123 +2843,6 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } break; - case ISD::MEMSET: - case ISD::MEMCPY: - case ISD::MEMMOVE: { - Tmp1 = LegalizeOp(Node->getOperand(0)); // Chain - Tmp2 = LegalizeOp(Node->getOperand(1)); // Pointer - - if (Node->getOpcode() == ISD::MEMSET) { // memset = ubyte - switch (getTypeAction(Node->getOperand(2).getValueType())) { - case Expand: assert(0 && "Cannot expand a byte!"); - case Legal: - Tmp3 = LegalizeOp(Node->getOperand(2)); - break; - case Promote: - Tmp3 = PromoteOp(Node->getOperand(2)); - break; - } - } else { - Tmp3 = LegalizeOp(Node->getOperand(2)); // memcpy/move = pointer, - } - - SDOperand Tmp4; - switch (getTypeAction(Node->getOperand(3).getValueType())) { - case Expand: { - // Length is too big, just take the lo-part of the length. - SDOperand HiPart; - ExpandOp(Node->getOperand(3), Tmp4, HiPart); - break; - } - case Legal: - Tmp4 = LegalizeOp(Node->getOperand(3)); - break; - case Promote: - Tmp4 = PromoteOp(Node->getOperand(3)); - break; - } - - SDOperand Tmp5; - switch (getTypeAction(Node->getOperand(4).getValueType())) { // uint - case Expand: assert(0 && "Cannot expand this yet!"); - case Legal: - Tmp5 = LegalizeOp(Node->getOperand(4)); - break; - case Promote: - Tmp5 = PromoteOp(Node->getOperand(4)); - break; - } - - SDOperand Tmp6; - switch (getTypeAction(Node->getOperand(5).getValueType())) { // bool - case Expand: assert(0 && "Cannot expand this yet!"); - case Legal: - Tmp6 = LegalizeOp(Node->getOperand(5)); - break; - case Promote: - Tmp6 = PromoteOp(Node->getOperand(5)); - break; - } - - switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { - default: assert(0 && "This action not implemented for this operation!"); - case TargetLowering::Custom: - isCustom = true; - // FALLTHROUGH - case TargetLowering::Legal: { - SDOperand Ops[] = { Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6 }; - Result = DAG.UpdateNodeOperands(Result, Ops, 6); - if (isCustom) { - Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; - } - break; - } - case TargetLowering::Expand: { - // Otherwise, the target does not support this operation. Lower the - // operation to an explicit libcall as appropriate. - MVT::ValueType IntPtr = TLI.getPointerTy(); - const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(); - TargetLowering::ArgListTy Args; - TargetLowering::ArgListEntry Entry; - - const char *FnName = 0; - if (Node->getOpcode() == ISD::MEMSET) { - Entry.Node = Tmp2; Entry.Ty = IntPtrTy; - Args.push_back(Entry); - // Extend the (previously legalized) ubyte argument to be an int value - // for the call. - if (Tmp3.getValueType() > MVT::i32) - Tmp3 = DAG.getNode(ISD::TRUNCATE, MVT::i32, Tmp3); - else - Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Tmp3); - Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSExt = true; - Args.push_back(Entry); - Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSExt = false; - Args.push_back(Entry); - - FnName = "memset"; - } else if (Node->getOpcode() == ISD::MEMCPY || - Node->getOpcode() == ISD::MEMMOVE) { - Entry.Ty = IntPtrTy; - Entry.Node = Tmp2; Args.push_back(Entry); - Entry.Node = Tmp3; Args.push_back(Entry); - Entry.Node = Tmp4; Args.push_back(Entry); - FnName = Node->getOpcode() == ISD::MEMMOVE ? "memmove" : "memcpy"; - } else { - assert(0 && "Unknown op!"); - } - - std::pair<SDOperand,SDOperand> CallResult = - TLI.LowerCallTo(Tmp1, Type::VoidTy, - false, false, false, CallingConv::C, false, - DAG.getExternalSymbol(FnName, IntPtr), Args, DAG); - Result = CallResult.second; - break; - } - } - break; - } case ISD::SHL_PARTS: case ISD::SRA_PARTS: diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index 6511cff1c6..380c42220c 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -439,51 +439,6 @@ SDOperand DAGTypeLegalizer::CreateStackStoreLoad(SDOperand Op, return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0); } -/// HandleMemIntrinsic - This handles memcpy/memset/memmove with invalid -/// operands. This promotes or expands the operands as required. -SDOperand DAGTypeLegalizer::HandleMemIntrinsic(SDNode *N) { - // The chain and pointer [operands #0 and #1] are always valid types. - SDOperand Chain = N->getOperand(0); - SDOperand Ptr = N->getOperand(1); - SDOperand Op2 = N->getOperand(2); - - // Op #2 is either a value (memset) or a pointer. Promote it if required. - switch (getTypeAction(Op2.getValueType())) { - default: assert(0 && "Unknown action for pointer/value operand"); - case Legal: break; - case Promote: Op2 = GetPromotedOp(Op2); break; - } - - // The length could have any action required. - SDOperand Length = N->getOperand(3); - switch (getTypeAction(Length.getValueType())) { - default: assert(0 && "Unknown action for memop operand"); - case Legal: break; - case Promote: Length = GetPromotedZExtOp(Length); break; - case Expand: - SDOperand Dummy; // discard the high part. - GetExpandedOp(Length, Length, Dummy); - break; - } - - SDOperand Align = N->getOperand(4); - switch (getTypeAction(Align.getValueType())) { - default: assert(0 && "Unknown action for memop operand"); - case Legal: break; - case Promote: Align = GetPromotedZExtOp(Align); break; - } - - SDOperand AlwaysInline = N->getOperand(5); - switch (getTypeAction(AlwaysInline.getValueType())) { - default: assert(0 && "Unknown action for memop operand"); - case Legal: break; - case Promote: AlwaysInline = GetPromotedZExtOp(AlwaysInline); break; - } - - SDOperand Ops[] = { Chain, Ptr, Op2, Length, Align, AlwaysInline }; - return DAG.UpdateNodeOperands(SDOperand(N, 0), Ops, 6); -} - /// JoinIntegers - Build an integer with low bits Lo and high bits Hi. SDOperand DAGTypeLegalizer::JoinIntegers(SDOperand Lo, SDOperand Hi) { MVT::ValueType LVT = Lo.getValueType(); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 7d245abed5..5b9879315f 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -165,7 +165,6 @@ private: // Common routines. SDOperand BitConvertToInteger(SDOperand Op); SDOperand CreateStackStoreLoad(SDOperand Op, MVT::ValueType DestVT); - SDOperand HandleMemIntrinsic(SDNode *N); SDOperand JoinIntegers(SDOperand Lo, SDOperand Hi); void SplitInteger(SDOperand Op, SDOperand &Lo, SDOperand &Hi); void SplitInteger(SDOperand Op, MVT::ValueType LoVT, MVT::ValueType HiVT, diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp index b872a44fec..fcde8f32d2 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesExpand.cpp @@ -946,9 +946,6 @@ bool DAGTypeLegalizer::ExpandOperand(SDNode *N, unsigned OpNo) { case ISD::STORE: Res = ExpandOperand_STORE(cast<StoreSDNode>(N), OpNo); break; - case ISD::MEMSET: - case ISD::MEMCPY: - case ISD::MEMMOVE: Res = HandleMemIntrinsic(N); break; case ISD::BUILD_VECTOR: Res = ExpandOperand_BUILD_VECTOR(N); break; } diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp index b8118eb039..93c8c60584 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesPromote.cpp @@ -447,9 +447,6 @@ bool DAGTypeLegalizer::PromoteOperand(SDNode *N, unsigned OpNo) { case ISD::STORE: Res = PromoteOperand_STORE(cast<StoreSDNode>(N), OpNo); break; - case ISD::MEMSET: - case ISD::MEMCPY: - case ISD::MEMMOVE: Res = HandleMemIntrinsic(N); break; case ISD::BUILD_VECTOR: Res = PromoteOperand_BUILD_VECTOR(N); break; case ISD::INSERT_VECTOR_ELT: diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index f096c70a3e..327a8fe897 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -17,6 +17,7 @@ #include "llvm/Intrinsics.h" #include "llvm/DerivedTypes.h" #include "llvm/Assembly/Writer.h" +#include "llvm/CallingConv.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -2385,28 +2386,357 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, return getNode(Opcode, VT, Ops, 5); } -SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dest, - SDOperand Src, SDOperand Size, - SDOperand Align, - SDOperand AlwaysInline) { - SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline }; - return getNode(ISD::MEMCPY, MVT::Other, Ops, 6); +/// getMemsetValue - Vectorized representation of the memset value +/// operand. +static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT, + SelectionDAG &DAG) { + MVT::ValueType CurVT = VT; + if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { + uint64_t Val = C->getValue() & 255; + unsigned Shift = 8; + while (CurVT != MVT::i8) { + Val = (Val << Shift) | Val; + Shift <<= 1; + CurVT = (MVT::ValueType)((unsigned)CurVT - 1); + } + return DAG.getConstant(Val, VT); + } else { + Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value); + unsigned Shift = 8; + while (CurVT != MVT::i8) { + Value = + DAG.getNode(ISD::OR, VT, + DAG.getNode(ISD::SHL, VT, Value, + DAG.getConstant(Shift, MVT::i8)), Value); + Shift <<= 1; + CurVT = (MVT::ValueType)((unsigned)CurVT - 1); + } + + return Value; + } } -SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dest, - SDOperand Src, SDOperand Size, - SDOperand Align, - SDOperand AlwaysInline) { - SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline }; - return getNode(ISD::MEMMOVE, MVT::Other, Ops, 6); +/// getMemsetStringVal - Similar to getMemsetValue. Except this is only +/// used when a memcpy is turned into a memset when the source is a constant +/// string ptr. +static SDOperand getMemsetStringVal(MVT::ValueType VT, + SelectionDAG &DAG, + const TargetLowering &TLI, + std::string &Str, unsigned Offset) { + uint64_t Val = 0; + unsigned MSB = MVT::getSizeInBits(VT) / 8; + if (TLI.isLittleEndian()) + Offset = Offset + MSB - 1; + for (unsigned i = 0; i != MSB; ++i) { + Val = (Val << 8) | (unsigned char)Str[Offset]; + Offset += TLI.isLittleEndian() ? -1 : 1; + } + return DAG.getConstant(Val, VT); +} + +/// getMemBasePlusOffset - Returns base and offset node for the +static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset, + SelectionDAG &DAG) { + MVT::ValueType VT = Base.getValueType(); + return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT)); } -SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dest, +/// MeetsMaxMemopRequirement - Determines if the number of memory ops required +/// to replace the memset / memcpy is below the threshold. It also returns the +/// types of the sequence of memory ops to perform memset / memcpy. +static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps, + unsigned Limit, uint64_t Size, + unsigned Align, + const TargetLowering &TLI) { + MVT::ValueType VT; + + if (TLI.allowsUnalignedMemoryAccesses()) { + VT = MVT::i64; + } else { + switch (Align & 7) { + case 0: + VT = MVT::i64; + break; + case 4: + VT = MVT::i32; + break; + case 2: + VT = MVT::i16; + break; + default: + VT = MVT::i8; + break; + } + } + + MVT::ValueType LVT = MVT::i64; + while (!TLI.isTypeLegal(LVT)) + LVT = (MVT::ValueType)((unsigned)LVT - 1); + assert(MVT::isInteger(LVT)); + + if (VT > LVT) + VT = LVT; + + unsigned NumMemOps = 0; + while (Size != 0) { + unsigned VTSize = MVT::getSizeInBits(VT) / 8; + while (VTSize > Size) { + VT = (MVT::ValueType)((unsigned)VT - 1); + VTSize >>= 1; + } + assert(MVT::isInteger(VT)); + + if (++NumMemOps > Limit) + return false; + MemOps.push_back(VT); + Size -= VTSize; + } + + return true; +} + +static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG, + SDOperand Chain, SDOperand Dst, + SDOperand Src, uint64_t Size, + unsigned Align, + bool AlwaysInline, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + + // Expand memcpy to a series of store ops if the size operand falls below + // a certain threshold. + std::vector<MVT::ValueType> MemOps; + uint64_t Limit = -1; + if (!AlwaysInline) + Limit = TLI.getMaxStoresPerMemcpy(); + if (!MeetsMaxMemopRequirement(MemOps, Limit, Size, Align, TLI)) + return SDOperand(); + + SmallVector<SDOperand, 8> OutChains; + + unsigned NumMemOps = MemOps.size(); + unsigned SrcDelta = 0; + GlobalAddressSDNode *G = NULL; + std::string Str; + bool CopyFromStr = false; + + if (Src.getOpcode() == ISD::GlobalAddress) + G = cast<GlobalAddressSDNode>(Src); + else if (Src.getOpcode() == ISD::ADD && + Src.getOperand(0).getOpcode() == ISD::GlobalAddress && + Src.getOperand(1).getOpcode() == ISD::Constant) { + G = cast<GlobalAddressSDNode>(Src.getOperand(0)); + SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue(); + } + if (G) { + GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); + if (GV && GV->isConstant()) { + Str = GV->getStringValue(false); + if (!Str.empty()) { + CopyFromStr = true; + SrcOff += SrcDelta; + } + } + } + + for (unsigned i = 0; i < NumMemOps; i++) { + MVT::ValueType VT = MemOps[i]; + unsigned VTSize = MVT::getSizeInBits(VT) / 8; + SDOperand Value, Store; + + if (CopyFromStr) { + Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff); + Store = + DAG.getStore(Chain, Value, + getMemBasePlusOffset(Dst, DstOff, DAG), + DstSV, DstOff); + } else { + Value = DAG.getLoad(VT, Chain, + getMemBasePlusOffset(Src, SrcOff, DAG), + SrcSV, SrcOff, false, Align); + Store = + DAG.getStore(Chain, Value, + getMemBasePlusOffset(Dst, DstOff, DAG), + DstSV, DstOff, false, Align); + } + OutChains.push_back(Store); + SrcOff += VTSize; + DstOff += VTSize; + } + + return DAG.getNode(ISD::TokenFactor, MVT::Other, + &OutChains[0], OutChains.size()); +} + +static SDOperand getMemsetStores(SelectionDAG &DAG, + SDOperand Chain, SDOperand Dst, + SDOperand Src, uint64_t Size, + unsigned Align, + Value *DstSV, uint64_t DstOff) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + + // Expand memset to a series of load/store ops if the size operand + // falls below a certain threshold. + std::vector<MVT::ValueType> MemOps; + if (!MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(), + Size, Align, TLI)) + return SDOperand(); + + SmallVector<SDOperand, 8> OutChains; + + unsigned NumMemOps = MemOps.size(); + for (unsigned i = 0; i < NumMemOps; i++) { + MVT::ValueType VT = MemOps[i]; + unsigned VTSize = MVT::getSizeInBits(VT) / 8; + SDOperand Value = getMemsetValue(Src, VT, DAG); + SDOperand Store = DAG.getStore(Chain, Value, + getMemBasePlusOffset(Dst, DstOff, DAG), + DstSV, DstOff); + OutChains.push_back(Store); + DstOff += VTSize; + } + + return DAG.getNode(ISD::TokenFactor, MVT::Other, + &OutChains[0], OutChains.size()); +} + +SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst, + SDOperand Src, SDOperand Size, + unsigned Align, bool AlwaysInline, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff) { + + // Check to see if we should lower the memcpy to loads and stores first. + // For cases within the target-specified limits, this is the best choice. + ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); + if (ConstantSize) { + // Memcpy with size zero? Just return the original chain. + if (ConstantSize->isNullValue()) + return Chain; + + SDOperand Result = + getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(), + Align, false, DstSV, DstOff, SrcSV, SrcOff); + if (Result.Val) + return Result; + } + + // Then check to see if we should lower the memcpy with target-specific + // code. If the target chooses to do this, this is the next best. + SDOperand Result = + TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align, + AlwaysInline, + DstSV, DstOff, SrcSV, SrcOff); + if (Result.Val) + return Result; + + // If we really need inline code and the target declined to provide it, + // use a (potentially long) sequence of loads and stores. + if (AlwaysInline) { + assert(ConstantSize && "AlwaysInline requires a constant size!"); + return getMemcpyLoadsAndStores(*this, Chain, Dst, Src, + ConstantSize->getValue(), Align, true, + DstSV, DstOff, SrcSV, SrcOff); + } + + // Emit a library call. + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Entry.Ty = TLI.getTargetData()->getIntPtrType(); + Entry.Node = Dst; Args.push_back(Entry); + Entry.Node = Src; Args.push_back(Entry); + Entry.Node = Size; Args.push_back(Entry); + std::pair<SDOperand,SDOperand> CallResult = + TLI.LowerCallTo(Chain, Type::VoidTy, + false, false, false, CallingConv::C, false, + getExternalSymbol("memcpy", TLI.getPointerTy()), + Args, *this); + return CallResult.second; +} + +SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst, + SDOperand Src, SDOperand Size, + unsigned Align, + Value *DstSV, uint64_t DstOff, + Value *SrcSV, uint64_t SrcOff) { + + // TODO: Optimize small memmove cases with simple loads and stores, + // ensuring that all loads precede all stores. This can cause severe + // register pressure, so targets should be careful with the size limit. + + // Then check to see if we should lower the memmove with target-specific + // code. If the target chooses to do this, this is the next best. + SDOperand Result = + TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align, + DstSV, DstOff, SrcSV, SrcOff); + if (Result.Val) + return Result; + + // Emit a library call. + TargetLowering::ArgListTy Args; + TargetLowering |