Merge ISD::TRUNCSTORE to ISD::STORE. Switch to using StoreSDNode.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@30945 91177308-0d34-0410-b5e6-96231b3b80d8

23 files changed, 455 insertions, 376 deletions

diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index ded775f778..2624125783 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -319,7 +319,10 @@ public:
   /// getStore - Helper function to build ISD::STORE nodes.
   ///
   SDOperand getStore(SDOperand Chain, SDOperand Value, SDOperand Ptr,
-                     SDOperand SV);
+                     const Value *SV, int SVOffset, bool isVolatile=false);
+  SDOperand getTruncStore(SDOperand Chain, SDOperand Value, SDOperand Ptr,
+                          const Value *SV, int SVOffset, MVT::ValueType TVT,
+                          bool isVolatile=false);
 
   // getSrcValue - construct a node to track a Value* through the backend
   SDOperand getSrcValue(const Value* I, int offset = 0);
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 45a01ac883..dea371e889 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1408,15 +1408,6 @@ protected:
     assert((Off.getOpcode() == ISD::UNDEF || AddrMode == ISD::POST_INDEXED) &&
            "Only post-indexed load has a non-undef offset operand");
   }
-  LoadSDNode(SDOperand Chain, SDOperand Ptr, SDOperand Off,
-             ISD::LoadExtType ETy, MVT::ValueType LVT,
-             const Value *SV, int O=0, unsigned Align=1, bool Vol=false)
-    : SDNode(ISD::LOAD, Chain, Ptr, Off),
-      AddrMode(ISD::UNINDEXED), ExtType(ETy), LoadedVT(LVT), SrcValue(SV),
-      SVOffset(O), Alignment(Align), IsVolatile(Vol) {
-    assert((Off.getOpcode() == ISD::UNDEF || AddrMode == ISD::POST_INDEXED) &&
-           "Only post-indexed load has a non-undef offset operand");
-  }
 public:
 
   const SDOperand getChain() const { return getOperand(0); }
@@ -1461,10 +1452,10 @@ class StoreSDNode : public SDNode {
   bool IsVolatile;
 protected:
   friend class SelectionDAG;
-  StoreSDNode(SDOperand Chain, SDOperand Ptr, SDOperand Off,
+  StoreSDNode(SDOperand Chain, SDOperand Value, SDOperand Ptr, SDOperand Off,
               ISD::MemOpAddrMode AM, bool isTrunc, MVT::ValueType SVT,
               const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
-    : SDNode(ISD::STORE, Chain, Ptr, Off),
+    : SDNode(ISD::STORE, Chain, Value, Ptr, Off),
       AddrMode(AM), IsTruncStore(isTrunc), StoredVT(SVT), SrcValue(SV),
       SVOffset(O), Alignment(Align), IsVolatile(Vol) {
     assert((Off.getOpcode() == ISD::UNDEF || AddrMode == ISD::POST_INDEXED) &&
@@ -1473,8 +1464,9 @@ protected:
 public:
 
   const SDOperand getChain() const { return getOperand(0); }
-  const SDOperand getBasePtr() const { return getOperand(1); }
-  const SDOperand getOffset() const { return getOperand(2); }
+  const SDOperand getValue() const { return getOperand(1); }
+  const SDOperand getBasePtr() const { return getOperand(2); }
+  const SDOperand getOffset() const { return getOperand(3); }
   ISD::MemOpAddrMode getAddressingMode() const { return AddrMode; }
   bool isTruncatingStore() const { return IsTruncStore; }
   MVT::ValueType getStoredVT() const { return StoredVT; }
@@ -1591,6 +1583,20 @@ namespace ISD {
     return N->getOpcode() == ISD::LOAD &&
       cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
   }
+
+  /// isNON_TRUNCStore - Returns true if the specified node is a non-truncating
+  /// store.
+  inline bool isNON_TRUNCStore(const SDNode *N) {
+    return N->getOpcode() == ISD::STORE &&
+      !cast<StoreSDNode>(N)->isTruncatingStore();
+  }
+
+  /// isTRUNCStore - Returns true if the specified node is a truncating
+  /// store.
+  inline bool isTRUNCStore(const SDNode *N) {
+    return N->getOpcode() == ISD::STORE &&
+      cast<StoreSDNode>(N)->isTruncatingStore();
+  }
 }
 
 
diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 87db38c0d7..6f2b07bbcd 100644
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -238,12 +238,26 @@ public:
   }
   
   /// isLoadXLegal - Return true if the specified load with extension is legal
-  /// is legal on this target.
+  /// on this target.
   bool isLoadXLegal(unsigned LType, MVT::ValueType VT) const {
     return getLoadXAction(LType, VT) == Legal ||
            getLoadXAction(LType, VT) == Custom;
   }
   
+  /// getStoreXAction - Return how this store with truncation should be treated:
+  /// either it is legal, needs to be promoted to a larger size, needs to be
+  /// expanded to some other code sequence, or the target has a custom expander
+  /// for it.
+  LegalizeAction getStoreXAction(MVT::ValueType VT) const {
+    return (LegalizeAction)((StoreXActions >> (2*VT)) & 3);
+  }
+  
+  /// isStoreXLegal - Return true if the specified store with truncation is
+  /// legal on this target.
+  bool isStoreXLegal(MVT::ValueType VT) const {
+    return getStoreXAction(VT) == Legal || getStoreXAction(VT) == Custom;
+  }
+
   /// getTypeToPromoteTo - If the action for this operation is to promote, this
   /// method returns the ValueType to promote to.
   MVT::ValueType getTypeToPromoteTo(unsigned Op, MVT::ValueType VT) const {
@@ -559,6 +573,14 @@ protected:
     LoadXActions[ExtType] |= (uint64_t)Action << VT*2;
   }
   
+  /// setStoreXAction - Indicate that the specified store with truncation does
+  /// not work with the with specified type and indicate what to do about it.
+  void setStoreXAction(MVT::ValueType VT, LegalizeAction Action) {
+    assert(VT < 32 && "Table isn't big enough!");
+    StoreXActions &= ~(uint64_t(3UL) << VT*2);
+    StoreXActions |= (uint64_t)Action << VT*2;
+  }
+
   /// AddPromotedToType - If Opc/OrigVT is specified as being promoted, the
   /// promotion code defaults to trying a larger integer/fp until it can find
   /// one that works.  If that default is insufficient, this method can be used
@@ -814,6 +836,11 @@ private:
   /// with the load.
   uint64_t LoadXActions[ISD::LAST_LOADX_TYPE];
   
+  /// StoreXActions - For each store with truncation of each value type, keep a
+  /// LegalizeAction that indicates how instruction selection should deal with
+  /// the store.
+  uint64_t StoreXActions;
+  
   ValueTypeActionImpl ValueTypeActions;
 
   std::vector<double> LegalFPImmediates;
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d6379dc858..06a1f0dd21 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -503,8 +503,6 @@ SDOperand DAGCombiner::visit(SDNode *N) {
   case ISD::BRCOND:             return visitBRCOND(N);
   case ISD::BR_CC:              return visitBR_CC(N);
   case ISD::LOAD:               return visitLOAD(N);
-  // FIXME - Switch over after StoreSDNode comes online.
-  case ISD::TRUNCSTORE:         // Fall thru
   case ISD::STORE:              return visitSTORE(N);
   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
   case ISD::VINSERT_VECTOR_ELT: return visitVINSERT_VECTOR_ELT(N);
@@ -2687,9 +2685,12 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) {
   // TODO: Handle store large -> read small portion.
   // TODO: Handle TRUNCSTORE/LOADEXT
   if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
-    if (Chain.getOpcode() == ISD::STORE && Chain.getOperand(2) == Ptr &&
-        Chain.getOperand(1).getValueType() == N->getValueType(0))
+    if (ISD::isNON_TRUNCStore(Chain.Val)) {
+      StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
+      if (PrevST->getBasePtr() == Ptr &&
+          PrevST->getValue().getValueType() == N->getValueType(0))
       return CombineTo(N, Chain.getOperand(1), Chain);
+    }
   }
     
   if (CombinerAA) {
@@ -2725,13 +2726,13 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) {
 }
 
 SDOperand DAGCombiner::visitSTORE(SDNode *N) {
-  SDOperand Chain    = N->getOperand(0);
-  SDOperand Value    = N->getOperand(1);
-  SDOperand Ptr      = N->getOperand(2);
-  SDOperand SrcValue = N->getOperand(3);
+  StoreSDNode *ST  = cast<StoreSDNode>(N);
+  SDOperand Chain = ST->getChain();
+  SDOperand Value = ST->getValue();
+  SDOperand Ptr   = ST->getBasePtr();
   
   // FIXME - Switch over after StoreSDNode comes online.
-  if (N->getOpcode() == ISD::TRUNCSTORE) {
+  if (ST->isTruncatingStore()) {
     if (CombinerAA) {
       // Walk up chain skipping non-aliasing memory nodes.
       SDOperand BetterChain = FindBetterChain(N, Chain);
@@ -2739,9 +2740,9 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) {
       // If there is a better chain.
       if (Chain != BetterChain) {
         // Replace the chain to avoid dependency.
-        SDOperand ReplTStore = DAG.getNode(ISD::TRUNCSTORE, MVT::Other,
-                                            BetterChain, Value, Ptr, SrcValue,
-                                            N->getOperand(4));
+        SDOperand ReplTStore =
+          DAG.getTruncStore(BetterChain, Value, Ptr, ST->getSrcValue(),
+                            ST->getSrcValueOffset(), ST->getStoredVT());
 
         // Create token to keep both nodes around.
         return DAG.getNode(ISD::TokenFactor, MVT::Other, Chain, ReplTStore);
@@ -2752,27 +2753,30 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) {
   }
  
   // If this is a store that kills a previous store, remove the previous store.
-  if (Chain.getOpcode() == ISD::STORE && Chain.getOperand(2) == Ptr &&
-      Chain.Val->hasOneUse() /* Avoid introducing DAG cycles */ &&
-      // Make sure that these stores are the same value type:
-      // FIXME: we really care that the second store is >= size of the first.
-      Value.getValueType() == Chain.getOperand(1).getValueType()) {
-    // Create a new store of Value that replaces both stores.
-    SDNode *PrevStore = Chain.Val;
-    if (PrevStore->getOperand(1) == Value) // Same value multiply stored.
-      return Chain;
-    SDOperand NewStore = DAG.getStore(PrevStore->getOperand(0), Value, Ptr,
-                                      SrcValue);
-    CombineTo(N, NewStore);                 // Nuke this store.
-    CombineTo(PrevStore, NewStore);  // Nuke the previous store.
-    return SDOperand(N, 0);
+  if (ISD::isNON_TRUNCStore(Chain.Val)) {
+    StoreSDNode *PrevST = cast<StoreSDNode>(Chain);
+    if (PrevST->getBasePtr() == Ptr &&
+        Chain.Val->hasOneUse() /* Avoid introducing DAG cycles */ &&
+        // Make sure that these stores are the same value type:
+        // FIXME: we really care that the second store is >= size of the first.
+        Value.getValueType() == PrevST->getValue().getValueType()) {
+      // Create a new store of Value that replaces both stores.
+      if (PrevST->getValue() == Value) // Same value multiply stored.
+        return Chain;
+      SDOperand NewStore = DAG.getStore(PrevST->getChain(), Value, Ptr,
+                                    ST->getSrcValue(), ST->getSrcValueOffset());
+      CombineTo(N, NewStore);                 // Nuke this store.
+      CombineTo(Chain.Val, NewStore);  // Nuke the previous store.
+      return SDOperand(N, 0);
+    }
   }
   
   // If this is a store of a bit convert, store the input value.
   // FIXME: This needs to know that the resultant store does not need a 
   // higher alignment than the original.
   if (0 && Value.getOpcode() == ISD::BIT_CONVERT) {
-    return DAG.getStore(Chain, Value.getOperand(0), Ptr, SrcValue);
+    return DAG.getStore(Chain, Value.getOperand(0), Ptr, ST->getSrcValue(),
+                        ST->getSrcValueOffset());
   }
   
   if (CombinerAA) { 
@@ -2789,7 +2793,8 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) {
     // If there is a better chain.
     if (Chain != BetterChain) {
       // Replace the chain to avoid dependency.
-      SDOperand ReplStore = DAG.getStore(BetterChain, Value, Ptr, SrcValue);
+      SDOperand ReplStore = DAG.getStore(BetterChain, Value, Ptr,
+                                    ST->getSrcValue(), ST->getSrcValueOffset());
       // Create token to keep both nodes around.
       return DAG.getNode(ISD::TokenFactor, MVT::Other, Chain, ReplStore);
     }
@@ -4050,20 +4055,9 @@ bool DAGCombiner::FindAliasInfo(SDNode *N,
     SrcValue = LD->getSrcValue();
     return true;
   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
-#if 1 // FIXME - Switch over after StoreSDNode comes online.
-    Ptr = ST->getOperand(2);
-    Size = MVT::getSizeInBits(ST->getOperand(1).getValueType()) >> 3;
-    SrcValue = 0;
-#else
     Ptr = ST->getBasePtr();
-    Size = MVT::getSizeInBits(ST->getOperand(1).getValueType()) >> 3;
+    Size = MVT::getSizeInBits(ST->getStoredVT()) >> 3;
     SrcValue = ST->getSrcValue();
-#endif
-  // FIXME - Switch over after StoreSDNode comes online.
-  } else if (N->getOpcode() == ISD::TRUNCSTORE) {
-    Ptr = N->getOperand(2);
-    Size = MVT::getSizeInBits(cast<VTSDNode>(N->getOperand(4))->getVT()) >> 3;
-    SrcValue = 0;
   } else {
     assert(0 && "FindAliasInfo expected a memory operand");
   }
@@ -4104,8 +4098,6 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain,
       break;
       
     case ISD::LOAD:
-    // FIXME - Switch over after StoreSDNode comes online.
-    case ISD::TRUNCSTORE:
     case ISD::STORE: {
       // Get alias information for Chain.
       SDOperand OpPtr;
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index f38b1a3455..723177f97d 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -917,8 +917,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
       MVT::ValueType PtrVT = TLI.getPointerTy();
       SDOperand StackPtr = CreateStackTemporary(VT);
       // Store the vector.
-      SDOperand Ch = DAG.getStore(DAG.getEntryNode(),
-                                  Tmp1, StackPtr, DAG.getSrcValue(NULL));
+      SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, NULL, 0);
 
       // Truncate or zero extend offset to target pointer type.
       unsigned CastOpc = (IdxVT > PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
@@ -928,7 +927,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
       Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
       SDOperand StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr);
       // Store the scalar value.
-      Ch = DAG.getStore(Ch, Tmp2, StackPtr2, DAG.getSrcValue(NULL));
+      Ch = DAG.getStore(Ch, Tmp2, StackPtr2, NULL, 0);
       // Load the updated vector.
       Result = DAG.getLoad(VT, Ch, StackPtr, NULL, 0);
       break;
@@ -1592,109 +1591,144 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
     }
     break;
   case ISD::STORE: {
-    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
-    Tmp2 = LegalizeOp(Node->getOperand(2));  // Legalize the pointer.
-
-    // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
-    // FIXME: We shouldn't do this for TargetConstantFP's.
-    // FIXME: move this to the DAG Combiner!
-    if (ConstantFPSDNode *CFP =dyn_cast<ConstantFPSDNode>(Node->getOperand(1))){
-      if (CFP->getValueType(0) == MVT::f32) {
-        Tmp3 = DAG.getConstant(FloatToBits(CFP->getValue()), MVT::i32);
-      } else {
-        assert(CFP->getValueType(0) == MVT::f64 && "Unknown FP type!");
-        Tmp3 = DAG.getConstant(DoubleToBits(CFP->getValue()), MVT::i64);
+    StoreSDNode *ST = cast<StoreSDNode>(Node);
+    Tmp1 = LegalizeOp(ST->getChain());    // Legalize the chain.
+    Tmp2 = LegalizeOp(ST->getBasePtr());  // Legalize the pointer.
+
+    if (!ST->isTruncatingStore()) {
+      // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
+      // FIXME: We shouldn't do this for TargetConstantFP's.
+      // FIXME: move this to the DAG Combiner!
+      if (ConstantFPSDNode *CFP =dyn_cast<ConstantFPSDNode>(ST->getValue())) {
+        if (CFP->getValueType(0) == MVT::f32) {
+          Tmp3 = DAG.getConstant(FloatToBits(CFP->getValue()), MVT::i32);
+        } else {
+          assert(CFP->getValueType(0) == MVT::f64 && "Unknown FP type!");
+          Tmp3 = DAG.getConstant(DoubleToBits(CFP->getValue()), MVT::i64);
+        }
+        Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+                              ST->getSrcValueOffset());
+        break;
       }
-      Result = DAG.getStore(Tmp1, Tmp3, Tmp2, Node->getOperand(3));
-      break;
-    }
 
-    switch (getTypeAction(Node->getOperand(1).getValueType())) {
-    case Legal: {
-      Tmp3 = LegalizeOp(Node->getOperand(1));
-      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 
-                                      Node->getOperand(3));
-
-      MVT::ValueType VT = Tmp3.getValueType();
-      switch (TLI.getOperationAction(ISD::STORE, VT)) {
-      default: assert(0 && "This action is not supported yet!");
-      case TargetLowering::Legal:  break;
-      case TargetLowering::Custom:
-        Tmp1 = TLI.LowerOperation(Result, DAG);
-        if (Tmp1.Val) Result = Tmp1;
-        break;
-      case TargetLowering::Promote:
-        assert(MVT::isVector(VT) && "Unknown legal promote case!");
-        Tmp3 = DAG.getNode(ISD::BIT_CONVERT, 
-                           TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
+      switch (getTypeAction(ST->getStoredVT())) {
+      case Legal: {
+        Tmp3 = LegalizeOp(ST->getValue());
         Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 
-                                        Node->getOperand(3));
+                                        ST->getOffset());
+
+        MVT::ValueType VT = Tmp3.getValueType();
+        switch (TLI.getOperationAction(ISD::STORE, VT)) {
+        default: assert(0 && "This action is not supported yet!");
+        case TargetLowering::Legal:  break;
+        case TargetLowering::Custom:
+          Tmp1 = TLI.LowerOperation(Result, DAG);
+          if (Tmp1.Val) Result = Tmp1;
+          break;
+        case TargetLowering::Promote:
+          assert(MVT::isVector(VT) && "Unknown legal promote case!");
+          Tmp3 = DAG.getNode(ISD::BIT_CONVERT, 
+                             TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
+          Result = DAG.getStore(Tmp1, Tmp3, Tmp2,
+                                ST->getSrcValue(), ST->getSrcValueOffset());
+          break;
+        }
         break;
       }
-      break;
-    }
-    case Promote:
-      // Truncate the value and store the result.
-      Tmp3 = PromoteOp(Node->getOperand(1));
-      Result = DAG.getNode(ISD::TRUNCSTORE, MVT::Other, Tmp1, Tmp3, Tmp2,
-                           Node->getOperand(3),
-                          DAG.getValueType(Node->getOperand(1).getValueType()));
-      break;
+      case Promote:
+        // Truncate the value and store the result.
+        Tmp3 = PromoteOp(ST->getValue());
+        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+                                   ST->getSrcValueOffset(), ST->getStoredVT());
+        break;
 
-    case Expand:
-      unsigned IncrementSize = 0;
-      SDOperand Lo, Hi;
+      case Expand:
+        unsigned IncrementSize = 0;
+        SDOperand Lo, Hi;
       
-      // If this is a vector type, then we have to calculate the increment as
-      // the product of the element size in bytes, and the number of elements
-      // in the high half of the vector.
-      if (Node->getOperand(1).getValueType() == MVT::Vector) {
-        SDNode *InVal = Node->getOperand(1).Val;
-        unsigned NumElems =
-          cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
-        MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-
-        // Figure out if there is a Packed type corresponding to this Vector
-        // type.  If so, convert to the packed type.
-        MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
-        if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
-          // Turn this into a normal store of the packed type.
-          Tmp3 = PackVectorOp(Node->getOperand(1), TVT);
-          Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 
-                                          Node->getOperand(3));
-          Result = LegalizeOp(Result);
-          break;
-        } else if (NumElems == 1) {
-          // Turn this into a normal store of the scalar type.
-          Tmp3 = PackVectorOp(Node->getOperand(1), EVT);
-          Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2, 
-                                          Node->getOperand(3));
-          // The scalarized value type may not be legal, e.g. it might require
-          // promotion or expansion.  Relegalize the scalar store.
-          Result = LegalizeOp(Result);
-          break;
+        // If this is a vector type, then we have to calculate the increment as
+        // the product of the element size in bytes, and the number of elements
+        // in the high half of the vector.
+        if (ST->getValue().getValueType() == MVT::Vector) {
+          SDNode *InVal = ST->getValue().Val;
+          unsigned NumElems =
+            cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
+          MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+
+          // Figure out if there is a Packed type corresponding to this Vector
+          // type.  If so, convert to the packed type.
+          MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+          if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
+            // Turn this into a normal store of the packed type.
+            Tmp3 = PackVectorOp(Node->getOperand(1), TVT);
+            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+                                  ST->getSrcValueOffset());
+            Result = LegalizeOp(Result);
+            break;
+          } else if (NumElems == 1) {
+            // Turn this into a normal store of the scalar type.
+            Tmp3 = PackVectorOp(Node->getOperand(1), EVT);
+            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+                                  ST->getSrcValueOffset());
+            // The scalarized value type may not be legal, e.g. it might require
+            // promotion or expansion.  Relegalize the scalar store.
+            Result = LegalizeOp(Result);
+            break;
+          } else {
+            SplitVectorOp(Node->getOperand(1), Lo, Hi);
+            IncrementSize = NumElems/2 * MVT::getSizeInBits(EVT)/8;
+          }
         } else {
-          SplitVectorOp(Node->getOperand(1), Lo, Hi);
-          IncrementSize = NumElems/2 * MVT::getSizeInBits(EVT)/8;
+          ExpandOp(Node->getOperand(1), Lo, Hi);
+          IncrementSize = MVT::getSizeInBits(Hi.getValueType())/8;
+
+          if (!TLI.isLittleEndian())
+            std::swap(Lo, Hi);
         }
-      } else {
-        ExpandOp(Node->getOperand(1), Lo, Hi);
-        IncrementSize = MVT::getSizeInBits(Hi.getValueType())/8;
 
-        if (!TLI.isLittleEndian())
-          std::swap(Lo, Hi);
+        Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
+                          ST->getSrcValueOffset());
+        Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
+                           getIntPtrConstant(IncrementSize));
+        assert(isTypeLegal(Tmp2.getValueType()) &&
+               "Pointers must be legal!");
+        // FIXME: This sets the srcvalue of both halves to be the same, which is
+        // wrong.
+        Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
+                          ST->getSrcValueOffset());
+        Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
+        break;
+      }
+    } else {
+      // Truncating store
+      assert(isTypeLegal(ST->getValue().getValueType()) &&
+             "Cannot handle illegal TRUNCSTORE yet!");
+      Tmp3 = LegalizeOp(ST->getValue());
+    
+      // The only promote case we handle is TRUNCSTORE:i1 X into
+      //   -> TRUNCSTORE:i8 (and X, 1)
+      if (ST->getStoredVT() == MVT::i1 &&
+          TLI.getStoreXAction(MVT::i1) == TargetLowering::Promote) {
+        // Promote the bool to a mask then store.
+        Tmp3 = DAG.getNode(ISD::AND, Tmp3.getValueType(), Tmp3,
+                           DAG.getConstant(1, Tmp3.getValueType()));
+        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+                                   ST->getSrcValueOffset(), MVT::i8);
+      } else if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
+                 Tmp2 != ST->getBasePtr()) {
+        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
+                                        ST->getOffset());
       }
 
-      Lo = DAG.getStore(Tmp1, Lo, Tmp2, Node->getOperand(3));
-      Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
-                         getIntPtrConstant(IncrementSize));
-      assert(isTypeLegal(Tmp2.getValueType()) &&
-             "Pointers must be legal!");
-      // FIXME: This sets the srcvalue of both halves to be the same, which is
-      // wrong.
-      Hi = DAG.getStore(Tmp1, Hi, Tmp2, Node->getOperand(3));
-      Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
-      break;
+      MVT::ValueType StVT = cast<StoreSDNode>(Result.Val)->getStoredVT();
+      switch (TLI.getStoreXAction(StVT)) {
+      default: assert(0 && "This action is not supported yet!");
+      case TargetLowering::Legal: break;
+      case TargetLowering::Custom:
+        Tmp1 = TLI.LowerOperation(Result, DAG);
+        if (Tmp1.Val) Result = Tmp1;
+        break;
+      }
     }
     break;
   }
@@ -1772,42 +1806,6 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
     AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
     return Result;
 
-  case ISD::TRUNCSTORE: {
-    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
-    Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the pointer.
-
-    assert(isTypeLegal(Node->getOperand(1).getValueType()) &&
-           "Cannot handle illegal TRUNCSTORE yet!");
-    Tmp2 = LegalizeOp(Node->getOperand(1));
-    
-    // The only promote case we handle is TRUNCSTORE:i1 X into
-    //   -> TRUNCSTORE:i8 (and X, 1)
-    if (cast<VTSDNode>(Node->getOperand(4))->getVT() == MVT::i1 &&
-        TLI.getOperationAction(ISD::TRUNCSTORE, MVT::i1) == 
-              TargetLowering::Promote) {
-      // Promote the bool to a mask then store.
-      Tmp2 = DAG.getNode(ISD::AND, Tmp2.getValueType(), Tmp2,
-                         DAG.getConstant(1, Tmp2.getValueType()));
-      Result = DAG.getNode(ISD::TRUNCSTORE, MVT::Other, Tmp1, Tmp2, Tmp3,
-                           Node->getOperand(3), DAG.getValueType(MVT::i8));
-
-    } else if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1) ||
-               Tmp3 != Node->getOperand(2)) {
-      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3,
-                                      Node->getOperand(3), Node->getOperand(4));
-    }
-
-    MVT::ValueType StVT = cast<VTSDNode>(Result.Val->getOperand(4))->getVT();
-    switch (TLI.getOperationAction(Result.Val->getOpcode(), StVT)) {
-    default: assert(0 && "This action is not supported yet!");
-    case TargetLowering::Legal: break;
-    case TargetLowering::Custom:
-      Tmp1 = TLI.LowerOperation(Result, DAG);
-      if (Tmp1.Val) Result = Tmp1;
-      break;
-    }
-    break;
-  }
   case ISD::SELECT:
     switch (getTypeAction(Node->getOperand(0).getValueType())) {
     case Expand: assert(0 && "It's impossible to expand bools");
@@ -2386,7 +2384,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
                          DAG.getConstant(MVT::getSizeInBits(VT)/8, 
                                          TLI.getPointerTy()));
       // Store the incremented VAList to the legalized pointer
-      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2,  Node->getOperand(2));
+      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
+                          SV->getOffset());
       // Load the actual argument out of the pointer VAList
       Result = DAG.getLoad(VT, Tmp3, VAList, NULL, 0);
       Tmp1 = LegalizeOp(Result.getValue(1));
@@ -2423,9 +2422,11 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
       // This defaults to loading a pointer from the input and storing it to the
       // output, returning the chain.
       SrcValueSDNode *SVD = cast<SrcValueSDNode>(Node->getOperand(3));
+      SrcValueSDNode *SVS = cast<SrcValueSDNode>(Node->getOperand(4));
       Tmp4 = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp3, SVD->getValue(),
                          SVD->getOffset());
-      Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, Node->getOperand(4));
+      Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, SVS->getValue(),
+                            SVS->getOffset());
       break;
     }
     break;
@@ -2864,9 +2865,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
         int SSFI =
           MF.getFrameInfo()->CreateStackObject((unsigned)TySize, Align);
         SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
-        Result = DAG.getNode(ISD::TRUNCSTORE, MVT::Other, DAG.getEntryNode(),
-                             Node->getOperand(0), StackSlot,
-                             DAG.getSrcValue(NULL), DAG.getValueType(ExtraVT));
+        Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
+                                   StackSlot, NULL, 0, ExtraVT);
         Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
                                 Result, StackSlot, NULL, 0, ExtraVT);
       } else {
@@ -3213,7 +3213,8 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
                          DAG.getConstant(MVT::getSizeInBits(VT)/8, 
                                          TLI.getPointerTy()));
       // Store the incremented VAList to the legalized pointer
-      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, Node->getOperand(2));
+      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
+                          SV->getOffset());
       // Load the actual argument out of the pointer VAList
       Result = DAG.getExtLoad(ISD::EXTLOAD, NVT, Tmp3, VAList, NULL, 0, VT);
     }
@@ -3351,8 +3352,7 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
   // If the target doesn't support this, store the value to a temporary
   // stack slot, then LOAD the scalar element back out.
   SDOperand StackPtr = CreateStackTemporary(Vector.getValueType());
-  SDOperand Ch = DAG.getStore(DAG.getEntryNode(),
-                              Vector, StackPtr, DAG.getSrcValue(NULL));
+  SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vector, StackPtr, NULL, 0);
   
   // Add the offset to the index.
   unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
@@ -3495,8 +3495,7 @@ SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT,
   SDOperand FIPtr = CreateStackTemporary(DestVT);
   
   // Emit a store to the stack slot.
-  SDOperand Store = DAG.getStore(DAG.getEntryNode(),
-                                 SrcOp, FIPtr, DAG.getSrcValue(NULL));
+  SDOperand Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, NULL, 0);
   // Result is a load from the stack slot.
   return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0);
 }
@@ -3506,7 +3505,7 @@ SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
   // then load the whole vector back out.
   SDOperand StackPtr = CreateStackTemporary(Node->getValueType(0));
   SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr,
-                              DAG.getSrcValue(NULL));
+                              NULL, 0);
   return D