More CellSPU refinement and progress:

- Cleaned up custom load/store logic, common code is now shared [see note
  below], cleaned up address modes

- More test cases: various intrinsics, structure element access (load/store
  test), updated target data strings, indirect function calls.

Note: This patch contains a refactoring of the LoadSDNode and StoreSDNode
structures: they now share a common base class, LSBaseSDNode, that
provides an interface to their common functionality. There is some hackery
to access the proper operand depending on the derived class; otherwise,
to do a proper job would require finding and rearranging the SDOperands
sent to StoreSDNode's constructor. The current refactor errs on the
side of being conservatively and backwardly compatible while providing
functionality that reduces redundant code for targets where loads and
stores are custom-lowered.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@45851 91177308-0d34-0410-b5e6-96231b3b80d8

32 files changed, 901 insertions, 395 deletions

diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index e222e1b463..2d801a1f8b 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1454,11 +1454,63 @@ public:
   }
 };
 
+/// LSBaseSDNode - Base class for LoadSDNode and StoreSDNode
+///
+class LSBaseSDNode : public SDNode {
+private:
+  //! SrcValue - Memory location for alias analysis.
+  const Value *SrcValue;
+
+  //! SVOffset - Memory location offset.
+  int SVOffset;
+
+  //! Alignment - Alignment of memory location in bytes.
+  unsigned Alignment;
+
+  //! IsVolatile - True if the store is volatile.
+  bool IsVolatile;
+protected:
+  //! Operand array for load and store
+  /*!
+    \note Moving this array to the base class captures more
+    common functionality shared between LoadSDNode and
+    StoreSDNode
+   */
+  SDOperand Ops[4];
+public:
+  LSBaseSDNode(ISD::NodeType NodeTy, SDVTList VTs, const Value *SV, int SVO,
+               unsigned Align, bool Vol)
+    : SDNode(NodeTy, VTs),
+      SrcValue(SV), SVOffset(SVO), Alignment(Align), IsVolatile(Vol)
+  { }
+
+  const SDOperand getChain() const {
+    return getOperand(0);
+  }
+  const SDOperand getBasePtr() const {
+    return getOperand(getOpcode() == ISD::LOAD ? 1 : 2);
+  }
+  const SDOperand getOffset() const {
+    return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
+  }
+  const SDOperand getValue() const {
+    assert(getOpcode() == ISD::STORE);
+    return getOperand(1);
+  }
+
+  const Value *getSrcValue() const { return SrcValue; }
+  int getSrcValueOffset() const { return SVOffset; }
+  unsigned getAlignment() const { return Alignment; }
+  bool isVolatile() const { return IsVolatile; }
+
+  static bool classof(const LSBaseSDNode *N) { return true; }
+  static bool classof(const SDNode *N) { return true; }
+};
+
 /// LoadSDNode - This class is used to represent ISD::LOAD nodes.
 ///
-class LoadSDNode : public SDNode {
+class LoadSDNode : public LSBaseSDNode {
   virtual void ANCHOR();  // Out-of-line virtual method to give class a home.
-  SDOperand Ops[3];
   
   // AddrMode - unindexed, pre-indexed, post-indexed.
   ISD::MemIndexedMode AddrMode;
@@ -1468,26 +1520,13 @@ class LoadSDNode : public SDNode {
 
   // LoadedVT - VT of loaded value before extension.
   MVT::ValueType LoadedVT;
-
-  // SrcValue - Memory location for alias analysis.
-  const Value *SrcValue;
-
-  // SVOffset - Memory location offset.
-  int SVOffset;
-
-  // Alignment - Alignment of memory location in bytes.
-  unsigned Alignment;
-
-  // IsVolatile - True if the load is volatile.
-  bool IsVolatile;
 protected:
   friend class SelectionDAG;
   LoadSDNode(SDOperand *ChainPtrOff, SDVTList VTs,
              ISD::MemIndexedMode AM, ISD::LoadExtType ETy, MVT::ValueType LVT,
              const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
-    : SDNode(ISD::LOAD, VTs),
-      AddrMode(AM), ExtType(ETy), LoadedVT(LVT), SrcValue(SV), SVOffset(O),
-      Alignment(Align), IsVolatile(Vol) {
+    : LSBaseSDNode(ISD::LOAD, VTs, SV, O, Align, Vol),
+      AddrMode(AM), ExtType(ETy), LoadedVT(LVT) {
     Ops[0] = ChainPtrOff[0]; // Chain
     Ops[1] = ChainPtrOff[1]; // Ptr
     Ops[2] = ChainPtrOff[2]; // Off
@@ -1499,18 +1538,12 @@ protected:
   }
 public:
 
-  const SDOperand getChain() const { return getOperand(0); }
-  const SDOperand getBasePtr() const { return getOperand(1); }
-  const SDOperand getOffset() const { return getOperand(2); }
   ISD::MemIndexedMode getAddressingMode() const { return AddrMode; }
   ISD::LoadExtType getExtensionType() const { return ExtType; }
   MVT::ValueType getLoadedVT() const { return LoadedVT; }
-  const Value *getSrcValue() const { return SrcValue; }
-  int getSrcValueOffset() const { return SVOffset; }
-  unsigned getAlignment() const { return Alignment; }
-  bool isVolatile() const { return IsVolatile; }
 
   static bool classof(const LoadSDNode *) { return true; }
+  static bool classof(const LSBaseSDNode *N) { return true; }
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::LOAD;
   }
@@ -1518,9 +1551,8 @@ public:
 
 /// StoreSDNode - This class is used to represent ISD::STORE nodes.
 ///
-class StoreSDNode : public SDNode {
+class StoreSDNode : public LSBaseSDNode {
   virtual void ANCHOR();  // Out-of-line virtual method to give class a home.
-  SDOperand Ops[4];
     
   // AddrMode - unindexed, pre-indexed, post-indexed.
   ISD::MemIndexedMode AddrMode;
@@ -1530,26 +1562,13 @@ class StoreSDNode : public SDNode {
 
   // StoredVT - VT of the value after truncation.
   MVT::ValueType StoredVT;
-
-  // SrcValue - Memory location for alias analysis.
-  const Value *SrcValue;
-
-  // SVOffset - Memory location offset.
-  int SVOffset;
-
-  // Alignment - Alignment of memory location in bytes.
-  unsigned Alignment;
-
-  // IsVolatile - True if the store is volatile.
-  bool IsVolatile;
 protected:
   friend class SelectionDAG;
   StoreSDNode(SDOperand *ChainValuePtrOff, SDVTList VTs,
               ISD::MemIndexedMode AM, bool isTrunc, MVT::ValueType SVT,
               const Value *SV, int O=0, unsigned Align=0, bool Vol=false)
-    : SDNode(ISD::STORE, VTs),
-      AddrMode(AM), IsTruncStore(isTrunc), StoredVT(SVT), SrcValue(SV),
-      SVOffset(O), Alignment(Align), IsVolatile(Vol) {
+    : LSBaseSDNode(ISD::STORE, VTs, SV, O, Align, Vol),
+      AddrMode(AM), IsTruncStore(isTrunc), StoredVT(SVT) {
     Ops[0] = ChainValuePtrOff[0]; // Chain
     Ops[1] = ChainValuePtrOff[1]; // Value
     Ops[2] = ChainValuePtrOff[2]; // Ptr
@@ -1562,19 +1581,12 @@ protected:
   }
 public:
 
-  const SDOperand getChain() const { return getOperand(0); }
-  const SDOperand getValue() const { return getOperand(1); }
-  const SDOperand getBasePtr() const { return getOperand(2); }
-  const SDOperand getOffset() const { return getOperand(3); }
   ISD::MemIndexedMode getAddressingMode() const { return AddrMode; }
   bool isTruncatingStore() const { return IsTruncStore; }
   MVT::ValueType getStoredVT() const { return StoredVT; }
-  const Value *getSrcValue() const { return SrcValue; }
-  int getSrcValueOffset() const { return SVOffset; }
-  unsigned getAlignment() const { return Alignment; }
-  bool isVolatile() const { return IsVolatile; }
 
   static bool classof(const StoreSDNode *) { return true; }
+  static bool classof(const LSBaseSDNode *N) { return true; }
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::STORE;
   }
diff --git a/include/llvm/IntrinsicsCellSPU.td b/include/llvm/IntrinsicsCellSPU.td
index 0e257c150a..7030278708 100644
--- a/include/llvm/IntrinsicsCellSPU.td
+++ b/include/llvm/IntrinsicsCellSPU.td
@@ -17,8 +17,8 @@
 //===----------------------------------------------------------------------===//
 
 // 7-bit integer type, used as an immediate:
-def cell_i7_ty: LLVMType<i16>;   // Note: This was i8
-def cell_i8_ty: LLVMType<i16>;   // Note: This was i8
+def cell_i7_ty: LLVMType<i8>;
+def cell_i8_ty: LLVMType<i8>;
 
 class v16i8_u7imm<string builtin_suffix> :
   GCCBuiltin<!strconcat("__builtin_si_", builtin_suffix)>,
@@ -27,7 +27,7 @@ class v16i8_u7imm<string builtin_suffix> :
 
 class v16i8_u8imm<string builtin_suffix> :
   GCCBuiltin<!strconcat("__builtin_si_", builtin_suffix)>,
-  Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty],
+  Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty],
             [IntrNoMem]>;
 
 class v16i8_s10imm<string builtin_suffix> :
diff --git a/lib/Target/CellSPU/CellSDKIntrinsics.td b/lib/Target/CellSPU/CellSDKIntrinsics.td
index cfa0089d84..2f453b1feb 100644
--- a/lib/Target/CellSPU/CellSDKIntrinsics.td
+++ b/lib/Target/CellSPU/CellSDKIntrinsics.td
@@ -108,18 +108,18 @@ def CellSDKmpyhhau:
 
 def CellSDKand:
         RRForm<0b1000011000, (outs VECREG:$rT), (ins VECREG:$rA, VECREG:$rB),
-          "add\t $rT, $rA, $rB", IntegerOp,
+          "and\t $rT, $rA, $rB", IntegerOp,
           [(set (v4i32 VECREG:$rT),
                 (int_spu_si_and (v4i32 VECREG:$rA), (v4i32 VECREG:$rB)))]>;
 
 def CellSDKandc:
         RRForm<0b10000011010, (outs VECREG:$rT), (ins VECREG:$rA, VECREG:$rB),
-          "addc\t $rT, $rA, $rB", IntegerOp,
+          "andc\t $rT, $rA, $rB", IntegerOp,
           [(set (v4i32 VECREG:$rT),
                 (int_spu_si_andc (v4i32 VECREG:$rA), (v4i32 VECREG:$rB)))]>;
 
 def CellSDKandbi:
-     RI10Form<0b01101000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+     RI10Form<0b01101000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
        "andbi\t $rT, $rA, $val", BranchResolv,
        [(set (v16i8 VECREG:$rT),
              (int_spu_si_andbi (v16i8 VECREG:$rA), immU8:$val))]>;
@@ -149,7 +149,7 @@ def CellSDKorc:
                 (int_spu_si_orc (v4i32 VECREG:$rA), (v4i32 VECREG:$rB)))]>;
 
 def CellSDKorbi:
-     RI10Form<0b01100000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+     RI10Form<0b01100000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
        "orbi\t $rT, $rA, $val", BranchResolv,
        [(set (v16i8 VECREG:$rT),
              (int_spu_si_orbi (v16i8 VECREG:$rA), immU8:$val))]>;
@@ -173,7 +173,7 @@ def CellSDKxor:
                 (int_spu_si_xor (v4i32 VECREG:$rA), (v4i32 VECREG:$rB)))]>;
 
 def CellSDKxorbi:
-     RI10Form<0b01100000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+     RI10Form<0b01100000, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
        "xorbi\t $rT, $rA, $val", BranchResolv,
        [(set (v16i8 VECREG:$rT), (int_spu_si_xorbi (v16i8 VECREG:$rA), immU8:$val))]>;
 
@@ -248,7 +248,7 @@ def CellSDKceqb:
               (int_spu_si_ceqb (v16i8 VECREG:$rA), (v16i8 VECREG:$rB)))]>;
 
 def CellSDKceqbi:
-  RI10Form<0b01111110, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+  RI10Form<0b01111110, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
         "ceqbi\t $rT, $rA, $val", BranchResolv,
     [(set (v16i8 VECREG:$rT), (int_spu_si_ceqbi (v16i8 VECREG:$rA), immU8:$val))]>;
 
@@ -294,7 +294,7 @@ def CellSDKcgtb:
               (int_spu_si_cgtb (v16i8 VECREG:$rA), (v16i8 VECREG:$rB)))]>;
 
 def CellSDKcgtbi:
-  RI10Form<0b01110010, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+  RI10Form<0b01110010, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
     "cgtbi\t $rT, $rA, $val", BranchResolv,
         [(set (v16i8 VECREG:$rT), (int_spu_si_cgtbi (v16i8 VECREG:$rA), immU8:$val))]>;
 
@@ -329,7 +329,7 @@ def CellSDKclgtb:
           (int_spu_si_clgtb (v16i8 VECREG:$rA), (v16i8 VECREG:$rB)))]>;
 
 def CellSDKclgtbi:
-  RI10Form<0b01111010, (outs VECREG:$rT), (ins VECREG:$rA, u10imm:$val),
+  RI10Form<0b01111010, (outs VECREG:$rT), (ins VECREG:$rA, u10imm_i8:$val),
     "clgtbi\t $rT, $rA, $val", BranchResolv,
     [(set (v16i8 VECREG:$rT),
           (int_spu_si_clgtbi (v16i8 VECREG:$rA), immU8:$val))]>;
diff --git a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
index 73e46fff1b..bb3b100d57 100644
--- a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
+++ b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
@@ -384,11 +384,17 @@ bool
 SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
 		    SDOperand &Index) {
   // These match the addr256k operand type:
-  MVT::ValueType PtrVT = SPUtli.getPointerTy();
   MVT::ValueType OffsVT = MVT::i16;
+  MVT::ValueType PtrVT = SPUtli.getPointerTy();
 
   switch (N.getOpcode()) {
   case ISD::Constant:
+  case ISD::ConstantPool:
+  case ISD::GlobalAddress:
+    cerr << "SPU SelectAFormAddr: Constant/Pool/Global not lowered.\n";
+    abort();
+    /*NOTREACHED*/
+
   case ISD::TargetConstant: {
     // Loading from a constant address.
     ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
@@ -400,23 +406,15 @@ SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       return true;
     }
   }
-  case ISD::ConstantPool:
-  case ISD::TargetConstantPool: {
-    // The constant pool address is N. Base is a dummy that will be ignored by
+  case ISD::TargetGlobalAddress:
+  case ISD::TargetConstantPool:
+  case SPUISD::AFormAddr: {
+    // The address is in Base. N is a dummy that will be ignored by
     // the assembly printer.
     Base = N;
     Index = CurDAG->getTargetConstant(0, OffsVT);
     return true;
   }
-
-  case ISD::GlobalAddress:
-  case ISD::TargetGlobalAddress: {
-    // The global address is N. Base is a dummy that is ignored by the
-    // assembly printer.
-    Base = N;
-    Index = CurDAG->getTargetConstant(0, OffsVT);
-    return true;
-  }
   }
 
   return false;
@@ -445,10 +443,9 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     Index = CurDAG->getTargetConstant(0, PtrTy);
     return true;
   } else if (Opc == ISD::FrameIndex) {
-    // Stack frame index must be less than 512 (divided by 16):
     FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
     DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
-	  << FI->getIndex() << "\n");
+	       << FI->getIndex() << "\n");
     if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
       Base = CurDAG->getTargetConstant(0, PtrTy);
       Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
@@ -458,45 +455,49 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     // Generated by getelementptr
     const SDOperand Op0 = N.getOperand(0); // Frame index/base
     const SDOperand Op1 = N.getOperand(1); // Offset within base
-    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op1);
 
-    // Not a constant?
-    if (CN == 0)
-      return false;
-
-    int32_t offset = (int32_t) CN->getSignExtended();
-    unsigned Opc0 = Op0.getOpcode();
-
-    if ((offset & 0xf) != 0) {
-      cerr << "SelectDFormAddr: unaligned offset = " << offset << "\n";
-      abort();
-      /*NOTREACHED*/
-    }
+    if (Op1.getOpcode() == ISD::Constant
+        || Op1.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op1);
+      assert(CN != 0 && "SelectDFormAddr: Expected a constant");
 
-    if (Opc0 == ISD::FrameIndex) {
-      FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0);
-      DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
-	    << " frame index = " << FI->getIndex() << "\n");
+      int32_t offset = (int32_t) CN->getSignExtended();
+      unsigned Opc0 = Op0.getOpcode();
 
-      if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
-	Base = CurDAG->getTargetConstant(offset, PtrTy);
-	Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
-	return true;
+      if ((offset & 0xf) != 0) {
+        // Unaligned offset: punt and let X-form address handle it.
+        // NOTE: This really doesn't have to be strictly 16-byte aligned,
+        // since the load/store quadword instructions will implicitly
+        // zero the lower 4 bits of the resulting address.
+        return false;
       }
-    } else if (offset > SPUFrameInfo::minFrameOffset()
-	       && offset < SPUFrameInfo::maxFrameOffset()) {
-      Base = CurDAG->getTargetConstant(offset, PtrTy);
-      if (Opc0 == ISD::GlobalAddress) {
-	// Convert global address to target global address
-	GlobalAddressSDNode *GV = dyn_cast<GlobalAddressSDNode>(Op0);
-	Index = CurDAG->getTargetGlobalAddress(GV->getGlobal(), PtrTy);
-	return true;
-      } else {
-	// Otherwise, just take operand 0
-	Index = Op0;
-	return true;
+
+      if (Opc0 == ISD::FrameIndex) {
+        FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0);
+        DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
+              << " frame index = " << FI->getIndex() << "\n");
+
+        if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+          Base = CurDAG->getTargetConstant(offset, PtrTy);
+          Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
+          return true;
+        }
+      } else if (offset > SPUFrameInfo::minFrameOffset()
+                 && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(offset, PtrTy);
+        if (Opc0 == ISD::GlobalAddress) {
+          // Convert global address to target global address
+          GlobalAddressSDNode *GV = dyn_cast<GlobalAddressSDNode>(Op0);
+          Index = CurDAG->getTargetGlobalAddress(GV->getGlobal(), PtrTy);
+          return true;
+        } else {
+          // Otherwise, just take operand 0
+          Index = Op0;
+          return true;
+        }
       }
-    }
+    } else
+      return false;
   } else if (Opc == SPUISD::DFormAddr) {
     // D-Form address: This is pretty straightforward, naturally...
     ConstantSDNode *CN = cast<ConstantSDNode>(N.getOperand(1));
@@ -504,6 +505,16 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
     Index = N.getOperand(0);
     return true;
+  } else if (Opc == ISD::FrameIndex) {
+    // Stack frame index must be less than 512 (divided by 16):
+    FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
+    DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
+	  << FI->getIndex() << "\n");
+    if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+      Base = CurDAG->getTargetConstant(0, PtrTy);
+      Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
+      return true;
+    }
   }
 
   return false;
@@ -535,7 +546,8 @@ SPUDAGToDAGISel::SelectXFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     unsigned N2Opc = N2.getOpcode();
 
     if ((N1Opc == SPUISD::Hi && N2Opc == SPUISD::Lo)
-	 || (N1Opc == SPUISD::Lo && N2Opc == SPUISD::Hi)) {
+	 || (N1Opc == SPUISD::Lo && N2Opc == SPUISD::Hi)
+        || (N1Opc == SPUISD::XFormAddr)) {
       Base = N.getOperand(0);
       Index = N.getOperand(1);
       return true;
@@ -548,6 +560,10 @@ SPUDAGToDAGISel::SelectXFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       abort();
       /*UNREACHED*/
     }
+  } else if (Opc == SPUISD::XFormAddr) {
+    Base = N;
+    Index = N.getOperand(1);
+    return true;
   } else if (N.getNumOperands() == 2) {
     SDOperand N1 = N.getOperand(0);
     SDOperand N2 = N.getOperand(1);
@@ -591,11 +607,14 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
   } else if (Opc == ISD::FrameIndex) {
     // Selects to AIr32 FI, 0 which in turn will become AIr32 SP, imm.
     int FI = cast<FrameIndexSDNode>(N)->getIndex();
-    SDOperand TFI = CurDAG->getTargetFrameIndex(FI, SPUtli.getPointerTy());
+    MVT::ValueType PtrVT = SPUtli.getPointerTy();
+    SDOperand Zero = CurDAG->getTargetConstant(0, PtrVT);
+    SDOperand TFI = CurDAG->getTargetFrameIndex(FI, PtrVT);
 
     DEBUG(cerr << "SPUDAGToDAGISel: Replacing FrameIndex with AI32 <FI>, 0\n");
-    return CurDAG->SelectNodeTo(N, SPU::AIr32, Op.getValueType(), TFI,
-				CurDAG->getTargetConstant(0, MVT::i32));
+    if (N->hasOneUse())
+      return CurDAG->SelectNodeTo(N, SPU::AIr32, Op.getValueType(), TFI, Zero);
+    CurDAG->getTargetNode(SPU::AIr32, Op.getValueType(), TFI, Zero);
   } else if (Opc == SPUISD::LDRESULT) {
     // Custom select instructions for LDRESULT
     unsigned VT = N->getValueType(0);
diff --git a/lib/Target/CellSPU/SPUISelLowering.cpp b/lib/Target/CellSPU/SPUISelLowering.cpp
index 7893e677fb..59e2068a7a 100644
--- a/lib/Target/CellSPU/SPUISelLowering.cpp
+++ b/lib/Target/CellSPU/SPUISelLowering.cpp
@@ -82,7 +82,7 @@ namespace {
   /*!
     \arg Op Operand to test
     \return true if the operand is a memory target (i.e., global
-    address, external symbol, constant pool) or an existing D-Form
+    address, external symbol, constant pool) or an A-form
     address.
    */
   bool isMemoryOperand(const SDOperand &Op)
@@ -90,17 +90,17 @@ namespace {
     const unsigned Opc = Op.getOpcode();
     return (Opc == ISD::GlobalAddress
             || Opc == ISD::GlobalTLSAddress
-            || Opc ==  ISD::FrameIndex
+            /* || Opc ==  ISD::FrameIndex */
             || Opc == ISD::JumpTable
             || Opc == ISD::ConstantPool
             || Opc == ISD::ExternalSymbol
             || Opc == ISD::TargetGlobalAddress
             || Opc == ISD::TargetGlobalTLSAddress
-            || Opc == ISD::TargetFrameIndex
+            /* || Opc == ISD::TargetFrameIndex */
             || Opc == ISD::TargetJumpTable
             || Opc == ISD::TargetConstantPool
             || Opc == ISD::TargetExternalSymbol
-	    || Opc == SPUISD::DFormAddr);
+            || Opc == SPUISD::AFormAddr);
   }
 }
 
@@ -356,7 +356,7 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   setOperationAction(ISD::OR,  MVT::v16i8, Custom);
   setOperationAction(ISD::XOR, MVT::v16i8, Custom);
   setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
-  
+
   setSetCCResultType(MVT::i32);
   setShiftAmountType(MVT::i32);
   setSetCCResultContents(ZeroOrOneSetCCResult);
@@ -377,6 +377,7 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
     node_names[(unsigned) SPUISD::Hi] = "SPUISD::Hi";
     node_names[(unsigned) SPUISD::Lo] = "SPUISD::Lo";
     node_names[(unsigned) SPUISD::PCRelAddr] = "SPUISD::PCRelAddr";
+    node_names[(unsigned) SPUISD::AFormAddr] = "SPUISD::AFormAddr";
     node_names[(unsigned) SPUISD::DFormAddr] = "SPUISD::DFormAddr";
     node_names[(unsigned) SPUISD::XFormAddr] = "SPUISD::XFormAddr";
     node_names[(unsigned) SPUISD::LDRESULT] = "SPUISD::LDRESULT";
@@ -430,6 +431,105 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
 //  LowerOperation implementation
 //===----------------------------------------------------------------------===//
 
+/// Aligned load common code for CellSPU
+/*!
+  \param[in] Op The SelectionDAG load or store operand
+  \param[in] DAG The selection DAG
+  \param[in] ST CellSPU subtarget information structure
+  \param[in,out] alignment Caller initializes this to the load or store node's
+  value from getAlignment(), may be updated while generating the aligned load
+  \param[in,out] alignOffs Aligned offset; set by AlignedLoad to the aligned
+  offset (divisible by 16, modulo 16 == 0)
+  \param[in,out] prefSlotOffs Preferred slot offset; set by AlignedLoad to the
+  offset of the preferred slot (modulo 16 != 0)
+  \param[in,out] VT Caller initializes this value type to the the load or store
+  node's loaded or stored value type; may be updated if an i1-extended load or
+  store.
+  \param[out] was16aligned true if the base pointer had 16-byte alignment,
+  otherwise false. Can help to determine if the chunk needs to be rotated.
+
+ Both load and store lowering load a block of data aligned on a 16-byte
+ boundary. This is the common aligned load code shared between both.
+ */
+static SDOperand
+AlignedLoad(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST,
+            LSBaseSDNode *LSN,
+            unsigned &alignment, int &alignOffs, int &prefSlotOffs,
+            unsigned &VT, bool &was16aligned)
+{
+  MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  const valtype_map_s *vtm = getValueTypeMapEntry(VT);
+  SDOperand basePtr = LSN->getBasePtr();
+  SDOperand chain = LSN->getChain();
+
+  if (basePtr.getOpcode() == ISD::ADD) {
+    SDOperand Op1 = basePtr.Val->getOperand(1);
+
+    if (Op1.getOpcode() == ISD::Constant || Op1.getOpcode() == ISD::TargetConstant) {
+      const ConstantSDNode *CN = cast<ConstantSDNode>(basePtr.Val->getOperand(1));
+
+      alignOffs = (int) CN->getValue();
+      prefSlotOffs = (int) (alignOffs & 0xf);
+
+      // Adjust the rotation amount to ensure that the final result ends up in
+      // the preferred slot:
+      prefSlotOffs -= vtm->prefslot_byte;
+      basePtr = basePtr.getOperand(0);
+
+      // Modify alignment, since the ADD is likely from getElementPtr:
+      switch (basePtr.getOpcode()) {
+      case ISD::GlobalAddress:
+      case ISD::TargetGlobalAddress: {
+        GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(basePtr.Val);
+        const GlobalValue *GV = GN->getGlobal();
+        alignment = GV->getAlignment();
+        break;
+      }
+      }
+    } else {
+      alignOffs = 0;
+      prefSlotOffs = -vtm->prefslot_byte;
+    }
+  } else {
+    alignOffs = 0;
+    prefSlotOffs = -vtm->prefslot_byte;
+  }
+
+  if (alignment == 16) {
+    // Realign the base pointer as a D-Form address:
+    if (!isMemoryOperand(basePtr) || (alignOffs & ~0xf) != 0) {
+      if (isMemoryOperand(basePtr)) {
+        SDOperand Zero = DAG.getConstant(0, PtrVT);
+        unsigned Opc = (!ST->usingLargeMem()
+                        ? SPUISD::AFormAddr
+                        : SPUISD::XFormAddr);
+        basePtr = DAG.getNode(Opc, PtrVT, basePtr, Zero);
+      }
+      basePtr = DAG.getNode(SPUISD::DFormAddr, PtrVT,
+                          basePtr, DAG.getConstant((alignOffs & ~0xf), PtrVT));
+    }
+
+    // Emit the vector load:
+    was16aligned = true;
+    return DAG.getLoad(MVT::v16i8, chain, basePtr,
+                       LSN->getSrcValue(), LSN->getSrcValueOffset(),
+                       LSN->isVolatile(), 16);
+  }
+
+  // Unaligned load or we're using the "large memory" model, which means that
+  // we have to be very pessimistic:
+  if (isMemoryOperand(basePtr)) {
+    basePtr = DAG.getNode(SPUISD::XFormAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
+  }
+
+  // Add the offset
+  basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, DAG.getConstant(alignOffs, PtrVT));
+  was16aligned = false;
+  return DAG.getLoad(MVT::v16i8, chain, basePtr,
+                     LSN->getSrcValue(), LSN->getSrcValueOffset(),
+                     LSN->isVolatile(), 16);
+}
+
 /// Custom lower loads for CellSPU
 /*!
  All CellSPU loads and stores are aligned to 16-byte boundaries, so for elements
@@ -438,22 +538,13 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
 static SDOperand
 LowerLOAD(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
   LoadSDNode *LN = cast<LoadSDNode>(Op);
-  SDOperand basep = LN->getBasePtr();
   SDOperand the_chain = LN->getChain();
-  MVT::ValueType BasepOpc = basep.Val->getOpcode();
   MVT::ValueType VT = LN->getLoadedVT();
   MVT::ValueType OpVT = Op.Val->getValueType(0);
-  MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
   ISD::LoadExtType ExtType = LN->getExtensionType();
   unsigned alignment = LN->getAlignment();
-  const valtype_map_s *vtm = getValueTypeMapEntry(VT);
   SDOperand Ops[8];
 
-  if (BasepOpc == ISD::FrameIndex) {
-    // Loading from a frame index is always properly aligned. Always.
-    return SDOperand();
-  }
-
   // For an extending load of an i1 variable, just call it i8 (or whatever we
   // were passed) and make it zero-extended:
   if (VT == MVT::i1) {
@@ -463,178 +554,76 @@ LowerLOAD(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
 
   switch (LN->getAddressingMode()) {
   case ISD::UNINDEXED: {
-    SDOperand result;
-    SDOperand rot_op, rotamt;
-    SDOperand ptrp;
-    int c_offset;
-    int c_rotamt;
-
-    // The vector type we really want to be when we load the 16-byte chunk
-    MVT::ValueType vecVT, opVecVT;
-    
-    vecVT = MVT::v16i8;
-    if (VT != MVT::i1)
-      vecVT =