Overhaul Cell SPU's addressing mode internals so that there are now

only two addressing mode nodes, SPUaform and SPUindirect (vice the
three previous ones, SPUaform, SPUdform and SPUxform). This improves
code somewhat because we now avoid using reg+reg addressing when
it can be avoided. It also simplifies the address selection logic,
which was the main point for doing this.

Also, for various global variables that would be loaded using SPU's
A-form addressing, prefer D-form offs[reg] addressing, keeping the
base in a register if the variable is used more than once.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@46483 91177308-0d34-0410-b5e6-96231b3b80d8

14 files changed, 645 insertions, 874 deletions

diff --git a/lib/Target/CellSPU/SPUAsmPrinter.cpp b/lib/Target/CellSPU/SPUAsmPrinter.cpp
index 224616eb93..4bd8673a5b 100644
--- a/lib/Target/CellSPU/SPUAsmPrinter.cpp
+++ b/lib/Target/CellSPU/SPUAsmPrinter.cpp
@@ -195,12 +195,18 @@ namespace {
     void
     printAddr256K(const MachineInstr *MI, unsigned OpNo)
     {
-      /* Note: operand 1 is an offset or symbol name. Operand 2 is
-	 ignored. */
+      /* Note: operand 1 is an offset or symbol name. */
       if (MI->getOperand(OpNo).isImmediate()) {
         printS16ImmOperand(MI, OpNo);
       } else {
         printOp(MI->getOperand(OpNo));
+        if (MI->getOperand(OpNo+1).isImmediate()) {
+          int displ = int(MI->getOperand(OpNo+1).getImm());
+          if (displ > 0)
+            O << "+" << displ;
+          else if (displ < 0)
+            O << displ;
+        }
       }
     }
 
@@ -309,7 +315,6 @@ void SPUAsmPrinter::printOp(const MachineOperand &MO) {
   case MachineOperand::MO_JumpTableIndex:
     O << TAI->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber()
       << '_' << MO.getIndex();
-    // FIXME: PIC relocation model
     return;
   case MachineOperand::MO_ConstantPoolIndex:
     O << TAI->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber()
@@ -521,7 +526,8 @@ bool LinuxAsmPrinter::doFinalization(Module &M) {
         O << "\t.zero\t" << Size;
       } else if (I->hasInternalLinkage()) {
         SwitchToDataSection("\t.data", I);
-        O << TAI->getLCOMMDirective() << name << "," << Size << "," << Align;
+        O << ".local " << name << "\n";
+        O << TAI->getCOMMDirective() << name << "," << Size << "," << Align << "\n";
       } else {
         SwitchToDataSection("\t.data", I);
         O << ".comm " << name << "," << Size;
diff --git a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
index 3a50e3bcf0..3905d55302 100644
--- a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
+++ b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
@@ -150,6 +150,16 @@ namespace {
     return false;
   }
 
+  bool
+  isHighLow(const SDOperand &Op) 
+  {
+    return (Op.getOpcode() == SPUISD::IndirectAddr
+            && ((Op.getOperand(0).getOpcode() == SPUISD::Hi
+                 && Op.getOperand(1).getOpcode() == SPUISD::Lo)
+                || (Op.getOperand(0).getOpcode() == SPUISD::Lo
+                    && Op.getOperand(1).getOpcode() == SPUISD::Hi)));
+  }
+
   //===------------------------------------------------------------------===//
   //! MVT::ValueType to "useful stuff" mapping structure:
 
@@ -157,40 +167,24 @@ namespace {
     MVT::ValueType VT;
     unsigned ldresult_ins;	/// LDRESULT instruction (0 = undefined)
     int prefslot_byte;		/// Byte offset of the "preferred" slot
-    unsigned brcc_eq_ins;	/// br_cc equal instruction
-    unsigned brcc_neq_ins;	/// br_cc not equal instruction
-    unsigned load_aform;        /// A-form load instruction for this VT
-    unsigned store_aform;       /// A-form store instruction for this VT
+    unsigned insmask_ins;       /// Insert mask instruction for a-form
   };
 
   const valtype_map_s valtype_map[] = {
-    { MVT::i1,    0,            3, 0,         0,          0,
-      0 },
-    { MVT::i8,    SPU::ORBIr8,  3, 0,         0,          SPU::LQAr8,
-      SPU::STQAr8 },
-    { MVT::i16,   SPU::ORHIr16, 2, SPU::BRHZ, SPU::BRHNZ, SPU::LQAr16,
-      SPU::STQAr16 },
-    { MVT::i32,   SPU::ORIr32,  0, SPU::BRZ,  SPU::BRNZ,  SPU::LQAr32,
-      SPU::STQAr32 },
-    { MVT::i64,   SPU::ORIr64,  0, 0,         0,          SPU::LQAr64,
-      SPU::STQAr64 },
-    { MVT::f32,   0,            0, 0,         0,          SPU::LQAf32,
-      SPU::STQAf32 },
-    { MVT::f64,   0,            0, 0,         0,          SPU::LQAf64,
-      SPU::STQAf64 },
+    { MVT::i1,    0,            3, 0 },
+    { MVT::i8,    SPU::ORBIr8,  3, 0 },
+    { MVT::i16,   SPU::ORHIr16, 2, 0 },
+    { MVT::i32,   SPU::ORIr32,  0, 0 },
+    { MVT::i64,   SPU::ORIr64,  0, 0 },
+    { MVT::f32,   0,            0, 0 },
+    { MVT::f64,   0,            0, 0 },
     // vector types... (sigh!)
-    { MVT::v16i8, 0,            0, 0,         0,          SPU::LQAv16i8,
-      SPU::STQAv16i8 },
-    { MVT::v8i16, 0,            0, 0,         0,          SPU::LQAv8i16,
-      SPU::STQAv8i16 },
-    { MVT::v4i32, 0,            0, 0,         0,          SPU::LQAv4i32,
-      SPU::STQAv4i32 },
-    { MVT::v2i64, 0,            0, 0,         0,          SPU::LQAv2i64,
-      SPU::STQAv2i64 },
-    { MVT::v4f32, 0,            0, 0,         0,          SPU::LQAv4f32,
-      SPU::STQAv4f32 },
-    { MVT::v2f64, 0,            0, 0,         0,          SPU::LQAv2f64,
-      SPU::STQAv2f64 },
+    { MVT::v16i8, 0,            0, SPU::CBD },
+    { MVT::v8i16, 0,            0, SPU::CHD },
+    { MVT::v4i32, 0,            0, SPU::CWD },
+    { MVT::v2i64, 0,            0, 0 },
+    { MVT::v4f32, 0,            0, SPU::CWD },
+    { MVT::v2f64, 0,            0, 0 }
   };
 
   const size_t n_valtype_map = sizeof(valtype_map) / sizeof(valtype_map[0]);
@@ -381,18 +375,20 @@ SPUDAGToDAGISel::SelectDForm2Addr(SDOperand Op, SDOperand N, SDOperand &Disp,
     Disp = CurDAG->getTargetConstant(0, VT);
     Base = N;
     return true;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // D-Form address: This is pretty straightforward, naturally...
-    CN = cast<ConstantSDNode>(N.getOperand(1));
-    assert(CN != 0 && "SelectDFormAddr/SPUISD::DForm2Addr expecting constant");
-    Imm = unsigned(CN->getValue());
-    if (Imm < 0xff) {
-      Disp = CurDAG->getTargetConstant(CN->getValue(), PtrVT);
-      Base = N.getOperand(0);
-      return true;
+  } else if (Opc == SPUISD::IndirectAddr) {
+    SDOperand Op1 = N.getOperand(1);
+    if (Op1.getOpcode() == ISD::TargetConstant
+        || Op1.getOpcode() == ISD::Constant) {
+      CN = cast<ConstantSDNode>(N.getOperand(1));
+      assert(CN != 0 && "SelectIndirectAddr/SPUISD::DForm2Addr expecting constant");
+      Imm = unsigned(CN->getValue());
+      if (Imm < 0xff) {
+        Disp = CurDAG->getTargetConstant(CN->getValue(), PtrVT);
+        Base = N.getOperand(0);
+        return true;
+      }
     }
   }
-
   return false;
 }
 
@@ -407,7 +403,7 @@ SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
 		    SDOperand &Index) {
   // These match the addr256k operand type:
   MVT::ValueType OffsVT = MVT::i16;
-  MVT::ValueType PtrVT = SPUtli.getPointerTy();
+  SDOperand Zero = CurDAG->getTargetConstant(0, OffsVT);
 
   switch (N.getOpcode()) {
   case ISD::Constant:
@@ -417,28 +413,40 @@ SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     abort();
     /*NOTREACHED*/
 
-  case ISD::TargetConstant: {
-    // Loading from a constant address.
-    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
-    int Imm = (int)CN->getValue();
-    if (Imm < 0x3ffff && (Imm & 0x3) == 0) {
-      Base = CurDAG->getTargetConstant(Imm, PtrVT);
-      // Note that this operand will be ignored by the assembly printer...
-      Index = CurDAG->getTargetConstant(0, OffsVT);
-      return true;
-    }
-  }
+  case ISD::TargetConstant:
   case ISD::TargetGlobalAddress:
-  case ISD::TargetConstantPool:
-  case SPUISD::AFormAddr: {
-    // The address is in Base. N is a dummy that will be ignored by
-    // the assembly printer.
-    Base = N;
-    Index = CurDAG->getTargetConstant(0, OffsVT);
-    return true;
-  }
-  }
+  case ISD::TargetJumpTable:
+    cerr << "SPUSelectAFormAddr: Target Constant/Pool/Global not wrapped as "
+         << "A-form address.\n";
+    abort();
+    /*NOTREACHED*/
 
+  case SPUISD::AFormAddr: 
+    // Just load from memory if there's only a single use of the location,
+    // otherwise, this will get handled below with D-form offset addresses
+    if (N.hasOneUse()) {
+      SDOperand Op0 = N.getOperand(0);
+      switch (Op0.getOpcode()) {
+      case ISD::TargetConstantPool:
+      case ISD::TargetJumpTable:
+        Base = Op0;
+        Index = Zero;
+        return true;
+
+      case ISD::TargetGlobalAddress: {
+        GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op0);
+        GlobalValue *GV = GSDN->getGlobal();
+        if (GV->getAlignment() == 16) {
+          Base = Op0;
+          Index = Zero;
+          return true;
+        }
+        break;
+      }
+      }
+    }
+    break;
+  }
   return false;
 }
 
@@ -460,14 +468,11 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
   unsigned Opc = N.getOpcode();
   unsigned PtrTy = SPUtli.getPointerTy();
 
-  if (Opc == ISD::Register) {
-    Base = N;
-    Index = CurDAG->getTargetConstant(0, PtrTy);
-    return true;
-  } else if (Opc == ISD::FrameIndex) {
+  if (Opc == ISD::FrameIndex) {
+    // Stack frame index must be less than 512 (divided by 16):
     FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
     DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
-	       << FI->getIndex() << "\n");
+	  << FI->getIndex() << "\n");
     if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
       Base = CurDAG->getTargetConstant(0, PtrTy);
       Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
@@ -475,19 +480,20 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     }
   } else if (Opc == ISD::ADD) {
     // Generated by getelementptr
-    const SDOperand Op0 = N.getOperand(0); // Frame index/base
-    const SDOperand Op1 = N.getOperand(1); // Offset within base
+    const SDOperand Op0 = N.getOperand(0);
+    const SDOperand Op1 = N.getOperand(1);
 
-    if ((Op1.getOpcode() == ISD::Constant
-	 || Op1.getOpcode() == ISD::TargetConstant)
-	&& Op0.getOpcode() != SPUISD::XFormAddr) {
+    if ((Op0.getOpcode() == SPUISD::Hi && Op1.getOpcode() == SPUISD::Lo)
+        || (Op1.getOpcode() == SPUISD::Hi && Op0.getOpcode() == SPUISD::Lo)) {
+      Base = CurDAG->getTargetConstant(0, PtrTy);
+      Index = N;
+      return true;
+    } else if (Op1.getOpcode() == ISD::Constant
+               || Op1.getOpcode() == ISD::TargetConstant) {
       ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op1);
-      assert(CN != 0 && "SelectDFormAddr: Expected a constant");
-
-      int32_t offset = (int32_t) CN->getSignExtended();
-      unsigned Opc0 = Op0.getOpcode();
+      int32_t offset = int32_t(CN->getSignExtended());
 
-      if (Opc0 == ISD::FrameIndex) {
+      if (Op0.getOpcode() == ISD::FrameIndex) {
         FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0);
         DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
               << " frame index = " << FI->getIndex() << "\n");
@@ -500,51 +506,69 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       } else if (offset > SPUFrameInfo::minFrameOffset()
                  && offset < SPUFrameInfo::maxFrameOffset()) {
         Base = CurDAG->getTargetConstant(offset, PtrTy);
-        if (Opc0 == ISD::GlobalAddress) {
-          // Convert global address to target global address
-          GlobalAddressSDNode *GV = dyn_cast<GlobalAddressSDNode>(Op0);
-          Index = CurDAG->getTargetGlobalAddress(GV->getGlobal(), PtrTy);
-          return true;
-        } else {
-          // Otherwise, just take operand 0
-          Index = Op0;
+        Index = Op0;
+        return true;
+      }
+    } else if (Op0.getOpcode() == ISD::Constant
+               || Op0.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op0);
+      int32_t offset = int32_t(CN->getSignExtended());
+
+      if (Op1.getOpcode() == ISD::FrameIndex) {
+        FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op1);
+        DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
+              << " frame index = " << FI->getIndex() << "\n");
+
+        if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+          Base = CurDAG->getTargetConstant(offset, PtrTy);
+          Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
           return true;
         }
+      } else if (offset > SPUFrameInfo::minFrameOffset()
+                 && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(offset, PtrTy);
+        Index = Op1;
+        return true;
       }
-    } else
-      return false;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // D-Form address: This is pretty straightforward,
-    // naturally... but make sure that this isn't a D-form address
-    // with a X-form address embedded within:
-    const SDOperand Op0 = N.getOperand(0); // Frame index/base
-    const SDOperand Op1 = N.getOperand(1); // Offset within base
-
-    if (Op0.getOpcode() == ISD::Constant
-        || Op0.getOpcode() == ISD::TargetConstant) {
-      ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
-      assert(CN != 0 && "SelectDFormAddr/SPUISD::DFormAddr expecting constant"); 
-      Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
-      Index = Op0;
-      return true;
     }
-  } else if (Opc == ISD::FrameIndex) {
-    // Stack frame index must be less than 512 (divided by 16):
-    FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
-    DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
-	  << FI->getIndex() << "\n");
-    if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+  } else if (Opc == SPUISD::IndirectAddr) {
+    // Indirect with constant offset -> D-Form address
+    const SDOperand Op0 = N.getOperand(0);
+    const SDOperand Op1 = N.getOperand(1);
+    SDOperand Zero = CurDAG->getTargetConstant(0, N.getValueType());
+
+    if (Op1.getOpcode() == ISD::Constant
+        || Op1.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
+      int32_t offset = int32_t(CN->getSignExtended());
+      if (offset > SPUFrameInfo::minFrameOffset()
+          && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
+        Index = Op0;
+        return true;
+      }
+    } else if (Op0.getOpcode() == ISD::Constant
+               || Op0.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op0);
+      int32_t offset = int32_t(CN->getSignExtended());
+      if (offset > SPUFrameInfo::minFrameOffset()
+          && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
+        Index = Op1;
+        return true;
+      }
+    } else if (Op0.getOpcode() == SPUISD::Hi
+               && Op1.getOpcode() == SPUISD::Lo) {
+      // (SPUindirect (SPUhi <arg>, 0), (SPUlo <arg>, 0))
       Base = CurDAG->getTargetConstant(0, PtrTy);
-      Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
+      Index = N;
       return true;
     }
-  } else if (Opc == SPUISD::LDRESULT) {
-    // It's a load result dereference
-    Base = CurDAG->getTargetConstant(0, PtrTy);
-    Index = N.getOperand(0);
+  } else if (Opc == SPUISD::AFormAddr) {
+    Base = CurDAG->getTargetConstant(0, N.getValueType());
+    Index = N;
     return true;
   }
-
   return false;
 }
 
@@ -565,108 +589,10 @@ SPUDAGToDAGISel::SelectXFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       || SelectDFormAddr(Op, N, Base, Index))
     return false;
 
-  unsigned Opc = N.getOpcode();
-
-  if (Opc == ISD::ADD) {
-    SDOperand N1 = N.getOperand(0);
-    SDOperand N2 = N.getOperand(1);
-    Base = N.getOperand(0);
-    Index = N.getOperand(1);
-    return true;
-  } else if (Opc == SPUISD::XFormAddr) {
-    Base = N;
-    Index = N.getOperand(1);
-    return true;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // Must be a D-form address with an X-form address embedded
-    // within:
-    Base = N.getOperand(0);
-    Index = N.getOperand(1);
-    return true;
-  } else if (N.getNumOperands() == 2) {
-    SDOperand N1 = N.getOperand(0);
-    SDOperand N2 = N.getOperand(1);
-    unsigned N1Opc = N1.getOpcode();
-    unsigned N2Opc = N2.getOpcode();
-
-    if ((N1Opc == ISD::CopyToReg || N1Opc == ISD::Register)
-	&& (N2Opc == ISD::CopyToReg || N2Opc == ISD::Register)) {
-      Base = N.getOperand(0);
-      Index = N.getOperand(1);
-      return true;
-      /*UNREACHED*/
-    } else {
-      cerr << "SelectXFormAddr: 2-operand unhandled operand:\n";
-      N.Val->dump(CurDAG);
-      cerr << "\n";
-      abort();
-    /*UNREACHED*/
-    }
-  } else {
-    cerr << "SelectXFormAddr: Unhandled operand type:\n";
-    N.Val->dump(CurDAG);
-    cerr << "\n";
-    abort();
-    /*UNREACHED*/
-  }
-
-  return false;
-}
-
-//! Emit load for A-form addresses
-/*
- */
-SDNode *
-Emit_LOAD_AFormAddr(SDOperand Op, SelectionDAG &CurDAG, SPUDAGToDAGISel &ISel)
-{
-  SDNode *Result;
-  MVT::ValueType OpVT = Op.getValueType();
-  SDOperand Chain = Op.getOperand(0);
-  SDOperand Ptr = Op.getOperand(1);
-  SDOperand PtrArg = Ptr.getOperand(0);
-  SDOperand PtrOffs = Ptr.getOperand(1);
-  const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
-
-  if (PtrOffs.getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(PtrOffs);
-    MVT::ValueType PVT = PtrOffs.getValueType();
-    PtrOffs = CurDAG.getTargetConstant(CN->getValue(), PVT);
-  }
-  ISel.AddToISelQueue(PtrArg);
-  ISel.AddToISelQueue(PtrOffs);
-  ISel.AddToISelQueue(Chain);
-  Result = CurDAG.getTargetNode(vtm->load_aform, OpVT, MVT::Other, PtrArg, PtrOffs, Chain);
-  Chain = SDOperand(Result, 1);
-  return Result;
-}
-
-//! Emit store for A-form addresses
-/*
- */
-SDNode *
-Emit_STORE_AFormAddr(SDOperand Op, SelectionDAG &CurDAG, SPUDAGToDAGISel &ISel)
-{
-  SDNode *Result;
-  SDOperand Chain = Op.getOperand(0);
-  SDOperand Val = Op.getOperand(1);
-  SDOperand Ptr = Op.getOperand(2);
-  SDOperand PtrArg = Ptr.getOperand(0);
-  SDOperand PtrOffs = Ptr.getOperand(1);
-  const valtype_map_s *vtm = getValueTypeMapEntry(Val.getValueType());
-
-  if (PtrOffs.getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(PtrOffs);
-    MVT::ValueType PVT = PtrOffs.getValueType();
-    PtrOffs = CurDAG.getTargetConstant(CN->getValue(), PVT);
-  }
-  ISel.AddToISelQueue(Val);
-  ISel.AddToISelQueue(PtrArg);
-  ISel.AddToISelQueue(PtrOffs);
-  ISel.AddToISelQueue(Chain);
-  SDOperand Ops[4] = { Val, PtrArg, PtrOffs, Chain };
-  Result = CurDAG.getTargetNode(vtm->store_aform, MVT::Other, Ops, 4);
-  Chain = SDOperand(Result, 1);
-  return Result;
+  // All else fails, punt and use an X-form address:
+  Base = N.getOperand(0);
+  Index = N.getOperand(1);
+  return true;
 }
 
 //! Convert the operand from a target-independent to a target-specific node
@@ -695,12 +621,6 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
     Ops[0] = TFI;
     Ops[1] = Zero;
     n_ops = 2;
-  } else if (Opc == ISD::LOAD
-             && Op.getOperand(1).getOpcode() == SPUISD::AFormAddr) {
-    return Emit_LOAD_AFormAddr(Op, *CurDAG, *this);
-  } else if (Opc == ISD::STORE
-             && Op.getOperand(2).getOpcode() == SPUISD::AFormAddr) {
-    return Emit_STORE_AFormAddr(Op, *CurDAG, *this);
   } else if (Opc == ISD::ZERO_EXTEND) {
     // (zero_extend:i16 (and:i8 <arg>, <const>))
     const SDOperand &Op1 = N->getOperand(0);
@@ -717,6 +637,38 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
         n_ops = 2;
       }
     }
+  } else if (Opc == SPUISD::INSERT_MASK) {
+    SDOperand Op0 = Op.getOperand(0);
+    if (Op0.getOpcode() == SPUISD::AFormAddr) {
+      // (SPUvecinsmask (SPUaform <arg>, 0)) ->
+      // (CBD|CHD|CWD 0, arg)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op0.getOperand(1));
+      assert(vtm->insmask_ins != 0 && "missing insert mask instruction");
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(CN->getValue(), Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+
+      AddToISelQueue(Op0);
+    } else if (Op0.getOpcode() == ISD::FrameIndex) {
+      // (SPUvecinsmask <fi>) ->
+      // (CBD|CHD|CWD 0, <fi>)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(0, Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+    } else if (isHighLow(Op0)) {
+      // (SPUvecinsmask (SPUindirect (SPUhi <arg>, 0), (SPUlow <arg>, 0))) ->
+      // (CBD|CHD|CWD 0, arg)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(0, Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+      AddToISelQueue(Op0);
+    }
   } else if (Opc == SPUISD::LDRESULT) {
     // Custom select instructions for LDRESULT
     unsigned VT = N->getValueType(0);
@@ -748,19 +700,19 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
     AddToISelQueue(Chain);
 
     return Result;
-  } else if (Opc == SPUISD::XFormAddr) {
+  } else if (Opc == SPUISD::IndirectAddr) {
     SDOperand Op0 = Op.getOperand(0);
     if (Op0.getOpcode() == SPUISD::LDRESULT
         || Op0.getOpcode() == SPUISD::AFormAddr) {
-      // (XFormAddr (LDRESULT|AFormAddr, imm))
+      // (IndirectAddr (LDRESULT|AFormAddr, imm))
       SDOperand Op1 = Op.getOperand(1);
       MVT::ValueType VT = Op.getValueType();
 
-      DEBUG(cerr << "CellSPU: XFormAddr("
-            << (Op0.getOpcode() == SPUISD::LDRESULT
-                ? "LDRESULT"
-                : "AFormAddr")
-            << ", imm):\nOp0 = ");
+      DEBUG(cerr << "CellSPU: IndirectAddr("
+                 << (Op0.getOpcode() == SPUISD::LDRESULT
+                     ? "LDRESULT"
+                     : "AFormAddr")
+                 << ", imm):\nOp0 = ");
       DEBUG(Op.getOperand(0).Val->dump(CurDAG));
       DEBUG(cerr << "\nOp1 = ");
       DEBUG(Op.getOperand(1).Val->dump(CurDAG));
diff --git a/lib/Target/CellSPU/SPUISelLowering.cpp b/lib/Target/CellSPU/SPUISelLowering.cpp
index 0f1d045280..33261a607b 100644
--- a/lib/Target/CellSPU/SPUISelLowering.cpp
+++ b/lib/Target/CellSPU/SPUISelLowering.cpp
@@ -289,14 +289,12 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   
   // We want to legalize GlobalAddress and ConstantPool nodes into the 
   // appropriate instructions to materialize the address.
-  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::i32, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::f32, Custom);
-  setOperationAction(ISD::JumpTable,     MVT::i32, Custom);
-  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::i64, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::f64, Custom);
-  setOperationAction(ISD::JumpTable,     MVT::i64, Custom);
+  for (unsigned sctype = (unsigned) MVT::i1; sctype < (unsigned) MVT::f128;
+       ++sctype) {
+    setOperationAction(ISD::GlobalAddress, sctype, Custom);
+    setOperationAction(ISD::ConstantPool,  sctype, Custom);
+    setOperationAction(ISD::JumpTable,     sctype, Custom);
+  }
 
   // RET must be custom lowered, to meet ABI requirements
   setOperationAction(ISD::RET,           MVT::Other, Custom);
@@ -377,7 +375,7 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   setStackPointerRegisterToSaveRestore(SPU::R1);
   
   // We have target-specific dag combine patterns for the following nodes:
-  // e.g., setTargetDAGCombine(ISD::SUB);
+  setTargetDAGCombine(ISD::ADD);
   
   computeRegisterProperties();
 }
@@ -391,8 +389,7 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
     node_names[(unsigned) SPUISD::Lo] = "SPUISD::Lo";
     node_names[(unsigned) SPUISD::PCRelAddr] = "SPUISD::PCRelAddr";
     node_names[(unsigned) SPUISD::AFormAddr] = "SPUISD::AFormAddr";
-    node_names[(unsigned) SPUISD::DFormAddr] = "SPUISD::DFormAddr";
-    node_names[(unsigned) SPUISD::XFormAddr] = "SPUISD::XFormAddr";
+    node_names[(unsigned) SPUISD::IndirectAddr] = "SPUISD::IndirectAddr";
     node_names[(unsigned) SPUISD::LDRESULT] = "SPUISD::LDRESULT";
     node_names[(unsigned) SPUISD::CALL] = "SPUISD::CALL";
     node_names[(unsigned) SPUISD::SHUFB] = "SPUISD::SHUFB";
@@ -524,11 +521,12 @@ AlignedLoad(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST,
   // Unaligned load or we're using the "large memory" model, which means that
   // we have to be very pessimistic:
   if (isMemoryOperand(basePtr) || isIndirectOperand(basePtr)) {
-    basePtr = DAG.getNode(SPUISD::XFormAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
+    basePtr = DAG.getNode(SPUISD::IndirectAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
   }
 
   // Add the offset
-  basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, DAG.getConstant(alignOffs, PtrVT));
+  basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr,
+			DAG.getConstant((alignOffs & ~0xf), PtrVT));
   was16aligned = false;
   return DAG.getLoad(MVT::v16i8, chain, basePtr,
                      LSN->getSrcValue(), LSN->getSrcValueOffset(),
@@ -706,21 +704,20 @@ LowerSTORE(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
     DEBUG(basePtr.Val->dump(&DAG));
     DEBUG(cerr << "\n");
 
-    if (basePtr.getOpcode() == SPUISD::DFormAddr) {
-      // Hmmmm... do we ever actually hit this code?
-      insertEltPtr = DAG.getNode(SPUISD::DFormAddr, PtrVT,
-				 basePtr.getOperand(0),
-				 insertEltOffs);
-    } else if (basePtr.getOpcode() == SPUISD::XFormAddr ||
-	       (basePtr.getOpcode() == ISD::ADD
-		&& basePtr.getOperand(0).getOpcode() == SPUISD::XFormAddr)) {
+    if (basePtr.getOpcode() == SPUISD::IndirectAddr ||
+        (basePtr.getOpcode() == ISD::ADD
+         && basePtr.getOperand(0).getOpcode() == SPUISD::IndirectAddr)) {
       insertEltPtr = basePtr;
     } else {
-      // $sp is always aligned, so use it instead of potentially loading an
-      // address into a new register:
-      insertEltPtr = DAG.getNode(SPUISD::DFormAddr, PtrVT,
-				 DAG.getRegister(SPU::R1, PtrVT),
-				 insertEltOffs);
+#if 0
+      // $sp is always aligned, so use it when necessary to avoid loading
+      // an address
+      SDOperand ptrP =
+        basePtr.Val->hasOneUse() ? DAG.getRegister(SPU::R1, PtrVT) : basePtr;
+      insertEltPtr = DAG.getNode(ISD::ADD, PtrVT, ptrP, insertEltOffs);
+#else
+      insertEltPtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, insertEltOffs);
+#endif
     }
 
     insertEltOp = DAG.getNode(SPUISD::INSERT_MASK, stVecVT, insertEltPtr);
@@ -772,7 +769,7 @@ LowerConstantPool(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
 
       return DAG.getNode(ISD::ADD, PtrVT, Lo, Hi);
 #else
-      return DAG.getNode(SPUISD::XFormAddr, PtrVT, CPI, Zero);
+      return DAG.getNode(SPUISD::IndirectAddr, PtrVT, CPI, Zero);
 #endif
     }
   }
@@ -791,9 +788,10 @@ LowerJumpTable(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
   const TargetMachine &TM = DAG.getTarget();
 
   if (TM.getRelocationModel() == Reloc::Static) {
+    SDOperand JmpAForm = DAG.getNode(SPUISD::AFormAddr, PtrVT, JTI, Zero);
     return (!ST->usingLargeMem()
-            ? DAG.getNode(SPUISD::AFormAddr, PtrVT, JTI, Zero)
-            : DAG.getNode(SPUISD::XFormAddr, PtrVT, JTI, Zero));
+            ? JmpAForm
+            : DAG.getNode(SPUISD::IndirectAddr, PtrVT, JmpAForm, Zero));
   }
 
   assert(0 &&
@@ -811,9 +809,13 @@ LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
   SDOperand Zero = DAG.getConstant(0, PtrVT);
   
   if (TM.getRelocationModel() == Reloc::Static) {
-    return (!ST->usingLargeMem()
-            ? DAG.getNode(SPUISD::AFormAddr, PtrVT, GA, Zero)
-            : DAG.getNode(SPUISD::XFormAddr, PtrVT, GA, Zero));
+    if (!ST->usingLargeMem()) {
+      return DAG.getNode(SPUISD::AFormAddr, PtrVT, GA, Zero);
+    } else {
+      SDOperand Hi = DAG.getNode(SPUISD::Hi, PtrVT, GA, Zero);
+      SDOperand Lo = DAG.getNode(SPUISD::Lo, PtrVT, GA, Zero);
+      return DAG.getNode(SPUISD::IndirectAddr, PtrVT, Hi, Lo);
+    }
   } else {
     cerr << "LowerGlobalAddress: Relocation model other than static not "
 	 << "supported.\n";
@@ -1202,7 +1204,7 @@ LowerCALL(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
     } else {
       // "Large memory" mode: Turn all calls into indirect calls with a X-form
       // address pairs:
-      Callee = DAG.getNode(SPUISD::XFormAddr, PtrVT, GA, Zero);
+      Callee = DAG.getNode(SPUISD::IndirectAddr, PtrVT, GA, Zero);
     }
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
     Callee = DAG.getExternalSymbol(S->getSymbol(), Callee.getValueType());
@@ -2553,16 +2555,80 @@ SPUTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
 {
 #if 0
   TargetMachine &TM = getTargetMachine();
+#endif
+  const SPUSubtarget *ST = SPUTM.getSubtargetImpl();
   SelectionDAG &DAG = DCI.DAG;
   SDOperand N0 = N->getOperand(0);	// everything has at least one operand
 
   switch (N->getOpcode()) {
   default: break;
-    // Do something creative here for ISD nodes that can be coalesced in unique
-    // ways.
+  case SPUISD::IndirectAddr: {
+    if (!ST->usingLargeMem() && N0.getOpcode() == SPUISD::AFormAddr) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(1));
+      if (CN->getValue() == 0) {
+        // (SPUindirect (SPUaform <addr>, 0), 0) ->
+        // (SPUaform <addr>, 0)
+
+        DEBUG(cerr << "Replace: ");
+        DEBUG(N->dump(&DAG));
+        DEBUG(cerr << "\nWith:    ");
+        DEBUG(N0.Val->dump(&DAG));
+        DEBUG(cerr << "\n");
+
+        return N0;
+      }
+    }
+  }
+  case ISD::ADD: {
+    SDOperand Op0 = N->getOperand(0);
+    SDOperand Op1 = N->getOperand(1);
+
+    if ((Op1.getOpcode() == ISD::Constant
+         || Op1.getOpcode() == ISD::TargetConstant)
+        && Op0.getOpcode() == SPUISD::IndirectAddr) {
+      SDOperand Op01 = Op0.getOperand(1);
+      if (Op01.getOpcode() == ISD::Constant
+          || Op01.getOpcode() == ISD::TargetConstant) {
+        // (add <const>, (SPUindirect <arg>, <const>)) ->
+        // (SPUindirect <arg>, <const + const>)
+        ConstantSDNode *CN0 = cast<ConstantSDNode>(Op1);
+        ConstantSDNode *CN1 = cast<ConstantSDNode>(Op01);
+        SDOperand combinedConst =
+          DAG.getConstant(CN0->getValue() + CN1->getValue(),
+                          Op0.getValueType());
+
+        DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
+                   << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
+        DEBUG(cerr << "With:    (SPUindirect <arg>, "
+                   << CN0->getValue() + CN1->getValue() << ")\n");
+        return DAG.getNode(SPUISD::IndirectAddr, Op0.getValueType(),
+                           Op0.getOperand(0), combinedConst);
+      }
+    } else if ((Op0.getOpcode() == ISD::Constant
+                || Op0.getOpcode() == ISD::TargetConstant)
+               && Op1.getOpcode() == SPUISD::IndirectAddr) {
+      SDOperand Op11 = Op1.getOperand(1);
+      if (Op11.getOpcode() == ISD::Constant
+          || Op11.getOpcode() == ISD::TargetConstant) {
+        // (add (SPUindirect <arg>, <const>), <const>) ->
+        // (SPUindirect <arg>, <const + const>)
+        ConstantSDNode *CN0 = cast<ConstantSDNode>(Op0);
+        ConstantSDNode *CN1 = cast<ConstantSDNode>(Op11);
+        SDOperand combinedConst =
+          DAG.getConstant(CN0->getValue() + CN1->getValue(),
+                          Op0.getValueType());
+
+        DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
+                   << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
+        DEBUG(cerr << "With:    (SPUindirect <arg>, "
+                   << CN0->getValue() + CN1->getValue() << ")\n");
+
+        return DAG.getNode(SPUISD::IndirectAddr, Op1.getValueType(),
+                           Op1.getOperand(0), combinedConst);
+      }
+    }
+  }
   }
-#endif
-  
   // Otherwise, return unchanged.
   return SDOperand();
 }
diff --git a/lib/Target/CellSPU/SPUISelLowering.h b/lib/Target/CellSPU/SPUISelLowering.h
index d9e4e7ed4e..916f2c931f 100644
--- a/lib/Target/CellSPU/SPUISelLowering.h
+++ b/lib/Target/CellSPU/SPUISelLowering.h
@@ -32,8 +32,7 @@ namespace llvm {
       Lo,                       ///< Low address component (lower 16)
       PCRelAddr,                ///< Program counter relative address
       AFormAddr,                ///< A-form address (local store)
-      DFormAddr,                ///< D-Form address "imm($r)"
-      XFormAddr,                ///< X-Form address