14 files changed, 645 insertions, 874 deletions

diff --git a/lib/Target/CellSPU/SPUAsmPrinter.cpp b/lib/Target/CellSPU/SPUAsmPrinter.cpp
index 224616eb93..4bd8673a5b 100644
--- a/lib/Target/CellSPU/SPUAsmPrinter.cpp
+++ b/lib/Target/CellSPU/SPUAsmPrinter.cpp
@@ -195,12 +195,18 @@ namespace {
     void
     printAddr256K(const MachineInstr *MI, unsigned OpNo)
     {
-      /* Note: operand 1 is an offset or symbol name. Operand 2 is
-	 ignored. */
+      /* Note: operand 1 is an offset or symbol name. */
       if (MI->getOperand(OpNo).isImmediate()) {
         printS16ImmOperand(MI, OpNo);
       } else {
         printOp(MI->getOperand(OpNo));
+        if (MI->getOperand(OpNo+1).isImmediate()) {
+          int displ = int(MI->getOperand(OpNo+1).getImm());
+          if (displ > 0)
+            O << "+" << displ;
+          else if (displ < 0)
+            O << displ;
+        }
       }
     }
 
@@ -309,7 +315,6 @@ void SPUAsmPrinter::printOp(const MachineOperand &MO) {
   case MachineOperand::MO_JumpTableIndex:
     O << TAI->getPrivateGlobalPrefix() << "JTI" << getFunctionNumber()
       << '_' << MO.getIndex();
-    // FIXME: PIC relocation model
     return;
   case MachineOperand::MO_ConstantPoolIndex:
     O << TAI->getPrivateGlobalPrefix() << "CPI" << getFunctionNumber()
@@ -521,7 +526,8 @@ bool LinuxAsmPrinter::doFinalization(Module &M) {
         O << "\t.zero\t" << Size;
       } else if (I->hasInternalLinkage()) {
         SwitchToDataSection("\t.data", I);
-        O << TAI->getLCOMMDirective() << name << "," << Size << "," << Align;
+        O << ".local " << name << "\n";
+        O << TAI->getCOMMDirective() << name << "," << Size << "," << Align << "\n";
       } else {
         SwitchToDataSection("\t.data", I);
         O << ".comm " << name << "," << Size;
diff --git a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
index 3a50e3bcf0..3905d55302 100644
--- a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
+++ b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
@@ -150,6 +150,16 @@ namespace {
     return false;
   }
 
+  bool
+  isHighLow(const SDOperand &Op) 
+  {
+    return (Op.getOpcode() == SPUISD::IndirectAddr
+            && ((Op.getOperand(0).getOpcode() == SPUISD::Hi
+                 && Op.getOperand(1).getOpcode() == SPUISD::Lo)
+                || (Op.getOperand(0).getOpcode() == SPUISD::Lo
+                    && Op.getOperand(1).getOpcode() == SPUISD::Hi)));
+  }
+
   //===------------------------------------------------------------------===//
   //! MVT::ValueType to "useful stuff" mapping structure:
 
@@ -157,40 +167,24 @@ namespace {
     MVT::ValueType VT;
     unsigned ldresult_ins;	/// LDRESULT instruction (0 = undefined)
     int prefslot_byte;		/// Byte offset of the "preferred" slot
-    unsigned brcc_eq_ins;	/// br_cc equal instruction
-    unsigned brcc_neq_ins;	/// br_cc not equal instruction
-    unsigned load_aform;        /// A-form load instruction for this VT
-    unsigned store_aform;       /// A-form store instruction for this VT
+    unsigned insmask_ins;       /// Insert mask instruction for a-form
   };
 
   const valtype_map_s valtype_map[] = {
-    { MVT::i1,    0,            3, 0,         0,          0,
-      0 },
-    { MVT::i8,    SPU::ORBIr8,  3, 0,         0,          SPU::LQAr8,
-      SPU::STQAr8 },
-    { MVT::i16,   SPU::ORHIr16, 2, SPU::BRHZ, SPU::BRHNZ, SPU::LQAr16,
-      SPU::STQAr16 },
-    { MVT::i32,   SPU::ORIr32,  0, SPU::BRZ,  SPU::BRNZ,  SPU::LQAr32,
-      SPU::STQAr32 },
-    { MVT::i64,   SPU::ORIr64,  0, 0,         0,          SPU::LQAr64,
-      SPU::STQAr64 },
-    { MVT::f32,   0,            0, 0,         0,          SPU::LQAf32,
-      SPU::STQAf32 },
-    { MVT::f64,   0,            0, 0,         0,          SPU::LQAf64,
-      SPU::STQAf64 },
+    { MVT::i1,    0,            3, 0 },
+    { MVT::i8,    SPU::ORBIr8,  3, 0 },
+    { MVT::i16,   SPU::ORHIr16, 2, 0 },
+    { MVT::i32,   SPU::ORIr32,  0, 0 },
+    { MVT::i64,   SPU::ORIr64,  0, 0 },
+    { MVT::f32,   0,            0, 0 },
+    { MVT::f64,   0,            0, 0 },
     // vector types... (sigh!)
-    { MVT::v16i8, 0,            0, 0,         0,          SPU::LQAv16i8,
-      SPU::STQAv16i8 },
-    { MVT::v8i16, 0,            0, 0,         0,          SPU::LQAv8i16,
-      SPU::STQAv8i16 },
-    { MVT::v4i32, 0,            0, 0,         0,          SPU::LQAv4i32,
-      SPU::STQAv4i32 },
-    { MVT::v2i64, 0,            0, 0,         0,          SPU::LQAv2i64,
-      SPU::STQAv2i64 },
-    { MVT::v4f32, 0,            0, 0,         0,          SPU::LQAv4f32,
-      SPU::STQAv4f32 },
-    { MVT::v2f64, 0,            0, 0,         0,          SPU::LQAv2f64,
-      SPU::STQAv2f64 },
+    { MVT::v16i8, 0,            0, SPU::CBD },
+    { MVT::v8i16, 0,            0, SPU::CHD },
+    { MVT::v4i32, 0,            0, SPU::CWD },
+    { MVT::v2i64, 0,            0, 0 },
+    { MVT::v4f32, 0,            0, SPU::CWD },
+    { MVT::v2f64, 0,            0, 0 }
   };
 
   const size_t n_valtype_map = sizeof(valtype_map) / sizeof(valtype_map[0]);
@@ -381,18 +375,20 @@ SPUDAGToDAGISel::SelectDForm2Addr(SDOperand Op, SDOperand N, SDOperand &Disp,
     Disp = CurDAG->getTargetConstant(0, VT);
     Base = N;
     return true;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // D-Form address: This is pretty straightforward, naturally...
-    CN = cast<ConstantSDNode>(N.getOperand(1));
-    assert(CN != 0 && "SelectDFormAddr/SPUISD::DForm2Addr expecting constant");
-    Imm = unsigned(CN->getValue());
-    if (Imm < 0xff) {
-      Disp = CurDAG->getTargetConstant(CN->getValue(), PtrVT);
-      Base = N.getOperand(0);
-      return true;
+  } else if (Opc == SPUISD::IndirectAddr) {
+    SDOperand Op1 = N.getOperand(1);
+    if (Op1.getOpcode() == ISD::TargetConstant
+        || Op1.getOpcode() == ISD::Constant) {
+      CN = cast<ConstantSDNode>(N.getOperand(1));
+      assert(CN != 0 && "SelectIndirectAddr/SPUISD::DForm2Addr expecting constant");
+      Imm = unsigned(CN->getValue());
+      if (Imm < 0xff) {
+        Disp = CurDAG->getTargetConstant(CN->getValue(), PtrVT);
+        Base = N.getOperand(0);
+        return true;
+      }
     }
   }
-
   return false;
 }
 
@@ -407,7 +403,7 @@ SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
 		    SDOperand &Index) {
   // These match the addr256k operand type:
   MVT::ValueType OffsVT = MVT::i16;
-  MVT::ValueType PtrVT = SPUtli.getPointerTy();
+  SDOperand Zero = CurDAG->getTargetConstant(0, OffsVT);
 
   switch (N.getOpcode()) {
   case ISD::Constant:
@@ -417,28 +413,40 @@ SPUDAGToDAGISel::SelectAFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     abort();
     /*NOTREACHED*/
 
-  case ISD::TargetConstant: {
-    // Loading from a constant address.
-    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
-    int Imm = (int)CN->getValue();
-    if (Imm < 0x3ffff && (Imm & 0x3) == 0) {
-      Base = CurDAG->getTargetConstant(Imm, PtrVT);
-      // Note that this operand will be ignored by the assembly printer...
-      Index = CurDAG->getTargetConstant(0, OffsVT);
-      return true;
-    }
-  }
+  case ISD::TargetConstant:
   case ISD::TargetGlobalAddress:
-  case ISD::TargetConstantPool:
-  case SPUISD::AFormAddr: {
-    // The address is in Base. N is a dummy that will be ignored by
-    // the assembly printer.
-    Base = N;
-    Index = CurDAG->getTargetConstant(0, OffsVT);
-    return true;
-  }
-  }
+  case ISD::TargetJumpTable:
+    cerr << "SPUSelectAFormAddr: Target Constant/Pool/Global not wrapped as "
+         << "A-form address.\n";
+    abort();
+    /*NOTREACHED*/
 
+  case SPUISD::AFormAddr: 
+    // Just load from memory if there's only a single use of the location,
+    // otherwise, this will get handled below with D-form offset addresses
+    if (N.hasOneUse()) {
+      SDOperand Op0 = N.getOperand(0);
+      switch (Op0.getOpcode()) {
+      case ISD::TargetConstantPool:
+      case ISD::TargetJumpTable:
+        Base = Op0;
+        Index = Zero;
+        return true;
+
+      case ISD::TargetGlobalAddress: {
+        GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op0);
+        GlobalValue *GV = GSDN->getGlobal();
+        if (GV->getAlignment() == 16) {
+          Base = Op0;
+          Index = Zero;
+          return true;
+        }
+        break;
+      }
+      }
+    }
+    break;
+  }
   return false;
 }
 
@@ -460,14 +468,11 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
   unsigned Opc = N.getOpcode();
   unsigned PtrTy = SPUtli.getPointerTy();
 
-  if (Opc == ISD::Register) {
-    Base = N;
-    Index = CurDAG->getTargetConstant(0, PtrTy);
-    return true;
-  } else if (Opc == ISD::FrameIndex) {
+  if (Opc == ISD::FrameIndex) {
+    // Stack frame index must be less than 512 (divided by 16):
     FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
     DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
-	       << FI->getIndex() << "\n");
+	  << FI->getIndex() << "\n");
     if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
       Base = CurDAG->getTargetConstant(0, PtrTy);
       Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
@@ -475,19 +480,20 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
     }
   } else if (Opc == ISD::ADD) {
     // Generated by getelementptr
-    const SDOperand Op0 = N.getOperand(0); // Frame index/base
-    const SDOperand Op1 = N.getOperand(1); // Offset within base
+    const SDOperand Op0 = N.getOperand(0);
+    const SDOperand Op1 = N.getOperand(1);
 
-    if ((Op1.getOpcode() == ISD::Constant
-	 || Op1.getOpcode() == ISD::TargetConstant)
-	&& Op0.getOpcode() != SPUISD::XFormAddr) {
+    if ((Op0.getOpcode() == SPUISD::Hi && Op1.getOpcode() == SPUISD::Lo)
+        || (Op1.getOpcode() == SPUISD::Hi && Op0.getOpcode() == SPUISD::Lo)) {
+      Base = CurDAG->getTargetConstant(0, PtrTy);
+      Index = N;
+      return true;
+    } else if (Op1.getOpcode() == ISD::Constant
+               || Op1.getOpcode() == ISD::TargetConstant) {
       ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op1);
-      assert(CN != 0 && "SelectDFormAddr: Expected a constant");
-
-      int32_t offset = (int32_t) CN->getSignExtended();
-      unsigned Opc0 = Op0.getOpcode();
+      int32_t offset = int32_t(CN->getSignExtended());
 
-      if (Opc0 == ISD::FrameIndex) {
+      if (Op0.getOpcode() == ISD::FrameIndex) {
         FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op0);
         DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
               << " frame index = " << FI->getIndex() << "\n");
@@ -500,51 +506,69 @@ SPUDAGToDAGISel::SelectDFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       } else if (offset > SPUFrameInfo::minFrameOffset()
                  && offset < SPUFrameInfo::maxFrameOffset()) {
         Base = CurDAG->getTargetConstant(offset, PtrTy);
-        if (Opc0 == ISD::GlobalAddress) {
-          // Convert global address to target global address
-          GlobalAddressSDNode *GV = dyn_cast<GlobalAddressSDNode>(Op0);
-          Index = CurDAG->getTargetGlobalAddress(GV->getGlobal(), PtrTy);
-          return true;
-        } else {
-          // Otherwise, just take operand 0
-          Index = Op0;
+        Index = Op0;
+        return true;
+      }
+    } else if (Op0.getOpcode() == ISD::Constant
+               || Op0.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op0);
+      int32_t offset = int32_t(CN->getSignExtended());
+
+      if (Op1.getOpcode() == ISD::FrameIndex) {
+        FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op1);
+        DEBUG(cerr << "SelectDFormAddr: ISD::ADD offset = " << offset
+              << " frame index = " << FI->getIndex() << "\n");
+
+        if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+          Base = CurDAG->getTargetConstant(offset, PtrTy);
+          Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
           return true;
         }
+      } else if (offset > SPUFrameInfo::minFrameOffset()
+                 && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(offset, PtrTy);
+        Index = Op1;
+        return true;
       }
-    } else
-      return false;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // D-Form address: This is pretty straightforward,
-    // naturally... but make sure that this isn't a D-form address
-    // with a X-form address embedded within:
-    const SDOperand Op0 = N.getOperand(0); // Frame index/base
-    const SDOperand Op1 = N.getOperand(1); // Offset within base
-
-    if (Op0.getOpcode() == ISD::Constant
-        || Op0.getOpcode() == ISD::TargetConstant) {
-      ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
-      assert(CN != 0 && "SelectDFormAddr/SPUISD::DFormAddr expecting constant"); 
-      Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
-      Index = Op0;
-      return true;
     }
-  } else if (Opc == ISD::FrameIndex) {
-    // Stack frame index must be less than 512 (divided by 16):
-    FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N);
-    DEBUG(cerr << "SelectDFormAddr: ISD::FrameIndex = "
-	  << FI->getIndex() << "\n");
-    if (FI->getIndex() < SPUFrameInfo::maxFrameOffset()) {
+  } else if (Opc == SPUISD::IndirectAddr) {
+    // Indirect with constant offset -> D-Form address
+    const SDOperand Op0 = N.getOperand(0);
+    const SDOperand Op1 = N.getOperand(1);
+    SDOperand Zero = CurDAG->getTargetConstant(0, N.getValueType());
+
+    if (Op1.getOpcode() == ISD::Constant
+        || Op1.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
+      int32_t offset = int32_t(CN->getSignExtended());
+      if (offset > SPUFrameInfo::minFrameOffset()
+          && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
+        Index = Op0;
+        return true;
+      }
+    } else if (Op0.getOpcode() == ISD::Constant
+               || Op0.getOpcode() == ISD::TargetConstant) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op0);
+      int32_t offset = int32_t(CN->getSignExtended());
+      if (offset > SPUFrameInfo::minFrameOffset()
+          && offset < SPUFrameInfo::maxFrameOffset()) {
+        Base = CurDAG->getTargetConstant(CN->getValue(), PtrTy);
+        Index = Op1;
+        return true;
+      }
+    } else if (Op0.getOpcode() == SPUISD::Hi
+               && Op1.getOpcode() == SPUISD::Lo) {
+      // (SPUindirect (SPUhi <arg>, 0), (SPUlo <arg>, 0))
       Base = CurDAG->getTargetConstant(0, PtrTy);
-      Index = CurDAG->getTargetFrameIndex(FI->getIndex(), PtrTy);
+      Index = N;
       return true;
     }
-  } else if (Opc == SPUISD::LDRESULT) {
-    // It's a load result dereference
-    Base = CurDAG->getTargetConstant(0, PtrTy);
-    Index = N.getOperand(0);
+  } else if (Opc == SPUISD::AFormAddr) {
+    Base = CurDAG->getTargetConstant(0, N.getValueType());
+    Index = N;
     return true;
   }
-
   return false;
 }
 
@@ -565,108 +589,10 @@ SPUDAGToDAGISel::SelectXFormAddr(SDOperand Op, SDOperand N, SDOperand &Base,
       || SelectDFormAddr(Op, N, Base, Index))
     return false;
 
-  unsigned Opc = N.getOpcode();
-
-  if (Opc == ISD::ADD) {
-    SDOperand N1 = N.getOperand(0);
-    SDOperand N2 = N.getOperand(1);
-    Base = N.getOperand(0);
-    Index = N.getOperand(1);
-    return true;
-  } else if (Opc == SPUISD::XFormAddr) {
-    Base = N;
-    Index = N.getOperand(1);
-    return true;
-  } else if (Opc == SPUISD::DFormAddr) {
-    // Must be a D-form address with an X-form address embedded
-    // within:
-    Base = N.getOperand(0);
-    Index = N.getOperand(1);
-    return true;
-  } else if (N.getNumOperands() == 2) {
-    SDOperand N1 = N.getOperand(0);
-    SDOperand N2 = N.getOperand(1);
-    unsigned N1Opc = N1.getOpcode();
-    unsigned N2Opc = N2.getOpcode();
-
-    if ((N1Opc == ISD::CopyToReg || N1Opc == ISD::Register)
-	&& (N2Opc == ISD::CopyToReg || N2Opc == ISD::Register)) {
-      Base = N.getOperand(0);
-      Index = N.getOperand(1);
-      return true;
-      /*UNREACHED*/
-    } else {
-      cerr << "SelectXFormAddr: 2-operand unhandled operand:\n";
-      N.Val->dump(CurDAG);
-      cerr << "\n";
-      abort();
-    /*UNREACHED*/
-    }
-  } else {
-    cerr << "SelectXFormAddr: Unhandled operand type:\n";
-    N.Val->dump(CurDAG);
-    cerr << "\n";
-    abort();
-    /*UNREACHED*/
-  }
-
-  return false;
-}
-
-//! Emit load for A-form addresses
-/*
- */
-SDNode *
-Emit_LOAD_AFormAddr(SDOperand Op, SelectionDAG &CurDAG, SPUDAGToDAGISel &ISel)
-{
-  SDNode *Result;
-  MVT::ValueType OpVT = Op.getValueType();
-  SDOperand Chain = Op.getOperand(0);
-  SDOperand Ptr = Op.getOperand(1);
-  SDOperand PtrArg = Ptr.getOperand(0);
-  SDOperand PtrOffs = Ptr.getOperand(1);
-  const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
-
-  if (PtrOffs.getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(PtrOffs);
-    MVT::ValueType PVT = PtrOffs.getValueType();
-    PtrOffs = CurDAG.getTargetConstant(CN->getValue(), PVT);
-  }
-  ISel.AddToISelQueue(PtrArg);
-  ISel.AddToISelQueue(PtrOffs);
-  ISel.AddToISelQueue(Chain);
-  Result = CurDAG.getTargetNode(vtm->load_aform, OpVT, MVT::Other, PtrArg, PtrOffs, Chain);
-  Chain = SDOperand(Result, 1);
-  return Result;
-}
-
-//! Emit store for A-form addresses
-/*
- */
-SDNode *
-Emit_STORE_AFormAddr(SDOperand Op, SelectionDAG &CurDAG, SPUDAGToDAGISel &ISel)
-{
-  SDNode *Result;
-  SDOperand Chain = Op.getOperand(0);
-  SDOperand Val = Op.getOperand(1);
-  SDOperand Ptr = Op.getOperand(2);
-  SDOperand PtrArg = Ptr.getOperand(0);
-  SDOperand PtrOffs = Ptr.getOperand(1);
-  const valtype_map_s *vtm = getValueTypeMapEntry(Val.getValueType());
-
-  if (PtrOffs.getOpcode() == ISD::Constant) {
-    ConstantSDNode *CN = cast<ConstantSDNode>(PtrOffs);
-    MVT::ValueType PVT = PtrOffs.getValueType();
-    PtrOffs = CurDAG.getTargetConstant(CN->getValue(), PVT);
-  }
-  ISel.AddToISelQueue(Val);
-  ISel.AddToISelQueue(PtrArg);
-  ISel.AddToISelQueue(PtrOffs);
-  ISel.AddToISelQueue(Chain);
-  SDOperand Ops[4] = { Val, PtrArg, PtrOffs, Chain };
-  Result = CurDAG.getTargetNode(vtm->store_aform, MVT::Other, Ops, 4);
-  Chain = SDOperand(Result, 1);
-  return Result;
+  // All else fails, punt and use an X-form address:
+  Base = N.getOperand(0);
+  Index = N.getOperand(1);
+  return true;
 }
 
 //! Convert the operand from a target-independent to a target-specific node
@@ -695,12 +621,6 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
     Ops[0] = TFI;
     Ops[1] = Zero;
     n_ops = 2;
-  } else if (Opc == ISD::LOAD
-             && Op.getOperand(1).getOpcode() == SPUISD::AFormAddr) {
-    return Emit_LOAD_AFormAddr(Op, *CurDAG, *this);
-  } else if (Opc == ISD::STORE
-             && Op.getOperand(2).getOpcode() == SPUISD::AFormAddr) {
-    return Emit_STORE_AFormAddr(Op, *CurDAG, *this);
   } else if (Opc == ISD::ZERO_EXTEND) {
     // (zero_extend:i16 (and:i8 <arg>, <const>))
     const SDOperand &Op1 = N->getOperand(0);
@@ -717,6 +637,38 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
         n_ops = 2;
       }
     }
+  } else if (Opc == SPUISD::INSERT_MASK) {
+    SDOperand Op0 = Op.getOperand(0);
+    if (Op0.getOpcode() == SPUISD::AFormAddr) {
+      // (SPUvecinsmask (SPUaform <arg>, 0)) ->
+      // (CBD|CHD|CWD 0, arg)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      ConstantSDNode *CN = cast<ConstantSDNode>(Op0.getOperand(1));
+      assert(vtm->insmask_ins != 0 && "missing insert mask instruction");
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(CN->getValue(), Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+
+      AddToISelQueue(Op0);
+    } else if (Op0.getOpcode() == ISD::FrameIndex) {
+      // (SPUvecinsmask <fi>) ->
+      // (CBD|CHD|CWD 0, <fi>)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(0, Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+    } else if (isHighLow(Op0)) {
+      // (SPUvecinsmask (SPUindirect (SPUhi <arg>, 0), (SPUlow <arg>, 0))) ->
+      // (CBD|CHD|CWD 0, arg)
+      const valtype_map_s *vtm = getValueTypeMapEntry(OpVT);
+      NewOpc = vtm->insmask_ins;
+      Ops[0] = CurDAG->getTargetConstant(0, Op0.getValueType());
+      Ops[1] = Op0;
+      n_ops = 2;
+      AddToISelQueue(Op0);
+    }
   } else if (Opc == SPUISD::LDRESULT) {
     // Custom select instructions for LDRESULT
     unsigned VT = N->getValueType(0);
@@ -748,19 +700,19 @@ SPUDAGToDAGISel::Select(SDOperand Op) {
     AddToISelQueue(Chain);
 
     return Result;
-  } else if (Opc == SPUISD::XFormAddr) {
+  } else if (Opc == SPUISD::IndirectAddr) {
     SDOperand Op0 = Op.getOperand(0);
     if (Op0.getOpcode() == SPUISD::LDRESULT
         || Op0.getOpcode() == SPUISD::AFormAddr) {
-      // (XFormAddr (LDRESULT|AFormAddr, imm))
+      // (IndirectAddr (LDRESULT|AFormAddr, imm))
       SDOperand Op1 = Op.getOperand(1);
       MVT::ValueType VT = Op.getValueType();
 
-      DEBUG(cerr << "CellSPU: XFormAddr("
-            << (Op0.getOpcode() == SPUISD::LDRESULT
-                ? "LDRESULT"
-                : "AFormAddr")
-            << ", imm):\nOp0 = ");
+      DEBUG(cerr << "CellSPU: IndirectAddr("
+                 << (Op0.getOpcode() == SPUISD::LDRESULT
+                     ? "LDRESULT"
+                     : "AFormAddr")
+                 << ", imm):\nOp0 = ");
       DEBUG(Op.getOperand(0).Val->dump(CurDAG));
       DEBUG(cerr << "\nOp1 = ");
       DEBUG(Op.getOperand(1).Val->dump(CurDAG));
diff --git a/lib/Target/CellSPU/SPUISelLowering.cpp b/lib/Target/CellSPU/SPUISelLowering.cpp
index 0f1d045280..33261a607b 100644
--- a/lib/Target/CellSPU/SPUISelLowering.cpp
+++ b/lib/Target/CellSPU/SPUISelLowering.cpp
@@ -289,14 +289,12 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   
   // We want to legalize GlobalAddress and ConstantPool nodes into the 
   // appropriate instructions to materialize the address.
-  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::i32, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::f32, Custom);
-  setOperationAction(ISD::JumpTable,     MVT::i32, Custom);
-  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::i64, Custom);
-  setOperationAction(ISD::ConstantPool,  MVT::f64, Custom);
-  setOperationAction(ISD::JumpTable,     MVT::i64, Custom);
+  for (unsigned sctype = (unsigned) MVT::i1; sctype < (unsigned) MVT::f128;
+       ++sctype) {
+    setOperationAction(ISD::GlobalAddress, sctype, Custom);
+    setOperationAction(ISD::ConstantPool,  sctype, Custom);
+    setOperationAction(ISD::JumpTable,     sctype, Custom);
+  }
 
   // RET must be custom lowered, to meet ABI requirements
   setOperationAction(ISD::RET,           MVT::Other, Custom);
@@ -377,7 +375,7 @@ SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
   setStackPointerRegisterToSaveRestore(SPU::R1);
   
   // We have target-specific dag combine patterns for the following nodes:
-  // e.g., setTargetDAGCombine(ISD::SUB);
+  setTargetDAGCombine(ISD::ADD);
   
   computeRegisterProperties();
 }
@@ -391,8 +389,7 @@ SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
     node_names[(unsigned) SPUISD::Lo] = "SPUISD::Lo";
     node_names[(unsigned) SPUISD::PCRelAddr] = "SPUISD::PCRelAddr";
     node_names[(unsigned) SPUISD::AFormAddr] = "SPUISD::AFormAddr";
-    node_names[(unsigned) SPUISD::DFormAddr] = "SPUISD::DFormAddr";
-    node_names[(unsigned) SPUISD::XFormAddr] = "SPUISD::XFormAddr";
+    node_names[(unsigned) SPUISD::IndirectAddr] = "SPUISD::IndirectAddr";
     node_names[(unsigned) SPUISD::LDRESULT] = "SPUISD::LDRESULT";
     node_names[(unsigned) SPUISD::CALL] = "SPUISD::CALL";
     node_names[(unsigned) SPUISD::SHUFB] = "SPUISD::SHUFB";
@@ -524,11 +521,12 @@ AlignedLoad(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST,
   // Unaligned load or we're using the "large memory" model, which means that
   // we have to be very pessimistic:
   if (isMemoryOperand(basePtr) || isIndirectOperand(basePtr)) {
-    basePtr = DAG.getNode(SPUISD::XFormAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
+    basePtr = DAG.getNode(SPUISD::IndirectAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
   }
 
   // Add the offset
-  basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, DAG.getConstant(alignOffs, PtrVT));
+  basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr,
+			DAG.getConstant((alignOffs & ~0xf), PtrVT));
   was16aligned = false;
   return DAG.getLoad(MVT::v16i8, chain, basePtr,
                      LSN->getSrcValue(), LSN->getSrcValueOffset(),
@@ -706,21 +704,20 @@ LowerSTORE(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
     DEBUG(basePtr.Val->dump(&DAG));
     DEBUG(cerr << "\n");
 
-    if (basePtr.getOpcode() == SPUISD::DFormAddr) {
-      // Hmmmm... do we ever actually hit this code?
-      insertEltPtr = DAG.getNode(SPUISD::DFormAddr, PtrVT,
-				 basePtr.getOperand(0),
-				 insertEltOffs);
-    } else if (basePtr.getOpcode() == SPUISD::XFormAddr ||
-	       (basePtr.getOpcode() == ISD::ADD
-		&& basePtr.getOperand(0).getOpcode() == SPUISD::XFormAddr)) {
+    if (basePtr.getOpcode() == SPUISD::IndirectAddr ||
+        (basePtr.getOpcode() == ISD::ADD
+         && basePtr.getOperand(0).getOpcode() == SPUISD::IndirectAddr)) {
       insertEltPtr = basePtr;
     } else {
-      // $sp is always aligned, so use it instead of potentially loading an
-      // address into a new register:
-      insertEltPtr = DAG.getNode(SPUISD::DFormAddr, PtrVT,
-				 DAG.getRegister(SPU::R1, PtrVT),
-				 insertEltOffs);
+#if 0
+      // $sp is always aligned, so use it when necessary to avoid loading
+      // an address
+      SDOperand ptrP =
+        basePtr.Val->hasOneUse() ? DAG.getRegister(SPU::R1, PtrVT) : basePtr;
+      insertEltPtr = DAG.getNode(ISD::ADD, PtrVT, ptrP, insertEltOffs);
+#else
+      insertEltPtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, insertEltOffs);
+#endif
     }
 
     insertEltOp = DAG.getNode(SPUISD::INSERT_MASK, stVecVT, insertEltPtr);
@@ -772,7 +769,7 @@ LowerConstantPool(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
 
       return DAG.getNode(ISD::ADD, PtrVT, Lo, Hi);
 #else
-      return DAG.getNode(SPUISD::XFormAddr, PtrVT, CPI, Zero);
+      return DAG.getNode(SPUISD::IndirectAddr, PtrVT, CPI, Zero);
 #endif
     }
   }
@@ -791,9 +788,10 @@ LowerJumpTable(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
   const TargetMachine &TM = DAG.getTarget();
 
   if (TM.getRelocationModel() == Reloc::Static) {
+    SDOperand JmpAForm = DAG.getNode(SPUISD::AFormAddr, PtrVT, JTI, Zero);
     return (!ST->usingLargeMem()
-            ? DAG.getNode(SPUISD::AFormAddr, PtrVT, JTI, Zero)
-            : DAG.getNode(SPUISD::XFormAddr, PtrVT, JTI, Zero));
+            ? JmpAForm
+            : DAG.getNode(SPUISD::IndirectAddr, PtrVT, JmpAForm, Zero));
   }
 
   assert(0 &&
@@ -811,9 +809,13 @@ LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
   SDOperand Zero = DAG.getConstant(0, PtrVT);
   
   if (TM.getRelocationModel() == Reloc::Static) {
-    return (!ST->usingLargeMem()
-            ? DAG.getNode(SPUISD::AFormAddr, PtrVT, GA, Zero)
-            : DAG.getNode(SPUISD::XFormAddr, PtrVT, GA, Zero));
+    if (!ST->usingLargeMem()) {
+      return DAG.getNode(SPUISD::AFormAddr, PtrVT, GA, Zero);
+    } else {
+      SDOperand Hi = DAG.getNode(SPUISD::Hi, PtrVT, GA, Zero);
+      SDOperand Lo = DAG.getNode(SPUISD::Lo, PtrVT, GA, Zero);
+      return DAG.getNode(SPUISD::IndirectAddr, PtrVT, Hi, Lo);
+    }
   } else {
     cerr << "LowerGlobalAddress: Relocation model other than static not "
 	 << "supported.\n";
@@ -1202,7 +1204,7 @@ LowerCALL(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
     } else {
       // "Large memory" mode: Turn all calls into indirect calls with a X-form
       // address pairs:
-      Callee = DAG.getNode(SPUISD::XFormAddr, PtrVT, GA, Zero);
+      Callee = DAG.getNode(SPUISD::IndirectAddr, PtrVT, GA, Zero);
     }
   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
     Callee = DAG.getExternalSymbol(S->getSymbol(), Callee.getValueType());
@@ -2553,16 +2555,80 @@ SPUTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
 {
 #if 0
   TargetMachine &TM = getTargetMachine();
+#endif
+  const SPUSubtarget *ST = SPUTM.getSubtargetImpl();
   SelectionDAG &DAG = DCI.DAG;
   SDOperand N0 = N->getOperand(0);	// everything has at least one operand
 
   switch (N->getOpcode()) {
   default: break;
-    // Do something creative here for ISD nodes that can be coalesced in unique
-    // ways.
+  case SPUISD::IndirectAddr: {
+    if (!ST->usingLargeMem() && N0.getOpcode() == SPUISD::AFormAddr) {
+      ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(1));
+      if (CN->getValue() == 0) {
+        // (SPUindirect (SPUaform <addr>, 0), 0) ->
+        // (SPUaform <addr>, 0)
+
+        DEBUG(cerr << "Replace: ");
+        DEBUG(N->dump(&DAG));
+        DEBUG(cerr << "\nWith:    ");
+        DEBUG(N0.Val->dump(&DAG));
+        DEBUG(cerr << "\n");
+
+        return N0;
+      }
+    }
+  }
+  case ISD::ADD: {
+    SDOperand Op0 = N->getOperand(0);
+    SDOperand Op1 = N->getOperand(1);
+
+    if ((Op1.getOpcode() == ISD::Constant
+         || Op1.getOpcode() == ISD::TargetConstant)
+        && Op0.getOpcode() == SPUISD::IndirectAddr) {
+      SDOperand Op01 = Op0.getOperand(1);
+      if (Op01.getOpcode() == ISD::Constant
+          || Op01.getOpcode() == ISD::TargetConstant) {
+        // (add <const>, (SPUindirect <arg>, <const>)) ->
+        // (SPUindirect <arg>, <const + const>)
+        ConstantSDNode *CN0 = cast<ConstantSDNode>(Op1);
+        ConstantSDNode *CN1 = cast<ConstantSDNode>(Op01);
+        SDOperand combinedConst =
+          DAG.getConstant(CN0->getValue() + CN1->getValue(),
+                          Op0.getValueType());
+
+        DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
+                   << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
+        DEBUG(cerr << "With:    (SPUindirect <arg>, "
+                   << CN0->getValue() + CN1->getValue() << ")\n");
+        return DAG.getNode(SPUISD::IndirectAddr, Op0.getValueType(),
+                           Op0.getOperand(0), combinedConst);
+      }
+    } else if ((Op0.getOpcode() == ISD::Constant
+                || Op0.getOpcode() == ISD::TargetConstant)
+               && Op1.getOpcode() == SPUISD::IndirectAddr) {
+      SDOperand Op11 = Op1.getOperand(1);
+      if (Op11.getOpcode() == ISD::Constant
+          || Op11.getOpcode() == ISD::TargetConstant) {
+        // (add (SPUindirect <arg>, <const>), <const>) ->
+        // (SPUindirect <arg>, <const + const>)
+        ConstantSDNode *CN0 = cast<ConstantSDNode>(Op0);
+        ConstantSDNode *CN1 = cast<ConstantSDNode>(Op11);
+        SDOperand combinedConst =
+          DAG.getConstant(CN0->getValue() + CN1->getValue(),
+                          Op0.getValueType());
+
+        DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
+                   << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
+        DEBUG(cerr << "With:    (SPUindirect <arg>, "
+                   << CN0->getValue() + CN1->getValue() << ")\n");
+
+        return DAG.getNode(SPUISD::IndirectAddr, Op1.getValueType(),
+                           Op1.getOperand(0), combinedConst);
+      }
+    }
+  }
   }
-#endif
-  
   // Otherwise, return unchanged.
   return SDOperand();
 }
diff --git a/lib/Target/CellSPU/SPUISelLowering.h b/lib/Target/CellSPU/SPUISelLowering.h
index d9e4e7ed4e..916f2c931f 100644
--- a/lib/Target/CellSPU/SPUISelLowering.h
+++ b/lib/Target/CellSPU/SPUISelLowering.h
@@ -32,8 +32,7 @@ namespace llvm {
       Lo,                       ///< Low address component (lower 16)
       PCRelAddr,                ///< Program counter relative address
       AFormAddr,                ///< A-form address (local store)
-      DFormAddr,                ///< D-Form address "imm($r)"
-      XFormAddr,                ///< X-Form address "$r($r)"
+      IndirectAddr,             ///< D-Form "imm($r)" and X-form "$r($r)"
 
       LDRESULT,                 ///< Load result (value, chain)
       CALL,                     ///< CALL instruction
diff --git a/lib/Target/CellSPU/SPUInstrFormats.td b/lib/Target/CellSPU/SPUInstrFormats.td
index 7221ab2dc8..eda1ab3da4 100644
--- a/lib/Target/CellSPU/SPUInstrFormats.td
+++ b/lib/Target/CellSPU/SPUInstrFormats.td
@@ -247,6 +247,10 @@ let RT = 0 in {
   { }
 }
 
+//===----------------------------------------------------------------------===//
+// Specialized versions of RI16:
+//===----------------------------------------------------------------------===//
+