Clean up the atomic opcodes in SelectionDAG.

This removes all the _8, _16, _32, and _64 opcodes and replaces each
group with an unsuffixed opcode. The MemoryVT field of the AtomicSDNode
is now used to carry the size information. In tablegen, the size-specific
opcodes are replaced by size-independent opcodes that utilize the
ability to compose them with predicates.

This shrinks the per-opcode tables and makes the code that handles
atomics much more concise.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@61389 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 276 insertions, 817 deletions

diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 4cbe3b02e4..ea35e69a48 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -464,13 +464,13 @@ public:
 
   /// getAtomic - Gets a node for an atomic op, produces result and chain and 
   /// takes 3 operands
-  SDValue getAtomic(unsigned Opcode, SDValue Chain, SDValue Ptr, 
+  SDValue getAtomic(unsigned Opcode, MVT MemVT, SDValue Chain, SDValue Ptr, 
                     SDValue Cmp, SDValue Swp, const Value* PtrVal,
                     unsigned Alignment=0);
 
   /// getAtomic - Gets a node for an atomic op, produces result and chain and
   /// takes 2 operands.
-  SDValue getAtomic(unsigned Opcode, SDValue Chain, SDValue Ptr, 
+  SDValue getAtomic(unsigned Opcode, MVT MemVT, SDValue Chain, SDValue Ptr, 
                     SDValue Val, const Value* PtrVal,
                     unsigned Alignment = 0);
 
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 2fabe16cf8..74693dc8bf 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -628,64 +628,28 @@ namespace ISD {
     // this corresponds to the atomic.lcs intrinsic.
     // cmp is compared to *ptr, and if equal, swap is stored in *ptr.
     // the return is always the original value in *ptr
-    ATOMIC_CMP_SWAP_8,
-    ATOMIC_CMP_SWAP_16,
-    ATOMIC_CMP_SWAP_32,
-    ATOMIC_CMP_SWAP_64,
+    ATOMIC_CMP_SWAP,
 
     // Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
     // this corresponds to the atomic.swap intrinsic.
     // amt is stored to *ptr atomically.
     // the return is always the original value in *ptr
-    ATOMIC_SWAP_8,
-    ATOMIC_SWAP_16,
-    ATOMIC_SWAP_32,
-    ATOMIC_SWAP_64,
+    ATOMIC_SWAP,
 
     // Val, OUTCHAIN = ATOMIC_L[OpName]S(INCHAIN, ptr, amt)
     // this corresponds to the atomic.[OpName] intrinsic.
     // op(*ptr, amt) is stored to *ptr atomically.
     // the return is always the original value in *ptr
-    ATOMIC_LOAD_ADD_8,
-    ATOMIC_LOAD_SUB_8,
-    ATOMIC_LOAD_AND_8,
-    ATOMIC_LOAD_OR_8,
-    ATOMIC_LOAD_XOR_8,
-    ATOMIC_LOAD_NAND_8,
-    ATOMIC_LOAD_MIN_8,
-    ATOMIC_LOAD_MAX_8,
-    ATOMIC_LOAD_UMIN_8,
-    ATOMIC_LOAD_UMAX_8,
-    ATOMIC_LOAD_ADD_16,
-    ATOMIC_LOAD_SUB_16,
-    ATOMIC_LOAD_AND_16,
-    ATOMIC_LOAD_OR_16,
-    ATOMIC_LOAD_XOR_16,
-    ATOMIC_LOAD_NAND_16,
-    ATOMIC_LOAD_MIN_16,
-    ATOMIC_LOAD_MAX_16,
-    ATOMIC_LOAD_UMIN_16,
-    ATOMIC_LOAD_UMAX_16,
-    ATOMIC_LOAD_ADD_32,
-    ATOMIC_LOAD_SUB_32,
-    ATOMIC_LOAD_AND_32,
-    ATOMIC_LOAD_OR_32,
-    ATOMIC_LOAD_XOR_32,
-    ATOMIC_LOAD_NAND_32,
-    ATOMIC_LOAD_MIN_32,
-    ATOMIC_LOAD_MAX_32,
-    ATOMIC_LOAD_UMIN_32,
-    ATOMIC_LOAD_UMAX_32,
-    ATOMIC_LOAD_ADD_64,
-    ATOMIC_LOAD_SUB_64,
-    ATOMIC_LOAD_AND_64,
-    ATOMIC_LOAD_OR_64,
-    ATOMIC_LOAD_XOR_64,
-    ATOMIC_LOAD_NAND_64,
-    ATOMIC_LOAD_MIN_64,
-    ATOMIC_LOAD_MAX_64,
-    ATOMIC_LOAD_UMIN_64,
-    ATOMIC_LOAD_UMAX_64,
+    ATOMIC_LOAD_ADD,
+    ATOMIC_LOAD_SUB,
+    ATOMIC_LOAD_AND,
+    ATOMIC_LOAD_OR,
+    ATOMIC_LOAD_XOR,
+    ATOMIC_LOAD_NAND,
+    ATOMIC_LOAD_MIN,
+    ATOMIC_LOAD_MAX,
+    ATOMIC_LOAD_UMIN,
+    ATOMIC_LOAD_UMAX,
     
     // BUILTIN_OP_END - This must be the last enum value in this list.
     BUILTIN_OP_END
@@ -1615,58 +1579,18 @@ public:
     // with either an intrinsic or a target opcode.
     return N->getOpcode() == ISD::LOAD                ||
            N->getOpcode() == ISD::STORE               ||
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_8   ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_8       ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_8    ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_8  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_8  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_8  ||
-
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_16  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_16      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_16   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_16 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_16 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_16 ||
-
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_32  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_32      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_32   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_32 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_32 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_32 ||
-
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_64  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_64      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_64   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_64 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_64 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_64 ||
-           
+           N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
+           N->getOpcode() == ISD::ATOMIC_SWAP         ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
            N->getOpcode() == ISD::INTRINSIC_W_CHAIN   ||
            N->getOpcode() == ISD::INTRINSIC_VOID      ||
            N->isTargetOpcode();
@@ -1688,10 +1612,11 @@ class AtomicSDNode : public MemSDNode {
   // Swp:    swap value
   // SrcVal: address to update as a Value (used for MemOperand)
   // Align:  alignment of memory
-  AtomicSDNode(unsigned Opc, SDVTList VTL, SDValue Chain, SDValue Ptr, 
+  AtomicSDNode(unsigned Opc, SDVTList VTL, MVT MemVT,
+               SDValue Chain, SDValue Ptr,
                SDValue Cmp, SDValue Swp, const Value* SrcVal,
                unsigned Align=0)
-    : MemSDNode(Opc, VTL, Cmp.getValueType(), SrcVal, /*SVOffset=*/0,
+    : MemSDNode(Opc, VTL, MemVT, SrcVal, /*SVOffset=*/0,
                 Align, /*isVolatile=*/true) {
     Ops[0] = Chain;
     Ops[1] = Ptr;
@@ -1699,9 +1624,10 @@ class AtomicSDNode : public MemSDNode {
     Ops[3] = Swp;
     InitOperands(Ops, 4);
   }
-  AtomicSDNode(unsigned Opc, SDVTList VTL, SDValue Chain, SDValue Ptr, 
+  AtomicSDNode(unsigned Opc, SDVTList VTL, MVT MemVT,
+               SDValue Chain, SDValue Ptr, 
                SDValue Val, const Value* SrcVal, unsigned Align=0)
-    : MemSDNode(Opc, VTL, Val.getValueType(), SrcVal, /*SVOffset=*/0,
+    : MemSDNode(Opc, VTL, MemVT, SrcVal, /*SVOffset=*/0,
                 Align, /*isVolatile=*/true) {
     Ops[0] = Chain;
     Ops[1] = Ptr;
@@ -1714,63 +1640,24 @@ class AtomicSDNode : public MemSDNode {
 
   bool isCompareAndSwap() const { 
     unsigned Op = getOpcode(); 
-    return Op == ISD::ATOMIC_CMP_SWAP_8 ||
-           Op == ISD::ATOMIC_CMP_SWAP_16 ||
-           Op == ISD::ATOMIC_CMP_SWAP_32 ||
-           Op == ISD::ATOMIC_CMP_SWAP_64;
+    return Op == ISD::ATOMIC_CMP_SWAP;
   }
 
   // Methods to support isa and dyn_cast
   static bool classof(const AtomicSDNode *) { return true; }
   static bool classof(const SDNode *N) {
-    return N->getOpcode() == ISD::ATOMIC_CMP_SWAP_8   ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_8       ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_8    ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_8  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_8   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_8  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_8  ||
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_16  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_16      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_16   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_16 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_16  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_16 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_16 ||
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_32  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_32      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_32   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_32 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_32  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_32 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_32 ||
-           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_64  ||
-           N->getOpcode() == ISD::ATOMIC_SWAP_64      ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_ADD_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_SUB_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_AND_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_OR_64   ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_XOR_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_NAND_64 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MIN_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_MAX_64  ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_64 ||
-           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_64;
+    return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
+           N->getOpcode() == ISD::ATOMIC_SWAP         ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_UMAX;
   }
 };
 
diff --git a/include/llvm/Target/TargetLowering.h b/include/llvm/Target/TargetLowering.h
index 2c655881ef..1e14a11d59 100644
--- a/include/llvm/Target/TargetLowering.h
+++ b/include/llvm/Target/TargetLowering.h
@@ -1492,7 +1492,7 @@ private:
   MVT TransformToType[MVT::LAST_VALUETYPE];
 
   // Defines the capacity of the TargetLowering::OpActions table
-  static const int OpActionsCapacity = 222;
+  static const int OpActionsCapacity = 184;
 
   /// OpActions - For each operation and each value type, keep a LegalizeAction
   /// that indicates how instruction selection should deal with the operation.
diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td
index c936f7af4c..5d1b37d1c7 100644
--- a/include/llvm/Target/TargetSelectionDAG.td
+++ b/include/llvm/Target/TargetSelectionDAG.td
@@ -363,101 +363,29 @@ def prefetch   : SDNode<"ISD::PREFETCH"   , STDPrefetch,
 def membarrier : SDNode<"ISD::MEMBARRIER" , STDMemBarrier,
                         [SDNPHasChain, SDNPSideEffect]>;
 
-def atomic_cmp_swap_8 : SDNode<"ISD::ATOMIC_CMP_SWAP_8" , STDAtomic3,
+def atomic_cmp_swap : SDNode<"ISD::ATOMIC_CMP_SWAP" , STDAtomic3,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_add_8 : SDNode<"ISD::ATOMIC_LOAD_ADD_8" , STDAtomic2,
+def atomic_load_add : SDNode<"ISD::ATOMIC_LOAD_ADD" , STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_swap_8     : SDNode<"ISD::ATOMIC_SWAP_8", STDAtomic2,
+def atomic_swap     : SDNode<"ISD::ATOMIC_SWAP", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_sub_8 : SDNode<"ISD::ATOMIC_LOAD_SUB_8" , STDAtomic2,
+def atomic_load_sub : SDNode<"ISD::ATOMIC_LOAD_SUB" , STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_and_8 : SDNode<"ISD::ATOMIC_LOAD_AND_8" , STDAtomic2,
+def atomic_load_and : SDNode<"ISD::ATOMIC_LOAD_AND" , STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_or_8  : SDNode<"ISD::ATOMIC_LOAD_OR_8" , STDAtomic2,
+def atomic_load_or  : SDNode<"ISD::ATOMIC_LOAD_OR" , STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_xor_8 : SDNode<"ISD::ATOMIC_LOAD_XOR_8" , STDAtomic2,
+def atomic_load_xor : SDNode<"ISD::ATOMIC_LOAD_XOR" , STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_nand_8: SDNode<"ISD::ATOMIC_LOAD_NAND_8", STDAtomic2,
+def atomic_load_nand: SDNode<"ISD::ATOMIC_LOAD_NAND", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_min_8 : SDNode<"ISD::ATOMIC_LOAD_MIN_8", STDAtomic2,
+def atomic_load_min : SDNode<"ISD::ATOMIC_LOAD_MIN", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_max_8 : SDNode<"ISD::ATOMIC_LOAD_MAX_8", STDAtomic2,
+def atomic_load_max : SDNode<"ISD::ATOMIC_LOAD_MAX", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umin_8 : SDNode<"ISD::ATOMIC_LOAD_UMIN_8", STDAtomic2,
+def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umax_8 : SDNode<"ISD::ATOMIC_LOAD_UMAX_8", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_cmp_swap_16 : SDNode<"ISD::ATOMIC_CMP_SWAP_16" , STDAtomic3,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_add_16 : SDNode<"ISD::ATOMIC_LOAD_ADD_16" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_swap_16     : SDNode<"ISD::ATOMIC_SWAP_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_sub_16 : SDNode<"ISD::ATOMIC_LOAD_SUB_16" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_and_16 : SDNode<"ISD::ATOMIC_LOAD_AND_16" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_or_16  : SDNode<"ISD::ATOMIC_LOAD_OR_16" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_xor_16 : SDNode<"ISD::ATOMIC_LOAD_XOR_16" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_nand_16: SDNode<"ISD::ATOMIC_LOAD_NAND_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_min_16 : SDNode<"ISD::ATOMIC_LOAD_MIN_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_max_16 : SDNode<"ISD::ATOMIC_LOAD_MAX_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umin_16 : SDNode<"ISD::ATOMIC_LOAD_UMIN_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umax_16 : SDNode<"ISD::ATOMIC_LOAD_UMAX_16", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_cmp_swap_32 : SDNode<"ISD::ATOMIC_CMP_SWAP_32" , STDAtomic3,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_add_32 : SDNode<"ISD::ATOMIC_LOAD_ADD_32" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_swap_32     : SDNode<"ISD::ATOMIC_SWAP_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_sub_32 : SDNode<"ISD::ATOMIC_LOAD_SUB_32" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_and_32 : SDNode<"ISD::ATOMIC_LOAD_AND_32" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_or_32  : SDNode<"ISD::ATOMIC_LOAD_OR_32" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_xor_32 : SDNode<"ISD::ATOMIC_LOAD_XOR_32" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_nand_32: SDNode<"ISD::ATOMIC_LOAD_NAND_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_min_32 : SDNode<"ISD::ATOMIC_LOAD_MIN_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_max_32 : SDNode<"ISD::ATOMIC_LOAD_MAX_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umin_32 : SDNode<"ISD::ATOMIC_LOAD_UMIN_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umax_32 : SDNode<"ISD::ATOMIC_LOAD_UMAX_32", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_cmp_swap_64 : SDNode<"ISD::ATOMIC_CMP_SWAP_64" , STDAtomic3,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_add_64 : SDNode<"ISD::ATOMIC_LOAD_ADD_64" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_swap_64     : SDNode<"ISD::ATOMIC_SWAP_64", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_sub_64 : SDNode<"ISD::ATOMIC_LOAD_SUB_64" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_and_64 : SDNode<"ISD::ATOMIC_LOAD_AND_64" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_or_64  : SDNode<"ISD::ATOMIC_LOAD_OR_64" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_xor_64 : SDNode<"ISD::ATOMIC_LOAD_XOR_64" , STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_nand_64: SDNode<"ISD::ATOMIC_LOAD_NAND_64", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_min_64 : SDNode<"ISD::ATOMIC_LOAD_MIN_64", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_max_64 : SDNode<"ISD::ATOMIC_LOAD_MAX_64", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umin_64 : SDNode<"ISD::ATOMIC_LOAD_UMIN_64", STDAtomic2,
-                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
-def atomic_load_umax_64 : SDNode<"ISD::ATOMIC_LOAD_UMAX_64", STDAtomic2,
+def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", STDAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 
 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,
@@ -794,6 +722,58 @@ def setle  : PatFrag<(ops node:$lhs, node:$rhs),
 def setne  : PatFrag<(ops node:$lhs, node:$rhs),
                      (setcc node:$lhs, node:$rhs, SETNE)>;
 
+def atomic_cmp_swap_8 :
+  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
+          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
+}]>;
+def atomic_cmp_swap_16 :
+  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
+          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+def atomic_cmp_swap_32 :
+  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
+          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
+}]>;
+def atomic_cmp_swap_64 :
+  PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
+          (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
+  return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
+}]>;
+
+multiclass binary_atomic_op<SDNode atomic_op> {
+  def _8 : PatFrag<(ops node:$ptr, node:$val),
+                   (atomic_op node:$ptr, node:$val), [{
+    return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
+  }]>;
+  def _16 : PatFrag<(ops node:$ptr, node:$val),
+                   (atomic_op node:$ptr, node:$val), [{
+    return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
+  }]>;
+  def _32 : PatFrag<(ops node:$ptr, node:$val),
+                   (atomic_op node:$ptr, node:$val), [{
+    return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
+  }]>;
+  def _64 : PatFrag<(ops node:$ptr, node:$val),
+                   (atomic_op node:$ptr, node:$val), [{
+    return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
+  }]>;
+}
+
+defm atomic_load_add  : binary_atomic_op<atomic_load_add>;
+defm atomic_swap      : binary_atomic_op<atomic_swap>;
+defm atomic_load_sub  : binary_atomic_op<atomic_load_sub>;
+defm atomic_load_and  : binary_atomic_op<atomic_load_and>;
+defm atomic_load_or   : binary_atomic_op<atomic_load_or>;
+defm atomic_load_xor  : binary_atomic_op<atomic_load_xor>;
+defm atomic_load_nand : binary_atomic_op<atomic_load_nand>;
+defm atomic_load_min  : binary_atomic_op<atomic_load_min>;
+defm atomic_load_max  : binary_atomic_op<atomic_load_max>;
+defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
+defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
+
 //===----------------------------------------------------------------------===//
 // Selection DAG CONVERT_RNDSAT patterns
 
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index 8cff5b5558..c162d0e3dc 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -413,6 +413,7 @@ MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() {
   while (!BB->empty())
     BB->remove(BB->begin());
 
+  // Then re-insert them according to the given schedule.
   for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
     SUnit *SU = Sequence[i];
     if (!SU) {
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 4a2af06966..d79f6bb1e1 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1403,10 +1403,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     break;
   }
 
-  case ISD::ATOMIC_CMP_SWAP_8:
-  case ISD::ATOMIC_CMP_SWAP_16:
-  case ISD::ATOMIC_CMP_SWAP_32:
-  case ISD::ATOMIC_CMP_SWAP_64: {
+  case ISD::ATOMIC_CMP_SWAP: {
     unsigned int num_operands = 4;
     assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
     SDValue Ops[4];
@@ -1426,50 +1423,17 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
     AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1));
     return Result.getValue(Op.getResNo());
   }
-  case ISD::ATOMIC_LOAD_ADD_8:
-  case ISD::ATOMIC_LOAD_SUB_8:
-  case ISD::ATOMIC_LOAD_AND_8:
-  case ISD::ATOMIC_LOAD_OR_8:
-  case ISD::ATOMIC_LOAD_XOR_8:
-  case ISD::ATOMIC_LOAD_NAND_8:
-  case ISD::ATOMIC_LOAD_MIN_8:
-  case ISD::ATOMIC_LOAD_MAX_8:
-  case ISD::ATOMIC_LOAD_UMIN_8:
-  case ISD::ATOMIC_LOAD_UMAX_8:
-  case ISD::ATOMIC_SWAP_8: 
-  case ISD::ATOMIC_LOAD_ADD_16:
-  case ISD::ATOMIC_LOAD_SUB_16:
-  case ISD::ATOMIC_LOAD_AND_16:
-  case ISD::ATOMIC_LOAD_OR_16:
-  case ISD::ATOMIC_LOAD_XOR_16:
-  case ISD::ATOMIC_LOAD_NAND_16:
-  case ISD::ATOMIC_LOAD_MIN_16:
-  case ISD::ATOMIC_LOAD_MAX_16:
-  case ISD::ATOMIC_LOAD_UMIN_16:
-  case ISD::ATOMIC_LOAD_UMAX_16:
-  case ISD::ATOMIC_SWAP_16:
-  case ISD::ATOMIC_LOAD_ADD_32:
-  case ISD::ATOMIC_LOAD_SUB_32:
-  case ISD::ATOMIC_LOAD_AND_32:
-  case ISD::ATOMIC_LOAD_OR_32:
-  case ISD::ATOMIC_LOAD_XOR_32:
-  case ISD::ATOMIC_LOAD_NAND_32:
-  case ISD::ATOMIC_LOAD_MIN_32:
-  case ISD::ATOMIC_LOAD_MAX_32:
-  case ISD::ATOMIC_LOAD_UMIN_32:
-  case ISD::ATOMIC_LOAD_UMAX_32:
-  case ISD::ATOMIC_SWAP_32:
-  case ISD::ATOMIC_LOAD_ADD_64:
-  case ISD::ATOMIC_LOAD_SUB_64:
-  case ISD::ATOMIC_LOAD_AND_64:
-  case ISD::ATOMIC_LOAD_OR_64:
-  case ISD::ATOMIC_LOAD_XOR_64:
-  case ISD::ATOMIC_LOAD_NAND_64:
-  case ISD::ATOMIC_LOAD_MIN_64:
-  case ISD::ATOMIC_LOAD_MAX_64:
-  case ISD::ATOMIC_LOAD_UMIN_64:
-  case ISD::ATOMIC_LOAD_UMAX_64:
-  case ISD::ATOMIC_SWAP_64: {
+  case ISD::ATOMIC_LOAD_ADD:
+  case ISD::ATOMIC_LOAD_SUB:
+  case ISD::ATOMIC_LOAD_AND:
+  case ISD::ATOMIC_LOAD_OR:
+  case ISD::ATOMIC_LOAD_XOR:
+  case ISD::ATOMIC_LOAD_NAND:
+  case ISD::ATOMIC_LOAD_MIN:
+  case ISD::ATOMIC_LOAD_MAX:
+  case ISD::ATOMIC_LOAD_UMIN:
+  case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_SWAP: {
     unsigned int num_operands = 3;
     assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
     SDValue Ops[3];
@@ -4718,14 +4682,12 @@ SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) {
     break;
   }
     
-  case ISD::ATOMIC_CMP_SWAP_8:
-  case ISD::ATOMIC_CMP_SWAP_16:
-  case ISD::ATOMIC_CMP_SWAP_32:
-  case ISD::ATOMIC_CMP_SWAP_64: {
+  case ISD::ATOMIC_CMP_SWAP: {
     AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
     Tmp2 = PromoteOp(Node->getOperand(2));
     Tmp3 = PromoteOp(Node->getOperand(3));
-    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), 
+    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getMemoryVT(), 
+                           AtomNode->getChain(), 
                            AtomNode->getBasePtr(), Tmp2, Tmp3,
                            AtomNode->getSrcValue(),
                            AtomNode->getAlignment());
@@ -4733,53 +4695,21 @@ SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) {
     AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
     break;
   }
-  case ISD::ATOMIC_LOAD_ADD_8:
-  case ISD::ATOMIC_LOAD_SUB_8:
-  case ISD::ATOMIC_LOAD_AND_8:
-  case ISD::ATOMIC_LOAD_OR_8:
-  case ISD::ATOMIC_LOAD_XOR_8:
-  case ISD::ATOMIC_LOAD_NAND_8:
-  case ISD::ATOMIC_LOAD_MIN_8:
-  case ISD::ATOMIC_LOAD_MAX_8:
-  case ISD::ATOMIC_LOAD_UMIN_8:
-  case ISD::ATOMIC_LOAD_UMAX_8:
-  case ISD::ATOMIC_SWAP_8: 
-  case ISD::ATOMIC_LOAD_ADD_16:
-  case ISD::ATOMIC_LOAD_SUB_16:
-  case ISD::ATOMIC_LOAD_AND_16:
-  case ISD::ATOMIC_LOAD_OR_16:
-  case ISD::ATOMIC_LOAD_XOR_16:
-  case ISD::ATOMIC_LOAD_NAND_16:
-  case ISD::ATOMIC_LOAD_MIN_16:
-  case ISD::ATOMIC_LOAD_MAX_16:
-  case ISD::ATOMIC_LOAD_UMIN_16:
-  case ISD::ATOMIC_LOAD_UMAX_16:
-  case ISD::ATOMIC_SWAP_16:
-  case ISD::ATOMIC_LOAD_ADD_32:
-  case ISD::ATOMIC_LOAD_SUB_32:
-  case ISD::ATOMIC_LOAD_AND_32:
-  case ISD::ATOMIC_LOAD_OR_32:
-  case ISD::ATOMIC_LOAD_XOR_32:
-  case ISD::ATOMIC_LOAD_NAND_32:
-  case ISD::ATOMIC_LOAD_MIN_32:
-  case ISD::ATOMIC_LOAD_MAX_32:
-  case ISD::ATOMIC_LOAD_UMIN_32:
-  case ISD::ATOMIC_LOAD_UMAX_32:
-  case ISD::ATOMIC_SWAP_32:
-  case ISD::ATOMIC_LOAD_ADD_64:
-  case ISD::ATOMIC_LOAD_SUB_64:
-  case ISD::ATOMIC_LOAD_AND_64:
-  case ISD::ATOMIC_LOAD_OR_64:
-  case ISD::ATOMIC_LOAD_XOR_64:
-  case ISD::ATOMIC_LOAD_NAND_64:
-  case ISD::ATOMIC_LOAD_MIN_64:
-  case ISD::ATOMIC_LOAD_MAX_64:
-  case ISD::ATOMIC_LOAD_UMIN_64:
-  case ISD::ATOMIC_LOAD_UMAX_64:
-  case ISD::ATOMIC_SWAP_64: {
+  case ISD::ATOMIC_LOAD_ADD:
+  case ISD::ATOMIC_LOAD_SUB:
+  case ISD::ATOMIC_LOAD_AND:
+  case ISD::ATOMIC_LOAD_OR:
+  case ISD::ATOMIC_LOAD_XOR:
+  case ISD::ATOMIC_LOAD_NAND:
+  case ISD::ATOMIC_LOAD_MIN:
+  case ISD::ATOMIC_LOAD_MAX:
+  case ISD::ATOMIC_LOAD_UMIN:
+  case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_SWAP: {
     AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
     Tmp2 = PromoteOp(Node->getOperand(2));
-    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), 
+    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getMemoryVT(),
+                           AtomNode->getChain(), 
                            AtomNode->getBasePtr(), Tmp2,
                            AtomNode->getSrcValue(),
                            AtomNode->getAlignment());
@@ -6769,7 +6699,7 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
     break;
   }
 
-  case ISD::ATOMIC_CMP_SWAP_64: {
+  case ISD::ATOMIC_CMP_SWAP: {
     // This operation does not need a loop.
     SDValue Tmp = TLI.LowerOperation(Op, DAG);
     assert(Tmp.getNode() && "Node must be custom expanded!");
@@ -6779,13 +6709,13 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
     break;
   }
 
-  case ISD::ATOMIC_LOAD_ADD_64:
-  case ISD::ATOMIC_LOAD_SUB_64:
-  case ISD::ATOMIC_LOAD_AND_64:
-  case ISD::ATOMIC_LOAD_OR_64:
-  case ISD::ATOMIC_LOAD_XOR_64:
-  case ISD::ATOMIC_LOAD_NAND_64:
-  case ISD::ATOMIC_SWAP_64: {
+  case ISD::ATOMIC_LOAD_ADD:
+  case ISD::ATOMIC_LOAD_SUB:
+  case ISD::ATOMIC_LOAD_AND:
+  case ISD::ATOMIC_LOAD_OR:
+  case ISD::ATOMIC_LOAD_XOR:
+  case ISD::ATOMIC_LOAD_NAND:
+  case ISD::ATOMIC_SWAP: {
     // These operations require a loop to be generated.  We can't do that yet,
     // so substitute a target-dependent pseudo and expand that later.
     SDValue In2Lo, In2Hi, In2;
@@ -6793,7 +6723,8 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
     In2 = DAG.getNode(ISD::BUILD_PAIR, VT, In2Lo, In2Hi);
     AtomicSDNode* Anode = cast<AtomicSDNode>(Node);
     SDValue Replace = 
-      DAG.getAtomic(Op.getOpcode(), Op.getOperand(0), Op.getOperand(1), In2,
+      DAG.getAtomic(Op.getOpcode(), Anode->getMemoryVT(),
+                    Op.getOperand(0), Op.getOperand(1), In2,
                     Anode->getSrcValue(), Anode->getAlignment());
     SDValue Result = TLI.LowerOperation(Replace, DAG);
     ExpandOp(Result.getValue(0), Lo, Hi);
@@ -8318,54 +8249,18 @@ SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
                          Node->getOperand(2));
     break;
   }
-  case ISD::ATOMIC_CMP_SWAP_8:
-  case ISD::ATOMIC_CMP_SWAP_16:
-  case ISD::ATOMIC_CMP_SWAP_32:
-  case ISD::ATOMIC_CMP_SWAP_64:
-  case ISD::ATOMIC_LOAD_ADD_8:
-  case ISD::ATOMIC_LOAD_SUB_8:
-  case ISD::ATOMIC_LOAD_AND_8:
-  case ISD::ATOMIC_LOAD_OR_8:
-  case ISD::ATOMIC_LOAD_XOR_8:
-  case ISD::ATOMIC_LOAD_NAND_8:
-  case ISD::ATOMIC_LOAD_MIN_8:
-  case ISD::ATOMIC_LOAD_MAX_8:
-  case ISD::ATOMIC_LOAD_UMIN_8:
-  case ISD::ATOMIC_LOAD_UMAX_8:
-  case ISD::ATOMIC_SWAP_8: 
-  case ISD::ATOMIC_LOAD_ADD_16:
-  case ISD::ATOMIC_LOAD_SUB_16:
-  case ISD::ATOMIC_LOAD_AND_16:
-  case ISD::ATOMIC_LOAD_OR_16:
-  case ISD::ATOMIC_LOAD_XOR_16:
-  case ISD::ATOMIC_LOAD_NAND_16:
-  case ISD::ATOMIC_LOAD_MIN_16:
-  case ISD::ATOMIC_LOAD_MAX_16:
-  case ISD::ATOMIC_LOAD_UMIN_16:
-  case ISD::ATOMIC_LOAD_UMAX_16:
-  case ISD::ATOMIC_SWAP_16:
-  case ISD::ATOMIC_LOAD_ADD_32:
-  case ISD::ATOMIC_LOAD_SUB_32:
-  case ISD::ATOMIC_LOAD_AND_32:
-  case ISD::ATOMIC_LOAD_OR_32:
-  case ISD::ATOMIC_LOAD_XOR_32:
-  case ISD::ATOMIC_LOAD_NAND_32:
-  case ISD::ATOMIC_LOAD_MIN_32:
-  case ISD::ATOMIC_LOAD_MAX_32:
-  case ISD::ATOMIC_LOAD_UMIN_32:
-  case ISD::ATOMIC_LOAD_UMAX_32:
-  case ISD::ATOMIC_SWAP_32:
-  case ISD::ATOMIC_LOAD_ADD_64:
-  case ISD::ATOMIC_LOAD_SUB_64:
-  case ISD::ATOMIC_LOAD_AND_64:
-  case ISD::ATOMIC_LOAD_OR_64:
-  case ISD::ATOMIC_LOAD_XOR_64:
-  case ISD::ATOMIC_LOAD_NAND_64:
-  case ISD::ATOMIC_LOAD_MIN_64:
-  case ISD::ATOMIC_LOAD_MAX_64:
-  case ISD::ATOMIC_LOAD_UMIN_64:
-  case ISD::ATOMIC_LOAD_UMAX_64:
-  case ISD::ATOMIC_SWAP_64: {
+  case ISD::ATOMIC_CMP_SWAP:
+  case ISD::ATOMIC_LOAD_ADD:
+  case ISD::ATOMIC_LOAD_SUB:
+  case ISD::ATOMIC_LOAD_AND:
+  case ISD::ATOMIC_LOAD_OR:
+  case ISD::ATOMIC_LOAD_XOR:
+  case ISD::ATOMIC_LOAD_NAND:
+  case ISD::ATOMIC_LOAD_MIN:
+  case ISD::ATOMIC_LOAD_MAX:
+  case ISD::ATOMIC_LOAD_UMIN:
+  case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_SWAP: {
     // For now, we assume that using vectors for these operations don't make
     // much sense so we just split it.  We return an empty result
     SDValue X, Y;
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 5f23677880..7ae53eb7a9 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -98,56 +98,20 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::SMULO:
   case ISD::UMULO:       Result = PromoteIntRes_XMULO(N, ResNo); break;
 
-  case ISD::ATOMIC_LOAD_ADD_8:
-  case ISD::ATOMIC_LOAD_SUB_8:
-  case ISD::ATOMIC_LOAD_AND_8:
-  case ISD::ATOMIC_LOAD_OR_8:
-  case ISD::ATOMIC_LOAD_XOR_8:
-  case ISD::ATOMIC_LOAD_NAND_8:
-  case ISD::ATOMIC_LOAD_MIN_8:
-  case ISD::ATOMIC_LOAD_MAX_8:
-  case ISD::ATOMIC_LOAD_UMIN_8:
-  case ISD::ATOMIC_LOAD_UMAX_8:
-  c