Reduce code duplication on the TLS implementation.

This introduces a small regression on the generated code
quality in the case we are just computing addresses, not
loading values.

Will work on it and on X86-64 support.



git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@68552 91177308-0d34-0410-b5e6-96231b3b80d8

17 files changed, 231 insertions, 193 deletions

diff --git a/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.cpp b/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.cpp
index 3b6b0efec5..e9b62a8c14 100644
--- a/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.cpp
+++ b/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.cpp
@@ -566,9 +566,8 @@ void X86ATTAsmPrinter::printSSECC(const MachineInstr *MI, unsigned Op) {
   }
 }
 
-void X86ATTAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
-                                         const char *Modifier){
-  assert(isMem(MI, Op) && "Invalid memory reference!");
+void X86ATTAsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op,
+                                            const char *Modifier){
   MachineOperand BaseReg  = MI->getOperand(Op);
   MachineOperand IndexReg = MI->getOperand(Op+2);
   const MachineOperand &DispSpec = MI->getOperand(Op+3);
@@ -611,6 +610,17 @@ void X86ATTAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
   }
 }
 
+void X86ATTAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
+                                         const char *Modifier){
+  assert(isMem(MI, Op) && "Invalid memory reference!");
+  MachineOperand Segment = MI->getOperand(Op+4);
+  if (Segment.getReg()) {
+      printOperand(MI, Op+4, Modifier);
+      O << ':';
+    }
+  printLeaMemReference(MI, Op, Modifier);
+}
+
 void X86ATTAsmPrinter::printPICJumpTableSetLabel(unsigned uid,
                                            const MachineBasicBlock *MBB) const {
   if (!TAI->getSetDirective())
diff --git a/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.h b/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.h
index 4da8076af5..30630e9784 100644
--- a/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.h
+++ b/lib/Target/X86/AsmPrinter/X86ATTAsmPrinter.h
@@ -93,8 +93,14 @@ class VISIBILITY_HIDDEN X86ATTAsmPrinter : public AsmPrinter {
   void printf128mem(const MachineInstr *MI, unsigned OpNo) {
     printMemReference(MI, OpNo);
   }
+  void printlea32mem(const MachineInstr *MI, unsigned OpNo) {
+    printLeaMemReference(MI, OpNo);
+  }
+  void printlea64mem(const MachineInstr *MI, unsigned OpNo) {
+    printLeaMemReference(MI, OpNo);
+  }
   void printlea64_32mem(const MachineInstr *MI, unsigned OpNo) {
-    printMemReference(MI, OpNo, "subreg64");
+    printLeaMemReference(MI, OpNo, "subreg64");
   }
 
   bool printAsmMRegister(const MachineOperand &MO, const char Mode);
@@ -107,6 +113,8 @@ class VISIBILITY_HIDDEN X86ATTAsmPrinter : public AsmPrinter {
   void printSSECC(const MachineInstr *MI, unsigned Op);
   void printMemReference(const MachineInstr *MI, unsigned Op,
                          const char *Modifier=NULL);
+  void printLeaMemReference(const MachineInstr *MI, unsigned Op,
+                            const char *Modifier=NULL);
   void printPICJumpTableSetLabel(unsigned uid,
                                  const MachineBasicBlock *MBB) const;
   void printPICJumpTableSetLabel(unsigned uid, unsigned uid2,
diff --git a/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.cpp b/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.cpp
index 7823ca6da3..b694a18510 100644
--- a/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.cpp
+++ b/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.cpp
@@ -271,10 +271,9 @@ void X86IntelAsmPrinter::printOp(const MachineOperand &MO,
   }
 }
 
-void X86IntelAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
-                                           const char *Modifier) {
-  assert(isMem(MI, Op) && "Invalid memory reference!");
-
+void X86IntelAsmPrinter::printLeaMemReference(const MachineInstr *MI,
+                                              unsigned Op,
+                                              const char *Modifier) {
   const MachineOperand &BaseReg  = MI->getOperand(Op);
   int ScaleVal                   = MI->getOperand(Op+1).getImm();
   const MachineOperand &IndexReg = MI->getOperand(Op+2);
@@ -317,6 +316,17 @@ void X86IntelAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
   O << "]";
 }
 
+void X86IntelAsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
+                                           const char *Modifier) {
+  assert(isMem(MI, Op) && "Invalid memory reference!");
+  MachineOperand Segment = MI->getOperand(Op+4);
+  if (Segment.getReg()) {
+      printOperand(MI, Op+4, Modifier);
+      O << ':';
+    }
+  printLeaMemReference(MI, Op, Modifier);
+}
+
 void X86IntelAsmPrinter::printPICJumpTableSetLabel(unsigned uid,
                                            const MachineBasicBlock *MBB) const {
   if (!TAI->getSetDirective())
diff --git a/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.h b/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.h
index ae84df22b7..489d946790 100644
--- a/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.h
+++ b/lib/Target/X86/AsmPrinter/X86IntelAsmPrinter.h
@@ -88,9 +88,17 @@ struct VISIBILITY_HIDDEN X86IntelAsmPrinter : public AsmPrinter {
     O << "XMMWORD PTR ";
     printMemReference(MI, OpNo);
   }
+  void printlea32mem(const MachineInstr *MI, unsigned OpNo) {
+    O << "DWORD PTR ";
+    printLeaMemReference(MI, OpNo);
+  }
+  void printlea64mem(const MachineInstr *MI, unsigned OpNo) {
+    O << "QWORD PTR ";
+    printLeaMemReference(MI, OpNo);
+  }
   void printlea64_32mem(const MachineInstr *MI, unsigned OpNo) {
     O << "QWORD PTR ";
-    printMemReference(MI, OpNo, "subreg64");
+    printLeaMemReference(MI, OpNo, "subreg64");
   }
 
   bool printAsmMRegister(const MachineOperand &MO, const char Mode);
@@ -103,6 +111,8 @@ struct VISIBILITY_HIDDEN X86IntelAsmPrinter : public AsmPrinter {
   void printSSECC(const MachineInstr *MI, unsigned Op);
   void printMemReference(const MachineInstr *MI, unsigned Op,
                          const char *Modifier=NULL);
+  void printLeaMemReference(const MachineInstr *MI, unsigned Op,
+                            const char *Modifier=NULL);
   void printPICJumpTableSetLabel(unsigned uid,
                                  const MachineBasicBlock *MBB) const;
   void printPICJumpTableSetLabel(unsigned uid, unsigned uid2,
diff --git a/lib/Target/X86/X86CodeEmitter.cpp b/lib/Target/X86/X86CodeEmitter.cpp
index 7c998382fc..c54a996cb2 100644
--- a/lib/Target/X86/X86CodeEmitter.cpp
+++ b/lib/Target/X86/X86CodeEmitter.cpp
@@ -533,23 +533,6 @@ void Emitter::emitInstruction(const MachineInstr &MI,
     case X86::DWARF_LOC:
     case X86::FP_REG_KILL:
       break;
-    case X86::TLS_tp: {
-      MCE.emitByte(BaseOpcode);
-      unsigned RegOpcodeField = getX86RegNum(MI.getOperand(0).getReg());
-      MCE.emitByte(ModRMByte(0, RegOpcodeField, 5));
-      emitConstant(0, 4);
-      break;
-    }
-    case X86::TLS_gs_ri: {
-      MCE.emitByte(BaseOpcode);
-      unsigned RegOpcodeField = getX86RegNum(MI.getOperand(0).getReg());
-      MCE.emitByte(ModRMByte(0, RegOpcodeField, 5));
-      GlobalValue* GV = MI.getOperand(1).getGlobal();
-      unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
-        : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
-      emitGlobalAddress(GV, rt);
-      break;
-    }
     case X86::MOVPC32r: {
       // This emits the "call" portion of this pseudo instruction.
       MCE.emitByte(BaseOpcode);
@@ -661,13 +644,21 @@ void Emitter::emitInstruction(const MachineInstr &MI,
     break;
 
   case X86II::MRMSrcMem: {
-    intptr_t PCAdj = (CurOp + X86AddrNumOperands + 1 != NumOps) ?
+    // FIXME: Maybe lea should have its own form?
+    int AddrOperands;
+    if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r ||
+        Opcode == X86::LEA16r || Opcode == X86::LEA32r)
+      AddrOperands = X86AddrNumOperands - 1; // No segment register
+    else
+      AddrOperands = X86AddrNumOperands;
+
+    intptr_t PCAdj = (CurOp + AddrOperands + 1 != NumOps) ?
       X86InstrInfo::sizeOfImm(Desc) : 0;
 
     MCE.emitByte(BaseOpcode);
     emitMemModRMByte(MI, CurOp+1, getX86RegNum(MI.getOperand(CurOp).getReg()),
                      PCAdj);
-    CurOp += X86AddrNumOperands + 1;
+    CurOp += AddrOperands + 1;
     if (CurOp != NumOps)
       emitConstant(MI.getOperand(CurOp++).getImm(), X86InstrInfo::sizeOfImm(Desc));
     break;
diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index 7a10b123bc..7b114ddbc3 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -1490,7 +1490,7 @@ unsigned X86FastISel::TargetMaterializeConstant(Constant *C) {
       else
         Opc = X86::LEA64r;
       unsigned ResultReg = createResultReg(RC);
-      addFullAddress(BuildMI(MBB, DL, TII.get(Opc), ResultReg), AM);
+      addLeaAddress(BuildMI(MBB, DL, TII.get(Opc), ResultReg), AM);
       return ResultReg;
     }
     return 0;
@@ -1535,7 +1535,7 @@ unsigned X86FastISel::TargetMaterializeAlloca(AllocaInst *C) {
   unsigned Opc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r;
   TargetRegisterClass* RC = TLI.getRegClassFor(TLI.getPointerTy());
   unsigned ResultReg = createResultReg(RC);
-  addFullAddress(BuildMI(MBB, DL, TII.get(Opc), ResultReg), AM);
+  addLeaAddress(BuildMI(MBB, DL, TII.get(Opc), ResultReg), AM);
   return ResultReg;
 }
 
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index f402053e73..2cd6f74f27 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -69,6 +69,7 @@ namespace {
     unsigned Scale;
     SDValue IndexReg; 
     int32_t Disp;
+    SDValue Segment;
     GlobalValue *GV;
     Constant *CP;
     const char *ES;
@@ -77,7 +78,7 @@ namespace {
 
     X86ISelAddressMode()
       : BaseType(RegBase), isRIPRel(false), Scale(1), IndexReg(), Disp(0),
-        GV(0), CP(0), ES(0), JT(-1), Align(0) {
+        Segment(), GV(0), CP(0), ES(0), JT(-1), Align(0) {
     }
 
     bool hasSymbolicDisplacement() const {
@@ -159,20 +160,25 @@ namespace {
     SDNode *Select(SDValue N);
     SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
 
+    bool MatchSegmentBaseAddress(SDValue N, X86ISelAddressMode &AM);
+    bool MatchLoad(SDValue N, X86ISelAddressMode &AM);
     bool MatchAddress(SDValue N, X86ISelAddressMode &AM,
                       unsigned Depth = 0);
     bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM);
     bool SelectAddr(SDValue Op, SDValue N, SDValue &Base,
-                    SDValue &Scale, SDValue &Index, SDValue &Disp);
+                    SDValue &Scale, SDValue &Index, SDValue &Disp,
+                    SDValue &Segment);
     bool SelectLEAAddr(SDValue Op, SDValue N, SDValue &Base,
                        SDValue &Scale, SDValue &Index, SDValue &Disp);
     bool SelectScalarSSELoad(SDValue Op, SDValue Pred,
                              SDValue N, SDValue &Base, SDValue &Scale,
                              SDValue &Index, SDValue &Disp,
+                             SDValue &Segment,
                              SDValue &InChain, SDValue &OutChain);
     bool TryFoldLoad(SDValue P, SDValue N,
                      SDValue &Base, SDValue &Scale,
-                     SDValue &Index, SDValue &Disp);
+                     SDValue &Index, SDValue &Disp,
+                     SDValue &Segment);
     void PreprocessForRMW();
     void PreprocessForFPConvert();
 
@@ -186,7 +192,7 @@ namespace {
 
     inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 
                                    SDValue &Scale, SDValue &Index,
-                                   SDValue &Disp) {
+                                   SDValue &Disp, SDValue &Segment) {
       Base  = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ?
         CurDAG->getTargetFrameIndex(AM.Base.FrameIndex, TLI.getPointerTy()) :
         AM.Base.Reg;
@@ -205,6 +211,11 @@ namespace {
         Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32);
       else
         Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32);
+
+      if (AM.Segment.getNode())
+        Segment = AM.Segment;
+      else
+        Segment = CurDAG->getRegister(0, MVT::i32);
     }
 
     /// getI8Imm - Return a target constant with the specified value, of type
@@ -726,6 +737,33 @@ void X86DAGToDAGISel::EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) {
     EmitSpecialCodeForMain(BB, MF.getFrameInfo());
 }
 
+
+bool X86DAGToDAGISel::MatchSegmentBaseAddress(SDValue N,
+                                              X86ISelAddressMode &AM) {
+  assert(N.getOpcode() == X86ISD::SegmentBaseAddress);
+  SDValue Segment = N.getOperand(0);
+
+  if (AM.Segment.getNode() == 0) {
+    AM.Segment = Segment;
+    return false;
+  }
+
+  return true;
+}
+
+bool X86DAGToDAGISel::MatchLoad(SDValue N, X86ISelAddressMode &AM) {
+  // This optimization is valid because the GNU TLS model defines that
+  // gs:0 (or fs:0 on X86-64) contains its own address.
+  // For more information see http://people.redhat.com/drepper/tls.pdf
+
+  SDValue Address = N.getOperand(1);
+  if (Address.getOpcode() == X86ISD::SegmentBaseAddress &&
+      !MatchSegmentBaseAddress (Address, AM))
+    return false;
+
+  return true;
+}
+
 /// MatchAddress - Add the specified node to the specified addressing mode,
 /// returning true if it cannot be done.  This just pattern matches for the
 /// addressing mode.
@@ -761,6 +799,11 @@ bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM,
     break;
   }
 
+  case X86ISD::SegmentBaseAddress:
+    if (!MatchSegmentBaseAddress(N, AM))
+      return false;
+    break;
+
   case X86ISD::Wrapper: {
     DOUT << "Wrapper: 64bit " << is64Bit;
     DOUT << " AM "; DEBUG(AM.dump()); DOUT << "\n";
@@ -807,6 +850,11 @@ bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM,
     break;
   }
 
+  case ISD::LOAD:
+    if (!MatchLoad(N, AM))
+      return false;
+    break;
+
   case ISD::FrameIndex:
     if (AM.BaseType == X86ISelAddressMode::RegBase
         && AM.Base.Reg.getNode() == 0) {
@@ -1034,7 +1082,7 @@ bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) {
 /// match by reference.
 bool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base,
                                  SDValue &Scale, SDValue &Index,
-                                 SDValue &Disp) {
+                                 SDValue &Disp, SDValue &Segment) {
   X86ISelAddressMode AM;
   bool Done = false;
   if (AvoidDupAddrCompute && !N.hasOneUse()) {
@@ -1069,7 +1117,7 @@ bool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base,
   if (!AM.IndexReg.getNode())
     AM.IndexReg = CurDAG->getRegister(0, VT);
 
-  getAddressOperands(AM, Base, Scale, Index, Disp);
+  getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
   return true;
 }
 
@@ -1079,7 +1127,8 @@ bool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base,
 bool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred,
                                           SDValue N, SDValue &Base,
                                           SDValue &Scale, SDValue &Index,
-                                          SDValue &Disp, SDValue &InChain,
+                                          SDValue &Disp, SDValue &Segment,
+                                          SDValue &InChain,
                                           SDValue &OutChain) {
   if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) {
     InChain = N.getOperand(0).getValue(1);
@@ -1088,7 +1137,7 @@ bool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred,
         N.hasOneUse() &&
         IsLegalAndProfitableToFold(N.getNode(), Pred.getNode(), Op.getNode())) {
       LoadSDNode *LD = cast<LoadSDNode>(InChain);
-      if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp))
+      if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment))
         return false;
       OutChain = LD->getChain();
       return true;
@@ -1105,7 +1154,7 @@ bool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred,
       N.getOperand(0).getOperand(0).hasOneUse()) {
     // Okay, this is a zero extending load.  Fold it.
     LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0));
-    if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp))
+    if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp, Segment))
       return false;
     OutChain = LD->getChain();
     InChain = SDValue(LD, 1);
@@ -1124,6 +1173,11 @@ bool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N,
   if (MatchAddress(N, AM))
     return false;
 
+  //Is it better to set AM.Segment before calling MatchAddress to
+  //prevent it from adding a segment?
+  if (AM.Segment.getNode())
+    return false;
+
   MVT VT = N.getValueType();
   unsigned Complexity = 0;
   if (AM.BaseType == X86ISelAddressMode::RegBase)
@@ -1162,7 +1216,8 @@ bool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N,
     Complexity++;
 
   if (Complexity > 2) {
-    getAddressOperands(AM, Base, Scale, Index, Disp);
+    SDValue Segment;
+    getAddressOperands(AM, Base, Scale, Index, Disp, Segment);
     return true;
   }
   return false;
@@ -1170,11 +1225,12 @@ bool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N,
 
 bool X86DAGToDAGISel::TryFoldLoad(SDValue P, SDValue N,
                                   SDValue &Base, SDValue &Scale,
-                                  SDValue &Index, SDValue &Disp) {
+                                  SDValue &Index, SDValue &Disp,
+                                  SDValue &Segment) {
   if (ISD::isNON_EXTLoad(N.getNode()) &&
       N.hasOneUse() &&
       IsLegalAndProfitableToFold(N.getNode(), P.getNode(), P.getNode()))
-    return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp);
+    return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp, Segment);
   return false;
 }
 
@@ -1230,11 +1286,11 @@ SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
   SDValue In1 = Node->getOperand(1);
   SDValue In2L = Node->getOperand(2);
   SDValue In2H = Node->getOperand(3);
-  SDValue Tmp0, Tmp1, Tmp2, Tmp3;
-  if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3))
+  SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+  if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4))
     return NULL;
   SDValue LSI = Node->getOperand(4);    // MemOperand
-  const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, In2L, In2H, LSI, Chain };
+  const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, LSI, Chain};
   return CurDAG->getTargetNode(Opc, Node->getDebugLoc(),
                                MVT::i32, MVT::i32, MVT::Other, Ops,
                                array_lengthof(Ops));
@@ -1316,11 +1372,11 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
       case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break;
       }
 
-      SDValue Tmp0, Tmp1, Tmp2, Tmp3;
-      bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3);
+      SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+      bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
       // multiplty is commmutative
       if (!foldedLoad) {
-        foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3);
+        foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
         if (foldedLoad)
           std::swap(N0, N1);
       }
@@ -1329,7 +1385,8 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
                                               N0, SDValue()).getValue(1);
 
       if (foldedLoad) {
-        SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag };
+        SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
+                          InFlag };
         SDNode *CNode =
           CurDAG->getTargetNode(MOpc, dl, MVT::Other, MVT::Flag, Ops,
                                 array_lengthof(Ops));
@@ -1438,17 +1495,17 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
         break;
       }
 
-      SDValue Tmp0, Tmp1, Tmp2, Tmp3;
-      bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3);
+      SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+      bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
       bool signBitIsZero = CurDAG->SignBitIsZero(N0);
 
       SDValue InFlag;
       if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) {
         // Special case for div8, just use a move with zero extension to AX to
         // clear the upper 8 bits (AH).
-        SDValue Tmp0, Tmp1, Tmp2, Tmp3, Move, Chain;
-        if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3)) {
-          SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N0.getOperand(0) };
+        SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain;
+        if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
+          SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) };
           Move =
             SDValue(CurDAG->getTargetNode(X86::MOVZX16rm8, dl, MVT::i16, 
                                           MVT::Other, Ops,
@@ -1480,7 +1537,8 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
       }
 
       if (foldedLoad) {
-        SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag };
+        SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
+                          InFlag };
         SDNode *CNode =
           CurDAG->getTargetNode(MOpc, dl, MVT::Other, MVT::Flag, Ops,
                                 array_lengthof(Ops));
@@ -1649,13 +1707,13 @@ SDNode *X86DAGToDAGISel::Select(SDValue N) {
 bool X86DAGToDAGISel::
 SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
                              std::vector<SDValue> &OutOps) {
-  SDValue Op0, Op1, Op2, Op3;
+  SDValue Op0, Op1, Op2, Op3, Op4;
   switch (ConstraintCode) {
   case 'o':   // offsetable        ??
   case 'v':   // not offsetable    ??
   default: return true;
   case 'm':   // memory
-    if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3))
+    if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3, Op4))
       return true;
     break;
   }
@@ -1664,6 +1722,7 @@ SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
   OutOps.push_back(Op1);
   OutOps.push_back(Op2);
   OutOps.push_back(Op3);
+  OutOps.push_back(Op4);
   return false;
 }
 
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index c5a6acbf7a..6cdc54cfba 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -4834,8 +4834,13 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
                                    const MVT PtrVT, TLSModel::Model model) {
   DebugLoc dl = GA->getDebugLoc();
   // Get the Thread Pointer
-  SDValue ThreadPointer = DAG.getNode(X86ISD::THREAD_POINTER,
-                                      DebugLoc::getUnknownLoc(), PtrVT);
+  SDValue Base = DAG.getNode(X86ISD::SegmentBaseAddress,
+                             DebugLoc::getUnknownLoc(), PtrVT,
+                             DAG.getRegister(X86::GS, MVT::i32));
+
+  SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Base,
+                                      NULL, 0);
+
   // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial
   // exec)
   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(),
@@ -7149,7 +7154,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
   case X86ISD::FRCP:               return "X86ISD::FRCP";
   case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
-  case X86ISD::THREAD_POINTER:     return "X86ISD::THREAD_POINTER";
+  case X86ISD::SegmentBaseAddress: return "X86ISD::SegmentBaseAddress";
   case X86ISD::EH_RETURN:          return "X86ISD::EH_RETURN";
   case X86ISD::TC_RETURN:          return "X86ISD::TC_RETURN";
   case X86ISD::FNSTCW16m:          return "X86ISD::FNSTCW16m";
@@ -7473,7 +7478,7 @@ X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr,
   unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
   MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2);
   // add 4 to displacement.
-  for (int i=0; i <= lastAddrIndx-1; ++i)
+  for (int i=0; i <= lastAddrIndx-2; ++i)
     (*MIB).addOperand(*argOpers[i]);
   MachineOperand newOp3 = *(argOpers[3]);
   if (newOp3.isImm())
@@ -7481,6 +7486,7 @@ X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr,
   else
     newOp3.setOffset(newOp3.getOffset()+4);
   (*MIB).addOperand(newOp3);
+  (*MIB).addOperand(*argOpers[lastAddrIndx]);
 
   // t3/4 are defined later, at the bottom of the loop
   unsigned t3 = F->getRegInfo().createVirtualRegister(RC);
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index ca4af63422..406f45256f 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -188,8 +188,11 @@ namespace llvm {
       /// in order to obtain suitable precision.
       FRSQRT, FRCP,
 
-      // TLSADDR, THREAD_POINTER - Thread Local Storage.
-      TLSADDR, THREAD_POINTER,
+      // TLSADDR - Thread Local Storage.
+      TLSADDR,
+
+      // SegmentBaseAddress - The address segment:0
+      SegmentBaseAddress,
 
       // EH_RETURN - Exception Handling helpers.
       EH_RETURN,
diff --git a/lib/Target/X86/X86Instr64bit.td b/lib/Target/X86/X86Instr64bit.td
index ce5a8a3703..6f1f761abf 100644
--- a/lib/Target/X86/X86Instr64bit.td
+++ b/lib/Target/X86/X86Instr64bit.td
@@ -23,7 +23,7 @@ def i64i32imm  : Operand<i64>;
 def i64i8imm   : Operand<i64>;
 
 def lea64mem : Operand<i64> {
-  let PrintMethod = "printi64mem";
+  let PrintMethod = "printlea64mem";
   let MIOperandInfo = (ops GR64, i8imm, GR64, i32imm);
 }
 
diff --git a/lib/Target/X86/X86InstrBuilder.h b/lib/Target/X86/X86InstrBuilder.h
index 74f9e05ffe..623b1458c8 100644
--- a/lib/Target/X86/X86InstrBuilder.h
+++ b/lib/Target/X86/X86InstrBuilder.h
@@ -66,6 +66,15 @@ inline const MachineInstrBuilder &addDirectMem(const MachineInstrBuilder &MIB,
   return MIB.addReg(Reg).addImm(1).addReg(0).addImm(0);
 }
 
+inline const MachineInstrBuilder &addLeaOffset(const MachineInstrBuilder &MIB,
+                                            int Offset) {
+  return MIB.addImm(1).addReg(0).addImm(Offset);
+}
+
+inline const MachineInstrBuilder &addOffset(const MachineInstrBuilder &MIB,
+                                            int Offset) {
+  return addLeaOffset(MIB, Offset).addReg(0);
+}
 
 /// addRegOffset - This function is used to add a memory reference of the form
 /// [Reg + Offset], i.e., one with no scale or index, but with a
@@ -74,8 +83,13 @@ inline const MachineInstrBuilder &addDirectMem(const MachineInstrBuilder &MIB,
 inline const MachineInstrBuilder &addRegOffset(const MachineInstrBuilder &MIB,
                                                unsigned Reg, bool isKill,
                                                int Offset) {
-  return MIB.addReg(Reg, false, false, isKill)
-    .addImm(1).addReg(0).addImm(Offset);
+  return addOffset(MIB.addReg(Reg, false, false, isKill), Offset);
+}
+
+inline const MachineInstrBuilder &addLeaRegOffset(const MachineInstrBuilder &MIB,
+                                                  unsigned Reg, bool isKill,
+                                                  int Offset) {
+  return addLeaOffset(MIB.addReg(Reg, false, false, isKill), Offset);
 }
 
 /// addRegReg - This function is used to add a memory reference of the form:
@@ -87,8 +101,8 @@ inline const MachineInstrBuilder &addRegReg(const MachineInstrBuilder &MIB,
     .addReg(Reg2, false, false, isKill2).addImm(0);
 }
 
-inline const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
-                                                 const X86AddressMode &AM) {
+inline const MachineInstrBuilder &addLeaAddress(const MachineInstrBuilder &MIB,
+                                                const X86AddressMode &AM) {
   assert (AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8);
 
   if (AM.BaseType == X86AddressMode::RegBase)
@@ -104,6 +118,11 @@ inline const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
     return MIB.addImm(AM.Disp);
 }
 
+inline const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
+                                                 const X86AddressMode &AM) {
+  return addLeaAddress(MIB, AM).addReg(0);
+}
+
 /// addFrameReference - This function is used to add a reference to the base of
 /// an abstract object on the stack frame of the current function.  This
 /// reference has base register as the FrameIndex offset until it is resolved.
@@ -125,7 +144,7 @@ addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset = 0) {
                         MFI.getObjectOffset(FI) + Offset,
                         MFI.getObjectSize(FI),
                         MFI.getObjectAlignment(FI));
-  return MIB.addFrameIndex(FI).addImm(1).addReg(0).addImm(Offset)
+  return addOffset(MIB.addFrameIndex(FI), Offset)
             .addMemOperand(MMO);
 }
 
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index dd7346a30f..4d9e2bef74 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -1138,9 +1138,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       assert(MI->getNumOperands() >= 2 && "Unknown inc instruction!");
       unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r
         : (is64Bit ? X86::LEA64_32r : X86::LEA32r);
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                           .addReg(Dest, true, false, false, isDead),
-                           Src, isKill, 1);
+      NewMI = addLeaRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
+                              .addReg(Dest, true, false, false, isDead),
+                              Src, isKill, 1);
       break;
     }
     case X86::INC16r:
@@ -1157,9 +1157,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       assert(MI->getNumOperands() >= 2 && "Unknown dec instruction!");
       unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r
         : (is64Bit ? X86::LEA64_32r : X86::LEA32r);
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                           .addReg(Dest, true, false, false, isDead),
-                           Src, isKill, -1);
+      NewMI = addLeaRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
+                              .addReg(Dest, true, false, false, isDead),
+                              Src, isKill, -1);
       break;
     }
     case X86::DEC16r:
@@ -1200,18 +1200,18 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     case X86::ADD64ri8:
       assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
       if (MI->getOperand(2).isImm())
-        NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
-                             .addReg(Dest, true, false, false, isDead),
-                             Src, isKill, MI->getOperand(2).getImm());
+        NewMI = addLeaRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
+                                .addReg(Dest, true, false, false, isDead),
+                                Src, isKill, MI->getOperand(2).getImm());
       break;
     case X86::ADD32ri:
     case X86::ADD32ri8:
       assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
       if (MI->getOperand(2).isImm()) {
         unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
-        NewMI = add