14 files changed, 366 insertions, 332 deletions

diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index a19765a644..01aba3f985 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -36,13 +36,6 @@ class MachineFunction;
 class MachineConstantPoolValue;
 class FunctionLoweringInfo;
 
-/// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use
-/// pool allocation with recycling.
-///
-typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode),
-                           AlignOf<MostAlignedSDNode>::Alignment>
-  NodeAllocatorType;
-
 template<> class ilist_traits<SDNode> : public ilist_default_traits<SDNode> {
   mutable SDNode Sentinel;
 public:
@@ -77,21 +70,31 @@ class SelectionDAG {
   FunctionLoweringInfo &FLI;
   MachineModuleInfo *MMI;
 
-  /// Root - The root of the entire DAG.  EntryNode - The starting token.
-  SDValue Root, EntryNode;
+  /// EntryNode - The starting token.
+  SDNode EntryNode;
+
+  /// Root - The root of the entire DAG.
+  SDValue Root;
 
   /// AllNodes - A linked list of nodes in the current DAG.
   ilist<SDNode> AllNodes;
 
-  /// NodeAllocator - Pool allocation for nodes. The allocator isn't
-  /// allocated inside this class because we want to reuse a single
-  /// recycler across multiple SelectionDAG runs.
-  NodeAllocatorType &NodeAllocator;
+  /// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use
+  /// pool allocation with recycling.
+  typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode),
+                             AlignOf<MostAlignedSDNode>::Alignment>
+    NodeAllocatorType;
+
+  /// NodeAllocator - Pool allocation for nodes.
+  NodeAllocatorType NodeAllocator;
 
   /// CSEMap - This structure is used to memoize nodes, automatically performing
   /// CSE with existing nodes with a duplicate is requested.
   FoldingSet<SDNode> CSEMap;
 
+  /// OperandAllocator - Pool allocation for machine-opcode SDNode operands.
+  BumpPtrAllocator OperandAllocator;
+
   /// Allocator - Pool allocation for misc. objects that are created once per
   /// SelectionDAG.
   BumpPtrAllocator Allocator;
@@ -101,10 +104,14 @@ class SelectionDAG {
 
 public:
   SelectionDAG(TargetLowering &tli, MachineFunction &mf, 
-               FunctionLoweringInfo &fli, MachineModuleInfo *mmi,
-               NodeAllocatorType &nodeallocator);
+               FunctionLoweringInfo &fli, MachineModuleInfo *mmi);
   ~SelectionDAG();
 
+  /// reset - Clear state and free memory necessary to make this
+  /// SelectionDAG ready to process a new block.
+  ///
+  void reset();
+
   MachineFunction &getMachineFunction() const { return MF; }
   const TargetMachine &getTarget() const;
   TargetLowering &getTargetLoweringInfo() const { return TLI; }
@@ -152,7 +159,9 @@ public:
 
   /// getEntryNode - Return the token chain corresponding to the entry of the
   /// function.
-  const SDValue &getEntryNode() const { return EntryNode; }
+  SDValue getEntryNode() const {
+    return SDValue(const_cast<SDNode *>(&EntryNode), 0);
+  }
 
   /// setRoot - Set the current root tag of the SelectionDAG.
   ///
@@ -721,6 +730,8 @@ private:
   void DeleteNodeNotInCSEMaps(SDNode *N);
 
   unsigned getMVTAlignment(MVT MemoryVT) const;
+
+  void allnodes_clear();
   
   // List of non-single value types.
   std::vector<SDVTList> VTList;
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index 951efe1975..64c3b25591 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_CODEGEN_SELECTIONDAG_ISEL_H
 #define LLVM_CODEGEN_SELECTIONDAG_ISEL_H
 
+#include "llvm/BasicBlock.h"
 #include "llvm/Pass.h"
 #include "llvm/Constant.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -58,7 +59,7 @@ public:
   unsigned MakeReg(MVT VT);
 
   virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) {}
-  virtual void InstructionSelect(SelectionDAG &SD) = 0;
+  virtual void InstructionSelect() = 0;
   virtual void InstructionSelectPostProcessing() {}
   
   void SelectRootInit() {
@@ -72,8 +73,7 @@ public:
   /// OutOps vector.
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                             char ConstraintCode,
-                                            std::vector<SDValue> &OutOps,
-                                            SelectionDAG &DAG) {
+                                            std::vector<SDValue> &OutOps) {
     return true;
   }
 
@@ -168,8 +168,7 @@ protected:
 
   /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
   /// by tblgen.  Others should not call it.
-  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
-                                     SelectionDAG &DAG);
+  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops);
 
   // Calls to these predicates are generated by tblgen.
   bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
@@ -180,26 +179,28 @@ protected:
 private:
   void SelectAllBasicBlocks(Function &Fn, MachineFunction &MF,
                             FunctionLoweringInfo &FuncInfo);
-  void SelectBasicBlock(BasicBlock *BB, MachineFunction &MF,
-                        FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                        NodeAllocatorType &NodeAllocator);
-  void FinishBasicBlock(BasicBlock *BB, MachineFunction &MF,
-                        FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                        NodeAllocatorType &NodeAllocator);
+  void FinishBasicBlock(FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate);
 
-  void BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
+  void SelectBasicBlock(BasicBlock *LLVMBB,
+                        BasicBlock::iterator Begin,
+                        BasicBlock::iterator End,
+                        bool DoArgs,
              std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                         FunctionLoweringInfo &FuncInfo);
-  void CodeGenAndEmitDAG(SelectionDAG &DAG);
+                        FunctionLoweringInfo &FuncInfo);
+  void CodeGenAndEmitDAG();
   void LowerArguments(BasicBlock *BB, SelectionDAGLowering &SDL);
   
-  void ComputeLiveOutVRegInfo(SelectionDAG &DAG);
+  void ComputeLiveOutVRegInfo();
+
+  void HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB,
+                                       FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                       SelectionDAGLowering &SDL);
 
   /// Pick a safe ordering for instructions for each target node in the
   /// graph.
-  ScheduleDAG *Schedule(SelectionDAG &DAG);
+  ScheduleDAG *Schedule();
 
   /// SwitchCases - Vector of CaseBlock structures used to communicate
   /// SwitchInst code generation information.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 3e3890142f..9275a21f9f 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -547,11 +547,8 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
   // Drop all of the operands and decrement used nodes use counts.
   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
     I->getVal()->removeUser(std::distance(N->op_begin(), I), N);
-  if (N->OperandsNeedDelete) {
+  if (N->OperandsNeedDelete)
     delete[] N->OperandList;
-  }
-  N->OperandList = 0;
-  N->NumOperands = 0;
   
   AllNodes.remove(N);
 }
@@ -563,6 +560,9 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
 void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
   bool Erased = false;
   switch (N->getOpcode()) {
+  case ISD::EntryToken:
+    assert(0 && "EntryToken should not be in CSEMaps!");
+    return;
   case ISD::HANDLENODE: return;  // noop.
   case ISD::CONDCODE:
     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
@@ -764,24 +764,44 @@ unsigned SelectionDAG::getMVTAlignment(MVT VT) const {
 }
 
 SelectionDAG::SelectionDAG(TargetLowering &tli, MachineFunction &mf,
-                           FunctionLoweringInfo &fli, MachineModuleInfo *mmi,
-                           NodeAllocatorType &nodeallocator)
-  : TLI(tli), MF(mf), FLI(fli), MMI(mmi), NodeAllocator(nodeallocator) {
-  EntryNode = Root = getNode(ISD::EntryToken, MVT::Other);
+                           FunctionLoweringInfo &fli, MachineModuleInfo *mmi)
+  : TLI(tli), MF(mf), FLI(fli), MMI(mmi),
+    EntryNode(ISD::EntryToken, getVTList(MVT::Other)), 
+    Root(getEntryNode()) {
+  AllNodes.push_back(&EntryNode);
 }
 
 SelectionDAG::~SelectionDAG() {
+  allnodes_clear();
+}
+
+void SelectionDAG::allnodes_clear() {
   while (!AllNodes.empty()) {
     SDNode *N = AllNodes.remove(AllNodes.begin());
     N->SetNextInBucket(0);
-    if (N->OperandsNeedDelete) {
+    if (N->OperandsNeedDelete)
       delete [] N->OperandList;
-    }
-    N->OperandList = 0;
-    N->NumOperands = 0;
   }
 }
 
+void SelectionDAG::reset() {
+  allnodes_clear();
+  OperandAllocator.Reset();
+  CSEMap.clear();
+
+  ExtendedValueTypeNodes.clear();
+  ExternalSymbols.clear();
+  TargetExternalSymbols.clear();
+  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
+            static_cast<CondCodeSDNode*>(0));
+  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
+            static_cast<SDNode*>(0));
+
+  EntryNode.Uses = 0;
+  AllNodes.push_back(&EntryNode);
+  Root = getEntryNode();
+}
+
 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, MVT VT) {
   if (Op.getValueType() == VT) return Op;
   APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(),
@@ -3988,9 +4008,9 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
       delete[] N->OperandList;
     if (N->isMachineOpcode()) {
       // We're creating a final node that will live unmorphed for the
-      // remainder of this SelectionDAG's duration, so we can allocate the
-      // operands directly out of the pool with no recycling metadata.
-      N->OperandList = Allocator.Allocate<SDUse>(NumOps);
+      // remainder of the current SelectionDAG iteration, so we can allocate
+      // the operands directly out of a pool with no recycling metadata.
+      N->OperandList = OperandAllocator.Allocate<SDUse>(NumOps);
       N->OperandsNeedDelete = false;
     } else {
       N->OperandList = new SDUse[NumOps];
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 147f45c8b5..e789c4e41c 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -394,6 +394,11 @@ static bool isUsedOutsideOfDefiningBlock(Instruction *I) {
 /// entry block, return true.  This includes arguments used by switches, since
 /// the switch may expand into multiple basic blocks.
 static bool isOnlyUsedInEntryBlock(Argument *A) {
+  // With FastISel active, we may be splitting blocks, so force creation
+  // of virtual registers for all non-dead arguments.
+  if (EnableFastISel)
+    return A->use_empty();
+
   BasicBlock *Entry = A->getParent()->begin();
   for (Value::use_iterator UI = A->use_begin(), E = A->use_end(); UI != E; ++UI)
     if (cast<Instruction>(*UI)->getParent() != Entry || isa<SwitchInst>(*UI))
@@ -5094,119 +5099,18 @@ static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG,
   }
 }
 
-void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
-       std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                         FunctionLoweringInfo &FuncInfo) {
-  SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GFI);
-  BB = FuncInfo.MBBMap[LLVMBB];
-
-  // Before doing SelectionDAG ISel, see if FastISel has been requested.
-  // FastISel doesn't currently support entry blocks, because that
-  // requires special handling for arguments. And it doesn't support EH
-  // landing pads, which also require special handling.
-  // For now, also exclude blocks with terminators that aren't
-  // unconditional branches.
-  BasicBlock::iterator Begin = LLVMBB->begin();
-  if (EnableFastISel &&
-      LLVMBB != &LLVMBB->getParent()->getEntryBlock() &&
-      !BB->isLandingPad() &&
-      isa<BranchInst>(LLVMBB->getTerminator()) &&
-      cast<BranchInst>(LLVMBB->getTerminator())->isUnconditional()) {
-    if (FastISel *F = TLI.createFastISel(FuncInfo.MF)) {
-      Begin = F->SelectInstructions(Begin, LLVMBB->end(),
-                                    FuncInfo.ValueMap, FuncInfo.MBBMap, BB);
-
-      // Clean up the FastISel object. TODO: Reorganize what data is
-      // stored in the FastISel class itself and what is merely passed
-      // to the SelectInstructions method, and then move the creation
-      // and deletion of the FastISel object up so that it is only
-      // done once per MachineFunction.
-      delete F;
-
-      if (Begin == LLVMBB->end())
-        // The "fast" selector selected the entire block, so we're done.
-        return;
-
-      if (!DisableFastISelAbort) {
-        // The "fast" selector couldn't handle something and bailed.
-        // For the purpose of debugging, just abort.
-#ifndef NDEBUG
-        Begin->dump();
-#endif
-        assert(0 && "FastISel didn't select the entire block");
-        abort();
-      }
-    }
-  }
-
-  // Lower any arguments needed in this block if this is the entry block.
-  if (LLVMBB == &LLVMBB->getParent()->getEntryBlock())
-    LowerArguments(LLVMBB, SDL);
-
-  SDL.setCurrentBasicBlock(BB);
-
-  MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
-
-  if (MMI && BB->isLandingPad()) {
-    // Add a label to mark the beginning of the landing pad.  Deletion of the
-    // landing pad can thus be detected via the MachineModuleInfo.
-    unsigned LabelID = MMI->addLandingPad(BB);
-    DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, DAG.getEntryNode(), LabelID));
-
-    // Mark exception register as live in.
-    unsigned Reg = TLI.getExceptionAddressRegister();
-    if (Reg) BB->addLiveIn(Reg);
-
-    // Mark exception selector register as live in.
-    Reg = TLI.getExceptionSelectorRegister();
-    if (Reg) BB->addLiveIn(Reg);
-
-    // FIXME: Hack around an exception handling flaw (PR1508): the personality
-    // function and list of typeids logically belong to the invoke (or, if you
-    // like, the basic block containing the invoke), and need to be associated
-    // with it in the dwarf exception handling tables.  Currently however the
-    // information is provided by an intrinsic (eh.selector) that can be moved
-    // to unexpected places by the optimizers: if the unwind edge is critical,
-    // then breaking it can result in the intrinsics being in the successor of
-    // the landing pad, not the landing pad itself.  This results in exceptions
-    // not being caught because no typeids are associated with the invoke.
-    // This may not be the only way things can go wrong, but it is the only way
-    // we try to work around for the moment.
-    BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
-
-    if (Br && Br->isUnconditional()) { // Critical edge?
-      BasicBlock::iterator I, E;
-      for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I)
-        if (isSelector(I))
-          break;
-
-      if (I == E)
-        // No catch info found - try to extract some from the successor.
-        copyCatchInfo(Br->getSuccessor(0), LLVMBB, MMI, FuncInfo);
-    }
-  }
-
-  // Lower all of the non-terminator instructions.
-  for (BasicBlock::iterator I = Begin, E = --LLVMBB->end();
-       I != E; ++I)
-    SDL.visit(*I);
-
-  // Ensure that all instructions which are used outside of their defining
-  // blocks are available as virtual registers.  Invoke is handled elsewhere.
-  for (BasicBlock::iterator I = Begin, E = LLVMBB->end(); I != E;++I)
-    if (!I->use_empty() && !isa<PHINode>(I) && !isa<InvokeInst>(I)) {
-      DenseMap<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I);
-      if (VMI != FuncInfo.ValueMap.end())
-        SDL.CopyValueToVirtualRegister(I, VMI->second);
-    }
-
-  // Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
-  // ensure constants are generated when needed.  Remember the virtual registers
-  // that need to be added to the Machine PHI nodes as input.  We cannot just
-  // directly add them, because expansion might result in multiple MBB's for one
-  // BB.  As such, the start of the BB might correspond to a different MBB than
-  // the end.
-  //
+/// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
+/// ensure constants are generated when needed.  Remember the virtual registers
+/// that need to be added to the Machine PHI nodes as input.  We cannot just
+/// directly add them, because expansion might result in multiple MBB's for one
+/// BB.  As such, the start of the BB might correspond to a different MBB than
+/// the end.
+///
+void
+SelectionDAGISel::HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB,
+                                                 FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                                  SelectionDAGLowering &SDL) {
   TerminatorInst *TI = LLVMBB->getTerminator();
 
   // Emit constants only once even if used by multiple PHI nodes.
@@ -5287,22 +5191,101 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB,
   JTCases = SDL.JTCases;
   BitTestCases.clear();
   BitTestCases = SDL.BitTestCases;
+}
+
+void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
+                                        BasicBlock::iterator Begin,
+                                        BasicBlock::iterator End,
+                                        bool DoArgs,
+       std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
+                                        FunctionLoweringInfo &FuncInfo) {
+  SelectionDAGLowering SDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
+
+  // Lower any arguments needed in this block if this is the entry block.
+  if (DoArgs)
+    LowerArguments(LLVMBB, SDL);
+
+  SDL.setCurrentBasicBlock(BB);
+
+  MachineModuleInfo *MMI = CurDAG->getMachineModuleInfo();
+
+  if (MMI && BB->isLandingPad()) {
+    // Add a label to mark the beginning of the landing pad.  Deletion of the
+    // landing pad can thus be detected via the MachineModuleInfo.
+    unsigned LabelID = MMI->addLandingPad(BB);
+    CurDAG->setRoot(CurDAG->getLabel(ISD::EH_LABEL,
+                                     CurDAG->getEntryNode(), LabelID));
+
+    // Mark exception register as live in.
+    unsigned Reg = TLI.getExceptionAddressRegister();
+    if (Reg) BB->addLiveIn(Reg);
+
+    // Mark exception selector register as live in.
+    Reg = TLI.getExceptionSelectorRegister();
+    if (Reg) BB->addLiveIn(Reg);
+
+    // FIXME: Hack around an exception handling flaw (PR1508): the personality
+    // function and list of typeids logically belong to the invoke (or, if you
+    // like, the basic block containing the invoke), and need to be associated
+    // with it in the dwarf exception handling tables.  Currently however the
+    // information is provided by an intrinsic (eh.selector) that can be moved
+    // to unexpected places by the optimizers: if the unwind edge is critical,
+    // then breaking it can result in the intrinsics being in the successor of
+    // the landing pad, not the landing pad itself.  This results in exceptions
+    // not being caught because no typeids are associated with the invoke.
+    // This may not be the only way things can go wrong, but it is the only way
+    // we try to work around for the moment.
+    BranchInst *Br = dyn_cast<BranchInst>(LLVMBB->getTerminator());
+
+    if (Br && Br->isUnconditional()) { // Critical edge?
+      BasicBlock::iterator I, E;
+      for (I = LLVMBB->begin(), E = --LLVMBB->end(); I != E; ++I)
+        if (isSelector(I))
+          break;
+
+      if (I == E)
+        // No catch info found - try to extract some from the successor.
+        copyCatchInfo(Br->getSuccessor(0), LLVMBB, MMI, FuncInfo);
+    }
+  }
+
+  // Lower all of the non-terminator instructions.
+  for (BasicBlock::iterator I = Begin; I != End; ++I)
+    if (!isa<TerminatorInst>(I))
+      SDL.visit(*I);
+
+  // Ensure that all instructions which are used outside of their defining
+  // blocks are available as virtual registers.  Invoke is handled elsewhere.
+  for (BasicBlock::iterator I = Begin; I != End; ++I)
+    if (!I->use_empty() && !isa<PHINode>(I) && !isa<InvokeInst>(I)) {
+      DenseMap<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I);
+      if (VMI != FuncInfo.ValueMap.end())
+        SDL.CopyValueToVirtualRegister(I, VMI->second);
+    }
+
+  // Handle PHI nodes in successor blocks.
+  if (Begin != End && End == LLVMBB->end())
+    HandlePHINodesInSuccessorBlocks(LLVMBB, FuncInfo, PHINodesToUpdate, SDL);
     
   // Make sure the root of the DAG is up-to-date.
-  DAG.setRoot(SDL.getControlRoot());
+  CurDAG->setRoot(SDL.getControlRoot());
 
   // Check whether calls in this block are real tail calls. Fix up CALL nodes
   // with correct tailcall attribute so that the target can rely on the tailcall
   // attribute indicating whether the call is really eligible for tail call
   // optimization.
-  CheckDAGForTailCallsAndFixThem(DAG, TLI);
+  CheckDAGForTailCallsAndFixThem(*CurDAG, TLI);
+
+  // Final step, emit the lowered DAG as machine code.
+  CodeGenAndEmitDAG();
+  CurDAG->reset();
 }
 
-void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
+void SelectionDAGISel::ComputeLiveOutVRegInfo() {
   SmallPtrSet<SDNode*, 128> VisitedNodes;
   SmallVector<SDNode*, 128> Worklist;
   
-  Worklist.push_back(DAG.getRoot().Val);
+  Worklist.push_back(CurDAG->getRoot().Val);
   
   APInt Mask;
   APInt KnownZero;
@@ -5335,14 +5318,14 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
     if (!SrcVT.isInteger() || SrcVT.isVector())
       continue;
     
-    unsigned NumSignBits = DAG.ComputeNumSignBits(Src);
+    unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
     Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits());
-    DAG.ComputeMaskedBits(Src, Mask, KnownZero, KnownOne);
+    CurDAG->ComputeMaskedBits(Src, Mask, KnownZero, KnownOne);
     
     // Only install this information if it tells us something.
     if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) {
       DestReg -= TargetRegisterInfo::FirstVirtualRegister;
-      FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo();
+      FunctionLoweringInfo &FLI = CurDAG->getFunctionLoweringInfo();
       if (DestReg >= FLI.LiveOutRegInfo.size())
         FLI.LiveOutRegInfo.resize(DestReg+1);
       FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[DestReg];
@@ -5353,102 +5336,102 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) {
   }
 }
 
-void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
+void SelectionDAGISel::CodeGenAndEmitDAG() {
   std::string GroupName;
   if (TimePassesIsEnabled)
     GroupName = "Instruction Selection and Scheduling";
   std::string BlockName;
   if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||
       ViewDAGCombine2 || ViewISelDAGs || ViewSchedDAGs || ViewSUnitDAGs)
-    BlockName = DAG.getMachineFunction().getFunction()->getName() + ':' +
+    BlockName = CurDAG->getMachineFunction().getFunction()->getName() + ':' +
                 BB->getBasicBlock()->getName();
 
   DOUT << "Initial selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewDAGCombine1) DAG.viewGraph("dag-combine1 input for " + BlockName);
+  if (ViewDAGCombine1) CurDAG->viewGraph("dag-combine1 input for " + BlockName);
 
   // Run the DAG combiner in pre-legalize mode.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Combining 1", GroupName);
-    DAG.Combine(false, *AA, Fast);
+    CurDAG->Combine(false, *AA, Fast);
   } else {
-    DAG.Combine(false, *AA, Fast);
+    CurDAG->Combine(false, *AA, Fast);
   }
   
   DOUT << "Optimized lowered selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
   
   // Second step, hack on the DAG until it only uses operations and types that
   // the target supports.
   if (EnableLegalizeTypes) {// Enable this some day.
-    if (ViewLegalizeTypesDAGs) DAG.viewGraph("legalize-types input for " +
-                                             BlockName);
+    if (ViewLegalizeTypesDAGs) CurDAG->viewGraph("legalize-types input for " +
+                                                 BlockName);
 
     if (TimePassesIsEnabled) {
       NamedRegionTimer T("Type Legalization", GroupName);
-      DAG.LegalizeTypes();
+      CurDAG->LegalizeTypes();
     } else {
-      DAG.LegalizeTypes();
+      CurDAG->LegalizeTypes();
     }
 
     DOUT << "Type-legalized selection DAG:\n";
-    DEBUG(DAG.dump());
+    DEBUG(CurDAG->dump());
 
     // TODO: enable a dag combine pass here.
   }
   
-  if (ViewLegalizeDAGs) DAG.viewGraph("legalize input for " + BlockName);
+  if (ViewLegalizeDAGs) CurDAG->viewGraph("legalize input for " + BlockName);
 
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Legalization", GroupName);
-    DAG.Legalize();
+    CurDAG->Legalize();
   } else {
-    DAG.Legalize();
+    CurDAG->Legalize();
   }
   
   DOUT << "Legalized selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
   
-  if (ViewDAGCombine2) DAG.viewGraph("dag-combine2 input for " + BlockName);
+  if (ViewDAGCombine2) CurDAG->viewGraph("dag-combine2 input for " + BlockName);
 
   // Run the DAG combiner in post-legalize mode.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("DAG Combining 2", GroupName);
-    DAG.Combine(true, *AA, Fast);
+    CurDAG->Combine(true, *AA, Fast);
   } else {
-    DAG.Combine(true, *AA, Fast);
+    CurDAG->Combine(true, *AA, Fast);
   }
   
   DOUT << "Optimized legalized selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewISelDAGs) DAG.viewGraph("isel input for " + BlockName);
+  if (ViewISelDAGs) CurDAG->viewGraph("isel input for " + BlockName);
   
   if (!Fast && EnableValueProp)
-    ComputeLiveOutVRegInfo(DAG);
+    ComputeLiveOutVRegInfo();
 
   // Third, instruction select all of the operations to machine code, adding the
   // code to the MachineBasicBlock.
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("Instruction Selection", GroupName);
-    InstructionSelect(DAG);
+    InstructionSelect();
   } else {
-    InstructionSelect(DAG);
+    InstructionSelect();
   }
 
   DOUT << "Selected selection DAG:\n";
-  DEBUG(DAG.dump());
+  DEBUG(CurDAG->dump());
 
-  if (ViewSchedDAGs) DAG.viewGraph("scheduler input for " + BlockName);
+  if (ViewSchedDAGs) CurDAG->viewGraph("scheduler input for " + BlockName);
 
   // Schedule machine code.
   ScheduleDAG *Scheduler;
   if (TimePassesIsEnabled) {
     NamedRegionTimer T("Instruction Scheduling", GroupName);
-    Scheduler = Schedule(DAG);
+    Scheduler = Schedule();
   } else {
-    Scheduler = Schedule(DAG);
+    Scheduler = Schedule();
   }
 
   if (ViewSUnitDAGs) Scheduler->viewGraph();
@@ -5470,57 +5453,94 @@ void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) {
     delete Scheduler;
   }
 
-  // Perform target specific isel post processing.
-  if (TimePassesIsEnabled) {
-    NamedRegionTimer T("Instruction Selection Post Processing", GroupName);
-    InstructionSelectPostProcessing();
-  } else {
-    InstructionSelectPostProcessing();
-  }
-  
   DOUT << "Selected machine code:\n";
   DEBUG(BB->dump());
 }  
 
 void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF,
                                             FunctionLoweringInfo &FuncInfo) {
-  // Define NodeAllocator here so that memory allocation is reused for
+  // Define the SelectionDAG here so that memory allocation is reused for
   // each basic block.
-  NodeAllocatorType NodeAllocator;
+  SelectionDAG DAG(TLI, MF, FuncInfo, 
+                   getAnalysisToUpdate<MachineModuleInfo>());
+  CurDAG = &DAG;
 
   std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate;
   for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
     BasicBlock *LLVMBB = &*I;
-    PHINodesToUpdate.clear();
+    BB = FuncInfo.MBBMap[LLVMBB];
+
+    BasicBlock::iterator Begin = LLVMBB->begin();
+    BasicBlock::iterator End = LLVMBB->end();
+    bool DoArgs = LLVMBB == &Fn.getEntryBlock();
+
+    // Before doing SelectionDAG ISel, see if FastISel has been requested.
+    // FastISel doesn't support EH landing pads, which require special handling.
+    if (EnableFastISel && !BB->isLandingPad()) {
+      if (FastISel *F = TLI.createFastISel(FuncInfo.MF)) {
+        while (Begin != End) {
+          Begin = F->SelectInstructions(Begin, End, FuncInfo.ValueMap,
+                                        FuncInfo.MBBMap, BB);
+
+          if (Begin == End)
+            // The "fast" selector selected the entire block, so we're done.
+            break;
+
+          // Handle certain instructions as single-LLVM-Instruction blocks.
+          if (isa<CallInst>(Begin) || isa<LoadInst>(Begin) ||
+              isa<StoreInst>(Begin)) {
+            if (Begin->getType() != Type::VoidTy) {
+              unsigned &R = FuncInfo.ValueMap[Begin];
+              if (!R)
+                R = FuncInfo.CreateRegForValue(Begin);
+            }
+
+            SelectBasicBlock(LLVMBB, Begin, next(Begin), DoArgs,
+                             PHINodesToUpdate, FuncInfo);
+
+            ++Begin;
+            DoArgs = false;
+            continue;
+          }
+
+          if (!DisableFastISelAbort &&
+              // For now, don't abort on non-conditional-branch terminators.
+              (!isa<TerminatorInst>(Begin) ||
+               (isa<BranchInst>(Begin) &&
+                cast<BranchInst>(Begin)->isUnconditional()))) {
+            // The "fast" selector couldn't handle something and bailed.
+            // For the purpose of debugging, just abort.
+#ifndef NDEBUG
+              Begin->dump();
+#endif
+            assert(0 && "FastISel didn't select the entire block");
+          }
+          break;
+        }
+        delete F;
+      }
+    }
+
+    if (Begin != End || DoArgs)
+      SelectBasicBlock(LLVMBB, Begin, End, DoArgs, PHINodesToUpdate, FuncInfo);
 
-    SelectBasicBlock(LLVMBB, MF, FuncInfo, PHINodesToUpdate, NodeAllocator);
-    FinishBasicBlock(LLVMBB, MF, FuncInfo, PHINodesToUpdate, NodeAllocator);
+    FinishBasicBlock(FuncInfo, PHINodesToUpdate);
+    PHINodesToUpdate.clear();
   }
+
+  CurDAG = 0;
 }
 
 void
-SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
-                                   FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                   NodeAllocatorType &NodeAllocator) {
-  SelectionDAG DAG(TLI, MF, FuncInfo, 
-                   getAnalysisToUpdate<MachineModuleInfo>(),
-                   NodeAllocator);
-  CurDAG = &DAG;
-  
-  // First step, lower LLVM code to some DAG.  This DAG may use operations and
-  // types that are not supported by the target.
-  BuildSelectionDAG(DAG, LLVMBB, PHINodesToUpdate, FuncInfo);
+SelectionDAGISel::FinishBasicBlock(FunctionLoweringInfo &FuncInfo,
+             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate) {
 
-  // Second step, emit the lowered DAG as machine code.
-  CodeGenAndEmitDAG(DAG);
-}
+  // Perform target specific isel post processing.
+  InstructionSelectPostProcessing();
+  
+  DOUT << "Target-post-processed machine code:\n";
+  DEBUG(BB->dump());
 
-void
-SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
-                                   FunctionLoweringInfo &FuncInfo,
-             std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate,
-                                   NodeAllocatorType &NodeAllocator) {
   DOUT << "Total amount of phi nodes to update: "
        << PHINodesToUpdate.size() << "\n";
   DEBUG(for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i)
@@ -5544,26 +5564,19 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,
   for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) {
     // Lower header first, if it wasn't already lowered
     if (!BitTestCases[i].Emitted) {
-      SelectionDAG HSDAG(TLI, MF, FuncInfo, 
-                         getAnalysisToUpdate<MachineModuleInfo>(),
-                         NodeAllocator);
-      CurDAG = &HSDAG;
-      SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GFI);
+      SelectionDAGLowering HSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Parent;
       HSDL.setCurrentBasicBlock(BB);
       // Emit the code
       HSDL.visitBitTestHeader(BitTestCases[i]);
-      HSDAG.setRoot(HSDL.getRoot());
-      CodeGenAndEmitDAG(HSDAG);
+      CurDAG->setRoot(HSDL.getRoot());
+      CodeGenAndEmitDAG();
+      CurDAG->reset();
     }    
 
     for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) {
-      SelectionDAG BSDAG(TLI, MF, FuncInfo, 
-                         getAnalysisToUpdate<MachineModuleInfo>(),
-                         NodeAllocator);
-      CurDAG = &BSDAG;
-      SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GFI);
+      SelectionDAGLowering BSDL(*CurDAG, TLI, *AA, FuncInfo, GFI);
       // Set the current basic block to the mbb we wish to insert the code into
       BB = BitTestCases[i].Cases[j].ThisBB;
       BSDL.setCurrentBasicBlock(BB);
@@ -5578,8 +5591,9 @@ SelectionDAGISel::FinishBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF,