Revise cleanup IR generation to fix a major bug with cleanups (PR7686)

as well as some significant asymptotic inefficiencies with threading
multiple jumps through deep cleanups.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@109274 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 449 insertions, 236 deletions

diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index dbebd136ba..1d0d6a8ab5 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -31,8 +31,9 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
   : BlockFunction(cgm, *this, Builder), CGM(cgm),
     Target(CGM.getContext().Target),
     Builder(cgm.getModule().getContext()),
+    NormalCleanupDest(0), EHCleanupDest(0), NextCleanupDestIndex(1),
     ExceptionSlot(0), DebugInfo(0), IndirectBranch(0),
-    SwitchInsn(0), CaseRangeBlock(0), InvokeDest(0),
+    SwitchInsn(0), CaseRangeBlock(0),
     DidCallStackSave(false), UnreachableBlock(0),
     CXXThisDecl(0), CXXThisValue(0), CXXVTTDecl(0), CXXVTTValue(0),
     ConditionalBranchLevel(0), TerminateLandingPad(0), TerminateHandler(0),
@@ -89,26 +90,26 @@ void CodeGenFunction::EmitReturnBlock() {
 
     // We have a valid insert point, reuse it if it is empty or there are no
     // explicit jumps to the return block.
-    if (CurBB->empty() || ReturnBlock.Block->use_empty()) {
-      ReturnBlock.Block->replaceAllUsesWith(CurBB);
-      delete ReturnBlock.Block;
+    if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
+      ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
+      delete ReturnBlock.getBlock();
     } else
-      EmitBlock(ReturnBlock.Block);
+      EmitBlock(ReturnBlock.getBlock());
     return;
   }
 
   // Otherwise, if the return block is the target of a single direct
   // branch then we can just put the code in that block instead. This
   // cleans up functions which started with a unified return block.
-  if (ReturnBlock.Block->hasOneUse()) {
+  if (ReturnBlock.getBlock()->hasOneUse()) {
     llvm::BranchInst *BI =
-      dyn_cast<llvm::BranchInst>(*ReturnBlock.Block->use_begin());
+      dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->use_begin());
     if (BI && BI->isUnconditional() &&
-        BI->getSuccessor(0) == ReturnBlock.Block) {
+        BI->getSuccessor(0) == ReturnBlock.getBlock()) {
       // Reset insertion point and delete the branch.
       Builder.SetInsertPoint(BI->getParent());
       BI->eraseFromParent();
-      delete ReturnBlock.Block;
+      delete ReturnBlock.getBlock();
       return;
     }
   }
@@ -117,7 +118,7 @@ void CodeGenFunction::EmitReturnBlock() {
   // unless it has uses. However, we still need a place to put the debug
   // region.end for now.
 
-  EmitBlock(ReturnBlock.Block);
+  EmitBlock(ReturnBlock.getBlock());
 }
 
 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
@@ -170,6 +171,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     }
   }
 
+  EmitIfUsed(*this, RethrowBlock.getBlock());
   EmitIfUsed(*this, TerminateLandingPad);
   EmitIfUsed(*this, TerminateHandler);
   EmitIfUsed(*this, UnreachableBlock);
@@ -585,7 +587,7 @@ llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelStmt *L) {
   if (IndirectBranch == 0)
     GetIndirectGotoBlock();
   
-  llvm::BasicBlock *BB = getJumpDestForLabel(L).Block;
+  llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
   
   // Make sure the indirect branch includes all of the address-taken blocks.
   IndirectBranch->addDestination(BB);
@@ -666,41 +668,75 @@ llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) {
 void CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old) {
   assert(Old.isValid());
 
-  EHScopeStack::iterator E = EHStack.find(Old);
-  while (EHStack.begin() != E)
-    PopCleanupBlock();
+  while (EHStack.stable_begin() != Old) {
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
+
+    // As long as Old strictly encloses the scope's enclosing normal
+    // cleanup, we're going to emit another normal cleanup which
+    // fallthrough can propagate through.
+    bool FallThroughIsBranchThrough =
+      Old.strictlyEncloses(Scope.getEnclosingNormalCleanup());
+
+    PopCleanupBlock(FallThroughIsBranchThrough);
+  }
 }
 
-/// Creates a switch instruction to thread branches out of the given
-/// block (which is the exit block of a cleanup).
-static void CreateCleanupSwitch(CodeGenFunction &CGF,
-                                llvm::BasicBlock *Block) {
-  if (Block->getTerminator()) {
-    assert(isa<llvm::SwitchInst>(Block->getTerminator()) &&
-           "cleanup block already has a terminator, but it isn't a switch");
-    return;
+static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
+                                           EHCleanupScope &Scope) {
+  assert(Scope.isNormalCleanup());
+  llvm::BasicBlock *Entry = Scope.getNormalBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("cleanup");
+    Scope.setNormalBlock(Entry);
   }
+  return Entry;
+}
 
-  llvm::Value *DestCodePtr
-    = CGF.CreateTempAlloca(CGF.Builder.getInt32Ty(), "cleanup.dst");
-  CGBuilderTy Builder(Block);
-  llvm::Value *DestCode = Builder.CreateLoad(DestCodePtr, "tmp");
+static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
+                                       EHCleanupScope &Scope) {
+  assert(Scope.isEHCleanup());
+  llvm::BasicBlock *Entry = Scope.getEHBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("eh.cleanup");
+    Scope.setEHBlock(Entry);
+  }
+  return Entry;
+}
 
-  // Create a switch instruction to determine where to jump next.
-  Builder.CreateSwitch(DestCode, CGF.getUnreachableBlock());
+/// Transitions the terminator of the given exit-block of a cleanup to
+/// be a cleanup switch.
+static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
+                                                   llvm::BasicBlock *Block) {
+  // If it's a branch, turn it into a switch whose default
+  // destination is its original target.
+  llvm::TerminatorInst *Term = Block->getTerminator();
+  assert(Term && "can't transition block without terminator");
+
+  if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
+    assert(Br->isUnconditional());
+    llvm::LoadInst *Load =
+      new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term);
+    llvm::SwitchInst *Switch =
+      llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
+    Br->eraseFromParent();
+    return Switch;
+  } else {
+    return cast<llvm::SwitchInst>(Term);
+  }
 }
 
 /// Attempts to reduce a cleanup's entry block to a fallthrough.  This
 /// is basically llvm::MergeBlockIntoPredecessor, except
-/// simplified/optimized for the tighter constraints on cleanup
-/// blocks.
-static void SimplifyCleanupEntry(CodeGenFunction &CGF,
-                                 llvm::BasicBlock *Entry) {
+/// simplified/optimized for the tighter constraints on cleanup blocks.
+///
+/// Returns the new block, whatever it is.
+static llvm::BasicBlock *SimplifyCleanupEntry(CodeGenFunction &CGF,
+                                              llvm::BasicBlock *Entry) {
   llvm::BasicBlock *Pred = Entry->getSinglePredecessor();
-  if (!Pred) return;
+  if (!Pred) return Entry;
 
   llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Pred->getTerminator());
-  if (!Br || Br->isConditional()) return;
+  if (!Br || Br->isConditional()) return Entry;
   assert(Br->getSuccessor(0) == Entry);
 
   // If we were previously inserting at the end of the cleanup entry
@@ -720,80 +756,8 @@ static void SimplifyCleanupEntry(CodeGenFunction &CGF,
 
   if (WasInsertBlock)
     CGF.Builder.SetInsertPoint(Pred);
-}
-
-/// Attempts to reduce an cleanup's exit switch to an unconditional
-/// branch.
-static void SimplifyCleanupExit(llvm::BasicBlock *Exit) {
-  llvm::TerminatorInst *Terminator = Exit->getTerminator();
-  assert(Terminator && "completed cleanup exit has no terminator");
-
-  llvm::SwitchInst *Switch = dyn_cast<llvm::SwitchInst>(Terminator);
-  if (!Switch) return;
-  if (Switch->getNumCases() != 2) return; // default + 1
-
-  llvm::LoadInst *Cond = cast<llvm::LoadInst>(Switch->getCondition());
-  llvm::AllocaInst *CondVar = cast<llvm::AllocaInst>(Cond->getPointerOperand());
-
-  // Replace the switch instruction with an unconditional branch.
-  llvm::BasicBlock *Dest = Switch->getSuccessor(1); // default is 0
-  Switch->eraseFromParent();
-  llvm::BranchInst::Create(Dest, Exit);
-
-  // Delete all uses of the condition variable.
-  Cond->eraseFromParent();
-  while (!CondVar->use_empty())
-    cast<llvm::StoreInst>(*CondVar->use_begin())->eraseFromParent();
-
-  // Delete the condition variable itself.
-  CondVar->eraseFromParent();
-}
-
-/// Threads a branch fixup through a cleanup block.
-static void ThreadFixupThroughCleanup(CodeGenFunction &CGF,
-                                      BranchFixup &Fixup,
-                                      llvm::BasicBlock *Entry,
-                                      llvm::BasicBlock *Exit) {
-  if (!Exit->getTerminator())
-    CreateCleanupSwitch(CGF, Exit);
-
-  // Find the switch and its destination index alloca.
-  llvm::SwitchInst *Switch = cast<llvm::SwitchInst>(Exit->getTerminator());
-  llvm::Value *DestCodePtr =
-    cast<llvm::LoadInst>(Switch->getCondition())->getPointerOperand();
-
-  // Compute the index of the new case we're adding to the switch.
-  unsigned Index = Switch->getNumCases();
-
-  const llvm::IntegerType *i32 = llvm::Type::getInt32Ty(CGF.getLLVMContext());
-  llvm::ConstantInt *IndexV = llvm::ConstantInt::get(i32, Index);
-
-  // Set the index in the origin block.
-  new llvm::StoreInst(IndexV, DestCodePtr, Fixup.Origin);
-
-  // Add a case to the switch.
-  Switch->addCase(IndexV, Fixup.Destination);
-
-  // Change the last branch to point to the cleanup entry block.
-  Fixup.LatestBranch->setSuccessor(Fixup.LatestBranchIndex, Entry);
-
-  // And finally, update the fixup.
-  Fixup.LatestBranch = Switch;
-  Fixup.LatestBranchIndex = Index;
-}
-
-/// Try to simplify both the entry and exit edges of a cleanup.
-static void SimplifyCleanupEdges(CodeGenFunction &CGF,
-                                 llvm::BasicBlock *Entry,
-                                 llvm::BasicBlock *Exit) {
-
-  // Given their current implementations, it's important to run these
-  // in this order: SimplifyCleanupEntry will delete Entry if it can
-  // be merged into its predecessor, which will then break
-  // SimplifyCleanupExit if (as is common) Entry == Exit.
 
-  SimplifyCleanupExit(Exit);
-  SimplifyCleanupEntry(CGF, Entry);  
+  return Pred;
 }
 
 static void EmitCleanup(CodeGenFunction &CGF,
@@ -805,42 +769,10 @@ static void EmitCleanup(CodeGenFunction &CGF,
   assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
 }
 
-static void SplitAndEmitCleanup(CodeGenFunction &CGF,
-                                EHScopeStack::Cleanup *Fn,
-                                bool ForEH,
-                                llvm::BasicBlock *Entry) {
-  assert(Entry && "no entry block for cleanup");
-
-  // Remove the switch and load from the end of the entry block.
-  llvm::Instruction *Switch = &Entry->getInstList().back();
-  Entry->getInstList().remove(Switch);
-  assert(isa<llvm::SwitchInst>(Switch));
-  llvm::Instruction *Load = &Entry->getInstList().back();
-  Entry->getInstList().remove(Load);
-  assert(isa<llvm::LoadInst>(Load));
-
-  assert(Entry->getInstList().empty() &&
-         "lazy cleanup block not empty after removing load/switch pair?");
-
-  // Emit the actual cleanup at the end of the entry block.
-  CGF.Builder.SetInsertPoint(Entry);
-  EmitCleanup(CGF, Fn, ForEH);
-
-  // Put the load and switch at the end of the exit block.
-  llvm::BasicBlock *Exit = CGF.Builder.GetInsertBlock();
-  Exit->getInstList().push_back(Load);
-  Exit->getInstList().push_back(Switch);
-
-  // Clean up the edges if possible.
-  SimplifyCleanupEdges(CGF, Entry, Exit);
-
-  CGF.Builder.ClearInsertionPoint();
-}
-
 /// Pops a cleanup block.  If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
-void CodeGenFunction::PopCleanupBlock() {
+void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   assert(!EHStack.empty() && "cleanup stack is empty!");
   assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
   EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
@@ -848,8 +780,7 @@ void CodeGenFunction::PopCleanupBlock() {
 
   // Check whether we need an EH cleanup.  This is only true if we've
   // generated a lazy EH cleanup block.
-  llvm::BasicBlock *EHEntry = Scope.getEHBlock();
-  bool RequiresEHCleanup = (EHEntry != 0);
+  bool RequiresEHCleanup = Scope.hasEHBranches();
 
   // Check the three conditions which might require a normal cleanup:
 
@@ -857,9 +788,8 @@ void CodeGenFunction::PopCleanupBlock() {
   unsigned FixupDepth = Scope.getFixupDepth();
   bool HasFixups = EHStack.getNumBranchFixups() != FixupDepth;
 
-  // - whether control has already been threaded through this cleanup
-  llvm::BasicBlock *NormalEntry = Scope.getNormalBlock();
-  bool HasExistingBranches = (NormalEntry != 0);
+  // - whether there are branch-throughs or branch-afters
+  bool HasExistingBranches = Scope.hasBranches();
 
   // - whether there's a fallthrough
   llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock();
@@ -873,7 +803,7 @@ void CodeGenFunction::PopCleanupBlock() {
 
   // If we don't need the cleanup at all, we're done.
   if (!RequiresNormalCleanup && !RequiresEHCleanup) {
-    EHStack.popCleanup();
+    EHStack.popCleanup(); // safe because there are no fixups
     assert(EHStack.getNumBranchFixups() == 0 ||
            EHStack.hasNormalCleanups());
     return;
@@ -890,157 +820,440 @@ void CodeGenFunction::PopCleanupBlock() {
   EHScopeStack::Cleanup *Fn =
     reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
 
-  // We're done with the scope; pop it off so we can emit the cleanups.
-  EHStack.popCleanup();
+  // We want to emit the EH cleanup after the normal cleanup, but go
+  // ahead and do the setup for the EH cleanup while the scope is still
+  // alive.
+  llvm::BasicBlock *EHEntry = 0;
+  llvm::SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
+  if (RequiresEHCleanup) {
+    EHEntry = CreateEHEntry(*this, Scope);
+
+    // Figure out the branch-through dest if necessary.
+    llvm::BasicBlock *EHBranchThroughDest = 0;
+    if (Scope.hasEHBranchThroughs()) {
+      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
+      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
+      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
+    }
+
+    // If we have exactly one branch-after and no branch-throughs, we
+    // can dispatch it without a switch.
+    if (!Scope.hasBranchThroughs() &&
+        Scope.getNumEHBranchAfters() == 1) {
+      assert(!EHBranchThroughDest);
+
+      // TODO: remove the spurious eh.cleanup.dest stores if this edge
+      // never went through any switches.
+      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
+    
+    // Otherwise, if we have any branch-afters, we need a switch.
+    } else if (Scope.getNumEHBranchAfters()) {
+      // The default of the switch belongs to the branch-throughs if
+      // they exist.
+      llvm::BasicBlock *Default =
+        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
+
+      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
+
+      llvm::LoadInst *Load =
+        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
+      llvm::SwitchInst *Switch =
+        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+      EHInstsToAppend.push_back(Load);
+      EHInstsToAppend.push_back(Switch);
+
+      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
+        Switch->addCase(Scope.getEHBranchAfterIndex(I),
+                        Scope.getEHBranchAfterBlock(I));
+
+    // Otherwise, we have only branch-throughs; jump to the next EH
+    // cleanup.
+    } else {
+      assert(EHBranchThroughDest);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
+    }
+  }
+
+  if (!RequiresNormalCleanup) {
+    EHStack.popCleanup();
+  } else {
+    // As a kindof crazy internal case, branch-through fall-throughs
+    // leave the insertion point set to the end of the last cleanup.
+    bool HasPrebranchedFallthrough =
+      (HasFallthrough && FallthroughSource->getTerminator());
+    assert(!HasPrebranchedFallthrough ||
+           FallthroughSource->getTerminator()->getSuccessor(0)
+             == Scope.getNormalBlock());
 
-  if (RequiresNormalCleanup) {
     // If we have a fallthrough and no other need for the cleanup,
     // emit it directly.
-    if (HasFallthrough && !HasFixups && !HasExistingBranches) {
+    if (HasFallthrough && !HasPrebranchedFallthrough &&
+        !HasFixups && !HasExistingBranches) {
+
+      // Fixups can cause us to optimistically create a normal block,
+      // only to later have no real uses for it.  Just delete it in
+      // this case.
+      // TODO: we can potentially simplify all the uses after this.
+      if (Scope.getNormalBlock()) {
+        Scope.getNormalBlock()->replaceAllUsesWith(getUnreachableBlock());
+        delete Scope.getNormalBlock();
+      }
+
+      EHStack.popCleanup();
+
       EmitCleanup(*this, Fn, /*ForEH*/ false);
 
     // Otherwise, the best approach is to thread everything through
     // the cleanup block and then try to clean up after ourselves.
     } else {
       // Force the entry block to exist.
-      if (!HasExistingBranches) {
-        NormalEntry = createBasicBlock("cleanup");
-        CreateCleanupSwitch(*this, NormalEntry);
-      }
+      llvm::BasicBlock *NormalEntry = CreateNormalEntry(*this, Scope);
 
+      // If there's a fallthrough, we need to store the cleanup
+      // destination index.  For fall-throughs this is always zero.
+      if (HasFallthrough && !HasPrebranchedFallthrough)
+        Builder.CreateStore(Builder.getInt32(0), getNormalCleanupDestSlot());
+
+      // Emit the entry block.  This implicitly branches to it if we
+      // have fallthrough.  All the fixups and existing branches should
+      // already be branched to it.
       EmitBlock(NormalEntry);
 
-      // Thread the fallthrough edge through the (momentarily trivial)
-      // cleanup.
-      llvm::BasicBlock *FallthroughDestination = 0;
-      if (HasFallthrough) {
-        assert(isa<llvm::BranchInst>(FallthroughSource->getTerminator()));
-        FallthroughDestination = createBasicBlock("cleanup.cont");
-
-        BranchFixup Fix;
-        Fix.Destination = FallthroughDestination;
-        Fix.LatestBranch = FallthroughSource->getTerminator();
-        Fix.LatestBranchIndex = 0;
-        Fix.Origin = Fix.LatestBranch;
-
-        // Restore fixup invariant.  EmitBlock added a branch to the
-        // cleanup which we need to redirect to the destination.
-        cast<llvm::BranchInst>(Fix.LatestBranch)
-          ->setSuccessor(0, Fix.Destination);
-
-        ThreadFixupThroughCleanup(*this, Fix, NormalEntry, NormalEntry);
+      bool HasEnclosingCleanups =
+        (Scope.getEnclosingNormalCleanup() != EHStack.stable_end());
+
+      // Compute the branch-through dest if we need it:
+      //   - if there are branch-throughs threaded through the scope
+      //   - if fall-through is a branch-through
+      //   - if there are fixups that will be optimistically forwarded
+      //     to the enclosing cleanup
+      llvm::BasicBlock *BranchThroughDest = 0;
+      if (Scope.hasBranchThroughs() ||
+          (HasFallthrough && FallthroughIsBranchThrough) ||
+          (HasFixups && HasEnclosingCleanups)) {
+        assert(HasEnclosingCleanups);
+        EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
+        BranchThroughDest = CreateNormalEntry(*this, cast<EHCleanupScope>(S));
       }
 
-      // Thread any "real" fixups we need to thread.
-      for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
-           I != E; ++I)
-        if (CGF.EHStack.getBranchFixup(I).Destination)
-          ThreadFixupThroughCleanup(*this, EHStack.getBranchFixup(I),
-                                    NormalEntry, NormalEntry);
+      llvm::BasicBlock *FallthroughDest = 0;
+      llvm::SmallVector<llvm::Instruction*, 2> InstsToAppend;
+
+      // If there's exactly one branch-after and no other threads,
+      // we can route it without a switch.
+      if (!Scope.hasBranchThroughs() && !HasFixups && !HasFallthrough &&
+          Scope.getNumBranchAfters() == 1) {
+        assert(!BranchThroughDest);
+
+        // TODO: clean up the possibly dead stores to the cleanup dest slot.
+        llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter));
+
+      // Build a switch-out if we need it:
+      //   - if there are branch-afters threaded through the scope
+      //   - if fall-through is a branch-after
+      //   - if there are fixups that have nowhere left to go and
+      //     so must be immediately resolved
+      } else if (Scope.getNumBranchAfters() ||
+                 (HasFallthrough && !FallthroughIsBranchThrough) ||
+                 (HasFixups && !HasEnclosingCleanups)) {
+
+        llvm::BasicBlock *Default =
+          (BranchThroughDest ? BranchThroughDest : getUnreachableBlock());
+
+        // TODO: base this on the number of branch-afters and fixups
+        const unsigned SwitchCapacity = 10;
+
+        llvm::LoadInst *Load =
+          new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest");
+        llvm::SwitchInst *Switch =
+          llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+        InstsToAppend.push_back(Load);
+        InstsToAppend.push_back(Switch);
+
+        // Branch-after fallthrough.
+        if (HasFallthrough && !FallthroughIsBranchThrough) {
+          FallthroughDest = createBasicBlock("cleanup.cont");
+          Switch->addCase(Builder.getInt32(0), FallthroughDest);
+        }
+
+        for (unsigned I = 0, E = Scope.getNumBranchAfters(); I != E; ++I) {
+          Switch->addCase(Scope.getBranchAfterIndex(I),
+                          Scope.getBranchAfterBlock(I));
+        }
+
+        if (HasFixups && !HasEnclosingCleanups)
+          ResolveAllBranchFixups(Switch);
+      } else {
+        // We should always have a branch-through destination in this case.
+        assert(BranchThroughDest);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchThroughDest));
+      }
 
-      SplitAndEmitCleanup(*this, Fn, /*ForEH*/ false, NormalEntry);
+      // We're finally ready to pop the cleanup.
+      EHStack.popCleanup();
+      assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups);
 
-      if (HasFallthrough)
-        EmitBlock(FallthroughDestination);
+      EmitCleanup(*this, Fn, /*ForEH*/ false);
+
+      // Append the prepared cleanup prologue from above.
+      llvm::BasicBlock *NormalExit = Builder.GetInsertBlock();
+      for (unsigned I = 0, E = InstsToAppend.size(); I != E; ++I)
+        NormalExit->getInstList().push_back(InstsToAppend[I]);
+
+      // Optimistically hope that any fixups will continue falling through.
+      for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+           I < E; ++I) {
+        BranchFixup &Fixup = CGF.EHStack.getBranchFixup(I);
+        if (!Fixup.Destination) continue;
+        if (!Fixup.OptimisticBranchBlock) {
+          new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex),
+                              getNormalCleanupDestSlot(),
+                              Fixup.InitialBranch);
+          Fixup.InitialBranch->setSuccessor(0, NormalEntry);
+        }
+        Fixup.OptimisticBranchBlock = NormalExit;
+      }
+      
+      if (FallthroughDest)
+        EmitBlock(FallthroughDest);
+      else if (!HasFallthrough)
+        Builder.ClearInsertionPoint();
+
+      // Check whether we can merge NormalEntry into a single predecessor.
+      // This might invalidate (non-IR) pointers to NormalEntry.
+      llvm::BasicBlock *NewNormalEntry =
+        SimplifyCleanupEntry(*this, NormalEntry);
+
+      // If it did invalidate those pointers, and NormalEntry was the same
+      // as NormalExit, go back and patch up the fixups.
+      if (NewNormalEntry != NormalEntry && NormalEntry == NormalExit)
+        for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+               I < E; ++I)
+          CGF.EHStack.getBranchFixup(I).OptimisticBranchBlock = NewNormalEntry;
     }
   }
 
+  assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
+
   // Emit the EH cleanup if required.
   if (RequiresEHCleanup) {
     CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+
     EmitBlock(EHEntry);
-    SplitAndEmitCleanup(*this, Fn, /*ForEH*/ true, EHEntry);
+    EmitCleanup(*this, Fn, /*ForEH*/ true);
+
+    // Append the prepared cleanup prologue from above.
+    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
+    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
+      EHExit->getInstList().push_back(EHInstsToAppend[I]);
+
     Builder.restoreIP(SavedIP);
+
+    SimplifyCleanupEntry(*this, EHEntry);
   }
 }
 
+/// Terminate the current block by emitting a branch which might leave
+/// the current cleanup-protected scope.  The target scope may not yet
+/// be known, in which case this will require a fixup.
+///
+/// As a side-effect, this method clears the insertion point.
 void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
   if (!HaveInsertPoint())
     return;
 
   // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.Block);
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
 
-  // If we're not in a cleanup scope, we don't need to worry about
+  // If we're not in a cleanup scope, or if the destination scope is
+  // the current normal-cleanup scope, we don't need to worry about
   // fixups.
-  if (!EHStack.hasNormalCleanups()) {
+  if (!EHStack.hasNormalCleanups() ||
+      Dest.getScopeDepth() == EHStack.getInnermostNormalCleanup()) {
     Builder.ClearInsertionPoint();
     return;
   }
 
-  // Initialize a fixup.
-  BranchFixup Fixup;
-  Fixup.Destination = Dest.Block;
-  Fixup.Origin = BI;
-  Fixup.LatestBranch = BI;
-  Fixup.LatestBranchIndex = 0;
-
   // If we can't resolve the destination cleanup scope, just add this
-  // to the current cleanup scope.
-  if (!Dest.ScopeDepth.isValid()) {
-    EHStack.addBranchFixup() = Fixup;
+  // to the current cleanup scope as a branch fixup.
+  if (!Dest.getScopeDepth().isValid()) {
+    BranchFixup &Fixup = EHStack.addBranchFixup();
+    Fixup.Destination = Dest.getBlock();
+    Fixup.DestinationIndex = Dest.getDestIndex();
+    Fixup.InitialBranch = BI;
+    Fixup.OptimisticBranchBlock = 0;
+
     Builder.ClearInsertionPoint();
     return;
   }
 
-  for (EHScopeStack::iterator I = EHStack.begin(),
-         E = EHStack.find(Dest.ScopeDepth); I != E; ++I) {
-    if (isa<EHCleanupScope>(*I)) {
-      EHCleanupScope &Scope = cast<EHCleanupScope>(*I);
-      if (Scope.isNormalCleanup()) {
-        llvm::BasicBlock *Block = Scope.getNormalBlock();
-        if (!Block) {
-          Block = createBasicBlock("cleanup");
-          Scope.setNormalBlock(Block);
-        }
-        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
+  // Otherwise, thread through all the normal cleanups in scope.
+
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI);
+
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(EHStack.getInnermostNormalCleanup()));
+    BI->setSuccessor(0, CreateNormalEntry(*this, Scope));
+  }
+
+  // Add this destination to all the scopes involved.
+  EHScopeStack::stable_iterator I = EHStack.getInnermostNormalCleanup();
+  EHScopeStack::stable_iterator E = Dest.getScopeDepth();
+  if (E.strictlyEncloses(I)) {
+    while (true) {
+      EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+      assert(Scope.isNormalCleanup());
+      I = Scope.getEnclosingNormalCleanup();
+
+      // If this is the last cleanup we're propagating through, tell it
+      // that there's a resolved jump moving through it.
+      if (!E.strictlyEncloses(I)) {
+        Scope.addBranchAfter(Index, Dest.getBlock());
+        break;
       }
+
+      // Otherwise, tell the scope that there's a jump propoagating
+      // through it.  If this isn't new information, all the rest of
+      // the work has been done before.
+      if (!Scope.addBranchThrough(Dest.getBlock()))
+        break;
     }
   }
   
   Builder.ClearInsertionPoint();
 }
 
-void CodeGenFunction::EmitBranchThroughEHCleanup(JumpDest Dest) {
+void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
+  // We should never get invalid scope depths for an UnwindDest; that
+  // implies that the destination wasn't set up correctly.
+  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
+
   if (!HaveInsertPoint())
     return;
 
   // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.Block);
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
 
-  // If we're not in a cleanup scope, we don't need to worry about
-  // fixups.
-  if (!EHStack.hasEHCleanups()) {
+  // If the destination is in the same EH cleanup scope as us, we
+  // don't need to thread through anything.
+  if (Dest.getScopeDepth() == EHStack.getInnermostEHCleanup()) {
     Builder.ClearInsertionPoint();
     return;
   }
+  assert(EHStack.hasEHCleanups());
 
-  // Initialize a fixup.
-  BranchFixup Fixup;
-  Fixup.Destination = Dest.Block;
-  Fixup.Origin = BI;
-  Fixup.LatestBranch = BI;
-  Fixup.LatestBranchIndex = 0;
-
-  // We should never get invalid scope depths for these: invalid scope
-  // depths only arise for as-yet-unemitted labels, and we can't do an
-  // EH-unwind to one of those.
-  assert(Dest.ScopeDepth.isValid() && "invalid scope depth on EH dest?");
-
-  for (EHScopeStack::iterator I = EHStack.begin(),
-         E = EHStack.find(Dest.ScopeDepth); I != E; ++I) {
-    if (isa<EHCleanupScope>(*I)) {
-      EHCleanupScope &Scope = cast<EHCleanupScope>(*I);
-      if (Scope.isEHCleanup()) {
-        llvm::BasicBlock *Block = Scope.getEHBlock();
-        if (!Block) {
-          Block = createBasicBlock("eh.cleanup");
-          Scope.setEHBlock(Block);
-        }
-        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
-      }
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
+
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(EHStack.getInnermostEHCleanup()));
+    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
+  }
+  
+  // Add this destination to all the scopes involved.
+  for (EHScopeStack::stable_iterator
+         I = EHStack.getInnermostEHCleanup(),
+         E = Dest.getScopeDepth(); ; ) {
+    assert(E.strictlyEncloses(I));
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+    assert(Scope.isEHCleanup());
+    I = Scope.getEnclosingEHCleanup();
+
+    // If this is the last cleanup we're propagating through, add this
+    // as a branch-after.
+    if (I == E) {
+      Scope.addEHBranchAfter(Index, Dest.getBlock());
+      break;
     }
+
+    // Otherwise, add it as a branch-through.  If this isn't new
+    // information, all the rest of the work has been done before.
+    if (!Scope.addEHBranchThrough(Dest.getBlock()))
+      break;
   }
   
   Builder.ClearInsertionPoint();
 }
+
+/// All the branch fixups on the EH stack have propagated out past the
+/// outermost normal cleanup; resolve them all by adding cases to the
+/// given switch instruction.
+void CodeGenFunction::ResolveAllBranchFixups(llvm::SwitchInst *Switch) {
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> CasesAdded;
+
+  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination isn't set or if we've
+    // already treated it.
+    BranchFixup &Fixup = EHStack.getBranchFixup(I);
+    if (Fixup.Destination == 0) continue;
+    if (!CasesAdded.insert(Fixup.Destination)) continue;
+
+    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex),
+                    Fixup.Destination);
+  }
+
+  EHStack.clearFixups();
+}
+
+void CodeGenFunction::ResolveBranchFixups(llvm::BasicBlock *Block) {
+  assert(Block && "resolving a null target block");
+  if (!EHStack.getNumBranchFixups()) return;
+
+  assert(EHStack.hasNormalCleanups() &&
+         "branch fixups exist with no normal cleanups on stack");
+
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> ModifiedOptimisticBlocks;
+  bool ResolvedAny = false;
+
+  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination doesn't match.
+    BranchFixup &Fixup = EHStack.getBranchFixup(I);
+    if (Fixup.Destination != Block) continue;
+
+    Fixup.Destination = 0;
+    ResolvedAny = true;
+
+    // If it doesn't have an optimistic branch block, LatestBranch is
+    // already pointing to the right place.
+    llvm::BasicBlock *BranchBB = Fixup.OptimisticBranchBlock;
+    if (!BranchBB)
+      continue;
+
+    // Don't process the same optimistic branch block twice.
+    if (!ModifiedOptimisticBlocks.insert(BranchBB))
+      continue;
+
+    llvm::SwitchInst *Switch = TransitionToCleanupSwitch(*this, BranchBB);
+
+    // Add a case to the switch.
+    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex), Block);
+  }
+
+  if (ResolvedAny)
+    EHStack.popNullFixups();
+}
+
+llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
+  if (!NormalCleanupDest)
+    NormalCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
+  return NormalCleanupDest;
+}
+
+llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
+  if (!EHCleanupDest)
+    EHCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
+  return EHCleanupDest;
+}