Simplify EH control flow by observing that EH scopes form a simple

hierarchy of delegation, and that EH selector values are meaningful
function-wide (good thing, too, or inlining wouldn't work).
2,3d
1a
hierarchy of delegation and that EH selector values have the same
meaning everywhere in the function instead of being meaningful only
in the context of a specific selector.

This removes the need for routing edges through EH cleanups,
since a cleanup simply always branches to its enclosing scope.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@137293 91177308-0d34-0410-b5e6-96231b3b80d8

13 files changed, 540 insertions, 656 deletions

diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp
index 2da10ca006..b2d0786cb6 100644
--- a/lib/CodeGen/CGCleanup.cpp
+++ b/lib/CodeGen/CGCleanup.cpp
@@ -119,16 +119,30 @@ char *EHScopeStack::allocate(size_t Size) {
 }
 
 EHScopeStack::stable_iterator
-EHScopeStack::getEnclosingEHCleanup(iterator it) const {
-  assert(it != end());
-  do {
-    if (isa<EHCleanupScope>(*it)) {
-      if (cast<EHCleanupScope>(*it).isEHCleanup())
-        return stabilize(it);
-      return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
+EHScopeStack::getInnermostActiveNormalCleanup() const {
+  for (stable_iterator si = getInnermostNormalCleanup(), se = stable_end();
+         si != se; ) {
+    EHCleanupScope &cleanup = cast<EHCleanupScope>(*find(si));
+    if (cleanup.isActive()) return si;
+    si = cleanup.getEnclosingNormalCleanup();
+  }
+  return stable_end();
+}
+
+EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveEHScope() const {
+  for (stable_iterator si = getInnermostEHScope(), se = stable_end();
+         si != se; ) {
+    // Skip over inactive cleanups.
+    EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*find(si));
+    if (cleanup && !cleanup->isActive()) {
+      si = cleanup->getEnclosingEHScope();
+      continue;
     }
-    ++it;
-  } while (it != end());
+
+    // All other scopes are always active.
+    return si;
+  }
+
   return stable_end();
 }
 
@@ -146,11 +160,11 @@ void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
                                 Size,
                                 BranchFixups.size(),
                                 InnermostNormalCleanup,
-                                InnermostEHCleanup);
+                                InnermostEHScope);
   if (IsNormalCleanup)
     InnermostNormalCleanup = stable_begin();
   if (IsEHCleanup)
-    InnermostEHCleanup = stable_begin();
+    InnermostEHScope = stable_begin();
 
   return Scope->getCleanupBuffer();
 }
@@ -161,11 +175,9 @@ void EHScopeStack::popCleanup() {
   assert(isa<EHCleanupScope>(*begin()));
   EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
   InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+  InnermostEHScope = Cleanup.getEnclosingEHScope();
   StartOfData += Cleanup.getAllocatedSize();
 
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
   // Destroy the cleanup.
   Cleanup.~EHCleanupScope();
 
@@ -182,37 +194,35 @@ void EHScopeStack::popCleanup() {
   }
 }
 
-EHFilterScope *EHScopeStack::pushFilter(unsigned NumFilters) {
-  char *Buffer = allocate(EHFilterScope::getSizeForNumFilters(NumFilters));
-  CatchDepth++;
-  return new (Buffer) EHFilterScope(NumFilters);
+EHFilterScope *EHScopeStack::pushFilter(unsigned numFilters) {
+  assert(getInnermostEHScope() == stable_end());
+  char *buffer = allocate(EHFilterScope::getSizeForNumFilters(numFilters));
+  EHFilterScope *filter = new (buffer) EHFilterScope(numFilters);
+  InnermostEHScope = stable_begin();
+  return filter;
 }
 
 void EHScopeStack::popFilter() {
   assert(!empty() && "popping exception stack when not empty");
 
-  EHFilterScope &Filter = cast<EHFilterScope>(*begin());
-  StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+  EHFilterScope &filter = cast<EHFilterScope>(*begin());
+  StartOfData += EHFilterScope::getSizeForNumFilters(filter.getNumFilters());
 
-  assert(CatchDepth > 0 && "mismatched filter push/pop");
-  CatchDepth--;
+  InnermostEHScope = filter.getEnclosingEHScope();
 }
 
-EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
-  char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
-  CatchDepth++;
-  EHCatchScope *Scope = new (Buffer) EHCatchScope(NumHandlers);
-  for (unsigned I = 0; I != NumHandlers; ++I)
-    Scope->getHandlers()[I].Index = getNextEHDestIndex();
-  return Scope;
+EHCatchScope *EHScopeStack::pushCatch(unsigned numHandlers) {
+  char *buffer = allocate(EHCatchScope::getSizeForNumHandlers(numHandlers));
+  EHCatchScope *scope =
+    new (buffer) EHCatchScope(numHandlers, InnermostEHScope);
+  InnermostEHScope = stable_begin();
+  return scope;
 }
 
 void EHScopeStack::pushTerminate() {
   char *Buffer = allocate(EHTerminateScope::getSize());
-  CatchDepth++;
-  new (Buffer) EHTerminateScope(getNextEHDestIndex());
+  new (Buffer) EHTerminateScope(InnermostEHScope);
+  InnermostEHScope = stable_begin();
 }
 
 /// Remove any 'null' fixups on the stack.  However, we can't pop more
@@ -384,17 +394,6 @@ static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
   return Entry;
 }
 
-static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
-                                       EHCleanupScope &Scope) {
-  assert(Scope.isEHCleanup());
-  llvm::BasicBlock *Entry = Scope.getEHBlock();
-  if (!Entry) {
-    Entry = CGF.createBasicBlock("eh.cleanup");
-    Scope.setEHBlock(Entry);
-  }
-  return Entry;
-}
-
 /// Attempts to reduce a cleanup's entry block to a fallthrough.  This
 /// is basically llvm::MergeBlockIntoPredecessor, except
 /// simplified/optimized for the tighter constraints on cleanup blocks.
@@ -544,7 +543,10 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
 
   // Check whether we need an EH cleanup.  This is only true if we've
   // generated a lazy EH cleanup block.
-  bool RequiresEHCleanup = Scope.hasEHBranches();
+  llvm::BasicBlock *EHEntry = Scope.getCachedEHDispatchBlock();
+  assert(Scope.hasEHBranches() == (EHEntry != 0));
+  bool RequiresEHCleanup = (EHEntry != 0);
+  EHScopeStack::stable_iterator EHParent = Scope.getEnclosingEHScope();
 
   // Check the three conditions which might require a normal cleanup:
 
@@ -580,12 +582,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     RequiresNormalCleanup = true;
   }
 
-  EHScopeStack::Cleanup::Flags cleanupFlags;
-  if (Scope.isNormalCleanup())
-    cleanupFlags.setIsNormalCleanupKind();
-  if (Scope.isEHCleanup())
-    cleanupFlags.setIsEHCleanupKind();
-
   // If we have a prebranched fallthrough into an inactive normal
   // cleanup, rewrite it so that it leads to the appropriate place.
   if (Scope.isNormalCleanup() && HasPrebranchedFallthrough && !IsActive) {
@@ -634,61 +630,11 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   EHScopeStack::Cleanup *Fn =
     reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
 
-  // We want to emit the EH cleanup after the normal cleanup, but go
-  // ahead and do the setup for the EH cleanup while the scope is still
-  // alive.
-  llvm::BasicBlock *EHEntry = 0;
-  SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
-  if (RequiresEHCleanup) {
-    EHEntry = CreateEHEntry(*this, Scope);
-
-    // Figure out the branch-through dest if necessary.
-    llvm::BasicBlock *EHBranchThroughDest = 0;
-    if (Scope.hasEHBranchThroughs()) {
-      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
-      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
-      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
-    }
-
-    // If we have exactly one branch-after and no branch-throughs, we
-    // can dispatch it without a switch.
-    if (!Scope.hasEHBranchThroughs() &&
-        Scope.getNumEHBranchAfters() == 1) {
-      assert(!EHBranchThroughDest);
-
-      // TODO: remove the spurious eh.cleanup.dest stores if this edge
-      // never went through any switches.
-      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
-    
-    // Otherwise, if we have any branch-afters, we need a switch.
-    } else if (Scope.getNumEHBranchAfters()) {
-      // The default of the switch belongs to the branch-throughs if
-      // they exist.
-      llvm::BasicBlock *Default =
-        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
-
-      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
-
-      llvm::LoadInst *Load =
-        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
-      llvm::SwitchInst *Switch =
-        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
-
-      EHInstsToAppend.push_back(Load);
-      EHInstsToAppend.push_back(Switch);
-
-      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
-        Switch->addCase(Scope.getEHBranchAfterIndex(I),
-                        Scope.getEHBranchAfterBlock(I));
-
-    // Otherwise, we have only branch-throughs; jump to the next EH
-    // cleanup.
-    } else {
-      assert(EHBranchThroughDest);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
-    }
-  }
+  EHScopeStack::Cleanup::Flags cleanupFlags;
+  if (Scope.isNormalCleanup())
+    cleanupFlags.setIsNormalCleanupKind();
+  if (Scope.isEHCleanup())
+    cleanupFlags.setIsEHCleanupKind();
 
   if (!RequiresNormalCleanup) {
     destroyOptimisticNormalEntry(*this, Scope);
@@ -890,10 +836,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     cleanupFlags.setIsForEHCleanup();
     EmitCleanup(*this, Fn, cleanupFlags, EHActiveFlag);
 
-    // Append the prepared cleanup prologue from above.
-    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
-    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
-      EHExit->getInstList().push_back(EHInstsToAppend[I]);
+    Builder.CreateBr(getEHDispatchBlock(EHParent));
 
     Builder.restoreIP(SavedIP);
 
@@ -1005,64 +948,6 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
   Builder.ClearInsertionPoint();
 }
 
-void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
-  // We should never get invalid scope depths for an UnwindDest; that
-  // implies that the destination wasn't set up correctly.
-  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
-
-  if (!HaveInsertPoint())
-    return;
-
-  // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
-
-  // Calculate the innermost active cleanup.
-  EHScopeStack::stable_iterator
-    InnermostCleanup = EHStack.getInnermostActiveEHCleanup();
-
-  // If the destination is in the same EH cleanup scope as us, we
-  // don't need to thread through anything.
-  if (InnermostCleanup.encloses(Dest.getScopeDepth())) {
-    Builder.ClearInsertionPoint();
-    return;
-  }
-  assert(InnermostCleanup != EHStack.stable_end());
-
-  // Store the index at the start.
-  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
-
-  // Adjust BI to point to the first cleanup block.
-  {
-    EHCleanupScope &Scope =
-      cast<EHCleanupScope>(*EHStack.find(InnermostCleanup));
-    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
-  }
-  
-  // Add this destination to all the scopes involved.
-  for (EHScopeStack::stable_iterator
-         I = InnermostCleanup, E = Dest.getScopeDepth(); ; ) {
-    assert(E.strictlyEncloses(I));
-    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
-    assert(Scope.isEHCleanup());
-    I = Scope.getEnclosingEHCleanup();
-
-    // If this is the last cleanup we're propagating through, add this
-    // as a branch-after.
-    if (I == E) {
-      Scope.addEHBranchAfter(Index, Dest.getBlock());
-      break;
-    }
-
-    // Otherwise, add it as a branch-through.  If this isn't new
-    // information, all the rest of the work has been done before.
-    if (!Scope.addEHBranchThrough(Dest.getBlock()))
-      break;
-  }
-  
-  Builder.ClearInsertionPoint();
-}
-
 static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack,
                                   EHScopeStack::stable_iterator C) {
   // If we needed a normal block for any reason, that counts.
@@ -1083,18 +968,21 @@ static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack,
 }
 
 static bool IsUsedAsEHCleanup(EHScopeStack &EHStack,
-                              EHScopeStack::stable_iterator C) {
+                              EHScopeStack::stable_iterator cleanup) {
   // If we needed an EH block for any reason, that counts.
-  if (cast<EHCleanupScope>(*EHStack.find(C)).getEHBlock())
+  if (EHStack.find(cleanup)->hasEHBranches())
     return true;
 
   // Check whether any enclosed cleanups were needed.
   for (EHScopeStack::stable_iterator
-         I = EHStack.getInnermostEHCleanup(); I != C; ) {
-    assert(C.strictlyEncloses(I));
-    EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
-    if (S.getEHBlock()) return true;
-    I = S.getEnclosingEHCleanup();
+         i = EHStack.getInnermostEHScope(); i != cleanup; ) {
+    assert(cleanup.strictlyEncloses(i));
+
+    EHScope &scope = *EHStack.find(i);
+    if (scope.hasEHBranches())
+      return true;
+
+    i = scope.getEnclosingEHScope();
   }
 
   return false;
@@ -1189,10 +1077,3 @@ llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
       CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
   return NormalCleanupDest;
 }
-
-llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
-  if (!EHCleanupDest)
-    EHCleanupDest =
-      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
-  return EHCleanupDest;
-}
diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h
index 9cf04bf79a..7726e442c0 100644
--- a/lib/CodeGen/CGCleanup.h
+++ b/lib/CodeGen/CGCleanup.h
@@ -29,25 +29,102 @@ namespace CodeGen {
 /// A protected scope for zero-cost EH handling.
 class EHScope {
   llvm::BasicBlock *CachedLandingPad;
+  llvm::BasicBlock *CachedEHDispatchBlock;
 
-  unsigned K : 2;
+  EHScopeStack::stable_iterator EnclosingEHScope;
+
+  class CommonBitFields {
+    friend class EHScope;
+    unsigned Kind : 2;
+  };
+  enum { NumCommonBits = 2 };
 
 protected:
-  enum { BitsRemaining = 30 };
+  class CatchBitFields {
+    friend class EHCatchScope;
+    unsigned : NumCommonBits;
+
+    unsigned NumHandlers : 32 - NumCommonBits;
+  };
+
+  class CleanupBitFields {
+    friend class EHCleanupScope;
+    unsigned : NumCommonBits;
+
+    /// Whether this cleanup needs to be run along normal edges.
+    unsigned IsNormalCleanup : 1;
+
+    /// Whether this cleanup needs to be run along exception edges.
+    unsigned IsEHCleanup : 1;
+
+    /// Whether this cleanup is currently active.
+    unsigned IsActive : 1;
+
+    /// Whether the normal cleanup should test the activation flag.
+    unsigned TestFlagInNormalCleanup : 1;
+
+    /// Whether the EH cleanup should test the activation flag.
+    unsigned TestFlagInEHCleanup : 1;
+
+    /// The amount of extra storage needed by the Cleanup.
+    /// Always a multiple of the scope-stack alignment.
+    unsigned CleanupSize : 12;
+
+    /// The number of fixups required by enclosing scopes (not including
+    /// this one).  If this is the top cleanup scope, all the fixups
+    /// from this index onwards belong to this scope.
+    unsigned FixupDepth : 32 - 17 - NumCommonBits; // currently 13    
+  };
+
+  class FilterBitFields {
+    friend class EHFilterScope;
+    unsigned : NumCommonBits;
+
+    unsigned NumFilters : 32 - NumCommonBits;
+  };
+
+  union {
+    CommonBitFields CommonBits;
+    CatchBitFields CatchBits;
+    CleanupBitFields CleanupBits;
+    FilterBitFields FilterBits;
+  };
 
 public:
   enum Kind { Cleanup, Catch, Terminate, Filter };
 
-  EHScope(Kind K) : CachedLandingPad(0), K(K) {}
+  EHScope(Kind kind, EHScopeStack::stable_iterator enclosingEHScope)
+    : CachedLandingPad(0), CachedEHDispatchBlock(0),
+      EnclosingEHScope(enclosingEHScope) {
+    CommonBits.Kind = kind;
+  }
 
-  Kind getKind() const { return static_cast<Kind>(K); }
+  Kind getKind() const { return static_cast<Kind>(CommonBits.Kind); }
 
   llvm::BasicBlock *getCachedLandingPad() const {
     return CachedLandingPad;
   }
 
-  void setCachedLandingPad(llvm::BasicBlock *Block) {
-    CachedLandingPad = Block;
+  void setCachedLandingPad(llvm::BasicBlock *block) {
+    CachedLandingPad = block;
+  }
+
+  llvm::BasicBlock *getCachedEHDispatchBlock() const {
+    return CachedEHDispatchBlock;
+  }
+
+  void setCachedEHDispatchBlock(llvm::BasicBlock *block) {
+    CachedEHDispatchBlock = block;
+  }
+
+  bool hasEHBranches() const {
+    if (llvm::BasicBlock *block = getCachedEHDispatchBlock())
+      return !block->use_empty();
+    return false;
+  }
+
+  EHScopeStack::stable_iterator getEnclosingEHScope() const {
+    return EnclosingEHScope;
   }
 };
 
@@ -57,8 +134,6 @@ public:
 /// Objective C @finally blocks are represented using a cleanup scope
 /// after the catch scope.
 class EHCatchScope : public EHScope {
-  unsigned NumHandlers : BitsRemaining;
-
   // In effect, we have a flexible array member
   //   Handler Handlers[0];
   // But that's only standard in C99, not C++, so we have to do
@@ -73,8 +148,7 @@ public:
     /// The catch handler for this type.
     llvm::BasicBlock *Block;
 
-    /// The unwind destination index for this handler.
-    unsigned Index;
+    bool isCatchAll() const { return Type == 0; }
   };
 
 private:
@@ -93,12 +167,14 @@ public:
     return sizeof(EHCatchScope) + N * sizeof(Handler);
   }
 
-  EHCatchScope(unsigned NumHandlers)
-    : EHScope(Catch), NumHandlers(NumHandlers) {
+  EHCatchScope(unsigned numHandlers,
+               EHScopeStack::stable_iterator enclosingEHScope)
+    : EHScope(Catch, enclosingEHScope) {
+    CatchBits.NumHandlers = numHandlers;
   }
 
   unsigned getNumHandlers() const {
-    return NumHandlers;
+    return CatchBits.NumHandlers;
   }
 
   void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) {
@@ -127,44 +203,16 @@ public:
 
 /// A cleanup scope which generates the cleanup blocks lazily.
 class EHCleanupScope : public EHScope {
-  /// Whether this cleanup needs to be run along normal edges.
-  bool IsNormalCleanup : 1;
-
-  /// Whether this cleanup needs to be run along exception edges.
-  bool IsEHCleanup : 1;
-
-  /// Whether this cleanup is currently active.
-  bool IsActive : 1;
-
-  /// Whether the normal cleanup should test the activation flag.
-  bool TestFlagInNormalCleanup : 1;
-
-  /// Whether the EH cleanup should test the activation flag.
-  bool TestFlagInEHCleanup : 1;
-
-  /// The amount of extra storage needed by the Cleanup.
-  /// Always a multiple of the scope-stack alignment.
-  unsigned CleanupSize : 12;
-
-  /// The number of fixups required by enclosing scopes (not including
-  /// this one).  If this is the top cleanup scope, all the fixups
-  /// from this index onwards belong to this scope.
-  unsigned FixupDepth : BitsRemaining - 17; // currently 13
-
   /// The nearest normal cleanup scope enclosing this one.
   EHScopeStack::stable_iterator EnclosingNormal;
 
-  /// The nearest EH cleanup scope enclosing this one.
+  /// The nearest EH scope enclosing this one.
   EHScopeStack::stable_iterator EnclosingEH;
 
   /// The dual entry/exit block along the normal edge.  This is lazily
   /// created if needed before the cleanup is popped.
   llvm::BasicBlock *NormalBlock;
 
-  /// The dual entry/exit block along the EH edge.  This is lazily
-  /// created if needed before the cleanup is popped.
-  llvm::BasicBlock *EHBlock;
-
   /// An optional i1 variable indicating whether this cleanup has been
   /// activated yet.
   llvm::AllocaInst *ActiveFlag;
@@ -180,15 +228,6 @@ class EHCleanupScope : public EHScope {
     /// Normal branch-afters.
     SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
       BranchAfters;
-
-    /// The destinations of EH branch-afters and branch-throughs.
-    /// TODO: optimize for the extremely common case of a single
-    /// branch-through.
-    llvm::SmallPtrSet<llvm::BasicBlock*, 4> EHBranches;
-
-    /// EH branch-afters.
-    SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
-    EHBranchAfters;
   };
   mutable struct ExtInfo *ExtInfo;
 
@@ -210,56 +249,64 @@ public:
   }
 
   size_t getAllocatedSize() const {
-    return sizeof(EHCleanupScope) + CleanupSize;
+    return sizeof(EHCleanupScope) + CleanupBits.CleanupSize;
   }
 
-  EHCleanupScope(bool IsNormal, bool IsEH, bool IsActive,
-                 unsigned CleanupSize, unsigned FixupDepth,
-                 EHScopeStack::stable_iterator EnclosingNormal,
-                 EHScopeStack::stable_iterator EnclosingEH)
-    : EHScope(EHScope::Cleanup),
-      IsNormalCleanup(IsNormal), IsEHCleanup(IsEH), IsActive(IsActive),
-      TestFlagInNormalCleanup(false), TestFlagInEHCleanup(false),
-      CleanupSize(CleanupSize), FixupDepth(FixupDepth),
-      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
-      NormalBlock(0), EHBlock(0), ActiveFlag(0), ExtInfo(0)
-  {
-    assert(this->CleanupSize == CleanupSize && "cleanup size overflow");
+  EHCleanupScope(bool isNormal, bool isEH, bool isActive,
+                 unsigned cleanupSize, unsigned fixupDepth,
+                 EHScopeStack::stable_iterator enclosingNormal,
+                 EHScopeStack::stable_iterator enclosingEH)
+    : EHScope(EHScope::Cleanup, enclosingEH), EnclosingNormal(enclosingNormal),
+      NormalBlock(0), ActiveFlag(0), ExtInfo(0) {
+    CleanupBits.IsNormalCleanup = isNormal;
+    CleanupBits.IsEHCleanup = isEH;
+    CleanupBits.IsActive = isActive;
+    CleanupBits.TestFlagInNormalCleanup = false;
+    CleanupBits.TestFlagInEHCleanup = false;
+    CleanupBits.CleanupSize = cleanupSize;
+    CleanupBits.FixupDepth = fixupDepth;
+
+    assert(CleanupBits.CleanupSize == cleanupSize && "cleanup size overflow");
   }
 
   ~EHCleanupScope() {
     delete ExtInfo;
   }
 
-  bool isNormalCleanup() const { return IsNormalCleanup; }
+  bool isNormalCleanup() const { return CleanupBits.IsNormalCleanup; }
   llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
   void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
 
-  bool isEHCleanup() const { return IsEHCleanup; }
-  llvm::BasicBlock *getEHBlock() const { return EHBlock; }
-  void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
+  bool isEHCleanup() const { return CleanupBits.IsEHCleanup; }
+  llvm::BasicBlock *getEHBlock() const { return getCachedEHDispatchBlock(); }
+  void setEHBlock(llvm::BasicBlock *BB) { setCachedEHDispatchBlock(BB); }
 
-  bool isActive() const { return IsActive; }
-  void setActive(bool A) { IsActive = A; }
+  bool isActive() const { return CleanupBits.IsActive; }
+  void setActive(bool A) { CleanupBits.IsActive = A; }
 
   llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; }
   void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; }
 
-  void setTestFlagInNormalCleanup() { TestFlagInNormalCleanup = true; }
-  bool shouldTestFlagInNormalCleanup() const { return TestFlagInNormalCleanup; }
+  void setTestFlagInNormalCleanup() {
+    CleanupBits.TestFlagInNormalCleanup = true;
+  }
+  bool shouldTestFlagInNormalCleanup() const {
+    return CleanupBits.TestFlagInNormalCleanup;
+  }
 
-  void setTestFlagInEHCleanup() { TestFlagInEHCleanup = true; }
-  bool shouldTestFlagInEHCleanup() const { return TestFlagInEHCleanup; }
+  void setTestFlagInEHCleanup() {
+    CleanupBits.TestFlagInEHCleanup = true;
+  }
+  bool shouldTestFlagInEHCleanup() const {
+    return CleanupBits.TestFlagInEHCleanup;
+  }
 
-  unsigned getFixupDepth() const { return FixupDepth; }
+  unsigned getFixupDepth() const { return CleanupBits.FixupDepth; }
   EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
     return EnclosingNormal;
   }
-  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
-    return EnclosingEH;
-  }
 
-  size_t getCleanupSize() const { return CleanupSize; }
+  size_t getCleanupSize() const { return CleanupBits.CleanupSize; }
   void *getCleanupBuffer() { return this + 1; }
 
   EHScopeStack::Cleanup *getCleanup() {
@@ -327,41 +374,6 @@ public:
     return (ExtInfo->BranchAfters.size() != ExtInfo->Branches.size());
   }
 
-  // Same stuff, only for EH branches instead of normal branches.
-  // It's quite possible that we could find a better representation
-  // for this.
-
-  bool hasEHBranches() const { return ExtInfo && !ExtInfo->EHBranches.empty(); }
-  void addEHBranchAfter(llvm::ConstantInt *Index,
-                        llvm::BasicBlock *Block) {
-    struct ExtInfo &ExtInfo = getExtInfo();
-    if (ExtInfo.EHBranches.insert(Block))
-      ExtInfo.EHBranchAfters.push_back(std::make_pair(Block, Index));
-  }
-
-  unsigned getNumEHBranchAfters() const {
-    return ExtInfo ? ExtInfo->EHBranchAfters.size() : 0;
-  }
-
-  llvm::BasicBlock *getEHBranchAfterBlock(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].first;
-  }
-
-  llvm::ConstantInt *getEHBranchAfterIndex(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].second;
-  }
-
-  bool addEHBranchThrough(llvm::BasicBlock *Block) {
-    return getExtInfo().EHBranches.insert(Block);
-  }
-
-  bool hasEHBranchThroughs() const {
-    if (!ExtInfo) return false;
-    return (ExtInfo->EHBranchAfters.size() != ExtInfo->EHBranches.size());
-  }
-
   static bool classof(const EHScope *Scope) {
     return (Scope->getKind() == Cleanup);
   }
@@ -373,8 +385,6 @@ public:
 ///
 /// This is used to implement C++ exception specifications.
 class EHFilterScope : public EHScope {
-  unsigned NumFilters : BitsRemaining;
-
   // Essentially ends in a flexible array member:
   // llvm::Value *FilterTypes[0];
 
@@ -387,42 +397,42 @@ class EHFilterScope : public EHScope {
   }
 
 public:
-  EHFilterScope(unsigned NumFilters) :
-    EHScope(Filter), NumFilters(NumFilters) {}
+  EHFilterScope(unsigned numFilters)
+    : EHScope(Filter, EHScopeStack::stable_end()) {
+    FilterBits.NumFilters = numFilters;
+  }
 
-  static size_t getSizeForNumFilters(unsigned NumFilters) {
-    return sizeof(EHFilterScope) + NumFilters * sizeof(llvm::Value*);
+  static size_t getSizeForNumFilters(unsigned numFilters) {
+    return sizeof(EHFilterScope) + numFilters * sizeof(llvm::Value*);
   }
 
-  unsigned getNumFilters() const { return NumFilters; }
+  unsigned getNumFilters() const { return FilterBits.NumFilters; }
 
-  void setFilter(unsigned I, llvm::Value *FilterValue) {
-    assert(I < getNumFilters());
-    getFilters()[I] = FilterValue;
+  void setFilter(unsigned i, llvm::Value *filterValue) {
+    assert(i < getNumFilters());
+    getFilters()[i] = filterValue;
   }
 
-  llvm::Value *getFilter(unsigned I) const {
-    assert(I < getNumFilters());
-    return getFilters()[I];
+  llvm::Value *getFilter(unsigned i) const {
+    assert(i < getNumFilters());
+    return getFilters()[i];
   }
 
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Filter;
+  static bool classof(const EHScope *scope) {
+    return scope->getKind() == Filter;
   }
 };
 
 /// An exceptions scope which calls std::terminate if any exception
 /// reaches it.
 class EHTerminateScope : public EHScope {
-  unsigned DestIndex : BitsRemaining;
 public:
-  EHTerminateScope(unsigned Index) : EHScope(Terminate), DestIndex(Index) {}
+  EHTerminateScope(EHScopeStack::stable_iterator enclosingEHScope)
+    : EHScope(Terminate, enclosingEHScope) {}
   static size_t getSize() { return sizeof(EHTerminateScope); }
 
-  unsigned getDestIndex() const { return DestIndex; }
-
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Terminate;
+  static bool classof(const EHScope *scope) {
+    return scope->getKind() == Terminate;
   }
 };
 
@@ -498,26 +508,17 @@ inline EHScopeStack::iterator EHScopeStack::end() const {
 inline void EHScopeStack::popCatch() {
   assert(!empty() && "popping exception stack when not empty");
 
-  assert(isa<EHCatchScope>(*begin()));
-  StartOfData += EHCatchScope::getSizeForNumHandlers(
-                          cast<EHCatchScope>(*begin()).getNumHandlers());
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
+  EHCatchScope &scope = cast<EHCatchScope>(*begin());
+  InnermostEHScope = scope.getEnclosingEHScope();
+  StartOfData += EHCatchScope::getSizeForNumHandlers(scope.getNumHandlers());
 }
 
 inline void EHScopeStack::popTerminate() {
   assert(!empty() && "popping exception stack when not empty");
 
-  assert(isa<EHTerminateScope>(*begin()));
+  EHTerminateScope &scope = cast<EHTerminateScope>(*begin());
+  InnermostEHScope = scope.getEnclosingEHScope();
   StartOfData += EHTerminateScope::getSize();
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
 }
 
 inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const {
@@ -532,28 +533,6 @@ EHScopeStack::stabilize(iterator ir) const {
   return stable_iterator(EndOfBuffer - ir.Ptr);
 }
 
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveNormalCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostNormalCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingNormalCleanup();
-  }
-  return stable_end();
-}
-
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveEHCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostEHCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingEHCleanup();
-  }
-  return stable_end();
-}
-
 }
 }
 
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index 46d127cf80..61c1739eab 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -467,6 +467,46 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
   }
 }
 
+/// Emit the dispatch block for a filter scope if necessary.
+static void emitFilterDispatchBlock(CodeGenFunction &CGF,
+                                    EHFilterScope &filterScope) {
+  llvm::BasicBlock *dispatchBlock = filterScope.getCachedEHDispatchBlock();
+  if (!dispatchBlock) return;
+  if (dispatchBlock->use_empty()) {
+    delete dispatchBlock;
+    return;
+  }
+
+  CGBuilderTy::InsertPoint savedIP = CGF.Builder.saveIP();
+  CGF.EmitBlockAfterUses(dispatchBlock);
+
+  // If this isn't a catch-all filter, we need to check whether we got
+  // here because the filter triggered.
+  if (filterScope.getNumFilters()) {
+    // Load the selector value.
+    llvm::Value *selector =
+      CGF.Builder.CreateLoad(CGF.getEHSelectorSlot(), "selector");
+
+    llvm::BasicBlock *unexpectedBB = CGF.createBasicBlock("ehspec.unexpected");
+
+    llvm::Value *zero = CGF.Builder.getInt32(0);
+    llvm::Value *failsFilter =
+      CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails");
+    CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock());
+
+    CGF.EmitBlock(unexpectedBB);
+  }
+
+  // Call __cxa_call_unexpected.  This doesn't need to be an invoke
+  // because __cxa_call_unexpected magically filters exceptions
+  // according to the last landing pad the exception was thrown
+  // into.  Seriously.
+  llvm::Value *exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot());
+  CGF.Builder.CreateCall(getUnexpectedFn(CGF), exn)
+    ->setDoesNotReturn();
+  CGF.Builder.CreateUnreachable();
+}
+
 void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
   if (!CGM.getLangOptions().CXXExceptions)
     return;
@@ -484,6 +524,8 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
       EHStack.popTerminate();
     }
   } else if (EST == EST_Dynamic || EST == EST_DynamicNone) {
+    EHFilterScope &filterScope = cast<EHFilterScope>(*EHStack.begin());
+    emitFilterDispatchBlock(*this, filterScope);
     EHStack.popFilter();
   }
 }
@@ -525,6 +567,50 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
   }
 }
 
+llvm::BasicBlock *
+CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) {
+  // The dispatch block for the end of the scope chain is a block that
+  // just resumes unwinding.
+  if (si == EHStack.stable_end())
+    return getEHResumeBlock();
+
+  // Otherwise, we should look at the actual scope.
+  EHScope &scope = *EHStack.find(si);
+
+  llvm::BasicBlock *dispatchBlock = scope.getCachedEHDispatc