Validated by nightly-test runs on x86 and x86-64 darwin, including after

self-host.  Hopefully these results hold up on different platforms.  

I tried to keep the GNU ObjC runtime happy, but it's hard for me to test.
Reimplement how clang generates IR for exceptions.  Instead of creating new
invoke destinations which sequentially chain to the previous destination,
push a more semantic representation of *why* we need the cleanup/catch/filter
behavior, then collect that information into a single landing pad upon request.

Also reorganizes how normal cleanups (i.e. cleanups triggered by non-exceptional
control flow) are generated, since it's actually fairly closely tied in with
the former.  Remove the need to track which cleanup scope a block is associated
with.

Document a lot of previously poorly-understood (by me, at least) behavior.

The new framework implements the Horrible Hack (tm), which requires every
landing pad to have a catch-all so that inlining will work.  Clang no longer
requires the Horrible Hack just to make exceptions flow correctly within
a function, however.  The HH is an unfortunate requirement of LLVM's EH IR.




git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@107631 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 1084 insertions, 409 deletions

diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index 182c2ea877..1645e66d55 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -14,11 +14,158 @@
 #include "clang/AST/StmtCXX.h"
 
 #include "llvm/Intrinsics.h"
+#include "llvm/Support/CallSite.h"
 
 #include "CodeGenFunction.h"
+#include "CGException.h"
+
 using namespace clang;
 using namespace CodeGen;
 
+/// Push an entry of the given size onto this protected-scope stack.
+char *EHScopeStack::allocate(size_t Size) {
+  if (!StartOfBuffer) {
+    unsigned Capacity = 1024;
+    while (Capacity < Size) Capacity *= 2;
+    StartOfBuffer = new char[Capacity];
+    StartOfData = EndOfBuffer = StartOfBuffer + Capacity;
+  } else if (static_cast<size_t>(StartOfData - StartOfBuffer) < Size) {
+    unsigned CurrentCapacity = EndOfBuffer - StartOfBuffer;
+    unsigned UsedCapacity = CurrentCapacity - (StartOfData - StartOfBuffer);
+
+    unsigned NewCapacity = CurrentCapacity;
+    do {
+      NewCapacity *= 2;
+    } while (NewCapacity < UsedCapacity + Size);
+
+    char *NewStartOfBuffer = new char[NewCapacity];
+    char *NewEndOfBuffer = NewStartOfBuffer + NewCapacity;
+    char *NewStartOfData = NewEndOfBuffer - UsedCapacity;
+    memcpy(NewStartOfData, StartOfData, UsedCapacity);
+    delete [] StartOfBuffer;
+    StartOfBuffer = NewStartOfBuffer;
+    EndOfBuffer = NewEndOfBuffer;
+    StartOfData = NewStartOfData;
+  }
+
+  assert(StartOfBuffer + Size <= StartOfData);
+  StartOfData -= Size;
+  return StartOfData;
+}
+
+EHScopeStack::stable_iterator
+EHScopeStack::getEnclosingEHCleanup(iterator it) const {
+  assert(it != end());
+  do {
+    if (isa<EHCleanupScope>(*it)) {
+      if (cast<EHCleanupScope>(*it).isEHCleanup())
+        return stabilize(it);
+      return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
+    }
+    ++it;
+  } while (it != end());
+  return stable_end();
+}
+
+
+void EHScopeStack::pushCleanup(llvm::BasicBlock *NormalEntry,
+                               llvm::BasicBlock *NormalExit,
+                               llvm::BasicBlock *EHEntry,
+                               llvm::BasicBlock *EHExit) {
+  char *Buffer = allocate(EHCleanupScope::getSize());
+  new (Buffer) EHCleanupScope(BranchFixups.size(),
+                              InnermostNormalCleanup,
+                              InnermostEHCleanup,
+                              NormalEntry, NormalExit, EHEntry, EHExit);
+  if (NormalEntry)
+    InnermostNormalCleanup = stable_begin();
+  if (EHEntry)
+    InnermostEHCleanup = stable_begin();
+}
+
+void EHScopeStack::popCleanup() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  assert(isa<EHCleanupScope>(*begin()));
+  EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
+  InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
+  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+  StartOfData += EHCleanupScope::getSize();
+
+  // Check whether we can shrink the branch-fixups stack.
+  if (!BranchFixups.empty()) {
+    // If we no longer have any normal cleanups, all the fixups are
+    // complete.
+    if (!hasNormalCleanups())
+      BranchFixups.clear();
+
+    // Otherwise we can still trim out unnecessary nulls.
+    else
+      popNullFixups();
+  }
+}
+
+EHFilterScope *EHScopeStack::pushFilter(unsigned NumFilters) {
+  char *Buffer = allocate(EHFilterScope::getSizeForNumFilters(NumFilters));
+  CatchDepth++;
+  return new (Buffer) EHFilterScope(NumFilters);
+}
+
+void EHScopeStack::popFilter() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  EHFilterScope &Filter = cast<EHFilterScope>(*begin());
+  StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
+
+  assert(CatchDepth > 0 && "mismatched filter push/pop");
+  CatchDepth--;
+}
+
+EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
+  char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
+  CatchDepth++;
+  return new (Buffer) EHCatchScope(NumHandlers);
+}
+
+void EHScopeStack::pushTerminate() {
+  char *Buffer = allocate(EHTerminateScope::getSize());
+  CatchDepth++;
+  new (Buffer) EHTerminateScope();
+}
+
+/// Remove any 'null' fixups on the stack.  However, we can't pop more
+/// fixups than the fixup depth on the innermost normal cleanup, or
+/// else fixups that we try to add to that cleanup will end up in the
+/// wrong place.  We *could* try to shrink fixup depths, but that's
+/// actually a lot of work for little benefit.
+void EHScopeStack::popNullFixups() {
+  // We expect this to only be called when there's still an innermost
+  // normal cleanup;  otherwise there really shouldn't be any fixups.
+  assert(hasNormalCleanups());
+
+  EHScopeStack::iterator it = find(InnermostNormalCleanup);
+  unsigned MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
+  assert(BranchFixups.size() >= MinSize && "fixup stack out of order");
+
+  while (BranchFixups.size() > MinSize &&
+         BranchFixups.back().Destination == 0)
+    BranchFixups.pop_back();
+}
+
+void EHScopeStack::resolveBranchFixups(llvm::BasicBlock *Dest) {
+  assert(Dest && "null block passed to resolveBranchFixups");
+
+  if (BranchFixups.empty()) return;
+  assert(hasNormalCleanups() &&
+         "branch fixups exist with no normal cleanups on stack");
+
+  for (unsigned I = 0, E = BranchFixups.size(); I != E; ++I)
+    if (BranchFixups[I].Destination == Dest)
+      BranchFixups[I].Destination = 0;
+
+  popNullFixups();
+}
+
 static llvm::Constant *getAllocateExceptionFn(CodeGenFunction &CGF) {
   // void *__cxa_allocate_exception(size_t thrown_size);
   const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
@@ -66,8 +213,19 @@ static llvm::Constant *getReThrowFn(CodeGenFunction &CGF) {
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_rethrow");
 }
 
+static llvm::Constant *getGetExceptionPtrFn(CodeGenFunction &CGF) {
+  // void *__cxa_get_exception_ptr(void*);
+  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  std::vector<const llvm::Type*> Args(1, Int8PtrTy);
+
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(Int8PtrTy, Args, false);
+
+  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr");
+}
+
 static llvm::Constant *getBeginCatchFn(CodeGenFunction &CGF) {
-  // void* __cxa_begin_catch();
+  // void *__cxa_begin_catch(void*);
 
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   std::vector<const llvm::Type*> Args(1, Int8PtrTy);
@@ -123,28 +281,87 @@ static llvm::Constant *getTerminateFn(CodeGenFunction &CGF) {
       CGF.CGM.getLangOptions().CPlusPlus ? "_ZSt9terminatev" : "abort");
 }
 
-static llvm::Constant *getPersonalityFn(CodeGenModule &CGM) {
-  const char *PersonalityFnName = "__gcc_personality_v0";
-  LangOptions Opts = CGM.getLangOptions();
-  if (Opts.CPlusPlus) {
-    if (Opts.SjLjExceptions)
-     PersonalityFnName = "__gxx_personality_sj0";
+static const char *getCPersonalityFn(CodeGenFunction &CGF) {
+  return "__gcc_personality_v0";
+}
+
+static const char *getObjCPersonalityFn(CodeGenFunction &CGF) {
+  if (CGF.CGM.getLangOptions().NeXTRuntime) {
+    if (CGF.CGM.getLangOptions().ObjCNonFragileABI)
+      return "__objc_personality_v0";
+    else
+      return getCPersonalityFn(CGF);
+  } else {
+    return "__gnu_objc_personality_v0";
+  }
+}
+
+static const char *getCXXPersonalityFn(CodeGenFunction &CGF) {
+  if (CGF.CGM.getLangOptions().SjLjExceptions)
+    return "__gxx_personality_sj0";
+  else
+    return "__gxx_personality_v0";
+}
+
+/// Determines the personality function to use when both C++
+/// and Objective-C exceptions are being caught.
+static const char *getObjCXXPersonalityFn(CodeGenFunction &CGF) {
+  // The ObjC personality defers to the C++ personality for non-ObjC
+  // handlers.  Unlike the C++ case, we use the same personality
+  // function on targets using (backend-driven) SJLJ EH.
+  if (CGF.CGM.getLangOptions().NeXTRuntime) {
+    if (CGF.CGM.getLangOptions().ObjCNonFragileABI)
+      return "__objc_personality_v0";
+
+    // In the fragile ABI, just use C++ exception handling and hope
+    // they're not doing crazy exception mixing.
     else
-     PersonalityFnName = "__gxx_personality_v0";
-  } else if (Opts.ObjC1) {
-    if (Opts.NeXTRuntime) {
-      if (Opts.ObjCNonFragileABI)
-        PersonalityFnName = "__gcc_personality_v0";
-    } else
-      PersonalityFnName = "__gnu_objc_personality_v0";
+      return getCXXPersonalityFn(CGF);
   }
 
+  // I'm pretty sure the GNU runtime doesn't support mixed EH.
+  // TODO: we don't necessarily need mixed EH here;  remember what
+  // kind of exceptions we actually try to catch in this function.
+  CGF.CGM.ErrorUnsupported(CGF.CurCodeDecl,
+                           "the GNU Objective C runtime does not support "
+                           "catching C++ and Objective C exceptions in the "
+                           "same function");
+  // Use the C++ personality just to avoid returning null.
+  return getCXXPersonalityFn(CGF);
+}
+
+static llvm::Constant *getPersonalityFn(CodeGenFunction &CGF) {
+  const char *Name;
+  const LangOptions &Opts = CGF.CGM.getLangOptions();
+  if (Opts.CPlusPlus && Opts.ObjC1)
+    Name = getObjCXXPersonalityFn(CGF);
+  else if (Opts.CPlusPlus)
+    Name = getCXXPersonalityFn(CGF);
+  else if (Opts.ObjC1)
+    Name = getObjCPersonalityFn(CGF);
+  else
+    Name = getCPersonalityFn(CGF);
+
   llvm::Constant *Personality =
-  CGM.CreateRuntimeFunction(llvm::FunctionType::get(llvm::Type::getInt32Ty(
-                                                        CGM.getLLVMContext()),
-                                                    true),
-      PersonalityFnName);
-  return llvm::ConstantExpr::getBitCast(Personality, CGM.PtrToInt8Ty);
+    CGF.CGM.CreateRuntimeFunction(llvm::FunctionType::get(
+                                    llvm::Type::getInt32Ty(
+                                      CGF.CGM.getLLVMContext()),
+                                    true),
+                            Name);
+  return llvm::ConstantExpr::getBitCast(Personality, CGF.CGM.PtrToInt8Ty);
+}
+
+/// Returns the value to inject into a selector to indicate the
+/// presence of a catch-all.
+static llvm::Constant *getCatchAllValue(CodeGenFunction &CGF) {
+  // Possibly we should use @llvm.eh.catch.all.value here.
+  return llvm::ConstantPointerNull::get(CGF.CGM.PtrToInt8Ty);
+}
+
+/// Returns the value to inject into a selector to indicate the
+/// presence of a cleanup.
+static llvm::Constant *getCleanupValue(CodeGenFunction &CGF) {
+  return llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
 }
 
 // Emits an exception expression into the given location.  This
@@ -169,9 +386,12 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
   llvm::AllocaInst *ExnLocVar =
     CGF.CreateTempAlloca(ExnLoc->getType(), "exnobj.var");
 
-  llvm::BasicBlock *SavedInvokeDest = CGF.getInvokeDest();
+  // Make sure the exception object is cleaned up if there's an
+  // exception during initialization.
+  // FIXME: StmtExprs probably force this to include a non-EH
+  // handler.
   {
-    CodeGenFunction::EHCleanupBlock Cleanup(CGF);
+    CodeGenFunction::CleanupBlock Cleanup(CGF, CodeGenFunction::EHCleanup);
     llvm::BasicBlock *FreeBB = CGF.createBasicBlock("free-exnobj");
     llvm::BasicBlock *DoneBB = CGF.createBasicBlock("free-exnobj.done");
 
@@ -180,10 +400,11 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
     CGF.Builder.CreateCondBr(ShouldFree, FreeBB, DoneBB);
     CGF.EmitBlock(FreeBB);
     llvm::Value *ExnLocLocal = CGF.Builder.CreateLoad(ExnLocVar, "exnobj");
-    CGF.Builder.CreateCall(getFreeExceptionFn(CGF), ExnLocLocal);
+    CGF.Builder.CreateCall(getFreeExceptionFn(CGF), ExnLocLocal)
+      ->setDoesNotThrow();
     CGF.EmitBlock(DoneBB);
   }
-  llvm::BasicBlock *Cleanup = CGF.getInvokeDest();
+  EHScopeStack::stable_iterator Cleanup = CGF.EHStack.stable_begin();
 
   CGF.Builder.CreateStore(ExnLoc, ExnLocVar);
   CGF.Builder.CreateStore(llvm::ConstantInt::getTrue(CGF.getLLVMContext()),
@@ -206,74 +427,38 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
   CGF.Builder.CreateStore(llvm::ConstantInt::getFalse(CGF.getLLVMContext()),
                           ShouldFreeVar);
 
-  // Pop the cleanup block if it's still the top of the cleanup stack.
-  // Otherwise, temporaries have been created and our cleanup will get
-  // properly removed in time.
-  // TODO: this is not very resilient.
-  if (CGF.getInvokeDest() == Cleanup)
-    CGF.setInvokeDest(SavedInvokeDest);
-}
-
-// CopyObject - Utility to copy an object.  Calls copy constructor as necessary.
-// N is casted to the right type.
-static void CopyObject(CodeGenFunction &CGF, QualType ObjectType,
-                       bool WasPointer, bool WasPointerReference,
-                       llvm::Value *E, llvm::Value *N) {
-  // Store the throw exception in the exception object.
-  if (WasPointer || !CGF.hasAggregateLLVMType(ObjectType)) {
-    llvm::Value *Value = E;
-    if (!WasPointer)
-      Value = CGF.Builder.CreateLoad(Value);
-    const llvm::Type *ValuePtrTy = Value->getType()->getPointerTo(0);
-    if (WasPointerReference) {
-      llvm::Value *Tmp = CGF.CreateTempAlloca(Value->getType(), "catch.param");
-      CGF.Builder.CreateStore(Value, Tmp);
-      Value = Tmp;
-      ValuePtrTy = Value->getType()->getPointerTo(0);
-    }
-    N = CGF.Builder.CreateBitCast(N, ValuePtrTy);
-    CGF.Builder.CreateStore(Value, N);
-  } else {
-    const llvm::Type *Ty = CGF.ConvertType(ObjectType)->getPointerTo(0);
-    const CXXRecordDecl *RD;
-    RD = cast<CXXRecordDecl>(ObjectType->getAs<RecordType>()->getDecl());
-    llvm::Value *This = CGF.Builder.CreateBitCast(N, Ty);
-    if (RD->hasTrivialCopyConstructor()) {
-      CGF.EmitAggregateCopy(This, E, ObjectType);
-    } else if (CXXConstructorDecl *CopyCtor
-               = RD->getCopyConstructor(CGF.getContext(), 0)) {
-      llvm::Value *Src = E;
-
-      // Stolen from EmitClassAggrMemberwiseCopy
-      llvm::Value *Callee = CGF.CGM.GetAddrOfCXXConstructor(CopyCtor,
-                                                            Ctor_Complete);
-      CallArgList CallArgs;
-      CallArgs.push_back(std::make_pair(RValue::get(This),
-                                      CopyCtor->getThisType(CGF.getContext())));
-
-      // Push the Src ptr.
-      CallArgs.push_back(std::make_pair(RValue::get(Src),
-                                        CopyCtor->getParamDecl(0)->getType()));
-
-      const FunctionProtoType *FPT
-        = CopyCtor->getType()->getAs<FunctionProtoType>();
-      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(CallArgs, FPT),
-                   Callee, ReturnValueSlot(), CallArgs, CopyCtor);
-    } else
-      llvm_unreachable("uncopyable object");
+  // Technically, the exception object is like a temporary; it has to
+  // be cleaned up when its full-expression is complete.
+  // Unfortunately, the AST represents full-expressions by creating a
+  // CXXExprWithTemporaries, which it only does when there are actually
+  // temporaries.
+  //
+  // If any cleanups have been added since we pushed ours, they must
+  // be from temporaries;  this will get popped at the same time.
+  // Otherwise we need to pop ours off.  FIXME: this is very brittle.
+  if (Cleanup == CGF.EHStack.stable_begin())
+    CGF.PopCleanupBlock();
+}
+
+llvm::Value *CodeGenFunction::getExceptionSlot() {
+  if (!ExceptionSlot) {
+    const llvm::Type *i8p = llvm::Type::getInt8PtrTy(getLLVMContext());
+    ExceptionSlot = CreateTempAlloca(i8p, "exn.slot");
   }
+  return ExceptionSlot;
 }
 
 void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
   if (!E->getSubExpr()) {
     if (getInvokeDest()) {
-      llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
-      Builder.CreateInvoke(getReThrowFn(*this), Cont, getInvokeDest())
+      Builder.CreateInvoke(getReThrowFn(*this),
+                           getUnreachableBlock(),
+                           getInvokeDest())
         ->setDoesNotReturn();
-      EmitBlock(Cont);
-    } else
+    } else {
       Builder.CreateCall(getReThrowFn(*this))->setDoesNotReturn();
-    Builder.CreateUnreachable();
+      Builder.CreateUnreachable();
+    }
 
     // Clear the insertion point to indicate we are in unreachable code.
     Builder.ClearInsertionPoint();
@@ -287,10 +472,11 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
   uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity();
 
   llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(*this);
-  llvm::Value *ExceptionPtr =
+  llvm::CallInst *ExceptionPtr =
     Builder.CreateCall(AllocExceptionFn,
                        llvm::ConstantInt::get(SizeTy, TypeSize),
                        "exception");
+  ExceptionPtr->setDoesNotThrow();
   
   EmitAnyExprToExn(*this, E->getSubExpr(), ExceptionPtr);
 
@@ -312,18 +498,17 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
   if (!Dtor) Dtor = llvm::Constant::getNullValue(Int8PtrTy);
 
   if (getInvokeDest()) {
-    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
     llvm::InvokeInst *ThrowCall =
-      Builder.CreateInvoke3(getThrowFn(*this), Cont, getInvokeDest(),
+      Builder.CreateInvoke3(getThrowFn(*this),
+                            getUnreachableBlock(), getInvokeDest(),
                             ExceptionPtr, TypeInfo, Dtor);
     ThrowCall->setDoesNotReturn();
-    EmitBlock(Cont);
   } else {
     llvm::CallInst *ThrowCall =
       Builder.CreateCall3(getThrowFn(*this), ExceptionPtr, TypeInfo, Dtor);
     ThrowCall->setDoesNotReturn();
+    Builder.CreateUnreachable();
   }
-  Builder.CreateUnreachable();
 
   // Clear the insertion point to indicate we are in unreachable code.
   Builder.ClearInsertionPoint();
@@ -349,79 +534,15 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
   if (!Proto->hasExceptionSpec())
     return;
 
-  llvm::Constant *Personality = getPersonalityFn(CGM);
-  llvm::Value *llvm_eh_exception =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_exception);
-  llvm::Value *llvm_eh_selector =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_selector);
-  const llvm::IntegerType *Int8Ty;
-  const llvm::PointerType *PtrToInt8Ty;
-  Int8Ty = llvm::Type::getInt8Ty(VMContext);
-  // C string type.  Used in lots of places.
-  PtrToInt8Ty = llvm::PointerType::getUnqual(Int8Ty);
-  llvm::Constant *Null = llvm::ConstantPointerNull::get(PtrToInt8Ty);
-  llvm::SmallVector<llvm::Value*, 8> SelectorArgs;
-
-  llvm::BasicBlock *PrevLandingPad = getInvokeDest();
-  llvm::BasicBlock *EHSpecHandler = createBasicBlock("ehspec.handler");
-  llvm::BasicBlock *Match = createBasicBlock("match");
-  llvm::BasicBlock *Unwind = 0;
-
-  assert(PrevLandingPad == 0 && "EHSpec has invoke context");
-  (void)PrevLandingPad;
-
-  llvm::BasicBlock *Cont = createBasicBlock("cont");
-
-  EmitBranchThroughCleanup(Cont);
-
-  // Emit the statements in the try {} block
-  setInvokeDest(EHSpecHandler);
-
-  EmitBlock(EHSpecHandler);
-  // Exception object
-  llvm::Value *Exc = Builder.CreateCall(llvm_eh_exception, "exc");
-  llvm::Value *RethrowPtr = CreateTempAlloca(Exc->getType(), "_rethrow");
-
-  SelectorArgs.push_back(Exc);
-  SelectorArgs.push_back(Personality);
-  SelectorArgs.push_back(llvm::ConstantInt::get(Int32Ty,
-                                                Proto->getNumExceptions()+1));
-
-  for (unsigned i = 0; i < Proto->getNumExceptions(); ++i) {
-    QualType Ty = Proto->getExceptionType(i);
-    QualType ExceptType
-      = Ty.getNonReferenceType().getUnqualifiedType();
-    llvm::Value *EHType = CGM.GetAddrOfRTTIDescriptor(ExceptType, true);
-    SelectorArgs.push_back(EHType);
-  }
-  if (Proto->getNumExceptions())
-    SelectorArgs.push_back(Null);
-
-  // Find which handler was matched.
-  llvm::Value *Selector
-    = Builder.CreateCall(llvm_eh_selector, SelectorArgs.begin(),
-                         SelectorArgs.end(), "selector");
-  if (Proto->getNumExceptions()) {
-    Unwind = createBasicBlock("Unwind");
-
-    Builder.CreateStore(Exc, RethrowPtr);
-    Builder.CreateCondBr(Builder.CreateICmpSLT(Selector,
-                                           llvm::ConstantInt::get(Int32Ty, 0)),
-                         Match, Unwind);
-
-    EmitBlock(Match);
-  }
-  Builder.CreateCall(getUnexpectedFn(*this), Exc)->setDoesNotReturn();
-  Builder.CreateUnreachable();
+  unsigned NumExceptions = Proto->getNumExceptions();
+  EHFilterScope *Filter = EHStack.pushFilter(NumExceptions);
 
-  if (Proto->getNumExceptions()) {
-    EmitBlock(Unwind);
-    Builder.CreateCall(getUnwindResumeOrRethrowFn(),
-                       Builder.CreateLoad(RethrowPtr));
-    Builder.CreateUnreachable();
+  for (unsigned I = 0; I != NumExceptions; ++I) {
+    QualType Ty = Proto->getExceptionType(I);
+    QualType ExceptType = Ty.getNonReferenceType().getUnqualifiedType();
+    llvm::Value *EHType = CGM.GetAddrOfRTTIDescriptor(ExceptType, true);
+    Filter->setFilter(I, EHType);
   }
-
-  EmitBlock(Cont);
 }
 
 void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
@@ -438,7 +559,7 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
   if (!Proto->hasExceptionSpec())
     return;
 
-  setInvokeDest(0);
+  EHStack.popFilter();
 }
 
 void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) {
@@ -449,304 +570,858 @@ void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) {
 
 CodeGenFunction::CXXTryStmtInfo
 CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S) {
-  CXXTryStmtInfo Info;
-  Info.SavedLandingPad = getInvokeDest();
-  Info.HandlerBlock = createBasicBlock("try.handler");
-  Info.FinallyBlock = createBasicBlock("finally");
+  unsigned NumHandlers = S.getNumHandlers();
+  EHCatchScope *CatchScope = EHStack.pushCatch(NumHandlers);
+
+  for (unsigned I = 0; I != NumHandlers; ++I) {
+    const CXXCatchStmt *C = S.getHandler(I);
+
+    llvm::BasicBlock *Handler = createBasicBlock("catch");
+    if (C->getExceptionDecl()) {
+      // FIXME: Dropping the reference type on the type into makes it
+      // impossible to correctly implement catch-by-reference
+      // semantics for pointers.  Unfortunately, this is what all
+      // existing compilers do, and it's not clear that the standard
+      // personality routine is capable of doing this right.  See C++ DR 388:
+      //   http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_active.html#388
+      QualType CaughtType = C->getCaughtType();
+      CaughtType = CaughtType.getNonReferenceType().getUnqualifiedType();
+      llvm::Value *TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, true);
+      CatchScope->setHandler(I, TypeInfo, Handler);
+    } else {
+      // No exception decl indicates '...', a catch-all.
+      CatchScope->setCatchAllHandler(I, Handler);
+    }
+  }
+  
+  return CXXTryStmtInfo();
+}
+
+/// Check whether this is a non-EH scope, i.e. a scope which doesn't
+/// affect exception handling.  Currently, the only non-EH scopes are
+/// normal-only cleanup scopes.
+static bool isNonEHScope(const EHScope &S) {
+  return isa<EHCleanupScope>(S) && !cast<EHCleanupScope>(S).isEHCleanup();
+}
 
-  PushCleanupBlock(Info.FinallyBlock);
-  setInvokeDest(Info.HandlerBlock);
+llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
+  assert(EHStack.requiresLandingPad());
+  assert(!EHStack.empty());
 
-  return Info;
+  // Check the innermost scope for a cached landing pad.  If this is
+  // a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad.
+  llvm::BasicBlock *LP = EHStack.begin()->getCachedLandingPad();
+  if (LP) return LP;
+
+  // Build the landing pad for this scope.
+  LP = EmitLandingPad();
+  assert(LP);
+
+  // Cache the landing pad on the innermost scope.  If this is a
+  // non-EH scope, cache the landing pad on the enclosing scope, too.
+  for (EHScopeStack::iterator ir = EHStack.begin(); true; ++ir) {
+    ir->setCachedLandingPad(LP);
+    if (!isNonEHScope(*ir)) break;
+  }
+
+  return LP;
 }
 
-void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S,
-                                     CXXTryStmtInfo TryInfo) {
-  // Pointer to the personality function
-  llvm::Constant *Personality = getPersonalityFn(CGM);
-  llvm::Value *llvm_eh_exception =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_exception);
-  llvm::Value *llvm_eh_selector =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_selector);
-
-  llvm::BasicBlock *PrevLandingPad = TryInfo.SavedLandingPad;
-  llvm::BasicBlock *TryHandler = TryInfo.HandlerBlock;
-  llvm::BasicBlock *FinallyBlock = TryInfo.FinallyBlock;
-  llvm::BasicBlock *FinallyRethrow = createBasicBlock("finally.throw");
-  llvm::BasicBlock *FinallyEnd = createBasicBlock("finally.end");
-
-  // Jump to end if there is no exception
-  EmitBranchThroughCleanup(FinallyEnd);
-
-  llvm::BasicBlock *TerminateHandler = getTerminateHandler();
-
-  // Emit the handlers
-  EmitBlock(TryHandler);
-
-  const llvm::IntegerType *Int8Ty;
-  const llvm::PointerType *PtrToInt8Ty;
-  Int8Ty = llvm::Type::getInt8Ty(VMContext);
-  // C string type.  Used in lots of places.
-  PtrToInt8Ty = llvm::PointerType::getUnqual(Int8Ty);
-  llvm::Constant *Null = llvm::ConstantPointerNull::get(PtrToInt8Ty);
-  llvm::SmallVector<llvm::Value*, 8> SelectorArgs;
-  llvm::Value *llvm_eh_typeid_for =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for);
-  // Exception object
-  llvm::Value *Exc = Builder.CreateCall(llvm_eh_exception, "exc");
-  llvm::Value *RethrowPtr = CreateTempAlloca(Exc->getType(), "_rethrow");
-
-  SelectorArgs.push_back(Exc);
-  SelectorArgs.push_back(Personality);
-
-  bool HasCatchAll = false;
-  for (unsigned i = 0; i<S.getNumHandlers(); ++i) {
-    const CXXCatchStmt *C = S.getHandler(i);
-    VarDecl *CatchParam = C->getExceptionDecl();
-    if (CatchParam) {
-      // C++ [except.handle]p3 indicates that top-level cv-qualifiers
-      // are ignored.
-      QualType CaughtType = C->getCaughtType().getNonReferenceType();
-      llvm::Value *EHTypeInfo
-        = CGM.GetAddrOfRTTIDescriptor(CaughtType.getUnqualifiedType(), true);
-      SelectorArgs.push_back(EHTypeInfo);
-    } else {
-      // null indicates catch all
-      SelectorArgs.push_back(Null);
-      HasCatchAll = true;
+llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
+  assert(EHStack.requiresLandingPad());
+
+  // This function contains a hack to work around a design flaw in
+  // LLVM's EH IR which breaks semantics after inlining.  This same
+  // hack is implemented in llvm-gcc.
+  //
+  // The LLVM EH abstraction is basically a thin veneer over the
+  // traditional GCC zero-cost design: for each range of instructions
+  // in the function, there is (at most) one "landing pad" with an
+  // associated chain of EH actions.  A language-specific personality
+  // function interprets this chain of actions and (1) decides whether
+  // or not to resume execution at the landing pad and (2) if so,
+  // provides an integer indicating why it's stopping.  In LLVM IR,
+  // the association of a landing pad with a range of instructions is
+  // achieved via an invoke instruction, the chain of actions becomes
+  // the arguments to the @llvm.eh.selector call, and the selector
+  // call returns the integer indicator.  Other than the required
+  // presence of two intrinsic function calls in the landing pad,
+  // the IR exactly describes the layout of the output code.
+  //
+  // A principal advantage of this design is that it is completely
+  // language-agnostic; in theory, the LLVM optimizers can treat
+  // landing pads neutrally, and targets need only know how to lower
+  // the intrinsics to have a functioning exceptions system (assuming
+  // that platform exceptions follow something approximately like the
+  // GCC design).  Unfortunately, landing pads cannot be combined in a
+  // language-agnostic way: given selectors A and B, there is no way
+  // to make a single landing pad which faithfully represents the
+  // semantics of propagating an exception first through A, then
+  // through B, without knowing how the personality will interpret the
+  // (lowered form of the) selectors.  This means that inlining has no
+  // choice but to crudely chain invokes (i.e., to ignore invokes in
+  // the inlined function, but to turn all unwindable calls into
+  // invokes), which is only semantically valid if every unwind stops
+  // at every landing pad.
+  //
+  // Therefore, the invoke-inline hack is to guarantee that every
+  // landing pad has a catch-all.
+  const bool UseInvokeInlineHack = true;
+
+  for (EHScopeStack::iterator ir = EHStack.begin(); ; ) {
+    assert(ir != EHStack.end() &&
+           "stack requiring landing pad is nothing but non-EH scopes?");
+
+    // If this is a terminate scope, just use the singleton terminate
+    // landing pad.
+    if (isa<EHTerminateScope>(*ir))
+      return getTerminateLandingPad();
+
+    // If this isn't an EH scope, iterate; otherwise break out.
+    if (!isNonEHScope(*ir)) break;
+    ++ir;
+
+    // We haven't checked this scope for a cached landing pad yet.
+    if (llvm::BasicBlock *LP = ir->getCachedLandingPad())
+      return LP;
+  }
+
+  // Save the current IR generation state.
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+
+  // Create and configure the landing pad.
+  llvm::BasicBlock *LP = createBasicBlock("lpad");
+  EmitBlock(LP);
+
+  // Save the exception pointer.  It's safe to use a single exception
+  // pointer per function because EH cleanups can never have nested
+  // try/catches.
+  llvm::CallInst *Exn =
+    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_exception), "exn");
+  Exn->setDoesNotThrow();
+  Builder.CreateStore(Exn, getExceptionSlot());
+  
+  // Build the selector arguments.
+  llvm::SmallVector<llvm::Value*, 8> EHSelector;
+  EHSelector.push_back(Exn);
+  EHSelector.push_back(getPersonalityFn(*this));
+
+  // Accumulate all the handlers in scope.
+  llvm::DenseMap<llvm::Value*, JumpDest> EHHandlers;
+  JumpDest CatchAll;
+  bool HasEHCleanup = false;
+  bool HasEHFilter = false;
+  llvm::SmallVector<llvm::Value*, 8> EHFilters;
+  for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end();
+         I != E; ++I) {
+
+    switch (I->getKind()) {
+    case EHScope::Cleanup:
+      if (!HasEHCleanup)
+        HasEHCleanup = cast<EHCleanupScope>(*I).isEHCleanup();
+      // We otherwise don't care about cleanups.
+      continue;
+
+    case EHScope::Filter: {
+      assert(I.next() == EHStack.end() && "EH filter is not end of EH stack");
+      assert(!CatchAll.Block && "EH filter reached after catch-all");
+
+      // Filter scopes get added to the selector in wierd ways.
+      EHFilterScope &Filter = cast<EHFilterScope>(*I);
+      HasEHFilter = true;
+
+      // Add all the filter values which we aren't already explicitly
+      // catching.
+      for (unsigned I = 0, E = Filter.getNumFilters(); I != E; ++I) {
+        llvm::Value *FV = Filter.getFilter(I);
+        if (!EHHandlers.count(FV))
+          EHFilters.push_back(FV);
+      }
+      goto done;
+    }
+
+    case EHScope::Terminate:
+      // Terminate scopes are basically catch-alls.
+      assert(!CatchAll.Block);
+      CatchAll.Block = getTerminateHandler();
+      CatchAll.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
+      goto done;
+
+    case EHScope::Catch:
+      break;
+    }
+
+    EHCatchScope &Catch = cast<EHCatchScope>(*I);
+    for (unsigned HI = 0, HE = Catch.getNumHandlers(); HI != HE; ++HI) {
+      EHCatchScope::Handler Handler = Catch.getHandler(HI);
+
+      // Catch-all.  We should only have one of these per catch.
+      if (!Handler.Type) {
+        assert(!CatchAll.Block);
+        CatchAll.Block = Handler.Block;
+        CatchAll.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
+        continue;
+      }
+
+      // Check whether we already have a handler for this type.
+      JumpDest &Dest = EHHandlers[Handler.Type];
+      if (Dest.Block) continue;
+
+      EHSelector.push_back(Handler.Type);
+      Dest.Block = Handler.Block;
+      Dest.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
     }
+
+    // Stop if we found a catch-all.
+    if (CatchAll.Block) break;
   }
 
-  // We use a cleanup unless there was already a catch all.
-  if (!HasCatchAll) {
-    SelectorArgs.push_back(Null);
+ done:
+  unsigned LastToEmitInLoop = EHSelector.size();
+
+  // If we have a catch-all, add null to the selector.
+  if (CatchAll.Block) {
+    EHSelector.push_back(getCatchAllValue(CGF));
+
+  // If we have an EH filter, we need to add those handlers in the
+  // right place in the selector, which is to say, at the end.
+  } else if (HasEHFilter) {
+    // Create a filter expression: an integer constant saying how many
+    // filters there are (+1 to avoid ambiguity with 0 for cleanup),
+    // followed by the filter types.  The personality routine only
+    // lands here if the filter doesn't match.
+    EHSelector.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(),
+                                                EHFilters.size() + 1));
+    EHSelector.append(EHFilters.begin(), EHFilters.end());
+
+    // Also check whether we need a cleanup.
+    if (UseInvokeInlineHack || HasEHCleanup)
+      EHSelector.push_back(UseInvokeInlineHack
+                           ? getCatchAllValue(CGF)
+                           : getCleanupValue(CGF));
+
+  // Otherwise, signal that we at least have cleanups.
+  } else if (UseInvokeInlineHack || HasEHCleanup) {
+    EHSelector.push_back(UseInvokeInlineHack
+                         ? getCatchAllValue(CGF)
+                         : getCleanupValue(CGF));
+  } else {
+    assert(LastToEmitInLoop > 2);
+    LastToEmitInLoop--;
   }
 
-  // Find which handler was matched.
-  llvm::Value *Selector
-    = Builder.CreateCall(llvm_eh_selector, SelectorArgs.begin(),
-                         SelectorArgs.end(), "selector");
-  for (unsigned i = 0; i<S.getNumHandlers(); ++i) {
-    const CXXCatchStmt *C = S.getHandler(i);
-    VarDecl *CatchParam = C->getExceptionDecl();
-    Stmt *CatchBody = C->getHandlerBlock();
-
-    llvm::BasicBlock *Next = 0;
-
-    if (SelectorArgs[i+2] != Null) {
-      llvm::BasicBlock *Match = createBasicBlock("match");
-      Next = createBasicBlock("catch.next");
-      const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
-      llvm::Value *Id
-        = Builder.CreateCall(llvm_eh_typeid_for,
-                             Builder.CreateBitCast(SelectorArgs[i+2],
-                                                   Int8PtrTy));
-      Builder.CreateCondBr(Builder.CreateICmpEQ(Selector, Id),
-                           Match, Next);
+  assert(EHSelector.size() >= 3 && "selector call has only two arguments!");
+
+  // Tell the backend how to generate the landing pad.
+  llvm::CallInst *Selection =
+    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
+                       EHSelector.begin(), EHSelector.end(), "eh.selector");
+  Selection->setDoesNotThrow();
+  
+  // Select the right handler.
+  llvm::Value *llvm_eh_typeid_for =
+    CGM.getIntrinsic(llvm::Intrinsic::eh_typeid_for);
+
+  // The results of llvm_eh_typeid_for aren't reliable --- at least
+  // not locally --- so we basically have to do this as an 'if' chain.
+  // We walk through the first N-1 catch clauses, testing and chaining,
+  // and then fall into the final clause (which is either a cleanup, a
+  // filter (possibly with a cleanup), a catch-all, or another catch).
+  for (unsigned I = 2; I != LastToEmitInLoop; ++I) {
+    llvm::Value *Type = EHSelector[I];
+    JumpDest Dest = EHHandlers[Type];
+    assert(Dest.Block && "no handler entry for value in selector?");
+
+    // Figure out where to branch on a match.  As a debug code-size
+    // optimization, if the scope depth matches the innermost cleanup,
+    // we branch directly to the catch handler.
+    llvm::BasicBlock *Match = Dest.Block;
+    bool MatchNeedsCleanup = Dest.ScopeDepth != EHStack.getInnermostEHCleanup();
+    if (MatchNeedsCleanup)
+      Match = createBasicBlock("eh.match");
+
+    llvm::BasicBlock *Next = createBasicBlock("eh.next");
+
+    // Check whether the exception matches.
+    llvm::CallInst *Id
+      = Builder.CreateCall(llvm_eh_typeid_for,
+                           Builder.CreateBitCast(Type, CGM.PtrToInt8Ty));
+    Id->setDoesNotThrow();
+    Builder.CreateCondBr(Builder.CreateICmpEQ(Selection, Id),
+                         Match, Next);
+    
+    // Emit match code if necessary.
+    if (MatchNeedsCleanup) {
       EmitBlock(Match);
+      EmitBranchThroughEHCleanup(Dest);
     }
 
-    llvm::BasicBlock *MatchEnd = createBasicBlock("match.end");
-    llvm::BasicBlock *MatchHandler = createBasicBlock("match.handler");
-
-    PushCleanupBlock(MatchEnd);
-    setInvokeDest(MatchHandler);
-
-    llvm::Value *ExcObject = Builder.CreateCall(getBeginCatchFn(*this), Exc);
-
-    {
-      CleanupScope CatchScope(*this);
-      // Bind the catch parameter if it exists.
-      if (CatchParam) {
-        QualType CatchType = CatchParam->getType().getNonReferenceType();
-        setInvokeDest(TerminateHandler);
-        bool WasPointer = true;
-        bool WasPointerReference = false;
-        CatchType = CGM.getContext().getCanonicalType(CatchType);
-        if (CatchType.getTypePtr()->isPointerType()) {
-          if (isa<ReferenceType>(CatchParam->getType()))
-            WasPointerReference = true;
-        } else {
-          if (!isa<ReferenceType>(CatchParam->getType()))
-            WasPointer = false;
-          CatchType = getContext().getPointerType(CatchType);
-        }
-        ExcObject = Builder.CreateBitCast(ExcObject, ConvertType(CatchType));
-        EmitLocalBlockVarDecl(*CatchParam);
-        // FIXME: we need to do this sooner so that the EH region for the
-        // cleanup doesn't start until after the ctor completes, use a decl
-        // init?
-        CopyObject(*this, CatchParam->getType().getNonReferenceType(),
-                   WasPointer, WasPointerReference, ExcObject,
-                   GetAddrOfLocalVar(CatchParam));
-        setInvokeDest(MatchHandler);
+    // Continue to the next match.
+    EmitBlock(Next);
+  }
+
+  // Emit the final case in the selector.
+  // This might be a catch-all....
+  if (CatchAll.Block) {
+    assert(isa<llvm::ConstantPointerNull>(EHSelector.back()));
+    EmitBranchThroughEHCleanup(CatchAll);
+
+  // ...or an EH filter...
+  } else if (HasEHFilter) {
+    llvm::Value *SavedSelection = Selection;
+
+    // First, unwind out to the outermost scope if necessary.
+    if (EHStack.hasEHCleanups()) {
+      // The end here might not dominate the beginning, so we might need to
+      // save the selector if we need it.
+      llvm::AllocaInst *SelectorVar = 0;
+      if (HasEHCleanup) {
+        SelectorVar = CreateTempAlloca(Builder.getInt32Ty(), "selector.var");
+        Builder.CreateStore(Selection, SelectorVar);
       }
 
-      EmitStmt(CatchBody);
+      llvm::BasicBlock *CleanupContBB = createBasicBlock("ehspec.cleanup.cont");
+      EmitBranchThroughEHCleanup(JumpDest(CleanupContBB, EHStack.stable_end()));
+      EmitBlock(CleanupContBB);
+
+      if (HasEHCleanup)
+        SavedSelection = Builder.CreateLoad(SelectorVar, "ehspec.saved-selector");
     }
 
-    EmitBranchThroughCleanup(FinallyEnd);
+    // If there was a cleanup, we'll need to actually check whether we
+    // landed here because the filter triggered.
+    if (UseInvokeInlineHack || HasEHCleanup) {
+      llvm::BasicBlock *RethrowBB = createBasicBlock("cleanup");
+      llvm::BasicBlock *UnexpectedBB = createBasicBlock("ehspec.unexpected");
+
+      llvm::Constant *Zero = llvm::ConstantInt::get(Builder.getInt32Ty(), 0);
+      llvm::Value *FailsFilter =
+        Builder.CreateICmpSLT(SavedSelection, Zero, "ehspec.fails");
+      Builder.CreateCondBr(FailsFilter, UnexpectedBB, RethrowBB);
+
+      // The rethrow block is where we land if this was a cleanup.
+      // TODO: can this be _Unwind_Resume if the InvokeInlineHack is off?
+      EmitBlock(RethrowBB);
+      Builder.CreateCall(getUnwindResumeOrRethrowFn(),
+                         Builder.CreateLoad(getExceptionSlot()))
+        ->setDoesNotReturn();
+      Builder.CreateUnreachable();
 
-    EmitBlock(MatchHandler);
+      EmitBlock(UnexpectedBB);
+    }
 
-    llvm::Value *Exc = Builder.CreateCall(llvm_eh_exception, "exc");
-    // We are required to emit this call to satisfy LLVM, even
-    // though we don't use the result.
-    llvm::Value *Args[] = {
-      Exc, Personality, llvm::ConstantInt::getNullValue(Int32Ty)
-    };
-    Builder.CreateCall(llvm_eh_selector, &Args[0], llvm::array_endof(Args));
-    Builder.CreateStore(Exc, RethrowPtr);
-    EmitBranchThroughCleanup(FinallyRethrow);
+    // Call __cxa_call_unexpected.  This doesn't need to be an invoke
+    // because __cxa_call_unexpected magically filters exceptions
+    // according to the last landing pad the exception was thrown
+    // into.  Seriously.
+    Builder.CreateCall(getUnexpectedFn(*this),
+                       Builder.CreateLoad(getExceptionSlot()))
+      ->setDoesNotReturn();
+    Builder.CreateUnreachable();
 
-    CodeGenFunction::CleanupBlockInfo Info = PopCleanupBlock();
+  // ...or a normal catch handler...
+  } else if (!UseInvokeInlineHack && !HasEHCleanup) {
+    llvm::Value *Type = EHSelector.back();
+    EmitBranchThroughEHCleanup(EHHandlers[Type]);
 
-    EmitBlock(MatchEnd);
+  // ...or a cleanup.
+  } else {
+    // We emit a jump to a notional label at the outermost unwind state.
+    llvm::BasicBlock *Unwind = createBasicBlock("eh.resume");
+    JumpDest Dest(Unwind, EHStack.stable_end());
+    EmitBranchThroughEHCleanup(Dest);
+
+    // The unwind block.  We have to reload the exception here because
+    // we might have unwound through arbitrary blocks, so the landing
+    // pad might not dominate.
+    EmitBlock(Unwind);
 
-    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
-    Builder.CreateInvoke(getEndCatchFn(*this),
-                         Cont, TerminateHandler,
-                         &Args[0], &Args[0]);
-    EmitBlock(Cont);
-    if (Info.SwitchBlock)
-      EmitBlock(Info.SwitchBlock);
-    if (Info.EndBlock)
-      EmitBlock(Info.EndBlock);
+    // This can always be a call because we necessarily didn't find
+    // anything on the EH stack which needs our help.
+    Builder.CreateCall(getUnwindResumeOrRethrowFn(),
+                       Builder.CreateLoad(getExceptionSlot()))
+      ->setDoesNotReturn();
+    Builder.CreateUnreachable();
+  }
+
+  // Restore the old IR generation state.
+  Builder.restoreIP(SavedIP);
+
+  return LP;
+}
 
-    Exc = Builder.CreateCall(llvm_eh_exception, "exc");
-    Builder.CreateStore(Exc, RethrowPtr);
-    EmitBranchThroughCleanup(FinallyRethrow);
+/// Emits a call to __cxa_begin_catch and enters a cleanup to call
+/// __cxa_end_catch.
+static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, llvm::Value *Exn) {
+  llvm::CallInst *Call = CGF.Builder.CreateCall(getBeginCatchFn(CGF), Exn);
+  Call->setDoesNotThrow();
 
-    if (Next)
-      EmitBlock(Next);
+  {
+    CodeGenFunction::CleanupBlock EndCatchCleanup(CGF,
+                                  CodeGenFunction::NormalAndEHCleanup);
+
+    // __cxa_end_catch never throws, so this can just be a call.
+    CGF.Builder.CreateCall(getEndCatchFn(CGF))->setDoesNotThrow();
   }
-  if (!HasCatchAll) {
-    Builder.CreateStore(Exc, RethrowPtr);
-    EmitBranchThroughCleanup(FinallyRethrow);
+
+  return Call;
+}
+
+/// A "special initializer" callback for initializing a catch
+/// parameter during catch initialization.
+static void InitCatchParam(CodeGenFunction &CGF,
+                           const VarDecl &CatchParam,
+                           llvm::Value *ParamAddr) {
+  // Load the exception from where the landing pad saved it.
+  llvm::Value *Exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot(), "exn");
+
+  CanQualType CatchType =
+    CGF.CGM.getContext().getCanonicalType(CatchParam.getType());
+  const llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType);
+
+  // If we're catching by reference, we can just cast the object
+  // pointer to the appropriate pointer.
+  if (isa<ReferenceType>(CatchType)) {
+    // __cxa_begin_catch returns the adjusted object pointer.
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    llvm::Value *ExnCast =
+      CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref");
+    CGF.Builder.CreateStore(ExnCast, ParamAddr);
+    return;
   }
 
-  CodeGenFunction::CleanupBlockInfo Info = PopCleanupBlock();
+  // Non-aggregates (plus complexes).
+  bool IsComplex = false;
+  if (!CGF.hasAggregateLLVMType(CatchType) ||
+      (IsComplex = CatchType->isAnyComplexType())) {
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    
+    // If the catch type is a pointer type, __cxa_begin_catch returns
+    // the pointer by value.
+    if (CatchType->hasPointerRepresentation()) {
+      llvm::Value *CastExn =
+        CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted");
+      CGF.Builder.CreateStore(CastExn, ParamAddr);
+      return;
+    }
 
-  setInvokeDest(PrevLandingPad);
+    // Otherwise, it returns a pointer into the exception object.
 
-  EmitBlock(FinallyBlock);
+    const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
+    llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
 
-  if (Info.SwitchBlock)
-    EmitBlock(Info.SwitchBlock);
-  if (Info.EndBlock)
-    EmitBlock(Info.EndBlock);
+    if (IsComplex) {
+      CGF.StoreComplexToAddr(CGF.LoadComplexFromAddr(Cast, /*volatile*/ false),
+                             ParamAddr, /*volatile*/ false);
+    } else {
+      llvm::Value *ExnLoad = CGF.Builder.CreateLoad(Cast, "exn.scalar");
+      CGF.EmitStoreOfScalar(ExnLoad, ParamAddr, /*volatile*/ false, CatchType);
+    }
+    return;
+  }
 
-  // Branch around the rethrow code.
-  EmitBranch(FinallyEnd);
+  // FIXME: this *really* needs to be done via a proper, Sema-emitted
+  // initializer expression.
 
-  EmitBlock(FinallyRethrow);
-  // FIXME: Eventually we can chain the handlers together and just do a call
-  // here.
-  if (getInvokeDest()) {
-    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
-    Builder.CreateInvoke(getUnwindResumeOrRethrowFn(), Cont,
-                         getInvokeDest(),
-                         Builder.CreateLoad(RethrowPtr));
-    EmitBlock(Cont);
-  } else
-    Builder.CreateCall(getUnwindResumeOrRethrowFn(),
-                       Builder.CreateLoad(RethrowPtr));
+  CXXRecordDecl *RD = CatchType.getTypePtr()->getAsCXXRecordDecl();
+  assert(RD && "aggregate catch type was not a record!");
 
-  Builder.CreateUnreachable();
+  const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
+
+  if (RD->hasTrivialCopyConstructor()) {
+    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn);
+    llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
+    CGF.EmitAggregateCopy(ParamAddr, Cast, CatchType);
+    return;
+  }
+
+  // We have to call __cxa_get_exception_ptr to get the adjusted
+  // pointer before copying.
+  llvm::CallInst *AdjustedExn =
+    CGF.Builder.CreateCall(getGetExceptionPtrFn(CGF), Exn);
+  AdjustedExn->setDoesNotThrow();
+  llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
+
+  CXXConstructorDecl *CD = RD->getCopyConstructor(CGF.getContext(), 0);
+  assert(CD && "record has no copy constructor!");
+  llvm::Value *CopyCtor = CGF.CGM.GetAddrOfCXXConstructor(CD, Ctor_Complete);
+
+  CallArgList CallArgs;
+  CallArgs.push_back(std::make_pair(RValue::get(ParamAddr),
+                                    CD->getThisType(CGF.getContext())));
+  CallArgs.push_back(std::make_pair(RValue::get(Cast),
+                                    CD->getParamDecl(0)->getType()));
+
+  const FunctionProtoType *FPT
+    = CD->getType()->getAs<FunctionProtoType>();
+
+  // Call the copy ctor in a terminate scope.
+  CGF.EHStack.pushTerminate();
+  CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(CallArgs, FPT),
+               CopyCtor, ReturnValueSlot(), CallArgs, CD);
+  CGF.EHStack.popTerminate();
+
+  // Finally we can call __cxa_begin_catch.
+  CallBeginCatch(CGF, Exn);
+}
+
+/// Begins a catch statement by initializing the catch variable and
+/// calling __cxa_begin_catch.
+static void BeginCatch(CodeGenFunction &CGF,
+                       const CXXCatchStmt *S) {
+  // We have to be very careful with the ordering of cleanups here:
+  //   C++ [except.throw]p4:
+  //     The destruction [of the exception temporary] occurs
+  //     immediately after the destruction of the object declared in
+  //     the exception-declaration in the handler.
+  //
+  // So the precise ordering is:
+  //   1.  Construct catch variable.
+  //   2.  __cxa_begin_catch
+  //   3.  Enter __cxa_end_catch cleanup
+  //   4.  Enter dtor cleanup
+  //
+  // We do this by initializing the exception variable with a
+  // "special initializer", InitCatchParam.  Delegation sequence:
+  //   - ExitCXXTryStmt opens a RunCleanupsScope
+  //     - EmitLocalBlockVarDecl creates the variable and debug info
+  //       - InitCatchParam initializes the variable from the exception
+  //         - CallBeginCatch calls __cxa_begin_catch
+  //         - CallBeginCatch enters the __cxa_end_catch cleanup
+  //     - EmitLocalBlockVarDecl enters the variable destructor cleanup
+  //   - EmitCXXTryStmt emits the code for the catch body
+  //   - EmitCXXTryStmt close the RunCleanupsScope
+
+  VarDecl *CatchParam = S->getExceptionDecl();
+  if (!CatchParam) {
+    llvm::Value *Exn = CGF.Builder.CreateLoad(CGF.getExceptionSlot(), "exn");
+    CallBeginCatch(CGF, Exn);
+    return;
+  }
+
+  // Emit the local.
+  CGF.EmitLocalBlockVarDecl(*CatchParam, &InitCatchParam);
+}
+
+void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S,
+                                     CXXTryStmtInfo TryInfo) {
+  unsigned NumHandlers = S.getNumHandlers();
+  EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin());
+  assert(CatchScope.getNumHandlers() == NumHandlers);
+
+  // Copy the handler blocks off before we pop the EH stack.  Emitting
+  // the handlers might scribble on this memory.
+  llvm::SmallVector<EHCatchScope::Handler, 8> Handlers(NumHandlers);
+  memcpy(Handlers.data(), CatchScope.begin(),
+         NumHandlers * sizeof(EHCatchScope::Handler));
+  EHStack.popCatch();
+
+  // The fall-through block.
+  llvm::BasicBlock *ContBB = createBasicBlock("try.cont");
+
+  // We just emitted the body of the try; jump to the continue block.
+  if (HaveInsertPoint())
+    Builder.CreateBr(ContBB);
+
+  for (unsigned I = 0; I != NumHandlers; ++I) {
+    llvm::BasicBlock *CatchBlock = Handlers[I].Block;
+    EmitBlock(CatchBlock);
+
+    // Catch the exception if this isn't a catch-all.
+    const CXXCatchStmt *C = S.getHandler(I);
+
+    // Enter a cleanup scope, including the catch variable and the
+    // end-catch.
+    RunCleanupsScope CatchScope(*this);
+
+    // Initialize the catch variable and set up the cleanups.
+    BeginCatch(*this, C);
+
+    // Perform the body of the catch.
+    EmitStmt(C->getHandlerBlock());
+
+    // Fall out through the catch cleanups.
+    CatchScope.ForceCleanup();
+
+    // Branch out of the try.
+    if (HaveInsertPoint())
+      Builder.CreateBr(ContBB);
+  }
 
-  EmitBlock(FinallyEnd);
-}
-
-CodeGenFunction::EHCleanupBlock::~EHCleanupBlock() {
-  CGF.setInvokeDest(PreviousInvokeDest);
-
-  llvm::BasicBlock *EndOfCleanup = CGF.Builder.GetInsertBlock();
-
-  // Jump to the beginning of the cleanup.
-  CGF.Builder.SetInsertPoint(CleanupHandler, CleanupHandler->begin());
- 
-  // The libstdc++ personality function.
-  // TODO: generalize to work with other libraries.
-  llvm::Constant *Personality = getPersonalityFn(CGF.CGM);
-
-  // %exception = call i8* @llvm.eh.exception()
-  //   Magic intrinsic which tells gives us a handle to the caught
-  //   exception.
-  llvm::Value *llvm_eh_exception =
-    CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_exception);
-  llvm::Value *Exc = CGF.Builder.CreateCall(llvm_eh_exception, "exc");
-
-  llvm::Constant *Null = llvm::ConstantPointerNull::get(CGF.PtrToInt8Ty);
-
-  // %ignored = call i32 @llvm.eh.selector(i8* %exception,
-  //                                       i8* @__gxx_personality_v0,
-  //                                       i8* null)
-  //   Magic intrinsic which tells LLVM that this invoke landing pad is
-  //   just a cleanup block.
-  llvm::Value *Args[] = { Exc, Personality, Null };
-  llvm::Value *llvm_eh_selector =
-    CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_selector);
-  CGF.Builder.CreateCall(llvm_eh_selector, &Args[0], llvm::array_endof(Args));
-
-  // And then we fall through into the code that the user put there.
-  // Jump back to the end of the cleanup.
-  CGF.Builder.SetInsertPoint(EndOfCleanup);
-
-  // Rethrow the exception.
-  if (CGF.getInvokeDest()) {
-    llvm::BasicBlock *Cont = CGF.createBasicBlock("invoke.cont");
-    CGF.Builder.CreateInvoke(CGF.getUnwindResumeOrRethrowFn(), Cont,
-                             CGF.getInvokeDest(), Exc);
-    CGF.EmitBlock(Cont);
-  } else
-    CGF.Builder.CreateCall(CGF.getUnwindResumeOrRethrowFn(), Exc);
+  EmitBlock(ContBB);
+}
+
+/// Enters a finally block for an implementation using zero-cost
+/// exceptions.  This is mostly general, but hard-codes some
+/// language/ABI-specific behavior in the catch-all sections.
+CodeGenFunction::FinallyInfo
+CodeGenFunction::EnterFinallyBlock(const Stmt *Body,
+                                   llvm::Constant *BeginCatchFn,
+                                   llvm::Constant *EndCatchFn,
+                                   llvm::Constant *RethrowFn) {
+  assert((BeginCatchFn != 0) == (EndCatchFn != 0) &&
+         "begin/end catch functions not paired");
+  assert(RethrowFn && "rethrow function is required");
+
+  // The rethrow function has one of the following two types:
+  //   void (*)()
+  //   void (*)(void*)
+  // In the latter case we need to pass it the exception object.
+  // But we can't use the exception slot because the @finally might
+  // have a landing pad (which would overwrite the exception slot).
+  const llvm::FunctionType *RethrowFnTy =
+    cast<llvm::FunctionType>(
+      cast<llvm::PointerType>(RethrowFn->getType())
+      ->getElementType());
+  llvm::Value *SavedExnVar = 0;
+  if (RethrowFnTy->getNumParams())
+    SavedExnVar = CreateTempAlloca(Builder.getInt8PtrTy(), "finally.exn");
+
+  // A finally block is a statement which must be executed on any edge
+  // out of a given scope.  Unlike a cleanup, the finally block may
+  // contain arbitrary control flow leading out of itself.  In
+  // addition, finally blocks should always be executed, even if there
+  // are no catch handlers higher on the stack.  Therefore, we
+  // surround the protected scope with a combination of a normal
+  // cleanup (to catch attempts to break out of the block via normal
+  // control flow) and an EH catch-all (semantically "outside" any try
+  // statement to which the finally block might have been attached).
+  // The finally block itself is generated in the context of a cleanup
+  // which conditionally leaves the catch-all.
+
+  FinallyInfo Info;
+
+  // Jump destination for performing the finally block on an exception
+  // edge.  We'll never actually reach this block, so unreachable is
+  // fine.
+  JumpDest RethrowDest = getJumpDestInCurrentScope(getUnreachableBlock());
+
+  // Whether the finally block is being executed for EH purposes.
+  llvm::AllocaInst *ForEHVar = CreateTempAlloca(CGF.Builder.getInt1Ty(),
+                                                "finally.for-eh");
+  InitTempAlloca(ForEHVar, llvm::ConstantInt::getFalse(getLLVMContext()));
+
+  // Enter a normal cleanup which will perform the @finally block.
+  {
+    CodeGenFunction::CleanupBlock
+      NormalCleanup(*this, CodeGenFunction::NormalCleanup);
+
+    // Enter a cleanup to call the end-catch function if one was provided.
+    if (EndCatchFn) {
+      CodeGenFunction::CleanupBlock
+        FinallyExitCleanup(CGF, CodeGenFunction::NormalAndEHCleanup);
+
+      llvm::BasicBlock *EndCatchBB = createBasicBlock("finally.endcatch");
+      llvm::BasicBlock *CleanupContBB = createBasicBlock("finally.cleanup.cont");
+
+      llvm::Value *ShouldEndCatch =
+        Builder.CreateLoad(ForEHVar, "finally.endcatch");
+      Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB);
+      EmitBlock(EndCatchBB);
+      Builder.CreateCall(EndCatchFn)->setDoesNotThrow();
+      EmitBlock(CleanupContBB);
+    }
+
+    // Emit the finally block.
+    EmitStmt(Body);
+
+    // If the end of the finally is reachable, check whether this was
+    // for EH.  If so, rethrow.
+    if (HaveInsertPoint()) {
+      llvm::BasicBlock *RethrowBB = createBasicBlock("finally.rethrow");
+      llvm::BasicBlock *ContBB = createBasicBlock("finally.cont");
+
+      llvm::Value *ShouldRethrow =
+        Builder.CreateLoad(ForEHVar, "finally.shouldthrow");
+      Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB);
+
+      EmitBlock(RethrowBB);
+      if (SavedExnVar) {
+        llvm::Value *Args[] = { Builder.CreateLoad(SavedExnVar) };
+        EmitCallOrInvoke(RethrowFn, Args, Args+1);
+      } else {
+        EmitCallOrInvoke(RethrowFn, 0, 0);
+      }
+      Builder.CreateUnreachable();
+
+      EmitBlock(ContBB);
+    }
+
+    // Leave the end-catch cleanup.  As an optimization, pretend that
+    // the fallthrough path was inaccessible; we've dynamically proven
+    // that we're not in the EH case along that path.
+    if (EndCatchFn) {
+      CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+      PopCleanupBlock();
+      Builder.restoreIP(SavedIP);
+    }
+    
+    // Now make sure we actually have an insertion point or the
+    // cleanup gods will hate us.
+    EnsureInsertPoint();
+  }
+
+  // Enter a catch-all scope.
+  llvm::BasicBlock *CatchAllBB = createBasicBlock("finally.catchall");
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveIP();
+  Builder.SetInsertPoint(CatchAllBB);
+
+  // If there's a begin-catch function, call it.
+  if (BeginCatchFn) {
+    Builder.CreateCall(BeginCatchFn, Builder.CreateLoad(getExceptionSlot()))
+      ->setDoesNotThrow();
+  }
+
+  // If we need to remember the exception pointer to rethrow later, do so.
+  if (SavedExnVar) {
+    llvm::Value *SavedExn = Builder.CreateLoad(getExceptionSlot());
+    Builder.CreateStore(SavedExn, SavedExnVar);
+  }
+
+  // Tell the finally block that we're in EH.
+  Builder.CreateStore(llvm::ConstantInt::getTrue(getLLVMContext()), ForEHVar);
+
+  // Thread a jump through the finally cleanup.
+  EmitBranchThroughCleanup(RethrowDest);
+
+  Builder.restoreIP(SavedIP);
+
+  EHCatchScope *CatchScope = EHStack.pushCatch(1);
+  CatchScope->setCatchAllHandler(0, CatchAllBB);
+
+  return Info;
+}
+
+void CodeGenFunction::ExitFinallyBlock(FinallyInfo &Info) {
+  // Leave the finally catch-all.
+  EHCatchScope &Catch = cast<EHCatchScope>(*EHStack.begin());
+  llvm::BasicBlock *CatchAllBB = Catch.getHandler(0).Block;
+  EHStack.popCatch();
+
+  // And leave the normal cleanup.
+  PopCleanupBlock();
+
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+  EmitBlock(CatchAllBB, true);
+
+  Builder.restoreIP(SavedIP);
+}
+
+llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
+  if (TerminateLandingPad)
+    return TerminateLandingPad;
+
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+
+  // This will get inserted at the end of the function.
+  TerminateLandingPad = createBasicBlock("terminate.lpad");
+  Builder.SetInsertPoint(TerminateLandingPad);
+
+  // Tell the backend that this is a landing pad.
+  llvm::CallInst *Exn =
+    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_exception), "exn");
+  Exn->setDoesNotThrow();
+  
+  // Tell the backend what the exception table should be:
+  // nothing but a catch-all.
+  llvm::Value *Args[3] = { Exn, getPersonalityFn(*this),
+                           getCatchAllValue(*this) };
+  Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
+                     Args, Args+3, "eh.selector")
+    ->setDoesNotThrow();
+
+  llvm::CallInst *TerminateCall = Builder.CreateCall(getTerminateFn(*this));
+  TerminateCall->setDoesNotReturn();
+  TerminateCall->setDoesNotThrow();
   CGF.Builder.CreateUnreachable();
 
-  // Resume inserting where we started, but put the new cleanup
-  // handler in place.
-  if (PreviousInsertionBlock)
-    CGF.Builder.SetInsertPoint(PreviousInsertionBlock);
-  else
-    CGF.Builder.ClearInsertionPoint();
+  // Restore the saved insertion state.
+  Builder.restoreIP(SavedIP);
 
-  if (CGF.Exceptions)
-    CGF.setInvokeDest(CleanupHandler);
+  return TerminateLandingPad;
 }
 
 llvm::BasicBlock *CodeGenFunction::getTerminateHandler() {
   if (TerminateHandler)
     return TerminateHandler;
 
-  // We don't want to change anything at the current location, so
-  // save it aside and clear the insert point.
-  llvm::BasicBlock *SavedInsertBlock = Builder.GetInsertBlock();
-  llvm::BasicBlock::iterator SavedInsertPoint = Builder.GetInsertPoint();
-  Builder.ClearInsertionPoint();
-
-  llvm::Constant *Personality = getPersonalityFn(CGM);
-  llvm::Value *llvm_eh_exception =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_exception);
-  llvm::Value *llvm_eh_selector =
-    CGM.getIntrinsic(llvm::Intrinsic::eh_selector);
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
 
-  // Set up terminate handler
+  // Set up the terminate handler.  This block is inserted at the very
+  // end of the function by FinishFunction.
   TerminateHandler = createBasicBlock("terminate.handler");
-  EmitBlock(TerminateHandler);
-  llvm::Value *Exc = Builder.CreateCall(llvm_eh_exception, "exc");
-  // We are required to emit this call to satisfy LLVM, even
-  // though we don't use the result.
-  llvm::Value *Args[] = {
-    Exc, Personality, llvm::ConstantInt::get(Int32Ty, 1)
-  };
-  Builder.CreateCall(llvm_eh_selector, &Args[0], llvm::array_endof(Args));
-  llvm::CallInst *TerminateCall =
-    Builder.CreateCall(getTerminateFn(*this));
+  Builder.SetInsertPoint(TerminateHandler);
+  llvm::CallInst *TerminateCall = Builder.CreateCall(getTerminateFn(*this));
   TerminateCall->setDoesNotReturn();
   TerminateCall->setDoesNotThrow();
   Builder.CreateUnreachable();
 
   // Restore the saved insertion state.
-  Builder.SetInsertPoint(SavedInsertBlock, SavedInsertPoint);
+  Builder.restoreIP(SavedIP);
 
   return TerminateHandler;
 }
+
+CodeGenFunction::CleanupBlock::CleanupBlock(CodeGenFunction &CGF,
+                                            CleanupKind Kind)
+  : CGF(CGF), SavedIP(CGF.Builder.saveIP()), NormalCleanupExitBB(0) {
+  llvm::BasicBlock *EntryBB = CGF.createBasicBlock("cleanup");
+  CGF.Builder.SetInsertPoint(EntryBB);
+
+  switch (Kind) {
+  case NormalAndEHCleanup:
+    NormalCleanupEntryBB = EHCleanupEntryBB = EntryBB;
+    break;
+
+  case NormalCleanup:
+    NormalCleanupEntryBB = EntryBB;
+    EHCleanupEntryBB = 0;
+    break;        
+
+  case EHCleanup:
+    NormalCleanupEntryBB = 0;
+    EHCleanupEntryBB = EntryBB;
+    CGF.EHStack.pushTerminate();
+    break;
+  }
+}
+
+void CodeGenFunction::CleanupBlock::beginEHCleanup() {
+  assert(EHCleanupEntryBB == 0 && "already started an EH cleanup");
+  NormalCleanupExitBB = CGF.Builder.GetInsertBlock();
+  assert(NormalCleanupExitBB && "end of normal cleanup is unreachable");
+      
+  EHCleanupEntryBB = CGF.createBasicBlock("eh.cleanup");
+  CGF.Builder.SetInsertPoint(EHCleanupEntryBB);
+  CGF.EHStack.pushTerminate();
+}
+
+CodeGenFunction::CleanupBlock::~CleanupBlock() {
+  llvm::BasicBlock *EHCleanupExitBB = 0;
+
+  // If we're currently writing the EH cleanup...
+  if (EHCleanupEntryBB) {
+    // Set the EH cleanup exit block.
+    EHCleanupExitBB = CGF.Builder.GetInsertBlock();
+    assert(EHCleanupExitBB && "end of EH cleanup is unreachable");
+
+    // If we're actually writing both at once, set the normal exit, too.
+    if (EHCleanupEntryBB == NormalCleanupEntryBB)
+      NormalCleanupExitBB = EHCleanupExitBB;
+
+    // Otherwise, we must have pushed a terminate handler.
+    else
+      CGF.EHStack.popTerminate();
+
+  // Otherwise, just set the normal cleanup exit block.
+  } else {
+    NormalCleanupExitBB = CGF.Builder.GetInsertBlock();
+    assert(NormalCleanupExitBB && "end of normal cleanup is unreachable");
+  }
+  
+  CGF.EHStack.pushCleanup(NormalCleanupEntryBB, NormalCleanupExitBB,
+                          EHCleanupEntryBB, EHCleanupExitBB);
+
+  CGF.Builder.restoreIP(SavedIP);
+}
+