Automatic Reference Counting.

Language-design credit goes to a lot of people, but I particularly want
to single out Blaine Garst and Patrick Beard for their contributions.

Compiler implementation credit goes to Argyrios, Doug, Fariborz, and myself,
in no particular order.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@133103 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 1362 insertions, 17 deletions

diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index cdb15bfd83..cdc2fffd94 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -15,15 +15,29 @@
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/StmtObjC.h"
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/InlineAsm.h"
 using namespace clang;
 using namespace CodeGen;
 
+typedef llvm::PointerIntPair<llvm::Value*,1,bool> TryEmitResult;
+static TryEmitResult
+tryEmitARCRetainScalarExpr(CodeGenFunction &CGF, const Expr *e);
+
+/// Given the address of a variable of pointer type, find the correct
+/// null to store into it.
+static llvm::Constant *getNullForVariable(llvm::Value *addr) {
+  const llvm::Type *type =
+    cast<llvm::PointerType>(addr->getType())->getElementType();
+  return llvm::ConstantPointerNull::get(cast<llvm::PointerType>(type));
+}
+
 /// Emits an instance of NSConstantString representing the object.
 llvm::Value *CodeGenFunction::EmitObjCStringLiteral(const ObjCStringLiteral *E)
 {
@@ -55,6 +69,7 @@ static RValue AdjustRelatedResultType(CodeGenFunction &CGF,
                                       RValue Result) {
   if (!Method)
     return Result;
+
   if (!Method->hasRelatedResultType() ||
       CGF.getContext().hasSameType(E->getType(), Method->getResultType()) ||
       !Result.isScalar())
@@ -71,6 +86,18 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
   // implementation vary between runtimes.  We can get the receiver and
   // arguments in generic code.
 
+  bool isDelegateInit = E->isDelegateInitCall();
+
+  // We don't retain the receiver in delegate init calls, and this is
+  // safe because the receiver value is always loaded from 'self',
+  // which we zero out.  We don't want to Block_copy block receivers,
+  // though.
+  bool retainSelf =
+    (!isDelegateInit &&
+     CGM.getLangOptions().ObjCAutoRefCount &&
+     E->getMethodDecl() &&
+     E->getMethodDecl()->hasAttr<NSConsumesSelfAttr>());
+
   CGObjCRuntime &Runtime = CGM.getObjCRuntime();
   bool isSuperMessage = false;
   bool isClassMessage = false;
@@ -80,8 +107,15 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
   llvm::Value *Receiver = 0;
   switch (E->getReceiverKind()) {
   case ObjCMessageExpr::Instance:
-    Receiver = EmitScalarExpr(E->getInstanceReceiver());
     ReceiverType = E->getInstanceReceiver()->getType();
+    if (retainSelf) {
+      TryEmitResult ter = tryEmitARCRetainScalarExpr(*this,
+                                                   E->getInstanceReceiver());
+      Receiver = ter.getPointer();
+      if (!ter.getInt())
+        Receiver = EmitARCRetainNonBlock(Receiver);
+    } else
+      Receiver = EmitScalarExpr(E->getInstanceReceiver());
     break;
 
   case ObjCMessageExpr::Class: {
@@ -92,6 +126,9 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     assert(OID && "Invalid Objective-C class message send");
     Receiver = Runtime.GetClass(Builder, OID);
     isClassMessage = true;
+
+    if (retainSelf)
+      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
   }
 
@@ -99,6 +136,9 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     ReceiverType = E->getSuperType();
     Receiver = LoadObjCSelf();
     isSuperMessage = true;
+
+    if (retainSelf)
+      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
 
   case ObjCMessageExpr::SuperClass:
@@ -106,14 +146,36 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
     Receiver = LoadObjCSelf();
     isSuperMessage = true;
     isClassMessage = true;
+
+    if (retainSelf)
+      Receiver = EmitARCRetainNonBlock(Receiver);
     break;
   }
 
+  QualType ResultType =
+    E->getMethodDecl() ? E->getMethodDecl()->getResultType() : E->getType();
+
   CallArgList Args;
   EmitCallArgs(Args, E->getMethodDecl(), E->arg_begin(), E->arg_end());
 
-  QualType ResultType =
-    E->getMethodDecl() ? E->getMethodDecl()->getResultType() : E->getType();
+  // For delegate init calls in ARC, do an unsafe store of null into
+  // self.  This represents the call taking direct ownership of that
+  // value.  We have to do this after emitting the other call
+  // arguments because they might also reference self, but we don't
+  // have to worry about any of them modifying self because that would
+  // be an undefined read and write of an object in unordered
+  // expressions.
+  if (isDelegateInit) {
+    assert(getLangOptions().ObjCAutoRefCount &&
+           "delegate init calls should only be marked in ARC");
+
+    // Do an unsafe store of null into self.
+    llvm::Value *selfAddr =
+      LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()];
+    assert(selfAddr && "no self entry for a delegate init call?");
+
+    Builder.CreateStore(getNullForVariable(selfAddr), selfAddr);
+  }
 
   RValue result;
   if (isSuperMessage) {
@@ -134,10 +196,52 @@ RValue CodeGenFunction::EmitObjCMessageExpr(const ObjCMessageExpr *E,
                                          Receiver, Args, OID,
                                          E->getMethodDecl());
   }
-  
+
+  // For delegate init calls in ARC, implicitly store the result of
+  // the call back into self.  This takes ownership of the value.
+  if (isDelegateInit) {
+    llvm::Value *selfAddr =
+      LocalDeclMap[cast<ObjCMethodDecl>(CurCodeDecl)->getSelfDecl()];
+    llvm::Value *newSelf = result.getScalarVal();
+
+    // The delegate return type isn't necessarily a matching type; in
+    // fact, it's quite likely to be 'id'.
+    const llvm::Type *selfTy =
+      cast<llvm::PointerType>(selfAddr->getType())->getElementType();
+    newSelf = Builder.CreateBitCast(newSelf, selfTy);
+
+    Builder.CreateStore(newSelf, selfAddr);
+  }
+
   return AdjustRelatedResultType(*this, E, E->getMethodDecl(), result);
 }
 
+namespace {
+struct FinishARCDealloc : EHScopeStack::Cleanup {
+  void Emit(CodeGenFunction &CGF, bool isForEH) {
+    const ObjCMethodDecl *method = cast<ObjCMethodDecl>(CGF.CurCodeDecl);
+    const ObjCImplementationDecl *impl
+      = cast<ObjCImplementationDecl>(method->getDeclContext());
+    const ObjCInterfaceDecl *iface = impl->getClassInterface();
+    if (!iface->getSuperClass()) return;
+
+    // Call [super dealloc] if we have a superclass.
+    llvm::Value *self = CGF.LoadObjCSelf();
+
+    CallArgList args;
+    CGF.CGM.getObjCRuntime().GenerateMessageSendSuper(CGF, ReturnValueSlot(),
+                                                      CGF.getContext().VoidTy,
+                                                      method->getSelector(),
+                                                      iface,
+                                                      /*is category*/ false,
+                                                      self,
+                                                      /*is class msg*/ false,
+                                                      args,
+                                                      method);
+  }
+};
+}
+
 /// StartObjCMethod - Begin emission of an ObjCMethod. This generates
 /// the LLVM function and sets the other context used by
 /// CodeGenFunction.
@@ -164,8 +268,21 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
   CurGD = OMD;
 
   StartFunction(OMD, OMD->getResultType(), Fn, FI, args, StartLoc);
+
+  // In ARC, certain methods get an extra cleanup.
+  if (CGM.getLangOptions().ObjCAutoRefCount &&
+      OMD->isInstanceMethod() &&
+      OMD->getSelector().isUnarySelector()) {
+    const IdentifierInfo *ident = 
+      OMD->getSelector().getIdentifierInfoForSlot(0);
+    if (ident->isStr("dealloc"))
+      EHStack.pushCleanup<FinishARCDealloc>(getARCCleanupKind());
+  }
 }
 
+static llvm::Value *emitARCRetainLoadOfScalar(CodeGenFunction &CGF,
+                                              LValue lvalue, QualType type);
+
 void CodeGenFunction::GenerateObjCGetterBody(ObjCIvarDecl *Ivar, 
                                              bool IsAtomic, bool IsStrong) {
   LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
@@ -269,6 +386,9 @@ void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
     RV = RValue::get(Builder.CreateBitCast(RV.getScalarVal(),
                                            Types.ConvertType(PD->getType())));
     EmitReturnOfRValue(RV, PD->getType());
+
+    // objc_getProperty does an autorelease, so we should suppress ours.
+    AutoreleaseResult = false;
   } else {
     const llvm::Triple &Triple = getContext().Target.getTriple();
     QualType IVART = Ivar->getType();
@@ -347,17 +467,23 @@ void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
     else {
         LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
                                     Ivar, 0);
-        if (PD->getType()->isReferenceType()) {
-          RValue RV = RValue::get(LV.getAddress());
-          EmitReturnOfRValue(RV, PD->getType());
-        }
-        else {
-          CodeGenTypes &Types = CGM.getTypes();
-          RValue RV = EmitLoadOfLValue(LV, IVART);
-          RV = RValue::get(Builder.CreateBitCast(RV.getScalarVal(),
-                                               Types.ConvertType(PD->getType())));
-          EmitReturnOfRValue(RV, PD->getType());
+        QualType propType = PD->getType();
+
+        llvm::Value *value;
+        if (propType->isReferenceType()) {
+          value = LV.getAddress();
+        } else {
+          // In ARC, we want to emit this retained.
+          if (getLangOptions().ObjCAutoRefCount &&
+              PD->getType()->isObjCRetainableType())
+            value = emitARCRetainLoadOfScalar(*this, LV, IVART);
+          else
+            value = EmitLoadOfLValue(LV, IVART).getScalarVal();
+
+          value = Builder.CreateBitCast(value, ConvertType(propType));
         }
+
+        EmitReturnOfRValue(RValue::get(value), propType);
     }
   }
 
@@ -597,6 +723,11 @@ namespace {
   };
 }
 
+static void pushReleaseForIvar(CodeGenFunction &CGF, ObjCIvarDecl *ivar,
+                               llvm::Value *self);
+static void pushWeakReleaseForIvar(CodeGenFunction &CGF, ObjCIvarDecl *ivar,
+                                   llvm::Value *self);
+
 static void emitCXXDestructMethod(CodeGenFunction &CGF,
                                   ObjCImplementationDecl *impl) {
   CodeGenFunction::RunCleanupsScope scope(CGF);
@@ -631,6 +762,14 @@ static void emitCXXDestructMethod(CodeGenFunction &CGF,
         CGF.EHStack.pushCleanup<CallIvarDtor>(NormalAndEHCleanup,
                                               ivar, self);
       break;
+
+    case QualType::DK_objc_strong_lifetime:
+      pushReleaseForIvar(CGF, ivar, self);
+      break;
+
+    case QualType::DK_objc_weak_lifetime:
+      pushWeakReleaseForIvar(CGF, ivar, self);
+      break;
     }
   }
 
@@ -645,6 +784,9 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
 
   // Emit .cxx_construct.
   if (ctor) {
+    // Suppress the final autorelease in ARC.
+    AutoreleaseResult = false;
+
     llvm::SmallVector<CXXCtorInitializer *, 8> IvarInitializers;
     for (ObjCImplementationDecl::init_const_iterator B = IMP->init_begin(),
            E = IMP->init_end(); B != E; ++B) {
@@ -747,6 +889,16 @@ RValue CodeGenFunction::EmitLoadOfPropertyRefLValue(LValue LV,
 
   llvm::Value *Receiver = LV.getPropertyRefBaseAddr();
 
+  if (CGM.getLangOptions().ObjCAutoRefCount) {
+    QualType receiverType;
+    if (E->isSuperReceiver())
+      receiverType = E->getSuperReceiverType();
+    else if (E->isClassReceiver())
+      receiverType = getContext().getObjCClassType();
+    else
+      receiverType = E->getBase()->getType();
+  }
+
   // Accesses to 'super' follow a different code path.
   if (E->isSuperReceiver())
     return AdjustRelatedResultType(*this, E, method,
@@ -757,9 +909,9 @@ RValue CodeGenFunction::EmitLoadOfPropertyRefLValue(LValue LV,
   const ObjCInterfaceDecl *ReceiverClass
     = (E->isClassReceiver() ? E->getClassReceiver() : 0);
   return AdjustRelatedResultType(*this, E, method,
-                                 CGM.getObjCRuntime().
-                 GenerateMessageSend(*this, Return, ResultType, S,
-                                     Receiver, CallArgList(), ReceiverClass));
+          CGM.getObjCRuntime().
+             GenerateMessageSend(*this, Return, ResultType, S,
+                                 Receiver, CallArgList(), ReceiverClass));
 }
 
 void CodeGenFunction::EmitStoreThroughPropertyRefLValue(RValue Src,
@@ -1072,4 +1224,1197 @@ void CodeGenFunction::EmitObjCAtSynchronizedStmt(
   CGM.getObjCRuntime().EmitSynchronizedStmt(*this, S);
 }
 
+/// Produce the code for a CK_ObjCProduceObject.  Just does a
+/// primitive retain.
+llvm::Value *CodeGenFunction::EmitObjCProduceObject(QualType type,
+                                                    llvm::Value *value) {
+  return EmitARCRetain(type, value);
+}
+
+namespace {
+  struct CallObjCRelease : EHScopeStack::Cleanup {
+    CallObjCRelease(QualType type, llvm::Value *ptr, llvm::Value *condition)
+      : type(type), ptr(ptr), condition(condition) {}
+    QualType type;
+    llvm::Value *ptr;
+    llvm::Value *condition;
+
+    void Emit(CodeGenFunction &CGF, bool forEH) {
+      llvm::Value *object;
+
+      // If we're in a conditional branch, we had to stash away in an
+      // alloca the pointer to be released.
+      llvm::BasicBlock *cont = 0;
+      if (condition) {
+        llvm::BasicBlock *release = CGF.createBasicBlock("release.yes");
+        cont = CGF.createBasicBlock("release.cont");
+
+        llvm::Value *cond = CGF.Builder.CreateLoad(condition);
+        CGF.Builder.CreateCondBr(cond, release, cont);
+        CGF.EmitBlock(release);
+        object = CGF.Builder.CreateLoad(ptr);
+      } else {
+        object = ptr;
+      }
+
+      CGF.EmitARCRelease(object, /*precise*/ true);
+
+      if (cont) CGF.EmitBlock(cont);
+    }
+  };
+}
+
+/// Produce the code for a CK_ObjCConsumeObject.  Does a primitive
+/// release at the end of the full-expression.
+llvm::Value *CodeGenFunction::EmitObjCConsumeObject(QualType type,
+                                                    llvm::Value *object) {
+  // If we're in a conditional branch, we need to make the cleanup
+  // conditional.  FIXME: this really needs to be supported by the
+  // environment.
+  llvm::AllocaInst *cond;
+  llvm::Value *ptr;
+  if (isInConditionalBranch()) {
+    cond = CreateTempAlloca(Builder.getInt1Ty(), "release.cond");
+    ptr = CreateTempAlloca(object->getType(), "release.value");
+
+    // The alloca is false until we get here.
+    // FIXME: er. doesn't this need to be set at the start of the condition?
+    InitTempAlloca(cond, Builder.getFalse());
+
+    // Then it turns true.
+    Builder.CreateStore(Builder.getTrue(), cond);
+    Builder.CreateStore(object, ptr);
+  } else {
+    cond = 0;
+    ptr = object;
+  }
+
+  EHStack.pushCleanup<CallObjCRelease>(getARCCleanupKind(), type, ptr, cond);
+  return object;
+}
+
+llvm::Value *CodeGenFunction::EmitObjCExtendObjectLifetime(QualType type,
+                                                           llvm::Value *value) {
+  return EmitARCRetainAutorelease(type, value);
+}
+
+
+static llvm::Constant *createARCRuntimeFunction(CodeGenModule &CGM,
+                                                const llvm::FunctionType *type,
+                                                llvm::StringRef fnName) {
+  llvm::Constant *fn = CGM.CreateRuntimeFunction(type, fnName);
+
+  // In -fobjc-no-arc-runtime, emit weak references to the runtime
+  // support library.
+  if (CGM.getLangOptions().ObjCNoAutoRefCountRuntime)
+    if (llvm::Function *f = dyn_cast<llvm::Function>(fn))
+      f->setLinkage(llvm::Function::ExternalWeakLinkage);
+
+  return fn;
+}
+
+/// Perform an operation having the signature
+///   i8* (i8*)
+/// where a null input causes a no-op and returns null.
+static llvm::Value *emitARCValueOperation(CodeGenFunction &CGF,
+                                          llvm::Value *value,
+                                          llvm::Constant *&fn,
+                                          llvm::StringRef fnName) {
+  if (isa<llvm::ConstantPointerNull>(value)) return value;
+
+  if (!fn) {
+    std::vector<const llvm::Type*> args(1, CGF.Int8PtrTy);
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(CGF.Int8PtrTy, args, false);
+    fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
+  }
+
+  // Cast the argument to 'id'.
+  const llvm::Type *origType = value->getType();
+  value = CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy);
+
+  // Call the function.
+  llvm::CallInst *call = CGF.Builder.CreateCall(fn, value);
+  call->setDoesNotThrow();
+
+  // Cast the result back to the original type.
+  return CGF.Builder.CreateBitCast(call, origType);
+}
+
+/// Perform an operation having the following signature:
+///   i8* (i8**)
+static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF,
+                                         llvm::Value *addr,
+                                         llvm::Constant *&fn,
+                                         llvm::StringRef fnName) {
+  if (!fn) {
+    std::vector<const llvm::Type*> args(1, CGF.Int8PtrPtrTy);
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(CGF.Int8PtrTy, args, false);
+    fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
+  }
+
+  // Cast the argument to 'id*'.
+  const llvm::Type *origType = addr->getType();
+  addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
+
+  // Call the function.
+  llvm::CallInst *call = CGF.Builder.CreateCall(fn, addr);
+  call->setDoesNotThrow();
+
+  // Cast the result back to a dereference of the original type.
+  llvm::Value *result = call;
+  if (origType != CGF.Int8PtrPtrTy)
+    result = CGF.Builder.CreateBitCast(result,
+                        cast<llvm::PointerType>(origType)->getElementType());
+
+  return result;
+}
+
+/// Perform an operation having the following signature:
+///   i8* (i8**, i8*)
+static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF,
+                                          llvm::Value *addr,
+                                          llvm::Value *value,
+                                          llvm::Constant *&fn,
+                                          llvm::StringRef fnName,
+                                          bool ignored) {
+  assert(cast<llvm::PointerType>(addr->getType())->getElementType()
+           == value->getType());
+
+  if (!fn) {
+    std::vector<const llvm::Type*> argTypes(2);
+    argTypes[0] = CGF.Int8PtrPtrTy;
+    argTypes[1] = CGF.Int8PtrTy;
+
+    const llvm::FunctionType *fnType
+      = llvm::FunctionType::get(CGF.Int8PtrTy, argTypes, false);
+    fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
+  }
+
+  const llvm::Type *origType = value->getType();
+
+  addr = CGF.Builder.CreateBitCast(addr, CGF.Int8PtrPtrTy);
+  value = CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy);
+    
+  llvm::CallInst *result = CGF.Builder.CreateCall2(fn, addr, value);
+  result->setDoesNotThrow();
+
+  if (ignored) return 0;
+
+  return CGF.Builder.CreateBitCast(result, origType);
+}
+
+/// Perform an operation having the following signature:
+///   void (i8**, i8**)
+static void emitARCCopyOperation(CodeGenFunction &CGF,
+                                 llvm::Value *dst,
+                                 llvm::Value *src,
+                                 llvm::Constant *&fn,
+                                 llvm::StringRef fnName) {
+  assert(dst->getType() == src->getType());
+
+  if (!fn) {
+    std::vector<const llvm::Type*> argTypes(2, CGF.Int8PtrPtrTy);
+    const llvm::FunctionType *fnType
+      = llvm::FunctionType::get(CGF.Builder.getVoidTy(), argTypes, false);
+    fn = createARCRuntimeFunction(CGF.CGM, fnType, fnName);
+  }
+
+  dst = CGF.Builder.CreateBitCast(dst, CGF.Int8PtrPtrTy);
+  src = CGF.Builder.CreateBitCast(src, CGF.Int8PtrPtrTy);
+    
+  llvm::CallInst *result = CGF.Builder.CreateCall2(fn, dst, src);
+  result->setDoesNotThrow();
+}
+
+/// Produce the code to do a retain.  Based on the type, calls one of:
+///   call i8* @objc_retain(i8* %value)
+///   call i8* @objc_retainBlock(i8* %value)
+llvm::Value *CodeGenFunction::EmitARCRetain(QualType type, llvm::Value *value) {
+  if (type->isBlockPointerType())
+    return EmitARCRetainBlock(value);
+  else
+    return EmitARCRetainNonBlock(value);
+}
+
+/// Retain the given object, with normal retain semantics.
+///   call i8* @objc_retain(i8* %value)
+llvm::Value *CodeGenFunction::EmitARCRetainNonBlock(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                               CGM.getARCEntrypoints().objc_retain,
+                               "objc_retain");
+}
+
+/// Retain the given block, with _Block_copy semantics.
+///   call i8* @objc_retainBlock(i8* %value)
+llvm::Value *CodeGenFunction::EmitARCRetainBlock(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                               CGM.getARCEntrypoints().objc_retainBlock,
+                               "objc_retainBlock");
+}
+
+/// Retain the given object which is the result of a function call.
+///   call i8* @objc_retainAutoreleasedReturnValue(i8* %value)
+///
+/// Yes, this function name is one character away from a different
+/// call with completely different semantics.
+llvm::Value *
+CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) {
+  // Fetch the void(void) inline asm which marks that we're going to
+  // retain the autoreleased return value.
+  llvm::InlineAsm *&marker
+    = CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker;
+  if (!marker) {
+    llvm::StringRef assembly
+      = CGM.getTargetCodeGenInfo()
+           .getARCRetainAutoreleasedReturnValueMarker();
+
+    // If we have an empty assembly string, there's nothing to do.
+    if (assembly.empty()) {
+
+    // Otherwise, at -O0, build an inline asm that we're going to call
+    // in a moment.
+    } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
+      llvm::FunctionType *type =
+        llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
+                                /*variadic*/ false);
+      
+      marker = llvm::InlineAsm::get(type, assembly, "", /*sideeffects*/ true);
+
+    // If we're at -O1 and above, we don't want to litter the code
+    // with this marker yet, so leave a breadcrumb for the ARC
+    // optimizer to pick up.
+    } else {
+      llvm::NamedMDNode *metadata =
+        CGM.getModule().getOrInsertNamedMetadata(
+                            "clang.arc.retainAutoreleasedReturnValueMarker");
+      assert(metadata->getNumOperands() <= 1);
+      if (metadata->getNumOperands() == 0) {
+        llvm::Value *string = llvm::MDString::get(getLLVMContext(), assembly);
+        llvm::Value *args[] = { string };
+        metadata->addOperand(llvm::MDNode::get(getLLVMContext(), args));
+      }
+    }
+  }
+
+  // Call the marker asm if we made one, which we do only at -O0.
+  if (marker) Builder.CreateCall(marker);
+
+  return emitARCValueOperation(*this, value,
+                     CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue,
+                               "objc_retainAutoreleasedReturnValue");
+}
+
+/// Release the given object.
+///   call void @objc_release(i8* %value)
+void CodeGenFunction::EmitARCRelease(llvm::Value *value, bool precise) {
+  if (isa<llvm::ConstantPointerNull>(value)) return;
+
+  llvm::Constant *&fn = CGM.getARCEntrypoints().objc_release;
+  if (!fn) {
+    std::vector<const llvm::Type*> args(1, Int8PtrTy);
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(Builder.getVoidTy(), args, false);
+    fn = createARCRuntimeFunction(CGM, fnType, "objc_release");
+  }
+
+  // Cast the argument to 'id'.
+  value = Builder.CreateBitCast(value, Int8PtrTy);
+
+  // Call objc_release.
+  llvm::CallInst *call = Builder.CreateCall(fn, value);
+  call->setDoesNotThrow();
+
+  if (!precise) {
+    llvm::SmallVector<llvm::Value*,1> args;
+    call->setMetadata("clang.imprecise_release",
+                      llvm::MDNode::get(Builder.getContext(), args));
+  }
+}
+
+/// Store into a strong object.  Always calls this:
+///   call void @objc_storeStrong(i8** %addr, i8* %value)
+llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr,
+                                                     llvm::Value *value,
+                                                     bool ignored) {
+  assert(cast<llvm::PointerType>(addr->getType())->getElementType()
+           == value->getType());
+
+  llvm::Constant *&fn = CGM.getARCEntrypoints().objc_storeStrong;
+  if (!fn) {
+    const llvm::Type *argTypes[] = { Int8PtrPtrTy, Int8PtrTy };
+    const llvm::FunctionType *fnType
+      = llvm::FunctionType::get(Builder.getVoidTy(), argTypes, false);
+    fn = createARCRuntimeFunction(CGM, fnType, "objc_storeStrong");
+  }
+
+  addr = Builder.CreateBitCast(addr, Int8PtrPtrTy);
+  llvm::Value *castValue = Builder.CreateBitCast(value, Int8PtrTy);
+  
+  Builder.CreateCall2(fn, addr, castValue)->setDoesNotThrow();
+
+  if (ignored) return 0;
+  return value;
+}
+
+/// Store into a strong object.  Sometimes calls this:
+///   call void @objc_storeStrong(i8** %addr, i8* %value)
+/// Other times, breaks it down into components.
+llvm::Value *CodeGenFunction::EmitARCStoreStrong(LValue dst, QualType type,
+                                                 llvm::Value *newValue,
+                                                 bool ignored) {
+  bool isBlock = type->isBlockPointerType();
+
+  // Use a store barrier at -O0 unless this is a block type or the
+  // lvalue is inadequately aligned.
+  if (shouldUseFusedARCCalls() &&
+      !isBlock &&
+      !(dst.getAlignment() && dst.getAlignment() < PointerAlignInBytes)) {
+    return EmitARCStoreStrongCall(dst.getAddress(), newValue, ignored);
+  }
+
+  // Otherwise, split it out.
+
+  // Retain the new value.
+  newValue = EmitARCRetain(type, newValue);
+
+  // Read the old value.
+  llvm::Value *oldValue =
+    EmitLoadOfScalar(dst.getAddress(), dst.isVolatileQualified(),
+                     dst.getAlignment(), type, dst.getTBAAInfo());
+
+  // Store.  We do this before the release so that any deallocs won't
+  // see the old value.
+  EmitStoreOfScalar(newValue, dst.getAddress(),
+                    dst.isVolatileQualified(), dst.getAlignment(),
+                    type, dst.getTBAAInfo());
+
+  // Finally, release the old value.
+  EmitARCRelease(oldValue, /*precise*/ false);
+
+  return newValue;
+}
+
+/// Autorelease the given object.
+///   call i8* @objc_autorelease(i8* %value)
+llvm::Value *CodeGenFunction::EmitARCAutorelease(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                               CGM.getARCEntrypoints().objc_autorelease,
+                               "objc_autorelease");
+}
+
+/// Autorelease the given object.
+///   call i8* @objc_autoreleaseReturnValue(i8* %value)
+llvm::Value *
+CodeGenFunction::EmitARCAutoreleaseReturnValue(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                            CGM.getARCEntrypoints().objc_autoreleaseReturnValue,
+                               "objc_autoreleaseReturnValue");
+}
+
+/// Do a fused retain/autorelease of the given object.
+///   call i8* @objc_retainAutoreleaseReturnValue(i8* %value)
+llvm::Value *
+CodeGenFunction::EmitARCRetainAutoreleaseReturnValue(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                     CGM.getARCEntrypoints().objc_retainAutoreleaseReturnValue,
+                               "objc_retainAutoreleaseReturnValue");
+}
+
+/// Do a fused retain/autorelease of the given object.
+///   call i8* @objc_retainAutorelease(i8* %value)
+/// or
+///   %retain = call i8* @objc_retainBlock(i8* %value)
+///   call i8* @objc_autorelease(i8* %retain)
+llvm::Value *CodeGenFunction::EmitARCRetainAutorelease(QualType type,
+                                                       llvm::Value *value) {
+  if (!type->isBlockPointerType())
+    return EmitARCRetainAutoreleaseNonBlock(value);
+
+  if (isa<llvm::ConstantPointerNull>(value)) return value;
+
+  const llvm::Type *origType = value->getType();
+  value = Builder.CreateBitCast(value, Int8PtrTy);
+  value = EmitARCRetainBlock(value);
+  value = EmitARCAutorelease(value);
+  return Builder.CreateBitCast(value, origType);
+}
+
+/// Do a fused retain/autorelease of the given object.
+///   call i8* @objc_retainAutorelease(i8* %value)
+llvm::Value *
+CodeGenFunction::EmitARCRetainAutoreleaseNonBlock(llvm::Value *value) {
+  return emitARCValueOperation(*this, value,
+                               CGM.getARCEntrypoints().objc_retainAutorelease,
+                               "objc_retainAutorelease");
+}
+
+/// i8* @objc_loadWeak(i8** %addr)
+/// Essentially objc_autorelease(objc_loadWeakRetained(addr)).
+llvm::Value *CodeGenFunction::EmitARCLoadWeak(llvm::Value *addr) {
+  return emitARCLoadOperation(*this, addr,
+                              CGM.getARCEntrypoints().objc_loadWeak,
+                              "objc_loadWeak");
+}
+
+/// i8* @objc_loadWeakRetained(i8** %addr)
+llvm::Value *CodeGenFunction::EmitARCLoadWeakRetained(llvm::Value *addr) {
+  return emitARCLoadOperation(*this, addr,
+                              CGM.getARCEntrypoints().objc_loadWeakRetained,
+                              "objc_loadWeakRetained");
+}
+
+/// i8* @objc_storeWeak(i8** %addr, i8* %value)
+/// Returns %value.
+llvm::Value *CodeGenFunction::EmitARCStoreWeak(llvm::Value *addr,
+                                               llvm::Value *value,
+                                               bool ignored) {
+  return emitARCStoreOperation(*this, addr, value,
+                               CGM.getARCEntrypoints().objc_storeWeak,
+                               "objc_storeWeak", ignored);
+}
+
+/// i8* @objc_initWeak(i8** %addr, i8* %value)
+/// Returns %value.  %addr is known to not have a current weak entry.
+/// Essentially equivalent to:
+///   *addr = nil; objc_storeWeak(addr, value);
+void CodeGenFunction::EmitARCInitWeak(llvm::Value *addr, llvm::Value *value) {
+  // If we're initializing to null, just write null to memory; no need
+  // to get the runtime involved.  But don't do this if optimization
+  // is enabled, because accounting for this would make the optimizer
+  // much more complicated.
+  if (isa<llvm::ConstantPointerNull>(value) &&
+      CGM.getCodeGenOpts().OptimizationLevel == 0) {
+    Builder.CreateStore(value, addr);
+    return;
+  }
+
+  emitARCStoreOperation(*this, addr, value,
+                        CGM.getARCEntrypoints().objc_initWeak,
+                        "objc_initWeak", /*ignored*/ true);
+}
+
+/// void @objc_destroyWeak(i8** %addr)
+/// Essentially objc_storeWeak(addr, nil).
+void CodeGenFunction::EmitARCDestroyWeak(llvm::Value *addr) {
+  llvm::Constant *&fn = CGM.getARCEntrypoints().objc_destroyWeak;
+  if (!fn) {
+    std::vector<const llvm::Type*> args(1, Int8PtrPtrTy);
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(Builder.getVoidTy(), args, false);
+    fn = createARCRuntimeFunction(CGM, fnType, "objc_destroyWeak");
+  }
+
+  // Cast the argument to 'id*'.
+  addr = Builder.CreateBitCast(addr, Int8PtrPtrTy);
+
+  llvm::CallInst *call = Builder.CreateCall(fn, addr);
+  call->setDoesNotThrow();
+}
+
+/// void @objc_moveWeak(i8** %dest, i8** %src)
+/// Disregards the current value in %dest.  Leaves %src pointing to nothing.
+/// Essentially (objc_copyWeak(dest, src), objc_destroyWeak(src)).
+void CodeGenFunction::EmitARCMoveWeak(llvm::Value *dst, llvm::Value *src) {
+  emitARCCopyOperation(*this, dst, src,
+                       CGM.getARCEntrypoints().objc_moveWeak,
+                       "objc_moveWeak");
+}
+
+/// void @objc_copyWeak(i8** %dest, i8** %src)
+/// Disregards the current value in %dest.  Essentially
+///   objc_release(objc_initWeak(dest, objc_readWeakRetained(src)))
+void CodeGenFunction::EmitARCCopyWeak(llvm::Value *dst, llvm::Value *src) {
+  emitARCCopyOperation(*this, dst, src,
+                       CGM.getARCEntrypoints().objc_copyWeak,
+                       "objc_copyWeak");
+}
+
+/// Produce the code to do a objc_autoreleasepool_push.
+///   call i8* @objc_autoreleasePoolPush(void)
+llvm::Value *CodeGenFunction::EmitObjCAutoreleasePoolPush() {
+  llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPush;
+  if (!fn) {
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(Int8PtrTy, false);
+    fn = createARCRuntimeFunction(CGM, fnType, "objc_autoreleasePoolPush");
+  }
+
+  llvm::CallInst *call = Builder.CreateCall(fn);
+  call->setDoesNotThrow();
+
+  return call;
+}
+
+/// Produce the code to do a primitive release.
+///   call void @objc_autoreleasePoolPop(i8* %ptr)
+void CodeGenFunction::EmitObjCAutoreleasePoolPop(llvm::Value *value) {
+  assert(value->getType() == Int8PtrTy);
+
+  llvm::Constant *&fn = CGM.getRREntrypoints().objc_autoreleasePoolPop;
+  if (!fn) {
+    std::vector<const llvm::Type*> args(1, Int8PtrTy);
+    const llvm::FunctionType *fnType =
+      llvm::FunctionType::get(Builder.getVoidTy(), args, false);
+
+    // We don't want to use a weak import here; instead we should not
+    // fall into this path.
+    fn = createARCRuntimeFunction(CGM, fnType, "objc_autoreleasePoolPop");
+  }
+
+  llvm::CallInst *call = Builder.CreateCall(fn, value);
+  call->setDoesNotThrow();
+}
+
+/// Produce the code to do an MRR version objc_autoreleasepool_push.
+/// Which is: [[NSAutoreleasePool alloc] init];
+/// Where alloc is declared as: + (id) alloc; in NSAutoreleasePool class.
+/// init is declared as: - (id) init; in its NSObject super class.
+///
+llvm::Value *CodeGenFunction::EmitObjCMRRAutoreleasePoolPush() {
+  CGObjCRuntime &Runtime = CGM.getObjCRuntime();
+  llvm::Value *Receiver = Runtime.EmitNSAutoreleasePoolClassRef(Builder);
+  // [NSAutoreleasePool alloc]
+  IdentifierInfo *II = &CGM.getContext().Idents.get("alloc");
+  Selector AllocSel = getContext().Selectors.getSelector(0, &II);
+  CallArgList Args;
+  RValue AllocRV =  
+    Runtime.GenerateMessageSend(*this, ReturnValueSlot(), 
+                                getContext().getObjCIdType(),
+                                AllocSel, Receiver, Args); 
+
+  // [Receiver init]
+  Receiver = AllocRV.getScalarVal();
+  II = &CGM.getContext().Idents.get("init");
+  Selector InitSel = getContext().Selectors.getSelector(0, &II);
+  RValue InitRV =
+    Runtime.GenerateMessageSend(*this, ReturnValueSlot(),
+                                getContext().getObjCIdType(),
+                                InitSel, Receiver, Args); 
+  return InitRV.getScalarVal();
+}
+
+/// Produce the code to do a primitive release.
+/// [tmp drain];
+void CodeGenFunction::EmitObjCMRRAutoreleasePoolPop(llvm::Value *Arg) {
+  IdentifierInfo *II = &CGM.getContext().Idents.get("drain");
+  Selector DrainSel = getContext().Selectors.getSelector(0, &II);
+  CallArgList Args;
+  CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
+                              getContext().VoidTy, DrainSel, Arg, Args); 
+}
+
+namespace {
+  struct ObjCReleasingCleanup : EHScopeStack::Cleanup {
+  private:
+    QualType type;
+    llvm::Value *addr;
+
+  protected:
+    ObjCReleasingCleanup(QualType type, llvm::Value *addr)
+      : type(type), addr(addr) {}
+
+    virtual llvm::Value *getAddress(CodeGenFunction &CGF,
+                                    llvm::Value *addr) {
+      return addr;
+    }
+
+    virtual void release(CodeGenFunction &CGF,
+                         QualType type,
+                         llvm::Value *addr) = 0;
+
+  public:
+    void Emit(CodeGenFunction &CGF, bool isForEH) {
+      const ArrayType *arrayType = CGF.getContext().getAsArrayType(type);
+
+      llvm::Value *addr = getAddress(CGF, this->addr);
+
+      // If we don't have an array type, this is easy.
+      if (!arrayType)
+        return release(CGF, type, addr);
+
+      llvm::Value *begin = addr;
+      QualType baseType;
+
+      // Otherwise, this is more painful.
+      llvm::Value *count = emitArrayLength(CGF, arrayType, baseType,
+                                           begin);
+
+      assert(baseType == CGF.getContext().getBaseElementType(arrayType));
+
+      llvm::BasicBlock