constexpr irgen: Add irgen support for APValue::Struct, APValue::Union,

APValue::Array and APValue::MemberPointer. All APValue values can now be emitted
as constants.

Add new CGCXXABI entry point for emitting an APValue MemberPointer. The other
entrypoints dealing with constant member pointers are no longer necessary and
will be removed in a later change.

Switch codegen from using EvaluateAsRValue/EvaluateAsLValue to
VarDecl::evaluateValue. This performs caching and deals with the nasty cases in
C++11 where a non-const object's initializer can refer indirectly to
previously-initialized fields within the same object.

Building the intermediate APValue object incurs a measurable performance hit on
pathological testcases with huge initializer lists, so we continue to build IR
directly from the Expr nodes for array and record types outside of C++11.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@148178 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 400 insertions, 223 deletions

diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 8fc4344201..e6ecb6a88b 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -215,6 +215,7 @@ static bool isSafeForCXXConstantCapture(QualType type) {
 /// acceptable because we make no promises about address stability of
 /// captured variables.
 static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
+                                            CodeGenFunction *CGF,
                                             const VarDecl *var) {
   QualType type = var->getType();
 
@@ -235,7 +236,7 @@ static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
   const Expr *init = var->getInit();
   if (!init) return 0;
 
-  return CGM.EmitConstantExpr(init, var->getType());
+  return CGM.EmitConstantInit(*var, CGF);
 }
 
 /// Get the low bit of a nonzero character count.  This is the
@@ -278,7 +279,8 @@ static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
 
 /// Compute the layout of the given block.  Attempts to lay the block
 /// out with minimal space requirements.
-static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
+static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
+                             CGBlockInfo &info) {
   ASTContext &C = CGM.getContext();
   const BlockDecl *block = info.getBlockDecl();
 
@@ -342,7 +344,7 @@ static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
 
     // Otherwise, build a layout chunk with the size and alignment of
     // the declaration.
-    if (llvm::Constant *constant = tryCaptureAsConstant(CGM, variable)) {
+    if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, variable)) {
       info.Captures[variable] = CGBlockInfo::Capture::makeConstant(constant);
       continue;
     }
@@ -497,7 +499,7 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) {
 
   // Compute information about the layout, etc., of this block,
   // pushing cleanups as necessary.
-  computeBlockInfo(CGF.CGM, blockInfo);
+  computeBlockInfo(CGF.CGM, &CGF, blockInfo);
 
   // Nothing else to do if it can be global.
   if (blockInfo.CanBeGlobal) return;
@@ -604,7 +606,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
   // layout for it.
   if (!blockExpr->getBlockDecl()->hasCaptures()) {
     CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
-    computeBlockInfo(CGM, blockInfo);
+    computeBlockInfo(CGM, this, blockInfo);
     blockInfo.BlockExpression = blockExpr;
     return EmitBlockLiteral(blockInfo);
   }
@@ -911,7 +913,7 @@ CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *blockExpr,
   blockInfo.BlockExpression = blockExpr;
 
   // Compute information about the layout, etc., of this block.
-  computeBlockInfo(*this, blockInfo);
+  computeBlockInfo(*this, 0, blockInfo);
 
   // Using that metadata, generate the actual block function.
   llvm::Constant *blockFn;
diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp
index 248448ccdc..702029dd34 100644
--- a/lib/CodeGen/CGCXXABI.cpp
+++ b/lib/CodeGen/CGCXXABI.cpp
@@ -110,6 +110,10 @@ llvm::Constant *CGCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
   return GetBogusMemberPointer(CGM, QualType(MPT, 0));
 }
 
+llvm::Constant *CGCXXABI::EmitMemberPointer(const APValue &MP, QualType MPT) {
+  return GetBogusMemberPointer(CGM, MPT);
+}
+
 bool CGCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
   // Fake answer.
   return true;
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index c2abf35832..e86e639627 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -125,6 +125,9 @@ public:
   virtual llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
                                                 CharUnits offset);
 
+  /// Create a member pointer for the given member pointer constant.
+  virtual llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT);
+
   /// Emit a comparison between two member pointers.  Returns an i1.
   virtual llvm::Value *
   EmitMemberPointerComparison(CodeGenFunction &CGF,
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 31e4bcf5f8..ba89d3756e 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -1112,9 +1112,8 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
   for (RecordDecl::decl_iterator I = RD->decls_begin(), E = RD->decls_end();
        I != E; ++I)
     if (const VarDecl *V = dyn_cast<VarDecl>(*I)) {
-      llvm::SmallVector<PartialDiagnosticAt, 8> Notes;
-      if (V->getInit() && V->evaluateValue(Notes)) {
-        APValue *Value = V->getEvaluatedValue();
+      if (V->getInit()) {
+        const APValue *Value = V->evaluateValue();
         if (Value && Value->isInt()) {
           llvm::ConstantInt *CI
             = llvm::ConstantInt::get(CGM.getLLVMContext(), Value->getInt());
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 05087f0e95..dd7cdb69a8 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -203,7 +203,7 @@ CodeGenFunction::CreateStaticVarDecl(const VarDecl &D,
 llvm::GlobalVariable *
 CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
                                                llvm::GlobalVariable *GV) {
-  llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(), D.getType(), this);
+  llvm::Constant *Init = CGM.EmitConstantInit(D, this);
 
   // If constant emission failed, then this should be a C++ static
   // initializer.
@@ -972,7 +972,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
   llvm::Constant *constant = 0;
   if (emission.IsConstantAggregate) {
     assert(!capturedByInit && "constant init contains a capturing block?");
-    constant = CGM.EmitConstantExpr(D.getInit(), type, this);
+    constant = CGM.EmitConstantInit(D, this);
   }
 
   if (!constant) {
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index a8399d719f..f27586da04 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -44,14 +44,16 @@ class ConstStructBuilder {
 public:
   static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF,
                                      InitListExpr *ILE);
-  
-private:  
+  static llvm::Constant *BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF,
+                                     const APValue &Value, QualType ValTy);
+
+private:
   ConstStructBuilder(CodeGenModule &CGM, CodeGenFunction *CGF)
     : CGM(CGM), CGF(CGF), Packed(false), 
     NextFieldOffsetInChars(CharUnits::Zero()),
     LLVMStructAlignment(CharUnits::One()) { }
 
-  bool AppendField(const FieldDecl *Field, uint64_t FieldOffset,
+  void AppendField(const FieldDecl *Field, uint64_t FieldOffset,
                    llvm::Constant *InitExpr);
 
   void AppendBitField(const FieldDecl *Field, uint64_t FieldOffset,
@@ -62,8 +64,10 @@ private:
   void AppendTailPadding(CharUnits RecordSize);
 
   void ConvertStructToPacked();
-                              
+
   bool Build(InitListExpr *ILE);
+  void Build(const APValue &Val, QualType ValTy);
+  llvm::Constant *Finalize(QualType Ty);
 
   CharUnits getAlignment(const llvm::Constant *C) const {
     if (Packed)  return CharUnits::One();
@@ -77,7 +81,7 @@ private:
   }
 };
 
-bool ConstStructBuilder::
+void ConstStructBuilder::
 AppendField(const FieldDecl *Field, uint64_t FieldOffset,
             llvm::Constant *InitCst) {
 
@@ -99,14 +103,13 @@ AppendField(const FieldDecl *Field, uint64_t FieldOffset,
 
     // Convert the struct to a packed struct.
     ConvertStructToPacked();
-    
+
     AlignedNextFieldOffsetInChars = NextFieldOffsetInChars;
   }
 
   if (AlignedNextFieldOffsetInChars < FieldOffsetInChars) {
     // We need to append padding.
-    AppendPadding(
-        FieldOffsetInChars - NextFieldOffsetInChars);
+    AppendPadding(FieldOffsetInChars - NextFieldOffsetInChars);
 
     assert(NextFieldOffsetInChars == FieldOffsetInChars &&
            "Did not add enough padding!");
@@ -118,14 +121,12 @@ AppendField(const FieldDecl *Field, uint64_t FieldOffset,
   Elements.push_back(InitCst);
   NextFieldOffsetInChars = AlignedNextFieldOffsetInChars +
                            getSizeInChars(InitCst);
-  
+
   if (Packed)
-    assert(LLVMStructAlignment == CharUnits::One() && 
+    assert(LLVMStructAlignment == CharUnits::One() &&
            "Packed struct not byte-aligned!");
   else
     LLVMStructAlignment = std::max(LLVMStructAlignment, FieldAlignment);
-
-  return true;
 }
 
 void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
@@ -382,8 +383,7 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
     
     if (!Field->isBitField()) {
       // Handle non-bitfield members.
-      if (!AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit))
-        return false;
+      AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit);
     } else {
       // Otherwise we have a bitfield.
       AppendBitField(*Field, Layout.getFieldOffset(FieldNo),
@@ -391,6 +391,77 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
     }
   }
 
+  return true;
+}
+
+void ConstStructBuilder::Build(const APValue &Val, QualType ValTy) {
+  RecordDecl *RD = ValTy->getAs<RecordType>()->getDecl();
+  const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+
+  if (CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(RD)) {
+    unsigned BaseNo = 0;
+    for (CXXRecordDecl::base_class_iterator Base = CD->bases_begin(),
+         BaseEnd = CD->bases_end(); Base != BaseEnd; ++Base, ++BaseNo) {
+      // Build the base class subobject at the appropriately-offset location
+      // within this object.
+      const CXXRecordDecl *BD = Base->getType()->getAsCXXRecordDecl();
+      CharUnits BaseOffset = Layout.getBaseClassOffset(BD);
+      NextFieldOffsetInChars -= BaseOffset;
+
+      Build(Val.getStructBase(BaseNo), Base->getType());
+
+      NextFieldOffsetInChars += BaseOffset;
+    }
+  }
+
+  unsigned FieldNo = 0;
+  const FieldDecl *LastFD = 0;
+  bool IsMsStruct = RD->hasAttr<MsStructAttr>();
+
+  for (RecordDecl::field_iterator Field = RD->field_begin(),
+       FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
+    if (IsMsStruct) {
+      // Zero-length bitfields following non-bitfield members are
+      // ignored:
+      if (CGM.getContext().ZeroBitfieldFollowsNonBitfield((*Field), LastFD)) {
+        --FieldNo;
+        continue;
+      }
+      LastFD = (*Field);
+    }
+
+    // If this is a union, skip all the fields that aren't being initialized.
+    if (RD->isUnion() && Val.getUnionField() != *Field)
+      continue;
+
+    // Don't emit anonymous bitfields, they just affect layout.
+    if (Field->isUnnamedBitfield()) {
+      LastFD = (*Field);
+      continue;
+    }
+
+    // Emit the value of the initializer.
+    const APValue &FieldValue =
+      RD->isUnion() ? Val.getUnionValue() : Val.getStructField(FieldNo);
+    llvm::Constant *EltInit =
+      CGM.EmitConstantValue(FieldValue, Field->getType(), CGF);
+    assert(EltInit && "EmitConstantValue can't fail");
+
+    if (!Field->isBitField()) {
+      // Handle non-bitfield members.
+      AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit);
+    } else {
+      // Otherwise we have a bitfield.
+      AppendBitField(*Field, Layout.getFieldOffset(FieldNo),
+                     cast<llvm::ConstantInt>(EltInit));
+    }
+  }
+}
+
+llvm::Constant *ConstStructBuilder::Finalize(QualType Ty) {
+  RecordDecl *RD = Ty->getAs<RecordType>()->getDecl();
+  const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
+
   CharUnits LayoutSizeInChars = Layout.getSize();
 
   if (NextFieldOffsetInChars > LayoutSizeInChars) {
@@ -398,62 +469,69 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
     // we must have a flexible array member at the end.
     assert(RD->hasFlexibleArrayMember() &&
            "Must have flexible array member if struct is bigger than type!");
-    
+
     // No tail padding is necessary.
-    return true;
-  }
+  } else {
+    // Append tail padding if necessary.
+    AppendTailPadding(LayoutSizeInChars);
 
-  CharUnits LLVMSizeInChars = 
-    NextFieldOffsetInChars.RoundUpToAlignment(LLVMStructAlignment);
+    CharUnits LLVMSizeInChars =
+      NextFieldOffsetInChars.RoundUpToAlignment(LLVMStructAlignment);
 
-  // Check if we need to convert the struct to a packed struct.
-  if (NextFieldOffsetInChars <= LayoutSizeInChars && 
-      LLVMSizeInChars > LayoutSizeInChars) {
-    assert(!Packed && "Size mismatch!");
-    
-    ConvertStructToPacked();
-    assert(NextFieldOffsetInChars <= LayoutSizeInChars &&
-           "Converting to packed did not help!");
-  }
+    // Check if we need to convert the struct to a packed struct.
+    if (NextFieldOffsetInChars <= LayoutSizeInChars &&
+        LLVMSizeInChars > LayoutSizeInChars) {
+      assert(!Packed && "Size mismatch!");
 
-  // Append tail padding if necessary.
-  AppendTailPadding(LayoutSizeInChars);
+      ConvertStructToPacked();
+      assert(NextFieldOffsetInChars <= LayoutSizeInChars &&
+             "Converting to packed did not help!");
+    }
 
-  assert(LayoutSizeInChars == NextFieldOffsetInChars &&
-         "Tail padding mismatch!");
+    assert(LayoutSizeInChars == NextFieldOffsetInChars &&
+           "Tail padding mismatch!");
+  }
 
-  return true;
-}
-  
-llvm::Constant *ConstStructBuilder::
-  BuildStruct(CodeGenModule &CGM, CodeGenFunction *CGF, InitListExpr *ILE) {
-  ConstStructBuilder Builder(CGM, CGF);
-  
-  if (!Builder.Build(ILE))
-    return 0;
-  
   // Pick the type to use.  If the type is layout identical to the ConvertType
   // type then use it, otherwise use whatever the builder produced for us.
   llvm::StructType *STy =
       llvm::ConstantStruct::getTypeForElements(CGM.getLLVMContext(),
-                                               Builder.Elements,Builder.Packed);
-  llvm::Type *ILETy = CGM.getTypes().ConvertType(ILE->getType());
-  if (llvm::StructType *ILESTy = dyn_cast<llvm::StructType>(ILETy)) {
-    if (ILESTy->isLayoutIdentical(STy))
-      STy = ILESTy;
+                                               Elements, Packed);
+  llvm::Type *ValTy = CGM.getTypes().ConvertType(Ty);
+  if (llvm::StructType *ValSTy = dyn_cast<llvm::StructType>(ValTy)) {
+    if (ValSTy->isLayoutIdentical(STy))
+      STy = ValSTy;
   }
-    
-  llvm::Constant *Result =
-    llvm::ConstantStruct::get(STy, Builder.Elements);
-  
-  assert(Builder.NextFieldOffsetInChars.RoundUpToAlignment(
-           Builder.getAlignment(Result)) ==
-         Builder.getSizeInChars(Result) && "Size mismatch!");
-  
+
+  llvm::Constant *Result = llvm::ConstantStruct::get(STy, Elements);
+
+  assert(NextFieldOffsetInChars.RoundUpToAlignment(getAlignment(Result)) ==
+         getSizeInChars(Result) && "Size mismatch!");
+
   return Result;
 }
 
-  
+llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM,
+                                                CodeGenFunction *CGF,
+                                                InitListExpr *ILE) {
+  ConstStructBuilder Builder(CGM, CGF);
+
+  if (!Builder.Build(ILE))
+    return 0;
+
+  return Builder.Finalize(ILE->getType());
+}
+
+llvm::Constant *ConstStructBuilder::BuildStruct(CodeGenModule &CGM,
+                                                CodeGenFunction *CGF,
+                                                const APValue &Val,
+                                                QualType ValTy) {
+  ConstStructBuilder Builder(CGM, CGF);
+  Builder.Build(Val, ValTy);
+  return Builder.Finalize(ValTy);
+}
+
+
 //===----------------------------------------------------------------------===//
 //                             ConstExprEmitter
 //===----------------------------------------------------------------------===//
@@ -863,6 +941,22 @@ public:
 
 }  // end anonymous namespace.
 
+llvm::Constant *CodeGenModule::EmitConstantInit(const VarDecl &D,
+                                                CodeGenFunction *CGF) {
+  if (const APValue *Value = D.evaluateValue())
+    return EmitConstantValue(*Value, D.getType(), CGF);
+
+  const Expr *E = D.getInit();
+  assert(E && "No initializer to emit");
+
+  llvm::Constant* C = ConstExprEmitter(*this, CGF).Visit(const_cast<Expr*>(E));
+  if (C && C->getType()->isIntegerTy(1)) {
+    llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
+    C = llvm::ConstantExpr::getZExt(C, BoolTy);
+  }
+  return C;
+}
+
 llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
                                                 QualType DestType,
                                                 CodeGenFunction *CGF) {
@@ -875,155 +969,192 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
   else
     Success = E->EvaluateAsRValue(Result, Context);
 
-  if (Success && !Result.HasSideEffects) {
-    switch (Result.Val.getKind()) {
-    case APValue::Uninitialized:
-      llvm_unreachable("Constant expressions should be initialized.");
-    case APValue::LValue: {
-      llvm::Type *DestTy = getTypes().ConvertTypeForMem(DestType);
-      llvm::Constant *Offset =
-        llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
-                               Result.Val.getLValueOffset().getQuantity());
-
-      llvm::Constant *C;
-      if (APValue::LValueBase LVBase = Result.Val.getLValueBase()) {
-        C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase);
-
-        // Apply offset if necessary.
-        if (!Offset->isNullValue()) {
-          llvm::Type *Type = llvm::Type::getInt8PtrTy(VMContext);
-          llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, Type);
-          Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset);
-          C = llvm::ConstantExpr::getBitCast(Casted, C->getType());
-        }
-
-        // Convert to the appropriate type; this could be an lvalue for
-        // an integer.
-        if (isa<llvm::PointerType>(DestTy))
-          return llvm::ConstantExpr::getBitCast(C, DestTy);
+  if (Success && !Result.HasSideEffects)
+    return EmitConstantValue(Result.Val, DestType, CGF);
 
-        return llvm::ConstantExpr::getPtrToInt(C, DestTy);
-      } else {
-        C = Offset;
+  llvm::Constant* C = ConstExprEmitter(*this, CGF).Visit(const_cast<Expr*>(E));
+  if (C && C->getType()->isIntegerTy(1)) {
+    llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
+    C = llvm::ConstantExpr::getZExt(C, BoolTy);
+  }
+  return C;
+}
 
-        // Convert to the appropriate type; this could be an lvalue for
-        // an integer.
-        if (isa<llvm::PointerType>(DestTy))
-          return llvm::ConstantExpr::getIntToPtr(C, DestTy);
+llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value,
+                                                 QualType DestType,
+                                                 CodeGenFunction *CGF) {
+  switch (Value.getKind()) {
+  case APValue::Uninitialized:
+    llvm_unreachable("Constant expressions should be initialized.");
+  case APValue::LValue: {
+    llvm::Type *DestTy = getTypes().ConvertTypeForMem(DestType);
+    llvm::Constant *Offset =
+      llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext),
+                             Value.getLValueOffset().getQuantity());
+
+    llvm::Constant *C;
+    if (APValue::LValueBase LVBase = Value.getLValueBase()) {
+      // An array can be represented as an lvalue referring to the base.
+      if (isa<llvm::ArrayType>(DestTy)) {
+        assert(Offset->isNullValue() && "offset on array initializer");
+        return ConstExprEmitter(*this, CGF).Visit(
+          const_cast<Expr*>(LVBase.get<const Expr*>()));
+      }
 
-        // If the types don't match this should only be a truncate.
-        if (C->getType() != DestTy)
-          return llvm::ConstantExpr::getTrunc(C, DestTy);
+      C = ConstExprEmitter(*this, CGF).EmitLValue(LVBase);
 
-        return C;
+      // Apply offset if necessary.
+      if (!Offset->isNullValue()) {
+        llvm::Type *Type = llvm::Type::getInt8PtrTy(VMContext);
+        llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, Type);
+        Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset);
+        C = llvm::ConstantExpr::getBitCast(Casted, C->getType());
       }
-    }
-    case APValue::Int: {
-      llvm::Constant *C = llvm::ConstantInt::get(VMContext,
-                                                 Result.Val.getInt());
 
-      if (C->getType()->isIntegerTy(1)) {
-        llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
-        C = llvm::ConstantExpr::getZExt(C, BoolTy);
-      }
-      return C;
-    }
-    case APValue::ComplexInt: {
-      llvm::Constant *Complex[2];
-
-      Complex[0] = llvm::ConstantInt::get(VMContext,
-                                          Result.Val.getComplexIntReal());
-      Complex[1] = llvm::ConstantInt::get(VMContext,
-                                          Result.Val.getComplexIntImag());
-
-      // FIXME: the target may want to specify that this is packed.
-      llvm::StructType *STy = llvm::StructType::get(Complex[0]->getType(),
-                                                    Complex[1]->getType(),
-                                                    NULL);
-      return llvm::ConstantStruct::get(STy, Complex);
-    }
-    case APValue::Float: {
-      const llvm::APFloat &Init = Result.Val.getFloat();
-      if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf)
-        return llvm::ConstantInt::get(VMContext, Init.bitcastToAPInt());
-      else
-        return llvm::ConstantFP::get(VMContext, Init);
-    }
-    case APValue::ComplexFloat: {
-      llvm::Constant *Complex[2];
-
-      Complex[0] = llvm::ConstantFP::get(VMContext,
-                                         Result.Val.getComplexFloatReal());
-      Complex[1] = llvm::ConstantFP::get(VMContext,
-                                         Result.Val.getComplexFloatImag());
-
-      // FIXME: the target may want to specify that this is packed.
-      llvm::StructType *STy = llvm::StructType::get(Complex[0]->getType(),
-                                                    Complex[1]->getType(),
-                                                    NULL);
-      return llvm::ConstantStruct::get(STy, Complex);
-    }
-    case APValue::Vector: {
-      SmallVector<llvm::Constant *, 4> Inits;
-      unsigned NumElts = Result.Val.getVectorLength();
+      // Convert to the appropriate type; this could be an lvalue for
+      // an integer.
+      if (isa<llvm::PointerType>(DestTy))
+        return llvm::ConstantExpr::getBitCast(C, DestTy);
 
-      if (Context.getLangOptions().AltiVec &&
-          isa<CastExpr>(E) &&
-          cast<CastExpr>(E)->getCastKind() == CK_VectorSplat) {
-        // AltiVec vector initialization with a single literal
-        APValue &Elt = Result.Val.getVectorElt(0);
+      return llvm::ConstantExpr::getPtrToInt(C, DestTy);
+    } else {
+      C = Offset;
 
-        llvm::Constant* InitValue = Elt.isInt()
-          ? cast<llvm::Constant>
-              (llvm::ConstantInt::get(VMContext, Elt.getInt()))
-          : cast<llvm::Constant>
-              (llvm::ConstantFP::get(VMContext, Elt.getFloat()));
+      // Convert to the appropriate type; this could be an lvalue for
+      // an integer.
+      if (isa<llvm::PointerType>(DestTy))
+        return llvm::ConstantExpr::getIntToPtr(C, DestTy);
 
-        for (unsigned i = 0; i != NumElts; ++i)
-          Inits.push_back(InitValue);
+      // If the types don't match this should only be a truncate.
+      if (C->getType() != DestTy)
+        return llvm::ConstantExpr::getTrunc(C, DestTy);
 
-      } else {
-        for (unsigned i = 0; i != NumElts; ++i) {
-          APValue &Elt = Result.Val.getVectorElt(i);
-          if (Elt.isInt())
-            Inits.push_back(llvm::ConstantInt::get(VMContext, Elt.getInt()));
-          else
-            Inits.push_back(llvm::ConstantFP::get(VMContext, Elt.getFloat()));
-        }
-      }
-      return llvm::ConstantVector::get(Inits);
+      return C;
     }
-    case APValue::AddrLabelDiff: {
-      const AddrLabelExpr *LHSExpr = Result.Val.getAddrLabelDiffLHS();
-      const AddrLabelExpr *RHSExpr = Result.Val.getAddrLabelDiffRHS();
-      llvm::Constant *LHS = EmitConstantExpr(LHSExpr, LHSExpr->getType(), CGF);
-      llvm::Constant *RHS = EmitConstantExpr(RHSExpr, RHSExpr->getType(), CGF);
-
-      // Compute difference
-      llvm::Type *ResultType = getTypes().ConvertType(E->getType());
-      LHS = llvm::ConstantExpr::getPtrToInt(LHS, IntPtrTy);
-      RHS = llvm::ConstantExpr::getPtrToInt(RHS, IntPtrTy);
-      llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(LHS, RHS);
-
-      // LLVM is a bit sensitive about the exact format of the
-      // address-of-label difference; make sure to truncate after
-      // the subtraction.
-      return llvm::ConstantExpr::getTruncOrBitCast(AddrLabelDiff, ResultType);
+  }
+  case APValue::Int: {
+    llvm::Constant *C = llvm::ConstantInt::get(VMContext,
+                                               Value.getInt());
+
+    if (C->getType()->isIntegerTy(1)) {
+      llvm::Type *BoolTy = getTypes().ConvertTypeForMem(DestType);
+      C = llvm::ConstantExpr::getZExt(C, BoolTy);
     }
-    case APValue::Array:
-    case APValue::Struct:
-    case APValue::Union:
-    case APValue::MemberPointer:
-      break;
+    return C;
+  }
+  case APValue::ComplexInt: {
+    llvm::Constant *Complex[2];
+
+    Complex[0] = llvm::ConstantInt::get(VMContext,
+                                        Value.getComplexIntReal());
+    Complex[1] = llvm::ConstantInt::get(VMContext,
+                                        Value.getComplexIntImag());
+
+    // FIXME: the target may want to specify that this is packed.
+    llvm::StructType *STy = llvm::StructType::get(Complex[0]->getType(),
+                                                  Complex[1]->getType(),
+                                                  NULL);
+    return llvm::ConstantStruct::get(STy, Complex);
+  }
+  case APValue::Float: {
+    const llvm::APFloat &Init = Value.getFloat();
+    if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf)
+      return llvm::ConstantInt::get(VMContext, Init.bitcastToAPInt());
+    else
+      return llvm::ConstantFP::get(VMContext, Init);
+  }
+  case APValue::ComplexFloat: {
+    llvm::Constant *Complex[2];
+
+    Complex[0] = llvm::ConstantFP::get(VMContext,
+                                       Value.getComplexFloatReal());
+    Complex[1] = llvm::ConstantFP::get(VMContext,
+                                       Value.getComplexFloatImag());
+
+    // FIXME: the target may want to specify that this is packed.
+    llvm::StructType *STy = llvm::StructType::get(Complex[0]->getType(),
+                                                  Complex[1]->getType(),
+                                                  NULL);
+    return llvm::ConstantStruct::get(STy, Complex);
+  }
+  case APValue::Vector: {
+    SmallVector<llvm::Constant *, 4> Inits;
+    unsigned NumElts = Value.getVectorLength();
+
+    for (unsigned i = 0; i != NumElts; ++i) {
+      const APValue &Elt = Value.getVectorElt(i);
+      if (Elt.isInt())
+        Inits.push_back(llvm::ConstantInt::get(VMContext, Elt.getInt()));
+      else
+        Inits.push_back(llvm::ConstantFP::get(VMContext, Elt.getFloat()));
     }
+    return llvm::ConstantVector::get(Inits);
+  }
+  case APValue::AddrLabelDiff: {
+    const AddrLabelExpr *LHSExpr = Value.getAddrLabelDiffLHS();
+    const AddrLabelExpr *RHSExpr = Value.getAddrLabelDiffRHS();
+    llvm::Constant *LHS = EmitConstantExpr(LHSExpr, LHSExpr->getType(), CGF);
+    llvm::Constant *RHS = EmitConstantExpr(RHSExpr, RHSExpr->getType(), CGF);
+
+    // Compute difference
+    llvm::Type *ResultType = getTypes().ConvertType(DestType);
+    LHS = llvm::ConstantExpr::getPtrToInt(LHS, IntPtrTy);
+    RHS = llvm::ConstantExpr::getPtrToInt(RHS, IntPtrTy);
+    llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(LHS, RHS);
+
+    // LLVM is a bit sensitive about the exact format of the
+    // address-of-label difference; make sure to truncate after
+    // the subtraction.
+    return llvm::ConstantExpr::getTruncOrBitCast(AddrLabelDiff, ResultType);
   }
+  case APValue::Struct:
+  case APValue::Union:
+    return ConstStructBuilder::BuildStruct(*this, CGF, Value, DestType);
+  case APValue::Array: {
+    const ArrayType *CAT = Context.getAsArrayType(DestType);
+    unsigned NumElements = Value.getArraySize();
+    unsigned NumInitElts = Value.getArrayInitializedElts();
 
-  llvm::Constant* C = ConstExprEmitter(*this, CGF).Visit(const_cast<Expr*>(E));
-  if (C && C->getType()->isIntegerTy(1)) {
-    llvm::Type *BoolTy = getTypes().ConvertTypeForMem(E->getType());
-    C = llvm::ConstantExpr::getZExt(C, BoolTy);
+    std::vector<llvm::Constant*> Elts;
+    Elts.reserve(NumElements);
+
+    // Emit array filler, if there is one.
+    llvm::Constant *Filler = 0;
+    if (Value.hasArrayFiller())
+      Filler = EmitConstantValue(Value.getArrayFiller(),
+                                 CAT->getElementType(), CGF);
+
+    // Emit initializer elements.
+    llvm::Type *CommonElementType = 0;
+    for (unsigned I = 0; I < NumElements; ++I) {
+      llvm::Constant *C = Filler;
+      if (I < NumInitElts)
+        C = EmitConstantValue(Value.getArrayInitializedElt(I),
+                              CAT->getElementType(), CGF);
+      if (I == 0)
+        CommonElementType = C->getType();
+      else if (C->getType() != CommonElementType)
+        CommonElementType = 0;
+      Elts.push_back(C);
+    }
+
+    if (!CommonElementType) {
+      // FIXME: Try to avoid packing the array
+      std::vector<llvm::Type*> Types;
+      for (unsigned i = 0; i < Elts.size(); ++i)
+        Types.push_back(Elts[i]->getType());
+      llvm::StructType *SType = llvm::StructType::get(VMContext, Types, true);
+      return llvm::ConstantStruct::get(SType, Elts);
+    }
+
+    llvm::ArrayType *AType =
+      llvm::ArrayType::get(CommonElementType, NumElements);
+    return llvm::ConstantArray::get(AType, Elts);
   }
-  return C;
+  case APValue::MemberPointer:
+    return getCXXABI().EmitMemberPointer(Value, DestType);
+  }
+  llvm_unreachable("Unknown APValue kind");
 }
 
 llvm::Constant *
@@ -1032,11 +1163,6 @@ CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
   return ConstExprEmitter(*this, 0).EmitLValue(E);
 }
 
-static uint64_t getFieldOffset(ASTContext &C, const FieldDecl *field) {
-  const ASTRecordLayout &layout = C.getASTRecordLayout(field->getParent());
-  return layout.getFieldOffset(field->getFieldIndex());
-}
-    
 llvm::Constant *
 CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
   // Member pointer constants always have a very particular form.
@@ -1048,18 +1174,7 @@ CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
     return getCXXABI().EmitMemberPointer(method);
 
   // Otherwise, a member data pointer.
-  uint64_t fieldOffset;
-  if (const FieldDecl *field = dyn_cast<FieldDecl>(decl))
-    fieldOffset = getFieldOffset(getContext(), field);
-  else {
-    const IndirectFieldDecl *ifield = cast<IndirectFieldDecl>(decl);
-
-    fieldOffset = 0;
-    for (IndirectFieldDecl::chain_iterator ci = ifield->chain_begin(),
-           ce = ifield->chain_end(); ci != ce; ++ci)
-      fieldOffset += getFieldOffset(getContext(), cast<FieldDecl>(*ci));
-  }
-
+  uint64_t fieldOffset = getContext().getFieldOffset(decl);
   CharUnits chars = getContext().toCharUnitsFromBits((int64_t) fieldOffset);
   return getCXXABI().EmitMemberDataPointer(type, chars);
 }
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index 76918dd448..c993abc2ac 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -1354,7 +1354,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
   QualType ASTTy = D->getType();
   bool NonConstInit = false;
 
-  const Expr *InitExpr = D->getAnyInitializer();
+  const VarDecl *InitDecl;
+  const Expr *InitExpr = D->getAnyInitializer(InitDecl);
   
   if (!InitExpr) {
     // This is a tentative definition; tentative definitions are
@@ -1369,12 +1370,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
     Init = EmitNullConstant(D->getType());
   } else {
-    Init = EmitConstantExpr(InitExpr, D->getType());       
+    Init = EmitConstantInit(*InitDecl);
     if (!Init) {
       QualType T = InitExpr->getType();
       if (D->getType()->isReferenceType())
         T = D->getType();
-      
+
       if (getLangOptions().CPlusPlus) {
         Init = EmitNullConstant(T);
         NonConstInit = true;
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index 4c839d1353..d94d10afc1 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -687,12 +687,22 @@ public:
 
   llvm::Constant *getMemberPointerConstant(const UnaryOperator *e);
 
+  /// EmitConstantInit - Try to emit the initializer for the given declaration
+  /// as a constant; returns 0 if the expression cannot be emitted as a
+  /// co