Almost complete implementation of rvalue references. One bug, and a few unclear areas. Maybe Doug can shed some light on some of the fixmes.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@67059 91177308-0d34-0410-b5e6-96231b3b80d8

31 files changed, 395 insertions, 128 deletions

diff --git a/include/clang/AST/ASTContext.h b/include/clang/AST/ASTContext.h
index 8a8be1e775..124f7db449 100644
--- a/include/clang/AST/ASTContext.h
+++ b/include/clang/AST/ASTContext.h
@@ -59,7 +59,8 @@ class ASTContext {
   llvm::FoldingSet<ComplexType> ComplexTypes;
   llvm::FoldingSet<PointerType> PointerTypes;
   llvm::FoldingSet<BlockPointerType> BlockPointerTypes;
-  llvm::FoldingSet<ReferenceType> ReferenceTypes;
+  llvm::FoldingSet<LValueReferenceType> LValueReferenceTypes;
+  llvm::FoldingSet<RValueReferenceType> RValueReferenceTypes;
   llvm::FoldingSet<MemberPointerType> MemberPointerTypes;
   llvm::FoldingSet<ConstantArrayType> ConstantArrayTypes;
   llvm::FoldingSet<IncompleteArrayType> IncompleteArrayTypes;
@@ -207,9 +208,13 @@ public:
   /// of the specified type.
   QualType getBlockPointerType(QualType T);
 
-  /// getReferenceType - Return the uniqued reference to the type for a
-  /// reference to the specified type.
-  QualType getReferenceType(QualType T);
+  /// getLValueReferenceType - Return the uniqued reference to the type for an
+  /// lvalue reference to the specified type.
+  QualType getLValueReferenceType(QualType T);
+
+  /// getRValueReferenceType - Return the uniqued reference to the type for an
+  /// rvalue reference to the specified type.
+  QualType getRValueReferenceType(QualType T);
 
   /// getMemberPointerType - Return the uniqued reference to the type for a
   /// member pointer to the specified type in the specified class. The class
diff --git a/include/clang/AST/Type.h b/include/clang/AST/Type.h
index 13668569ca..4e1672d435 100644
--- a/include/clang/AST/Type.h
+++ b/include/clang/AST/Type.h
@@ -337,6 +337,8 @@ public:
   bool isPointerType() const;
   bool isBlockPointerType() const;
   bool isReferenceType() const;
+  bool isLValueReferenceType() const;
+  bool isRValueReferenceType() const;
   bool isFunctionPointerType() const;
   bool isMemberPointerType() const;
   bool isMemberFunctionPointerType() const;
@@ -383,6 +385,8 @@ public:
   const PointerType *getAsPointerType() const;
   const BlockPointerType *getAsBlockPointerType() const;
   const ReferenceType *getAsReferenceType() const;
+  const LValueReferenceType *getAsLValueReferenceType() const;
+  const RValueReferenceType *getAsRValueReferenceType() const;
   const MemberPointerType *getAsMemberPointerType() const;
   const TagType *getAsTagType() const;
   const RecordType *getAsRecordType() const;
@@ -674,19 +678,17 @@ public:
     friend class Type;
 };
 
-/// ReferenceType - C++ 8.3.2 - Reference Declarators.
+/// ReferenceType - Base for LValueReferenceType and RValueReferenceType
 ///
 class ReferenceType : public Type, public llvm::FoldingSetNode {
   QualType PointeeType;
 
-  ReferenceType(QualType Referencee, QualType CanonicalRef) :
-    Type(Reference, CanonicalRef, Referencee->isDependentType()), 
+protected:
+  ReferenceType(TypeClass tc, QualType Referencee, QualType CanonicalRef) :
+    Type(tc, CanonicalRef, Referencee->isDependentType()),
     PointeeType(Referencee) {
   }
-  friend class ASTContext;  // ASTContext creates these.
 public:
-  virtual void getAsStringInternal(std::string &InnerString) const;
-
   QualType getPointeeType() const { return PointeeType; }
 
   void Profile(llvm::FoldingSetNodeID &ID) {
@@ -696,11 +698,52 @@ public:
     ID.AddPointer(Referencee.getAsOpaquePtr());
   }
 
-  static bool classof(const Type *T) { return T->getTypeClass() == Reference; }
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == LValueReference ||
+           T->getTypeClass() == RValueReference;
+  }
   static bool classof(const ReferenceType *) { return true; }
 
 protected:
   virtual void EmitImpl(llvm::Serializer& S) const;
+};
+
+/// LValueReferenceType - C++ [dcl.ref] - Lvalue reference
+///
+class LValueReferenceType : public ReferenceType {
+  LValueReferenceType(QualType Referencee, QualType CanonicalRef) :
+    ReferenceType(LValueReference, Referencee, CanonicalRef) {
+  }
+  friend class ASTContext; // ASTContext creates these
+public:
+  virtual void getAsStringInternal(std::string &InnerString) const;
+
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == LValueReference;
+  }
+  static bool classof(const LValueReferenceType *) { return true; }
+
+protected:
+  static Type* CreateImpl(ASTContext& Context, llvm::Deserializer& D);
+  friend class Type;
+};
+
+/// RValueReferenceType - C++0x [dcl.ref] - Rvalue reference
+///
+class RValueReferenceType : public ReferenceType {
+  RValueReferenceType(QualType Referencee, QualType CanonicalRef) :
+    ReferenceType(RValueReference, Referencee, CanonicalRef) {
+  }
+  friend class ASTContext; // ASTContext creates these
+public:
+  virtual void getAsStringInternal(std::string &InnerString) const;
+
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == RValueReference;
+  }
+  static bool classof(const RValueReferenceType *) { return true; }
+
+protected:
   static Type* CreateImpl(ASTContext& Context, llvm::Deserializer& D);
   friend class Type;
 };
@@ -1798,6 +1841,12 @@ inline bool Type::isBlockPointerType() const {
 inline bool Type::isReferenceType() const {
   return isa<ReferenceType>(CanonicalType.getUnqualifiedType());
 }
+inline bool Type::isLValueReferenceType() const {
+  return isa<LValueReferenceType>(CanonicalType.getUnqualifiedType());
+}
+inline bool Type::isRValueReferenceType() const {
+  return isa<RValueReferenceType>(CanonicalType.getUnqualifiedType());
+}
 inline bool Type::isFunctionPointerType() const {
   if (const PointerType* T = getAsPointerType())
     return T->getPointeeType()->isFunctionType();
diff --git a/include/clang/AST/TypeNodes.def b/include/clang/AST/TypeNodes.def
index e071f640ba..4c5c02d2a5 100644
--- a/include/clang/AST/TypeNodes.def
+++ b/include/clang/AST/TypeNodes.def
@@ -51,7 +51,9 @@ TYPE(FixedWidthInt, Type)
 TYPE(Complex, Type)
 TYPE(Pointer, Type)
 TYPE(BlockPointer, Type)
-TYPE(Reference, Type)
+ABSTRACT_TYPE(Reference, Type)
+TYPE(LValueReference, Reference)
+TYPE(RValueReference, Reference)
 TYPE(MemberPointer, Type)
 ABSTRACT_TYPE(Array, Type)
 TYPE(ConstantArray, ArrayType)
diff --git a/include/clang/Basic/DiagnosticSemaKinds.def b/include/clang/Basic/DiagnosticSemaKinds.def
index 4d721bc331..d415ee5392 100644
--- a/include/clang/Basic/DiagnosticSemaKinds.def
+++ b/include/clang/Basic/DiagnosticSemaKinds.def
@@ -304,9 +304,12 @@ DIAG(err_destructor_typedef_name, ERROR,
      "destructor cannot be declared using a typedef %0 of the class name")
 
 // C++ initialization
+DIAG(err_lvalue_to_rvalue_ref, ERROR,
+     "rvalue reference cannot bind to lvalue")
 // FIXME: passing in an English string as %1!
 DIAG(err_not_reference_to_const_init, ERROR,
-     "non-const reference to type %0 cannot be initialized with a %1 of type %2")
+     "non-const lvalue reference to type %0 cannot be initialized "
+     "with a %1 of type %2")
 // FIXME: passing in an English string as %1!
 DIAG(err_reference_init_drops_quals, ERROR,
      "initialization of reference to type %0 with a %1 of type %2 drops qualifiers")
diff --git a/include/clang/Basic/DiagnosticSemaKinds.td b/include/clang/Basic/DiagnosticSemaKinds.td
index dd2a17e791..a2ba0e9708 100644
--- a/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/include/clang/Basic/DiagnosticSemaKinds.td
@@ -271,9 +271,11 @@ def err_destructor_typedef_name : Error<
   "destructor cannot be declared using a typedef %0 of the class name">;
 
 // C++ initialization
+def err_lvalue_to_rvalue_ref : Error<"rvalue reference cannot bind to lvalue">;
 // FIXME: passing in an English string as %1!
 def err_not_reference_to_const_init : Error<
-  "non-const reference to type %0 cannot be initialized with a %1 of type %2">;
+  "non-const lvalue reference to type %0 cannot be initialized "
+  "with a %1 of type %2">;
 // FIXME: passing in an English string as %1!
 def err_reference_init_drops_quals : Error<
   "initialization of reference to type %0 with a %1 of type %2 drops "
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index 80e17746bb..c318107dab 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -85,9 +85,9 @@ void ASTContext::PrintStats() const {
   fprintf(stderr, "  %d types total.\n", (int)Types.size());
   unsigned NumBuiltin = 0, NumPointer = 0, NumArray = 0, NumFunctionP = 0;
   unsigned NumVector = 0, NumComplex = 0, NumBlockPointer = 0;
-  unsigned NumFunctionNP = 0, NumTypeName = 0, NumTagged = 0, NumReference = 0;
-  unsigned NumMemberPointer = 0;
-  
+  unsigned NumFunctionNP = 0, NumTypeName = 0, NumTagged = 0;
+  unsigned NumLValueReference = 0, NumRValueReference = 0, NumMemberPointer = 0;
+
   unsigned NumTagStruct = 0, NumTagUnion = 0, NumTagEnum = 0, NumTagClass = 0;
   unsigned NumObjCInterfaces = 0, NumObjCQualifiedInterfaces = 0;
   unsigned NumObjCQualifiedIds = 0;
@@ -101,8 +101,10 @@ void ASTContext::PrintStats() const {
       ++NumPointer;
     else if (isa<BlockPointerType>(T))
       ++NumBlockPointer;
-    else if (isa<ReferenceType>(T))
-      ++NumReference;
+    else if (isa<LValueReferenceType>(T))
+      ++NumLValueReference;
+    else if (isa<RValueReferenceType>(T))
+      ++NumRValueReference;
     else if (isa<MemberPointerType>(T))
       ++NumMemberPointer;
     else if (isa<ComplexType>(T))
@@ -145,7 +147,8 @@ void ASTContext::PrintStats() const {
   fprintf(stderr, "    %d builtin types\n", NumBuiltin);
   fprintf(stderr, "    %d pointer types\n", NumPointer);
   fprintf(stderr, "    %d block pointer types\n", NumBlockPointer);
-  fprintf(stderr, "    %d reference types\n", NumReference);
+  fprintf(stderr, "    %d lvalue reference types\n", NumLValueReference);
+  fprintf(stderr, "    %d rvalue reference types\n", NumRValueReference);
   fprintf(stderr, "    %d member pointer types\n", NumMemberPointer);
   fprintf(stderr, "    %d complex types\n", NumComplex);
   fprintf(stderr, "    %d array types\n", NumArray);
@@ -165,10 +168,12 @@ void ASTContext::PrintStats() const {
           NumObjCQualifiedIds);
   fprintf(stderr, "    %d typeof types\n", NumTypeOfTypes);
   fprintf(stderr, "    %d typeof exprs\n", NumTypeOfExprTypes);
-  
+
   fprintf(stderr, "Total bytes = %d\n", int(NumBuiltin*sizeof(BuiltinType)+
     NumPointer*sizeof(PointerType)+NumArray*sizeof(ArrayType)+
     NumComplex*sizeof(ComplexType)+NumVector*sizeof(VectorType)+
+    NumLValueReference*sizeof(LValueReferenceType)+
+    NumRValueReference*sizeof(RValueReferenceType)+
     NumMemberPointer*sizeof(MemberPointerType)+
     NumFunctionP*sizeof(FunctionProtoType)+
     NumFunctionNP*sizeof(FunctionNoProtoType)+
@@ -411,7 +416,8 @@ ASTContext::getTypeInfo(const Type *T) {
     Align = Target.getPointerAlign(AS);
     break;
   }
-  case Type::Reference:
+  case Type::LValueReference:
+  case Type::RValueReference:
     // "When applied to a reference or a reference type, the result is the size
     // of the referenced type." C++98 5.3.3p2: expr.sizeof.
     // FIXME: This is wrong for struct layout: a reference in a struct has
@@ -910,32 +916,65 @@ QualType ASTContext::getBlockPointerType(QualType T) {
   return QualType(New, 0);
 }
 
-/// getReferenceType - Return the uniqued reference to the type for a reference
-/// to the specified type.
-QualType ASTContext::getReferenceType(QualType T) {
+/// getLValueReferenceType - Return the uniqued reference to the type for an
+/// lvalue reference to the specified type.
+QualType ASTContext::getLValueReferenceType(QualType T) {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   ReferenceType::Profile(ID, T);
 
   void *InsertPos = 0;
-  if (ReferenceType *RT = ReferenceTypes.FindNodeOrInsertPos(ID, InsertPos))
+  if (LValueReferenceType *RT =
+        LValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(RT, 0);
-  
+
   // If the referencee type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.
   QualType Canonical;
   if (!T->isCanonical()) {
-    Canonical = getReferenceType(getCanonicalType(T));
-   
+    Canonical = getLValueReferenceType(getCanonicalType(T));
+
+    // Get the new insert position for the node we care about.
+    LValueReferenceType *NewIP =
+      LValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
+    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+  }
+
+  LValueReferenceType *New = new (*this,8) LValueReferenceType(T, Canonical);
+  Types.push_back(New);
+  LValueReferenceTypes.InsertNode(New, InsertPos);
+  return QualType(New, 0);
+}
+
+/// getRValueReferenceType - Return the uniqued reference to the type for an
+/// rvalue reference to the specified type.
+QualType ASTContext::getRValueReferenceType(QualType T) {
+  // Unique pointers, to guarantee there is only one pointer of a particular
+  // structure.
+  llvm::FoldingSetNodeID ID;
+  ReferenceType::Profile(ID, T);
+
+  void *InsertPos = 0;
+  if (RValueReferenceType *RT =
+        RValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(RT, 0);
+
+  // If the referencee type isn't canonical, this won't be a canonical type
+  // either, so fill in the canonical type field.
+  QualType Canonical;
+  if (!T->isCanonical()) {
+    Canonical = getRValueReferenceType(getCanonicalType(T));
+
     // Get the new insert position for the node we care about.
-    ReferenceType *NewIP = ReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
+    RValueReferenceType *NewIP =
+      RValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
   }
 
-  ReferenceType *New = new (*this,8) ReferenceType(T, Canonical);
+  RValueReferenceType *New = new (*this,8) RValueReferenceType(T, Canonical);
   Types.push_back(New);
-  ReferenceTypes.InsertNode(New, InsertPos);
+  RValueReferenceTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
@@ -2641,9 +2680,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS) {
   // C++ [expr]: If an expression initially has the type "reference to T", the
   // type is adjusted to "T" prior to any further analysis, the expression
   // designates the object or function denoted by the reference, and the
-  // expression is an lvalue.
+  // expression is an lvalue unless the reference is an rvalue reference and
+  // the expression is a function call (possibly inside parentheses).
   // FIXME: C++ shouldn't be going through here!  The rules are different
   // enough that they should be handled separately.
+  // FIXME: Merging of lvalue and rvalue references is incorrect. C++ *really*
+  // shouldn't be going through here!
   if (const ReferenceType *RT = LHS->getAsReferenceType())
     LHS = RT->getPointeeType();
   if (const ReferenceType *RT = RHS->getAsReferenceType())
@@ -2746,7 +2788,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS) {
     assert(false && "Non-canonical and dependent types shouldn't get here");
     return QualType();
 
-  case Type::Reference:
+  case Type::LValueReference:
+  case Type::RValueReference:
   case Type::MemberPointer:
     assert(false && "C++ should never be in mergeTypes");
     return QualType();
diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp
index 4655cf392d..7eab2679e8 100644
--- a/lib/AST/Builtins.cpp
+++ b/lib/AST/Builtins.cpp
@@ -183,7 +183,7 @@ static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
     if (Type->isArrayType()) {
       Type = Context.getArrayDecayedType(Type);
     } else {
-      Type = Context.getReferenceType(Type);
+      Type = Context.getLValueReferenceType(Type);
     }
     break;
   case 'V': {
@@ -224,8 +224,9 @@ static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
         Type = Context.getPointerType(Type);
         break;
       case '&':
-        Type = Context.getReferenceType(Type);
+        Type = Context.getLValueReferenceType(Type);
         break;
+      // FIXME: There's no way to have a built-in with an rvalue ref arg.
       case 'C':
         Type = Type.getQualifiedType(QualType::Const);
         break;
diff --git a/lib/AST/DeclCXX.cpp b/lib/AST/DeclCXX.cpp
index fd900834ca..a3c7997c6d 100644
--- a/lib/AST/DeclCXX.cpp
+++ b/lib/AST/DeclCXX.cpp
@@ -102,7 +102,7 @@ bool CXXRecordDecl::hasConstCopyAssignment(ASTContext &Context) const {
       continue;
     bool AcceptsConst = true;
     QualType ArgType = FnType->getArgType(0);
-    if (const ReferenceType *Ref = ArgType->getAsReferenceType()) {
+    if (const LValueReferenceType *Ref = ArgType->getAsLValueReferenceType()) {
       ArgType = Ref->getPointeeType();
       // Is it a non-const reference?
       if (!ArgType.isConstQualified())
@@ -152,7 +152,7 @@ void CXXRecordDecl::addedAssignmentOperator(ASTContext &Context,
   assert(FnType && "Overloaded operator has no proto function type.");
   assert(FnType->getNumArgs() == 1 && !FnType->isVariadic());
   QualType ArgType = FnType->getArgType(0);
-  if (const ReferenceType *Ref = ArgType->getAsReferenceType())
+  if (const LValueReferenceType *Ref = ArgType->getAsLValueReferenceType())
     ArgType = Ref->getPointeeType();
 
   ArgType = ArgType.getUnqualifiedType();
@@ -263,8 +263,9 @@ CXXConstructorDecl::isCopyConstructor(ASTContext &Context,
 
   const ParmVarDecl *Param = getParamDecl(0);
 
-  // Do we have a reference type?
-  const ReferenceType *ParamRefType = Param->getType()->getAsReferenceType();
+  // Do we have a reference type? Rvalue references don't count.
+  const LValueReferenceType *ParamRefType =
+    Param->getType()->getAsLValueReferenceType();
   if (!ParamRefType)
     return false;
 
diff --git a/lib/AST/Expr.cpp b/lib/AST/Expr.cpp
index 669f5c8225..2f7e3630fd 100644
--- a/lib/AST/Expr.cpp
+++ b/lib/AST/Expr.cpp
@@ -575,10 +575,7 @@ Expr::isLvalueResult Expr::isLvalue(ASTContext &Ctx) const {
   if (TR->isVoidType() && !Ctx.getCanonicalType(TR).getCVRQualifiers())
     return LV_IncompleteVoidType;
 
-  /// FIXME: Expressions can't have reference type, so the following
-  /// isn't needed.
-  if (TR->isReferenceType()) // C++ [expr]
-    return LV_Valid;
+  assert(!TR->isReferenceType() && "Expressions can't have reference type.");
 
   // the type looks fine, now check the expression
   switch (getStmtClass()) {
@@ -691,16 +688,16 @@ Expr::isLvalueResult Expr::isLvalue(ASTContext &Ctx) const {
   case CallExprClass: 
   case CXXOperatorCallExprClass:
   case CXXMemberCallExprClass: {
-    // C++ [expr.call]p10:
+    // C++0x [expr.call]p10
     //   A function call is an lvalue if and only if the result type
-    //   is a reference.
+    //   is an lvalue reference.
     QualType CalleeType = cast<CallExpr>(this)->getCallee()->getType();
     if (const PointerType *FnTypePtr = CalleeType->getAsPointerType())
       CalleeType = FnTypePtr->getPointeeType();
     if (const FunctionType *FnType = CalleeType->getAsFunctionType())
-      if (FnType->getResultType()->isReferenceType())
+      if (FnType->getResultType()->isLValueReferenceType())
         return LV_Valid;
-    
+
     break;
   }
   case CompoundLiteralExprClass: // C99 6.5.2.5p5
@@ -733,10 +730,11 @@ Expr::isLvalueResult Expr::isLvalue(ASTContext &Ctx) const {
   case CXXReinterpretCastExprClass:
   case CXXConstCastExprClass:
     // The result of an explicit cast is an lvalue if the type we are
-    // casting to is a reference type. See C++ [expr.cast]p1, 
+    // casting to is an lvalue reference type. See C++ [expr.cast]p1,
     // C++ [expr.static.cast]p2, C++ [expr.dynamic.cast]p2,
     // C++ [expr.reinterpret.cast]p1, C++ [expr.const.cast]p1.
-    if (cast<ExplicitCastExpr>(this)->getTypeAsWritten()->isReferenceType())
+    if (cast<ExplicitCastExpr>(this)->getTypeAsWritten()->
+          isLValueReferenceType())
       return LV_Valid;
     break;
   case CXXTypeidExprClass:
diff --git a/lib/AST/ExprConstant.cpp b/lib/AST/ExprConstant.cpp
index 6fb2abe7cd..a0ef7460c5 100644
--- a/lib/AST/ExprConstant.cpp
+++ b/lib/AST/ExprConstant.cpp
@@ -696,6 +696,7 @@ bool IntExprEvaluator::VisitDeclRefExpr(const DeclRefExpr *E) {
 /// as GCC.
 static int EvaluateBuiltinClassifyType(const CallExpr *E) {
   // The following enum mimics the values returned by GCC.
+  // FIXME: Does GCC differ between lvalue and rvalue references here?
   enum gcc_type_class {
     no_type_class = -1,
     void_type_class, integer_type_class, char_type_class,
diff --git a/lib/AST/Type.cpp b/lib/AST/Type.cpp
index fc71097c12..1714e84129 100644
--- a/lib/AST/Type.cpp
+++ b/lib/AST/Type.cpp
@@ -117,7 +117,8 @@ bool Type::isDerivedType() const {
   case IncompleteArray:
   case FunctionProto:
   case FunctionNoProto:
-  case Reference:
+  case LValueReference:
+  case RValueReference:
   case Record:
     return true;
   default:
@@ -264,9 +265,9 @@ const ReferenceType *Type::getAsReferenceType() const {
   // If this is directly a reference type, return it.
   if (const ReferenceType *RTy = dyn_cast<ReferenceType>(this))
     return RTy;
-  
+
   // If the canonical form of this type isn't the right kind, reject it.
-  if (!isa<ReferenceType>(CanonicalType)) {    
+  if (!isa<ReferenceType>(CanonicalType)) {
     // Look through type qualifiers
     if (isa<ReferenceType>(CanonicalType.getUnqualifiedType()))
       return CanonicalType.getUnqualifiedType()->getAsReferenceType();
@@ -278,6 +279,42 @@ const ReferenceType *Type::getAsReferenceType() const {
   return getDesugaredType()->getAsReferenceType();
 }
 
+const LValueReferenceType *Type::getAsLValueReferenceType() const {
+  // If this is directly an lvalue reference type, return it.
+  if (const LValueReferenceType *RTy = dyn_cast<LValueReferenceType>(this))
+    return RTy;
+
+  // If the canonical form of this type isn't the right kind, reject it.
+  if (!isa<LValueReferenceType>(CanonicalType)) {
+    // Look through type qualifiers
+    if (isa<LValueReferenceType>(CanonicalType.getUnqualifiedType()))
+      return CanonicalType.getUnqualifiedType()->getAsLValueReferenceType();
+    return 0;
+  }
+
+  // If this is a typedef for an lvalue reference type, strip the typedef off
+  // without losing all typedef information.
+  return getDesugaredType()->getAsLValueReferenceType();
+}
+
+const RValueReferenceType *Type::getAsRValueReferenceType() const {
+  // If this is directly an rvalue reference type, return it.
+  if (const RValueReferenceType *RTy = dyn_cast<RValueReferenceType>(this))
+    return RTy;
+
+  // If the canonical form of this type isn't the right kind, reject it.
+  if (!isa<RValueReferenceType>(CanonicalType)) {
+    // Look through type qualifiers
+    if (isa<RValueReferenceType>(CanonicalType.getUnqualifiedType()))
+      return CanonicalType.getUnqualifiedType()->getAsRValueReferenceType();
+    return 0;
+  }
+
+  // If this is a typedef for an rvalue reference type, strip the typedef off
+  // without losing all typedef information.
+  return getDesugaredType()->getAsRValueReferenceType();
+}
+
 const MemberPointerType *Type::getAsMemberPointerType() const {
   // If this is directly a member pointer type, return it.
   if (const MemberPointerType *MTy = dyn_cast<MemberPointerType>(this))
@@ -1116,14 +1153,25 @@ void BlockPointerType::getAsStringInternal(std::string &S) const {
   PointeeType.getAsStringInternal(S);
 }
 
-void ReferenceType::getAsStringInternal(std::string &S) const {
+void LValueReferenceType::getAsStringInternal(std::string &S) const {
   S = '&' + S;
-  
+
   // Handle things like 'int (&A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(getPointeeType()))
     S = '(' + S + ')';
-  
+
+  getPointeeType().getAsStringInternal(S);
+}
+
+void RValueReferenceType::getAsStringInternal(std::string &S) const {
+  S = "&&" + S;
+
+  // Handle things like 'int (&&A)[4];' correctly.
+  // FIXME: this should include vectors, but vectors use attributes I guess.
+  if (isa<ArrayType>(getPointeeType()))
+    S = '(' + S + ')';
+
   getPointeeType().getAsStringInternal(S);
 }
 
@@ -1133,7 +1181,7 @@ void MemberPointerType::getAsStringInternal(std::string &S) const {
   C += "::*";
   S = C + S;
 
-  // Handle things like 'int (&A)[4];' correctly.
+  // Handle things like 'int (Cls::*A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(getPointeeType()))
     S = '(' + S + ')';
diff --git a/lib/AST/TypeSerialization.cpp b/lib/AST/TypeSerialization.cpp
index 1b9fed4866..8498e0123e 100644
--- a/lib/AST/TypeSerialization.cpp
+++ b/lib/AST/TypeSerialization.cpp
@@ -107,8 +107,12 @@ void Type::Create(ASTContext& Context, unsigned i, Deserializer& D) {
       D.RegisterPtr(PtrID, BlockPointerType::CreateImpl(Context, D));
       break;
 
-    case Type::Reference:
-      D.RegisterPtr(PtrID, ReferenceType::CreateImpl(Context, D));
+    case Type::LValueReference:
+      D.RegisterPtr(PtrID, LValueReferenceType::CreateImpl(Context, D));
+      break;
+
+    case Type::RValueReference:
+      D.RegisterPtr(PtrID, RValueReferenceType::CreateImpl(Context, D));
       break;
 
     case Type::Record:
@@ -261,8 +265,12 @@ void ReferenceType::EmitImpl(Serializer& S) const {
   S.Emit(getPointeeType());
 }
 
-Type* ReferenceType::CreateImpl(ASTContext& Context, Deserializer& D) {
-  return Context.getReferenceType(QualType::ReadVal(D)).getTypePtr();
+Type* LValueReferenceType::CreateImpl(ASTContext& Context, Deserializer& D) {
+  return Context.getLValueReferenceType(QualType::ReadVal(D)).getTypePtr();
+}
+
+Type* RValueReferenceType::CreateImpl(ASTContext& Context, Deserializer& D) {
+  return Context.getRValueReferenceType(QualType::ReadVal(D)).getTypePtr();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 9b56b8449f..0dac6ba920 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -502,7 +502,8 @@ llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty,
     assert(false && "Dependent types cannot show up in debug information");
     
   case Type::Complex:
-  case Type::Reference:
+  case Type::LValueReference:
+  case Type::RValueReference:
   case Type::Vector:
   case Type::ExtVector:
   case Type::ExtQual:
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 8cd353408f..1a64e3c905 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -571,6 +571,7 @@ Value *ScalarExprEmitter::VisitImplicitCastExpr(const ImplicitCastExpr *E) {
     return V;
     
   } else if (E->getType()->isReferenceType()) {
+    // FIXME: An expression cannot have reference type.
     return EmitLValue(Op).getAddress();
   }
   
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index 2d4c27c2ac..85d7384c6c 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -266,7 +266,8 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
       ConvertTypeRecursive(cast<ComplexType>(Ty).getElementType());
     return llvm::StructType::get(EltTy, EltTy, NULL);
   }
-  case Type::Reference: {
+  case Type::LValueReference:
+  case Type::RValueReference: {
     const ReferenceType &RTy = cast<ReferenceType>(Ty);
     QualType ETy = RTy.getPointeeType();
     llvm::OpaqueType *PointeeType = llvm::OpaqueType::get();
diff --git a/lib/CodeGen/Mangle.cpp b/lib/CodeGen/Mangle.cpp
index 92eeb5db14..d54849bd78 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/CodeGen/Mangle.cpp
@@ -365,12 +365,17 @@ void CXXNameMangler::mangleType(QualType T) {
     mangleType(PT->getPointeeType());
   }
   //         ::= R <type>   # reference-to
-  //         ::= O <type>   # rvalue reference-to (C++0x)
-  else if (const ReferenceType *RT = dyn_cast<ReferenceType>(T.getTypePtr())) {
-    // FIXME: rvalue references
+  else if (const LValueReferenceType *RT =
+           dyn_cast<LValueReferenceType>(T.getTypePtr())) {
     Out << 'R';