Merge the "types" and "declarations" blocks in the precompiled header

format, so that we don't end up with multiple declaration and types
blocks. Also, fix a few obscure bugs with PCH loading and generation:

  - If the DeclIDs DenseMap reallocates while we are writing a
    declaration (due to recursively writing other declarations), we
    could end up writing a bad ID to ExternalDefinitions.
  - When loading an ArrayLoc (part of DeclaratorInfo), we need to set
    the size expression to NULL if no size expression was provided.

PCH -> AST rewriting is still partly broken, unfortunately.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@84293 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 128 insertions, 153 deletions

diff --git a/include/clang/Frontend/PCHBitCodes.h b/include/clang/Frontend/PCHBitCodes.h
index 6b3f20bdb8..93eac86ee2 100644
--- a/include/clang/Frontend/PCHBitCodes.h
+++ b/include/clang/Frontend/PCHBitCodes.h
@@ -86,38 +86,33 @@ namespace clang {
       PREPROCESSOR_BLOCK_ID,
 
       /// \brief The block containing the definitions of all of the
-      /// types used within the PCH file.
-      TYPES_BLOCK_ID,
-
-      /// \brief The block containing the definitions of all of the
-      /// declarations stored in the PCH file.
-      DECLS_BLOCK_ID
+      /// types and decls used within the PCH file.
+      DECLTYPES_BLOCK_ID
     };
 
     /// \brief Record types that occur within the PCH block itself.
     enum PCHRecordTypes {
-      /// \brief Offset of each type within the types block.
+      /// \brief Record code for the offsets of each type.
       ///
       /// The TYPE_OFFSET constant describes the record that occurs
-      /// within the block identified by TYPE_OFFSETS_BLOCK_ID within
-      /// the PCH file. The record itself is an array of offsets that
-      /// point into the types block (identified by TYPES_BLOCK_ID in
-      /// the PCH file). The index into the array is based on the ID
+      /// within the PCH block. The record itself is an array of offsets that
+      /// point into the declarations and types block (identified by 
+      /// DECLTYPES_BLOCK_ID). The index into the array is based on the ID
       /// of a type. For a given type ID @c T, the lower three bits of
       /// @c T are its qualifiers (const, volatile, restrict), as in
       /// the QualType class. The upper bits, after being shifted and
       /// subtracting NUM_PREDEF_TYPE_IDS, are used to index into the
       /// TYPE_OFFSET block to determine the offset of that type's
-      /// corresponding record within the TYPES_BLOCK_ID block.
+      /// corresponding record within the DECLTYPES_BLOCK_ID block.
       TYPE_OFFSET = 1,
 
       /// \brief Record code for the offsets of each decl.
       ///
       /// The DECL_OFFSET constant describes the record that occurs
-      /// within the block identifier by DECL_OFFSETS_BLOCK_ID within
-      /// the PCH file. The record itself is an array of offsets that
-      /// point into the declarations block (identified by
-      /// DECLS_BLOCK_ID). The declaration ID is an index into this
+      /// within the block identified by DECL_OFFSETS_BLOCK_ID within
+      /// the PCH block. The record itself is an array of offsets that
+      /// point into the declarations and types block (identified by
+      /// DECLTYPES_BLOCK_ID). The declaration ID is an index into this
       /// record, after subtracting one to account for the use of
       /// declaration ID 0 for a NULL declaration pointer. Index 0 is
       /// reserved for the translation unit declaration.
@@ -353,8 +348,8 @@ namespace clang {
 
     /// \brief Record codes for each kind of type.
     ///
-    /// These constants describe the records that can occur within a
-    /// block identified by TYPES_BLOCK_ID in the PCH file. Each
+    /// These constants describe the type records that can occur within a
+    /// block identified by DECLTYPES_BLOCK_ID in the PCH file. Each
     /// constant describes a record for a specific type class in the
     /// AST.
     enum TypeCode {
@@ -444,13 +439,13 @@ namespace clang {
 
     /// \brief Record codes for each kind of declaration.
     ///
-    /// These constants describe the records that can occur within a
-    /// declarations block (identified by DECLS_BLOCK_ID). Each
+    /// These constants describe the declaration records that can occur within
+    /// a declarations block (identified by DECLS_BLOCK_ID). Each
     /// constant describes a record for a specific declaration class
     /// in the AST.
     enum DeclCode {
       /// \brief Attributes attached to a declaration.
-      DECL_ATTR = 1,
+      DECL_ATTR = 50,
       /// \brief A TranslationUnitDecl record.
       DECL_TRANSLATION_UNIT,
       /// \brief A TypedefDecl record.
@@ -525,14 +520,14 @@ namespace clang {
     /// \brief Record codes for each kind of statement or expression.
     ///
     /// These constants describe the records that describe statements
-    /// or expressions. These records can occur within either the type
-    /// or declaration blocks, so they begin with record values of
-    /// 50.  Each constant describes a record for a specific
-    /// statement or expression class in the AST.
+    /// or expressions. These records  occur within type and declarations
+    /// block, so they begin with record values of 100.  Each constant 
+    /// describes a record for a specific statement or expression class in the
+    /// AST.
     enum StmtCode {
       /// \brief A marker record that indicates that we are at the end
       /// of an expression.
-      STMT_STOP = 50,
+      STMT_STOP = 100,
       /// \brief A NULL expression.
       STMT_NULL_PTR,
       /// \brief A NullStmt record.
diff --git a/include/clang/Frontend/PCHWriter.h b/include/clang/Frontend/PCHWriter.h
index 5ff2f10180..728e138d9e 100644
--- a/include/clang/Frontend/PCHWriter.h
+++ b/include/clang/Frontend/PCHWriter.h
@@ -73,6 +73,33 @@ private:
   /// \brief The bitstream writer used to emit this precompiled header.
   llvm::BitstreamWriter &Stream;
 
+  /// \brief Stores a declaration or a type to be written to the PCH file.
+  class DeclOrType {
+  public:
+    DeclOrType(Decl *D) : Stored(D), IsType(false) { }
+    DeclOrType(QualType T) : Stored(T.getAsOpaquePtr()), IsType(true) { }
+    
+    bool isType() const { return IsType; }
+    bool isDecl() const { return !IsType; }
+    
+    QualType getType() const {
+      assert(isType() && "Not a type!");
+      return QualType::getFromOpaquePtr(Stored);
+    }
+    
+    Decl *getDecl() const {
+      assert(isDecl() && "Not a decl!");
+      return static_cast<Decl *>(Stored);
+    }
+    
+  private:
+    void *Stored;
+    bool IsType;
+  };
+  
+  /// \brief The declarations and types to emit.
+  std::queue<DeclOrType> DeclTypesToEmit;
+  
   /// \brief Map that provides the ID numbers of each declaration within
   /// the output stream.
   ///
@@ -85,10 +112,6 @@ private:
   /// the declaration's ID.
   std::vector<uint32_t> DeclOffsets;
 
-  /// \brief Queue containing the declarations that we still need to
-  /// emit.
-  std::queue<Decl *> DeclsToEmit;
-
   /// \brief Map that provides the ID numbers of each type within the
   /// output stream.
   ///
@@ -107,10 +130,6 @@ private:
   /// \brief The type ID that will be assigned to the next new type.
   pch::TypeID NextTypeID;
 
-  /// \brief Queue containing the types that we still need to
-  /// emit.
-  std::queue<QualType> TypesToEmit;
-
   /// \brief Map that provides the ID numbers of each identifier in
   /// the output stream.
   ///
@@ -189,18 +208,17 @@ private:
   void WritePreprocessor(const Preprocessor &PP);
   void WriteComments(ASTContext &Context);
   void WriteType(QualType T);
-  void WriteTypesBlock(ASTContext &Context);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context, DeclContext *DC);
   uint64_t WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC);
 
-  void WriteDeclsBlock(ASTContext &Context);
   void WriteMethodPool(Sema &SemaRef);
   void WriteIdentifierTable(Preprocessor &PP);
   void WriteAttributeRecord(const Attr *Attr);
 
   unsigned ParmVarDeclAbbrev;
   void WriteDeclsBlockAbbrevs();
-
+  void WriteDecl(ASTContext &Context, Decl *D);
+  
 public:
   /// \brief Create a new precompiled header writer that outputs to
   /// the given bitstream.
diff --git a/lib/Frontend/PCHReader.cpp b/lib/Frontend/PCHReader.cpp
index ad9bfcbfd2..96fc7d69f1 100644
--- a/lib/Frontend/PCHReader.cpp
+++ b/lib/Frontend/PCHReader.cpp
@@ -382,7 +382,7 @@ Expr *PCHReader::ReadDeclExpr() {
 }
 
 Expr *PCHReader::ReadTypeExpr() {
-  return dyn_cast_or_null<Expr>(ReadStmt(Stream));
+  return dyn_cast_or_null<Expr>(ReadStmt(DeclsCursor));
 }
 
 
@@ -1157,15 +1157,7 @@ PCHReader::ReadPCHBlock() {
 
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
       switch (Stream.ReadSubBlockID()) {
-      case pch::TYPES_BLOCK_ID: // Skip types block (lazily loaded)
-      default:  // Skip unknown content.
-        if (Stream.SkipBlock()) {
-          Error("malformed block record in PCH file");
-          return Failure;
-        }
-        break;
-
-      case pch::DECLS_BLOCK_ID:
+      case pch::DECLTYPES_BLOCK_ID:
         // We lazily load the decls block, but we want to set up the
         // DeclsCursor cursor to point into it.  Clone our current bitcode
         // cursor to it, enter the block and read the abbrevs in that block.
@@ -1173,7 +1165,7 @@ PCHReader::ReadPCHBlock() {
         DeclsCursor = Stream;
         if (Stream.SkipBlock() ||  // Skip with the main cursor.
             // Read the abbrevs.
-            ReadBlockAbbrevs(DeclsCursor, pch::DECLS_BLOCK_ID)) {
+            ReadBlockAbbrevs(DeclsCursor, pch::DECLTYPES_BLOCK_ID)) {
           Error("malformed block record in PCH file");
           return Failure;
         }
@@ -1773,15 +1765,15 @@ void PCHReader::ReadComments(std::vector<SourceRange> &Comments) {
 QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
   // Keep track of where we are in the stream, then jump back there
   // after reading this type.
-  SavedStreamPosition SavedPosition(Stream);
+  SavedStreamPosition SavedPosition(DeclsCursor);
 
   // Note that we are loading a type record.
   LoadingTypeOrDecl Loading(*this);
 
-  Stream.JumpToBit(Offset);
+  DeclsCursor.JumpToBit(Offset);
   RecordData Record;
-  unsigned Code = Stream.ReadCode();
-  switch ((pch::TypeCode)Stream.ReadRecord(Code, Record)) {
+  unsigned Code = DeclsCursor.ReadCode();
+  switch ((pch::TypeCode)DeclsCursor.ReadRecord(Code, Record)) {
   case pch::TYPE_EXT_QUAL: {
     assert(Record.size() == 2 &&
            "Incorrect encoding of extended qualifier type");
@@ -2045,6 +2037,8 @@ void TypeLocReader::VisitArrayLoc(ArrayLoc TyLoc) {
   TyLoc.setRBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   if (Record[Idx++])
     TyLoc.setSizeExpr(Reader.ReadDeclExpr());
+  else
+    TyLoc.setSizeExpr(0);
 }
 
 DeclaratorInfo *PCHReader::GetDeclaratorInfo(const RecordData &Record,
diff --git a/lib/Frontend/PCHWriter.cpp b/lib/Frontend/PCHWriter.cpp
index 91c47a32b8..73ce4bef34 100644
--- a/lib/Frontend/PCHWriter.cpp
+++ b/lib/Frontend/PCHWriter.cpp
@@ -461,8 +461,8 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(PP_MACRO_FUNCTION_LIKE);
   RECORD(PP_TOKEN);
 
-  // Types block.
-  BLOCK(TYPES_BLOCK);
+  // Decls and Types block.
+  BLOCK(DECLTYPES_BLOCK);
   RECORD(TYPE_EXT_QUAL);
   RECORD(TYPE_FIXED_WIDTH_INT);
   RECORD(TYPE_COMPLEX);
@@ -486,11 +486,6 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(TYPE_OBJC_INTERFACE);
   RECORD(TYPE_OBJC_OBJECT_POINTER);
   RECORD(TYPE_OBJC_PROTOCOL_LIST);
-  // Statements and Exprs can occur in the Types block.
-  AddStmtsExprs(Stream, Record);
-
-  // Decls block.
-  BLOCK(DECLS_BLOCK);
   RECORD(DECL_ATTR);
   RECORD(DECL_TRANSLATION_UNIT);
   RECORD(DECL_TYPEDEF);
@@ -520,7 +515,7 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(DECL_BLOCK);
   RECORD(DECL_CONTEXT_LEXICAL);
   RECORD(DECL_CONTEXT_VISIBLE);
-  // Statements and Exprs can occur in the Decls block.
+  // Statements and Exprs can occur in the Decls and Types block.
   AddStmtsExprs(Stream, Record);
 #undef RECORD
 #undef BLOCK
@@ -1201,22 +1196,6 @@ void PCHWriter::WriteType(QualType T) {
   FlushStmts();
 }
 
-/// \brief Write a block containing all of the types.
-void PCHWriter::WriteTypesBlock(ASTContext &Context) {
-  // Enter the types block.
-  Stream.EnterSubblock(pch::TYPES_BLOCK_ID, 2);
-
-  // Emit all of the types that need to be emitted (so far).
-  while (!TypesToEmit.empty()) {
-    QualType T = TypesToEmit.front();
-    TypesToEmit.pop();
-    WriteType(T);
-  }
-
-  // Exit the types block
-  Stream.ExitBlock();
-}
-
 //===----------------------------------------------------------------------===//
 // Declaration Serialization
 //===----------------------------------------------------------------------===//
@@ -1859,7 +1838,7 @@ void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // The translation unit is the first declaration we'll emit.
   DeclIDs[Context.getTranslationUnitDecl()] = 1;
-  DeclsToEmit.push(Context.getTranslationUnitDecl());
+  DeclTypesToEmit.push(Context.getTranslationUnitDecl());
 
   // Make sure that we emit IdentifierInfos (and any attached
   // declarations) for builtins.
@@ -1928,13 +1907,18 @@ void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // Keep writing types and declarations until all types and
   // declarations have been written.
-  do {
-    if (!DeclsToEmit.empty())
-      WriteDeclsBlock(Context);
-    if (!TypesToEmit.empty())
-      WriteTypesBlock(Context);
-  } while (!(DeclsToEmit.empty() && TypesToEmit.empty()));
-
+  Stream.EnterSubblock(pch::DECLTYPES_BLOCK_ID, 3);
+  WriteDeclsBlockAbbrevs();
+  while (!DeclTypesToEmit.empty()) {
+    DeclOrType DOT = DeclTypesToEmit.front();
+    DeclTypesToEmit.pop();
+    if (DOT.isType())
+      WriteType(DOT.getType());
+    else
+      WriteDecl(Context, DOT.getDecl());
+  }
+  Stream.ExitBlock();
+  
   WriteMethodPool(SemaRef);
   WriteIdentifierTable(PP);
 
@@ -2068,7 +2052,7 @@ void PCHWriter::AddTypeRef(QualType T, RecordData &Record) {
       // Assign it a new ID.  This is the only time we enqueue a
       // qualified type, and it has no CV qualifiers.
       ID = NextTypeID++;
-      TypesToEmit.push(T);
+      DeclTypesToEmit.push(T);
     }
     
     // Encode the type qualifiers in the type reference.
@@ -2122,7 +2106,7 @@ void PCHWriter::AddTypeRef(QualType T, RecordData &Record) {
     // We haven't seen this type before. Assign it a new ID and put it
     // into the queue of types to emit.
     ID = NextTypeID++;
-    TypesToEmit.push(T);
+    DeclTypesToEmit.push(T);
   }
 
   // Encode the type qualifiers in the type reference.
@@ -2140,7 +2124,7 @@ void PCHWriter::AddDeclRef(const Decl *D, RecordData &Record) {
     // We haven't seen this declaration before. Give it a new ID and
     // enqueue it in the list of declarations to emit.
     ID = DeclIDs.size();
-    DeclsToEmit.push(const_cast<Decl *>(D));
+    DeclTypesToEmit.push(const_cast<Decl *>(D));
   }
 
   Record.push_back(ID);
diff --git a/lib/Frontend/PCHWriterDecl.cpp b/lib/Frontend/PCHWriterDecl.cpp
index 73598abf96..fbd9929e49 100644
--- a/lib/Frontend/PCHWriterDecl.cpp
+++ b/lib/Frontend/PCHWriterDecl.cpp
@@ -535,80 +535,64 @@ static bool isRequiredDecl(const Decl *D, ASTContext &Context) {
   }
 }
 
-/// \brief Write a block containing all of the declarations.
-void PCHWriter::WriteDeclsBlock(ASTContext &Context) {
-  // Enter the declarations block.
-  Stream.EnterSubblock(pch::DECLS_BLOCK_ID, 3);
-
-  // Output the abbreviations that we will use in this block.
-  WriteDeclsBlockAbbrevs();
-
-  // Emit all of the declarations.
+void PCHWriter::WriteDecl(ASTContext &Context, Decl *D) {
   RecordData Record;
   PCHDeclWriter W(*this, Context, Record);
-  while (!DeclsToEmit.empty()) {
-    // Pull the next declaration off the queue
-    Decl *D = DeclsToEmit.front();
-    DeclsToEmit.pop();
-
-    // If this declaration is also a DeclContext, write blocks for the
-    // declarations that lexically stored inside its context and those
-    // declarations that are visible from its context. These blocks
-    // are written before the declaration itself so that we can put
-    // their offsets into the record for the declaration.
-    uint64_t LexicalOffset = 0;
-    uint64_t VisibleOffset = 0;
-    DeclContext *DC = dyn_cast<DeclContext>(D);
-    if (DC) {
-      LexicalOffset = WriteDeclContextLexicalBlock(Context, DC);
-      VisibleOffset = WriteDeclContextVisibleBlock(Context, DC);
-    }
 
-    // Determine the ID for this declaration
-    pch::DeclID &ID = DeclIDs[D];
-    if (ID == 0)
-      ID = DeclIDs.size();
+  // If this declaration is also a DeclContext, write blocks for the
+  // declarations that lexically stored inside its context and those
+  // declarations that are visible from its context. These blocks
+  // are written before the declaration itself so that we can put
+  // their offsets into the record for the declaration.
+  uint64_t LexicalOffset = 0;
+  uint64_t VisibleOffset = 0;
+  DeclContext *DC = dyn_cast<DeclContext>(D);
+  if (DC) {
+    LexicalOffset = WriteDeclContextLexicalBlock(Context, DC);
+    VisibleOffset = WriteDeclContextVisibleBlock(Context, DC);
+  }
 
-    unsigned Index = ID - 1;
+  // Determine the ID for this declaration
+  pch::DeclID &ID = DeclIDs[D];
+  if (ID == 0)
+    ID = DeclIDs.size();
 
-    // Record the offset for this declaration
-    if (DeclOffsets.size() == Index)
-      DeclOffsets.push_back(Stream.GetCurrentBitNo());
-    else if (DeclOffsets.size() < Index) {
-      DeclOffsets.resize(Index+1);
-      DeclOffsets[Index] = Stream.GetCurrentBitNo();
-    }
+  unsigned Index = ID - 1;
 
-    // Build and emit a record for this declaration
-    Record.clear();
-    W.Code = (pch::DeclCode)0;
-    W.AbbrevToUse = 0;
-    W.Visit(D);
-    if (DC) W.VisitDeclContext(DC, LexicalOffset, VisibleOffset);
-
-    if (!W.Code) {
-      fprintf(stderr, "Cannot serialize declaration of kind %s\n",
-              D->getDeclKindName());
-      assert(false && "Unhandled declaration kind while generating PCH");
-      exit(-1);
-    }
-    Stream.EmitRecord(W.Code, Record, W.AbbrevToUse);
+  // Record the offset for this declaration
+  if (DeclOffsets.size() == Index)
+    DeclOffsets.push_back(Stream.GetCurrentBitNo());
+  else if (DeclOffsets.size() < Index) {
+    DeclOffsets.resize(Index+1);
+    DeclOffsets[Index] = Stream.GetCurrentBitNo();
+  }
 
-    // If the declaration had any attributes, write them now.
-    if (D->hasAttrs())
-      WriteAttributeRecord(D->getAttrs());
+  // Build and emit a record for this declaration
+  Record.clear();
+  W.Code = (pch::DeclCode)0;
+  W.AbbrevToUse = 0;
+  W.Visit(D);
+  if (DC) W.VisitDeclContext(DC, LexicalOffset, VisibleOffset);
+
+  if (!W.Code) {
+    fprintf(stderr, "Cannot serialize declaration of kind %s\n",
+            D->getDeclKindName());
+    assert(false && "Unhandled declaration kind while generating PCH");
+    exit(-1);
+  }
+  Stream.EmitRecord(W.Code, Record, W.AbbrevToUse);
 
-    // Flush any expressions that were written as part of this declaration.
-    FlushStmts();
+  // If the declaration had any attributes, write them now.
+  if (D->hasAttrs())
+    WriteAttributeRecord(D->getAttrs());
 
-    // Note "external" declarations so that we can add them to a record in the
-    // PCH file later.
-    //
-    // FIXME: This should be renamed, the predicate is much more complicated.
-    if (isRequiredDecl(D, Context))
-      ExternalDefinitions.push_back(ID);
-  }
+  // Flush any expressions that were written as part of this declaration.
+  FlushStmts();
 
-  // Exit the declarations block
-  Stream.ExitBlock();
+  // Note "external" declarations so that we can add them to a record in the
+  // PCH file later.
+  //
+  // FIXME: This should be renamed, the predicate is much more complicated.
+  if (isRequiredDecl(D, Context))
+    ExternalDefinitions.push_back(Index + 1);
 }