When using a precompiled preamble with detailed preprocessing records,

trap the serialized preprocessing records (macro definitions, macro
instantiations, macro definitions) from the generation of the
precompiled preamble, then replay those when walking the list of
preprocessed entities. This eliminates a bug where clang_getCursor()
wasn't able to find preprocessed-entity cursors in the preamble.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@120396 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 149 insertions, 40 deletions

diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index a43d0d3ca1..586002daef 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -27,6 +27,7 @@
 #include "clang/Frontend/FrontendOptions.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTSerializationListener.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
@@ -637,10 +638,11 @@ public:
   }
 };
 
-class PrecompilePreambleConsumer : public PCHGenerator {
+class PrecompilePreambleConsumer : public PCHGenerator, 
+                                   public ASTSerializationListener {
   ASTUnit &Unit;
   std::vector<Decl *> TopLevelDecls;
-
+                                     
 public:
   PrecompilePreambleConsumer(ASTUnit &Unit,
                              const Preprocessor &PP, bool Chaining,
@@ -672,6 +674,15 @@ public:
                                       getWriter().getDeclID(TopLevelDecls[I]));
     }
   }
+                                     
+  virtual void SerializedPreprocessedEntity(PreprocessedEntity *Entity,
+                                            uint64_t Offset) {
+    Unit.addPreprocessedEntityFromPreamble(Offset);
+  }
+                                     
+  virtual ASTSerializationListener *GetASTSerializationListener() {
+    return this;
+  }
 };
 
 class PrecompilePreambleAction : public ASTFrontendAction {
@@ -757,6 +768,7 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   
   // Clear out old caches and data.
   TopLevelDecls.clear();
+  PreprocessedEntities.clear();
   CleanTemporaryFiles();
   PreprocessedEntitiesByFile.clear();
 
@@ -765,6 +777,7 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
                     StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
                             StoredDiagnostics.end());
     TopLevelDeclsInPreamble.clear();
+    PreprocessedEntitiesInPreamble.clear();
   }
 
   // Create a file manager object to provide access to and cache the filesystem.
@@ -1237,6 +1250,8 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
                           StoredDiagnostics.end());
   TopLevelDecls.clear();
   TopLevelDeclsInPreamble.clear();
+  PreprocessedEntities.clear();
+  PreprocessedEntitiesInPreamble.clear();
   
   // Create a file manager object to provide access to and cache the filesystem.
   Clang.setFileManager(new FileManager(Clang.getFileSystemOpts()));
@@ -1269,6 +1284,8 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
     TopLevelDeclsInPreamble.clear();
+    PreprocessedEntities.clear();
+    PreprocessedEntitiesInPreamble.clear();
     PreambleRebuildCounter = DefaultPreambleRebuildInterval;
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
@@ -1321,6 +1338,55 @@ void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
   TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
 }
 
+void ASTUnit::RealizePreprocessedEntitiesFromPreamble() {
+  if (!PP)
+    return;
+  
+  PreprocessingRecord *PPRec = PP->getPreprocessingRecord();
+  if (!PPRec)
+    return;
+  
+  ExternalPreprocessingRecordSource *External = PPRec->getExternalSource();
+  if (!External)
+    return;
+
+  for (unsigned I = 0, N = PreprocessedEntitiesInPreamble.size(); I != N; ++I) {
+    if (PreprocessedEntity *PE
+          = External->ReadPreprocessedEntity(PreprocessedEntitiesInPreamble[I]))
+      PreprocessedEntities.push_back(PE);
+  }
+  
+  if (PreprocessedEntities.empty())
+    return;
+  
+  PreprocessedEntities.insert(PreprocessedEntities.end(), 
+                              PPRec->begin(true), PPRec->end(true));
+}
+
+ASTUnit::pp_entity_iterator ASTUnit::pp_entity_begin() {
+  if (!PreprocessedEntitiesInPreamble.empty() &&
+      PreprocessedEntities.empty())
+    RealizePreprocessedEntitiesFromPreamble();
+  
+  if (PreprocessedEntities.empty())
+    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
+      return PPRec->begin(true);
+  
+  return PreprocessedEntities.begin();
+}
+
+ASTUnit::pp_entity_iterator ASTUnit::pp_entity_end() {
+  if (!PreprocessedEntitiesInPreamble.empty() &&
+      PreprocessedEntities.empty())
+    RealizePreprocessedEntitiesFromPreamble();
+  
+  if (PreprocessedEntities.empty())
+    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
+      return PPRec->end(true);
+  
+  return PreprocessedEntities.end();
+}
+
 unsigned ASTUnit::getMaxPCHLevel() const {
   if (!getOnlyLocalDecls())
     return Decl::MaxPCHLevel;
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index 1cac948d03..57002eb5d0 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -1339,7 +1339,7 @@ bool ASTReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
   }
 }
 
-void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
+PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
   assert(PP && "Forgot to set Preprocessor ?");
   llvm::BitstreamCursor &Stream = F.MacroCursor;
 
@@ -1356,14 +1356,14 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
     unsigned Code = Stream.ReadCode();
     switch (Code) {
     case llvm::bitc::END_BLOCK:
-      return;
+      return 0;
 
     case llvm::bitc::ENTER_SUBBLOCK:
       // No known subblocks, always skip them.
       Stream.ReadSubBlockID();
       if (Stream.SkipBlock()) {
         Error("malformed block record in AST file");
-        return;
+        return 0;
       }
       continue;
 
@@ -1387,12 +1387,12 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
+        return 0;
 
       IdentifierInfo *II = DecodeIdentifierInfo(Record[0]);
       if (II == 0) {
         Error("macro must have a name in AST file");
-        return;
+        return 0;
       }
       SourceLocation Loc = ReadSourceLocation(F, Record[1]);
       bool isUsed = Record[2];
@@ -1459,16 +1459,16 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
+        return 0;
 
       if (!PP->getPreprocessingRecord()) {
         Error("missing preprocessing record in AST file");
-        return;
+        return 0;
       }
 
       PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-      if (PPRec.getPreprocessedEntity(Record[0]))
-        return;
+      if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+        return PE;
 
       MacroInstantiation *MI
         = new (PPRec) MacroInstantiation(DecodeIdentifierInfo(Record[3]),
@@ -1476,7 +1476,7 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
                                            ReadSourceLocation(F, Record[2])),
                                          getMacroDefinition(Record[4]));
       PPRec.SetPreallocatedEntity(Record[0], MI);
-      return;
+      return MI;
     }
 
     case PP_MACRO_DEFINITION: {
@@ -1484,20 +1484,20 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
+        return 0;
 
       if (!PP->getPreprocessingRecord()) {
         Error("missing preprocessing record in AST file");
-        return;
+        return 0;
       }
 
       PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-      if (PPRec.getPreprocessedEntity(Record[0]))
-        return;
+      if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+        return PE;
 
       if (Record[1] > MacroDefinitionsLoaded.size()) {
         Error("out-of-bounds macro definition record");
-        return;
+        return 0;
       }
 
       // Decode the identifier info and then check again; if the macro is
@@ -1518,7 +1518,7 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
           DeserializationListener->MacroDefinitionRead(Record[1], MD);
       }
 
-      return;
+      return MacroDefinitionsLoaded[Record[1] - 1];
     }
 
     case PP_INCLUSION_DIRECTIVE: {
@@ -1526,21 +1526,21 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
+        return 0;
 
       if (!PP->getPreprocessingRecord()) {
         Error("missing preprocessing record in AST file");
-        return;
+        return 0;
       }
 
       PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-      if (PPRec.getPreprocessedEntity(Record[0]))
-        return;
+      if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+        return PE;
 
       const char *FullFileNameStart = BlobStart + Record[3];
       const FileEntry *File
-      = PP->getFileManager().getFile(llvm::StringRef(FullFileNameStart,
-                                                     BlobLen - Record[3]));
+        = PP->getFileManager().getFile(llvm::StringRef(FullFileNameStart,
+                                                       BlobLen - Record[3]));
 
       // FIXME: Stable encoding
       InclusionDirective::InclusionKind Kind
@@ -1553,10 +1553,12 @@ void ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
                                  SourceRange(ReadSourceLocation(F, Record[1]),
                                              ReadSourceLocation(F, Record[2])));
       PPRec.SetPreallocatedEntity(Record[0], ID);
-      return;
+      return ID;
     }
     }
   }
+  
+  return 0;
 }
 
 void ASTReader::SetIdentifierIsMacro(IdentifierInfo *II, PerFileData &F,
@@ -2638,6 +2640,25 @@ void ASTReader::ReadPreprocessedEntities() {
   ReadDefinedMacros();
 }
 
+PreprocessedEntity *ASTReader::ReadPreprocessedEntity(uint64_t Offset) {
+  PerFileData *F = 0;  
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (Offset < Chain[I]->SizeInBits) {
+      F = Chain[I];
+      break;
+    }
+    
+    Offset -= Chain[I]->SizeInBits;
+  }
+  
+  if (!F) {
+    Error("Malformed preprocessed entity offset");
+    return 0;
+  }
+
+  return ReadMacroRecord(*F, Offset);
+}
+
 void ASTReader::ReadUserDiagnosticMappings(Diagnostic &Diag) {
   unsigned Idx = 0;
   while (Idx < UserDiagMappings.size()) {
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index b143ce4b1b..9a5085da27 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Serialization/ASTWriter.h"
+#include "clang/Serialization/ASTSerializationListener.h"
 #include "ASTCommon.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/IdentifierResolver.h"
@@ -1358,33 +1359,28 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
          E != EEnd; ++E) {
       Record.clear();
 
-      if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(*E)) {
-        Record.push_back(IndexBase + NumPreprocessingRecords++);
-        AddSourceLocation(MI->getSourceRange().getBegin(), Record);
-        AddSourceLocation(MI->getSourceRange().getEnd(), Record);
-        AddIdentifierRef(MI->getName(), Record);
-        Record.push_back(getMacroDefinitionID(MI->getDefinition()));
-        Stream.EmitRecord(PP_MACRO_INSTANTIATION, Record);
-        continue;
-      }
-
       if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
         // Record this macro definition's location.
         MacroID ID = getMacroDefinitionID(MD);
-
+        
         // Don't write the macro definition if it is from another AST file.
         if (ID < FirstMacroID)
           continue;
+        
+        // Notify the serialization listener that we're serializing this entity.
+        if (SerializationListener)
+          SerializationListener->SerializedPreprocessedEntity(*E, 
+                                                      Stream.GetCurrentBitNo());
 
         unsigned Position = ID - FirstMacroID;
         if (Position != MacroDefinitionOffsets.size()) {
           if (Position > MacroDefinitionOffsets.size())
             MacroDefinitionOffsets.resize(Position + 1);
-
+          
           MacroDefinitionOffsets[Position] = Stream.GetCurrentBitNo();
         } else
           MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo());
-
+        
         Record.push_back(IndexBase + NumPreprocessingRecords++);
         Record.push_back(ID);
         AddSourceLocation(MD->getSourceRange().getBegin(), Record);
@@ -1395,6 +1391,21 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
         continue;
       }
 
+      // Notify the serialization listener that we're serializing this entity.
+      if (SerializationListener)
+        SerializationListener->SerializedPreprocessedEntity(*E, 
+                                                      Stream.GetCurrentBitNo());
+
+      if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(*E)) {          
+        Record.push_back(IndexBase + NumPreprocessingRecords++);
+        AddSourceLocation(MI->getSourceRange().getBegin(), Record);
+        AddSourceLocation(MI->getSourceRange().getEnd(), Record);
+        AddIdentifierRef(MI->getName(), Record);
+        Record.push_back(getMacroDefinitionID(MI->getDefinition()));
+        Stream.EmitRecord(PP_MACRO_INSTANTIATION, Record);
+        continue;
+      }
+
       if (InclusionDirective *ID = dyn_cast<InclusionDirective>(*E)) {
         Record.push_back(PP_INCLUSION_DIRECTIVE);
         Record.push_back(IndexBase + NumPreprocessingRecords++);
@@ -1409,6 +1420,8 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
         Stream.EmitRecordWithBlob(InclusionAbbrev, Record, Buffer);
         continue;
       }
+      
+      llvm_unreachable("Unhandled PreprocessedEntity in ASTWriter");
     }
   }
 
@@ -2232,7 +2245,8 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
 }
 
 ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
-  : Stream(Stream), Chain(0), FirstDeclID(1), NextDeclID(FirstDeclID),
+  : Stream(Stream), Chain(0), SerializationListener(0), 
+    FirstDeclID(1), NextDeclID(FirstDeclID),
     FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID),
     FirstIdentID(1), NextIdentID(FirstIdentID), FirstSelectorID(1),
     NextSelectorID(FirstSelectorID), FirstMacroID(1), NextMacroID(FirstMacroID),
@@ -3415,3 +3429,5 @@ void ASTWriter::AddedCXXTemplateSpecialization(const ClassTemplateDecl *TD,
   Record.push_back(UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION);
   AddDeclRef(D, Record);
 }
+
+ASTSerializationListener::~ASTSerializationListener() { }
diff --git a/lib/Serialization/GeneratePCH.cpp b/lib/Serialization/GeneratePCH.cpp
index 4f6f5cae42..3b4f869c00 100644
--- a/lib/Serialization/GeneratePCH.cpp
+++ b/lib/Serialization/GeneratePCH.cpp
@@ -32,7 +32,6 @@ PCHGenerator::PCHGenerator(const Preprocessor &PP,
                            llvm::raw_ostream *OS)
   : PP(PP), isysroot(isysroot), Out(OS), SemaPtr(0),
     StatCalls(0), Stream(Buffer), Writer(Stream), Chaining(Chaining) {
-
   // Install a stat() listener to keep track of all of the stat()
   // calls.
   StatCalls = new MemorizeStatCalls();
@@ -46,6 +45,9 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   if (PP.getDiagnostics().hasErrorOccurred())
     return;
 
+  // Set up the serialization listener.
+  Writer.SetSerializationListener(GetASTSerializationListener());
+  
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
   Writer.WriteAST(*SemaPtr, StatCalls, isysroot);
@@ -66,6 +68,10 @@ ASTMutationListener *PCHGenerator::GetASTMutationListener() {
   return 0;
 }
 
+ASTSerializationListener *PCHGenerator::GetASTSerializationListener() {
+  return 0;
+}
+
 ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() {
   return &Writer;
 }