Implement serialization and lazy deserialization of the preprocessing

record (which includes all macro instantiations and definitions). As
with all lay deserialization, this introduces a new external source
(here, an external preprocessing record source) that loads all of the
preprocessed entities prior to iterating over the entities.

The preprocessing record is an optional part of the precompiled header
that is disabled by default (enabled with
-detailed-preprocessing-record). When the preprocessor given to the
PCH writer has a preprocessing record, that record is written into the
PCH file. When the PCH reader is given a PCH file that contains a
preprocessing record, it will be lazily loaded (which, effectively,
implicitly adds -detailed-preprocessing-record). This is the first
case where we have sections of the precompiled header that are
added/removed based on a compilation flag, which is
unfortunate. However, this data consumes ~550k in the PCH file for
Cocoa.h (out of ~9.9MB), and there is a non-trivial cost to gathering
this detailed preprocessing information, so it's too expensive to turn
on by default. In the future, we should investigate a better encoding
of this information.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@99002 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 206 insertions, 8 deletions

diff --git a/lib/Frontend/PCHReader.cpp b/lib/Frontend/PCHReader.cpp
index 36cddd32e5..7d8fee94f0 100644
--- a/lib/Frontend/PCHReader.cpp
+++ b/lib/Frontend/PCHReader.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Basic/OnDiskHashTable.h"
@@ -326,8 +327,9 @@ PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context,
     IdentifierOffsets(0),
     MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
     TotalSelectorsInMethodPool(0), SelectorOffsets(0),
-    TotalNumSelectors(0), Comments(0), NumComments(0), isysroot(isysroot),
-    NumStatHits(0), NumStatMisses(0),
+    TotalNumSelectors(0), MacroDefinitionOffsets(0), 
+    NumPreallocatedPreprocessingEntities(0), Comments(0), NumComments(0), 
+    isysroot(isysroot), NumStatHits(0), NumStatMisses(0),
     NumSLocEntriesRead(0), NumStatementsRead(0),
     NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
     NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0),
@@ -343,8 +345,9 @@ PCHReader::PCHReader(SourceManager &SourceMgr, FileManager &FileMgr,
     IdentifierOffsets(0),
     MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
     TotalSelectorsInMethodPool(0), SelectorOffsets(0),
-    TotalNumSelectors(0), Comments(0), NumComments(0), isysroot(isysroot),
-    NumStatHits(0), NumStatMisses(0),
+    TotalNumSelectors(0), MacroDefinitionOffsets(0), 
+    NumPreallocatedPreprocessingEntities(0), Comments(0), NumComments(0), 
+    isysroot(isysroot), NumStatHits(0), NumStatMisses(0),
     NumSLocEntriesRead(0), NumStatementsRead(0),
     NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
     NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0),
@@ -1047,12 +1050,14 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       MacroInfo *MI = PP->AllocateMacroInfo(Loc);
       MI->setIsUsed(isUsed);
 
+      unsigned NextIndex = 3;
       if (RecType == pch::PP_MACRO_FUNCTION_LIKE) {
         // Decode function-like macro info.
         bool isC99VarArgs = Record[3];
         bool isGNUVarArgs = Record[4];
         MacroArgs.clear();
         unsigned NumArgs = Record[5];
+        NextIndex = 6 + NumArgs;
         for (unsigned i = 0; i != NumArgs; ++i)
           MacroArgs.push_back(DecodeIdentifierInfo(Record[6+i]));
 
@@ -1070,6 +1075,13 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       // Remember that we saw this macro last so that we add the tokens that
       // form its body to it.
       Macro = MI;
+      
+      if (NextIndex + 1 == Record.size() && PP->getPreprocessingRecord()) {
+        // We have a macro definition. Load it now.
+        PP->getPreprocessingRecord()->RegisterMacroDefinition(Macro,
+                                        getMacroDefinition(Record[NextIndex]));
+      }
+      
       ++NumMacrosRead;
       break;
     }
@@ -1090,6 +1102,64 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       Macro->AddTokenToBody(Tok);
       break;
     }
+        
+    case pch::PP_MACRO_INSTANTIATION: {
+      // If we already have a macro, that means that we've hit the end
+      // of the definition of the macro we were looking for. We're
+      // done.
+      if (Macro)
+        return;
+      
+      if (!PP->getPreprocessingRecord()) {
+        Error("missing preprocessing record in PCH file");
+        return;
+      }
+        
+      PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
+      if (PPRec.getPreprocessedEntity(Record[0]))
+        return;
+
+      MacroInstantiation *MI
+        = new (PPRec) MacroInstantiation(DecodeIdentifierInfo(Record[3]),
+                               SourceRange(
+                                 SourceLocation::getFromRawEncoding(Record[1]),
+                                 SourceLocation::getFromRawEncoding(Record[2])),
+                                         getMacroDefinition(Record[4]));
+      PPRec.SetPreallocatedEntity(Record[0], MI);
+      return;
+    }
+
+    case pch::PP_MACRO_DEFINITION: {
+      // If we already have a macro, that means that we've hit the end
+      // of the definition of the macro we were looking for. We're
+      // done.
+      if (Macro)
+        return;
+      
+      if (!PP->getPreprocessingRecord()) {
+        Error("missing preprocessing record in PCH file");
+        return;
+      }
+      
+      PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
+      if (PPRec.getPreprocessedEntity(Record[0]))
+        return;
+        
+      if (Record[1] >= MacroDefinitionsLoaded.size()) {
+        Error("out-of-bounds macro definition record");
+        return;
+      }
+
+      MacroDefinition *MD
+        = new (PPRec) MacroDefinition(DecodeIdentifierInfo(Record[4]),
+                                SourceLocation::getFromRawEncoding(Record[5]),
+                              SourceRange(
+                                SourceLocation::getFromRawEncoding(Record[2]),
+                                SourceLocation::getFromRawEncoding(Record[3])));
+      PPRec.SetPreallocatedEntity(Record[0], MD);
+      MacroDefinitionsLoaded[Record[1]] = MD;
+      return;
+    }
   }
   }
 }
@@ -1139,16 +1209,32 @@ void PCHReader::ReadDefinedMacros() {
 
     case pch::PP_MACRO_OBJECT_LIKE:
     case pch::PP_MACRO_FUNCTION_LIKE:
-        DecodeIdentifierInfo(Record[0]);
+      DecodeIdentifierInfo(Record[0]);
       break;
 
     case pch::PP_TOKEN:
       // Ignore tokens.
       break;
+        
+    case pch::PP_MACRO_INSTANTIATION:
+    case pch::PP_MACRO_DEFINITION:
+      // Read the macro record.
+      ReadMacroRecord(Cursor.GetCurrentBitNo());
+      break;
     }
   }
 }
 
+MacroDefinition *PCHReader::getMacroDefinition(pch::IdentID ID) {
+  if (ID == 0 || ID >= MacroDefinitionsLoaded.size())
+    return 0;
+  
+  if (!MacroDefinitionsLoaded[ID])
+    ReadMacroRecord(MacroDefinitionOffsets[ID]);
+    
+  return MacroDefinitionsLoaded[ID];
+}
+
 /// \brief If we are loading a relocatable PCH file, and the filename is
 /// not an absolute path, add the system root to the beginning of the file
 /// name.
@@ -1431,6 +1517,19 @@ PCHReader::ReadPCHBlock() {
       }
       break;
     }
+        
+    case pch::MACRO_DEFINITION_OFFSETS:
+      MacroDefinitionOffsets = (const uint32_t *)BlobStart;
+      if (PP) {
+        if (!PP->getPreprocessingRecord())
+          PP->createPreprocessingRecord();
+        PP->getPreprocessingRecord()->SetExternalSource(*this, Record[0]);
+      } else {
+        NumPreallocatedPreprocessingEntities = Record[0];
+      }
+       
+      MacroDefinitionsLoaded.resize(Record[1]);
+      break;
     }
   }
   Error("premature end of bitstream in PCH file");
@@ -1562,6 +1661,18 @@ PCHReader::PCHReadResult PCHReader::ReadPCH(const std::string &FileName) {
   return Success;
 }
 
+void PCHReader::setPreprocessor(Preprocessor &pp) {
+  PP = &pp;
+  
+  if (NumPreallocatedPreprocessingEntities) {
+    if (!PP->getPreprocessingRecord())
+      PP->createPreprocessingRecord();
+    PP->getPreprocessingRecord()->SetExternalSource(*this, 
+                                          NumPreallocatedPreprocessingEntities);
+    NumPreallocatedPreprocessingEntities = 0;
+  }
+}
+
 void PCHReader::InitializeContext(ASTContext &Ctx) {
   Context = &Ctx;
   assert(Context && "Passed null context!");
@@ -1823,6 +1934,10 @@ bool PCHReader::ParseLanguageOptions(
   return false;
 }
 
+void PCHReader::ReadPreprocessedEntities() {
+  ReadDefinedMacros();
+}
+
 void PCHReader::ReadComments(std::vector<SourceRange> &Comments) {
   Comments.resize(NumComments);
   std::copy(this->Comments, this->Comments + NumComments,
diff --git a/lib/Frontend/PCHWriter.cpp b/lib/Frontend/PCHWriter.cpp
index e55599b7ec..df41ca1e59 100644
--- a/lib/Frontend/PCHWriter.cpp
+++ b/lib/Frontend/PCHWriter.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Basic/FileManager.h"
@@ -565,7 +566,9 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(EXT_VECTOR_DECLS);
   RECORD(COMMENT_RANGES);
   RECORD(VERSION_CONTROL_BRANCH_REVISION);
-
+  RECORD(UNUSED_STATIC_FUNCS);
+  RECORD(MACRO_DEFINITION_OFFSETS);
+  
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
   RECORD(SM_SLOC_FILE_ENTRY);
@@ -579,7 +582,9 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(PP_MACRO_OBJECT_LIKE);
   RECORD(PP_MACRO_FUNCTION_LIKE);
   RECORD(PP_TOKEN);
-
+  RECORD(PP_MACRO_INSTANTIATION);
+  RECORD(PP_MACRO_DEFINITION);
+  
   // Decls and Types block.
   BLOCK(DECLTYPES_BLOCK);
   RECORD(TYPE_EXT_QUAL);
@@ -1174,6 +1179,7 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
 
   // Loop over all the macro definitions that are live at the end of the file,
   // emitting each to the PP section.
+  PreprocessingRecord *PPRec = PP.getPreprocessingRecord();
   for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
        I != E; ++I) {
     // FIXME: This emits macros in hash table order, we should do it in a stable
@@ -1203,6 +1209,12 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
            I != E; ++I)
         AddIdentifierRef(*I, Record);
     }
+    
+    // If we have a detailed preprocessing record, record the macro definition
+    // ID that corresponds to this macro.
+    if (PPRec)
+      Record.push_back(getMacroDefinitionID(PPRec->findMacroDefinition(MI)));
+    
     Stream.EmitRecord(Code, Record);
     Record.clear();
 
@@ -1230,7 +1242,68 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
     }
     ++NumMacros;
   }
+  
+  // If the preprocessor has a preprocessing record, emit it.
+  unsigned NumPreprocessingRecords = 0;
+  if (PPRec) {
+    for (PreprocessingRecord::iterator E = PPRec->begin(), EEnd = PPRec->end();
+         E != EEnd; ++E) {
+      Record.clear();
+      
+      if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(*E)) {
+        Record.push_back(NumPreprocessingRecords++);
+        AddSourceLocation(MI->getSourceRange().getBegin(), Record);
+        AddSourceLocation(MI->getSourceRange().getEnd(), Record);
+        AddIdentifierRef(MI->getName(), Record);
+        Record.push_back(getMacroDefinitionID(MI->getDefinition()));
+        Stream.EmitRecord(pch::PP_MACRO_INSTANTIATION, Record);
+        continue;
+      }
+      
+      if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
+        // Record this macro definition's location.
+        pch::IdentID ID = getMacroDefinitionID(MD);
+        if (ID != MacroDefinitionOffsets.size()) {
+          if (ID > MacroDefinitionOffsets.size())
+            MacroDefinitionOffsets.resize(ID + 1);
+          
+          MacroDefinitionOffsets[ID] = Stream.GetCurrentBitNo();            
+        } else
+          MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo());
+        
+        Record.push_back(NumPreprocessingRecords++);
+        Record.push_back(ID);
+        AddSourceLocation(MD->getSourceRange().getBegin(), Record);
+        AddSourceLocation(MD->getSourceRange().getEnd(), Record);
+        AddIdentifierRef(MD->getName(), Record);
+        AddSourceLocation(MD->getLocation(), Record);
+        Stream.EmitRecord(pch::PP_MACRO_DEFINITION, Record);
+        continue;
+      }
+    }
+  }
+  
   Stream.ExitBlock();
+  
+  // Write the offsets table for the preprocessing record.
+  if (NumPreprocessingRecords > 0) {
+    // Write the offsets table for identifier IDs.
+    using namespace llvm;
+    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(pch::MACRO_DEFINITION_OFFSETS));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of records
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macro defs
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+    unsigned MacroDefOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+    
+    Record.clear();
+    Record.push_back(pch::MACRO_DEFINITION_OFFSETS);
+    Record.push_back(NumPreprocessingRecords);
+    Record.push_back(MacroDefinitionOffsets.size());
+    Stream.EmitRecordWithBlob(MacroDefOffsetAbbrev, Record,
+                              (const char *)&MacroDefinitionOffsets.front(),
+                              MacroDefinitionOffsets.size() * sizeof(uint32_t));
+  }
 }
 
 void PCHWriter::WriteComments(ASTContext &Context) {
@@ -2009,7 +2082,7 @@ void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // Write the remaining PCH contents.
   RecordData Record;
-  Stream.EnterSubblock(pch::PCH_BLOCK_ID, 4);
+  Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
   WriteMetadata(Context, isysroot);
   WriteLanguageOptions(Context.getLangOptions());
   if (StatCalls && !isysroot)
@@ -2149,6 +2222,16 @@ pch::IdentID PCHWriter::getIdentifierRef(const IdentifierInfo *II) {
   return ID;
 }
 
+pch::IdentID PCHWriter::getMacroDefinitionID(MacroDefinition *MD) {
+  if (MD == 0)
+    return 0;
+  
+  pch::IdentID &ID = MacroDefinitions[MD];
+  if (ID == 0)
+    ID = MacroDefinitions.size();
+  return ID;
+}
+
 void PCHWriter::AddSelectorRef(const Selector SelRef, RecordData &Record) {
   if (SelRef.getAsOpaquePtr() == 0) {
     Record.push_back(0);