Rework the (de-)serialization of macros, as stored in

MacroInfo*. Instead of simply dumping an offset into the current file,
give each macro definition a proper ID with all of the standard
modules-remapping facilities. Additionally, when a macro is modified
in a subsequent AST file (e.g., #undef'ing a macro loaded from another
module or from a precompiled header), provide a macro update record
rather than rewriting the entire macro definition. This gives us
greater consistency with the way we handle declarations, and ties
together macro definitions much more cleanly.

Note that we're still not actually deserializing macro history (we
never were), but it's far easy to do properly now.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@165560 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 110 insertions, 31 deletions

diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h
index 57d896ee41..a0c62bb296 100644
--- a/include/clang/Serialization/ASTBitCodes.h
+++ b/include/clang/Serialization/ASTBitCodes.h
@@ -126,7 +126,13 @@ namespace clang {
 
     /// \brief The number of predefined identifier IDs.
     const unsigned int NUM_PREDEF_IDENT_IDS = 1;
+
+    /// \brief An ID number that refers to a macro in an AST file.
+    typedef uint32_t MacroID;
     
+    /// \brief The number of predefined macro IDs.
+    const unsigned int NUM_PREDEF_MACRO_IDS = 1;
+
     /// \brief An ID number that refers to an ObjC selector in an AST file.
     typedef uint32_t SelectorID;
 
@@ -472,7 +478,18 @@ namespace clang {
       ///
       /// This array can only be interpreted properly using the Objective-C
       /// categories map.
-      OBJC_CATEGORIES = 54
+      OBJC_CATEGORIES = 54,
+
+      /// \brief Record code for the table of offsets of each macro ID.
+      ///
+      /// The offset table contains offsets into the blob stored in
+      /// the preprocessor block. Each offset points to the corresponding
+      /// macro definition.
+      MACRO_OFFSET = 55,
+
+      /// \brief Record of updates for a macro that was modified after
+      /// being deserialized.
+      MACRO_UPDATES = 56
     };
 
     /// \brief Record types used within a source manager block.
diff --git a/include/clang/Serialization/ASTDeserializationListener.h b/include/clang/Serialization/ASTDeserializationListener.h
index ab0d313a5c..f24e39d8e1 100644
--- a/include/clang/Serialization/ASTDeserializationListener.h
+++ b/include/clang/Serialization/ASTDeserializationListener.h
@@ -23,6 +23,7 @@ class Decl;
 class ASTReader;
 class QualType;
 class MacroDefinition;
+class MacroInfo;
 class Module;
   
 class ASTDeserializationListener {
@@ -37,6 +38,8 @@ public:
   /// \brief An identifier was deserialized from the AST file.
   virtual void IdentifierRead(serialization::IdentID ID,
                               IdentifierInfo *II) { }
+  /// \brief A macro was read from the AST file.
+  virtual void MacroRead(serialization::MacroID ID, MacroInfo *MI) { }
   /// \brief A type was deserialized from the AST file. The ID here has the
   ///        qualifier bits already removed, and T is guaranteed to be locally
   ///        unqualified.
diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h
index 5875a5052b..a5fb396a50 100644
--- a/include/clang/Serialization/ASTReader.h
+++ b/include/clang/Serialization/ASTReader.h
@@ -24,6 +24,7 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/PPMutationListener.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
@@ -368,11 +369,33 @@ private:
   typedef ContinuousRangeMap<serialization::IdentID, ModuleFile *, 4>
     GlobalIdentifierMapType;
 
-  /// \brief Mapping from global identifer IDs to the module in which the
+  /// \brief Mapping from global identifier IDs to the module in which the
   /// identifier resides along with the offset that should be added to the
   /// global identifier ID to produce a local ID.
   GlobalIdentifierMapType GlobalIdentifierMap;
 
+  /// \brief A vector containing macros that have already been
+  /// loaded.
+  ///
+  /// If the pointer at index I is non-NULL, then it refers to the
+  /// MacroInfo for the identifier with ID=I+1 that has already
+  /// been loaded.
+  std::vector<MacroInfo *> MacrosLoaded;
+
+  typedef ContinuousRangeMap<serialization::MacroID, ModuleFile *, 4>
+    GlobalMacroMapType;
+
+  /// \brief Mapping from global macro IDs to the module in which the
+  /// macro resides along with the offset that should be added to the
+  /// global macro ID to produce a local ID.
+  GlobalMacroMapType GlobalMacroMap;
+
+  typedef llvm::DenseMap<serialization::MacroID, MacroUpdate> MacroUpdatesMap;
+
+  /// \brief Mapping from (global) macro IDs to the set of updates to be
+  /// performed to the corresponding macro.
+  MacroUpdatesMap MacroUpdates;
+
   /// \brief A vector containing submodules that have already been loaded.
   ///
   /// This vector is indexed by the Submodule ID (-1). NULL submodule entries
@@ -441,9 +464,8 @@ private:
   llvm::DenseMap<Selector, unsigned> SelectorGeneration;
 
   /// \brief Mapping from identifiers that represent macros whose definitions
-  /// have not yet been deserialized to the global offset where the macro
-  /// record resides.
-  llvm::DenseMap<IdentifierInfo *, uint64_t> UnreadMacroRecordOffsets;
+  /// have not yet been deserialized to the global ID of the macro.
+  llvm::DenseMap<IdentifierInfo *, serialization::MacroID> UnreadMacroIDs;
 
   typedef ContinuousRangeMap<unsigned, ModuleFile *, 4>
     GlobalPreprocessedEntityMapType;
@@ -1065,6 +1087,11 @@ public:
     return static_cast<unsigned>(IdentifiersLoaded.size());
   }
 
+  /// \brief Returns the number of macros found in the chain.
+  unsigned getTotalNumMacros() const {
+    return static_cast<unsigned>(MacrosLoaded.size());
+  }
+
   /// \brief Returns the number of types found in the chain.
   unsigned getTotalNumTypes() const {
     return static_cast<unsigned>(TypesLoaded.size());
@@ -1366,6 +1393,13 @@ public:
   serialization::IdentifierID getGlobalIdentifierID(ModuleFile &M,
                                                     unsigned LocalID);
 
+  /// \brief Retrieve the macro with the given ID.
+  MacroInfo *getMacro(serialization::MacroID ID);
+
+  /// \brief Retrieve the global macro ID corresponding to the given local
+  /// ID within the given module file.
+  serialization::MacroID getGlobalMacroID(ModuleFile &M, unsigned LocalID);
+
   /// \brief Read the source location entry with index ID.
   virtual bool ReadSLocEntry(int ID);
 
@@ -1521,14 +1555,11 @@ public:
   ///
   /// \param II The name of the macro.
   ///
-  /// \param F The module file from which the macro definition was deserialized.
-  ///
-  /// \param Offset The offset into the module file at which the macro 
-  /// definition is located.
+  /// \param ID The global macro ID.
   ///
   /// \param Visible Whether the macro should be made visible.
-  void setIdentifierIsMacro(IdentifierInfo *II, ModuleFile &F,
-                            uint64_t Offset, bool Visible);
+  void setIdentifierIsMacro(IdentifierInfo *II, serialization::MacroID ID,
+                            bool Visible);
 
   /// \brief Read the set of macros defined by this external macro source.
   virtual void ReadDefinedMacros();
@@ -1541,11 +1572,11 @@ public:
 
   /// \brief Note that this identifier is up-to-date.
   void markIdentifierUpToDate(IdentifierInfo *II);
-  
+
   /// \brief Read the macro definition corresponding to this iterator
   /// into the unread macro record offsets table.
   void LoadMacroDefinition(
-                     llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos);
+         llvm::DenseMap<IdentifierInfo *,serialization::MacroID>::iterator Pos);
 
   /// \brief Load all external visible decls in the given DeclContext.
   void completeVisibleDeclsMap(const DeclContext *DC);
diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h
index d95b249a28..db5b8ca99e 100644
--- a/include/clang/Serialization/ASTWriter.h
+++ b/include/clang/Serialization/ASTWriter.h
@@ -18,6 +18,7 @@
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/ASTMutationListener.h"
+#include "clang/Lex/PPMutationListener.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Sema/SemaConsumer.h"
@@ -25,6 +26,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include <map>
@@ -51,6 +53,7 @@ class MemorizeStatCalls;
 class OpaqueValueExpr;
 class OpenCLOptions;
 class ASTReader;
+class MacroInfo;
 class Module;
 class PreprocessedEntity;
 class PreprocessingRecord;
@@ -70,6 +73,7 @@ namespace SrcMgr { class SLocEntry; }
 /// data structures. This bitstream can be de-serialized via an
 /// instance of the ASTReader class.
 class ASTWriter : public ASTDeserializationListener,
+                  public PPMutationListener,
                   public ASTMutationListener {
 public:
   typedef SmallVector<uint64_t, 64> RecordData;
@@ -214,6 +218,15 @@ private:
   /// IdentifierInfo.
   llvm::DenseMap<const IdentifierInfo *, serialization::IdentID> IdentifierIDs;
 
+  /// \brief The first ID number we can use for our own macros.
+  serialization::MacroID FirstMacroID;
+
+  /// \brief The identifier ID that will be assigned to the next new identifier.
+  serialization::MacroID NextMacroID;
+
+  /// \brief Map that provides the ID numbers of each macro.
+  llvm::DenseMap<MacroInfo *, serialization::MacroID> MacroIDs;
+
   /// @name FlushStmt Caches
   /// @{
 
@@ -249,17 +262,14 @@ private:
   /// table, indexed by the Selector ID (-1).
   std::vector<uint32_t> SelectorOffsets;
 
-  /// \brief Offsets of each of the macro identifiers into the
-  /// bitstream.
-  ///
-  /// For each identifier that is associated with a macro, this map
-  /// provides the offset into the bitstream where that macro is
-  /// defined.
-  llvm::DenseMap<const IdentifierInfo *, uint64_t> MacroOffsets;
-
   /// \brief The set of identifiers that had macro definitions at some point.
   std::vector<const IdentifierInfo *> DeserializedMacroNames;
 
+  typedef llvm::MapVector<MacroInfo *, MacroUpdate> MacroUpdatesMap;
+
+  /// \brief Updates to macro definitions that were loaded from an AST file.
+  MacroUpdatesMap MacroUpdates;
+
   /// \brief Mapping from macro definitions (as they occur in the preprocessing
   /// record) to the macro IDs.
   llvm::DenseMap<const MacroDefinition *, serialization::PreprocessedEntityID>
@@ -427,6 +437,7 @@ private:
   void WriteIdentifierTable(Preprocessor &PP, IdentifierResolver &IdResolver,
                             bool IsModule);
   void WriteAttributes(ArrayRef<const Attr*> Attrs, RecordDataImpl &Record);
+  void WriteMacroUpdates();
   void ResolveDeclUpdatesBlocks();
   void WriteDeclUpdatesBlocks();
   void WriteDeclReplacementsBlock();
@@ -500,6 +511,9 @@ public:
   /// \brief Emit a reference to an identifier.
   void AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Record);
 
+  /// \brief Emit a reference to a macro.
+  void addMacroRef(MacroInfo *MI, RecordDataImpl &Record);
+
   /// \brief Emit a Selector (which is a smart pointer reference).
   void AddSelectorRef(Selector, RecordDataImpl &Record);
 
@@ -517,15 +531,8 @@ public:
   /// \brief Get the unique number used to refer to the given identifier.
   serialization::IdentID getIdentifierRef(const IdentifierInfo *II);
 
-  /// \brief Retrieve the offset of the macro definition for the given
-  /// identifier.
-  ///
-  /// The identifier must refer to a macro.
-  uint64_t getMacroOffset(const IdentifierInfo *II) {
-    assert(MacroOffsets.find(II) != MacroOffsets.end() &&
-           "Identifier does not name a macro");
-    return MacroOffsets[II];
-  }
+  /// \brief Get the unique number used to refer to the given macro.
+  serialization::MacroID getMacroRef(MacroInfo *MI);
 
   /// \brief Emit a reference to a type.
   void AddTypeRef(QualType T, RecordDataImpl &Record);
@@ -688,13 +695,17 @@ public:
   // ASTDeserializationListener implementation
   void ReaderInitialized(ASTReader *Reader);
   void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II);
+  void MacroRead(serialization::MacroID ID, MacroInfo *MI);
   void TypeRead(serialization::TypeIdx Idx, QualType T);
   void SelectorRead(serialization::SelectorID ID, Selector Sel);
   void MacroDefinitionRead(serialization::PreprocessedEntityID ID,
                            MacroDefinition *MD);
   void MacroVisible(IdentifierInfo *II);
   void ModuleRead(serialization::SubmoduleID ID, Module *Mod);
-                    
+
+  // PPMutationListener implementation.
+  virtual void UndefinedMacro(MacroInfo *MI);
+
   // ASTMutationListener implementation.
   virtual void CompletedTagDefinition(const TagDecl *D);
   virtual void AddedVisibleDecl(const DeclContext *DC, const Decl *D);
@@ -737,6 +748,7 @@ public:
   ~PCHGenerator();
   virtual void InitializeSema(Sema &S) { SemaPtr = &S; }
   virtual void HandleTranslationUnit(ASTContext &Ctx);
+  virtual PPMutationListener *GetPPMutationListener();
   virtual ASTMutationListener *GetASTMutationListener();
   virtual ASTDeserializationListener *GetASTDeserializationListener();
 };
diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h
index 87a0e40e9e..ecf0ae1349 100644
--- a/include/clang/Serialization/Module.h
+++ b/include/clang/Serialization/Module.h
@@ -173,6 +173,22 @@ public:
   /// all of the macro definitions.
   llvm::BitstreamCursor MacroCursor;
 
+  /// \brief The number of macros in this AST file.
+  unsigned LocalNumMacros;
+
+  /// \brief Offsets of macros in the preprocessor block.
+  ///
+  /// This array is indexed by the macro ID (-1), and provides
+  /// the offset into the preprocessor block where macro definitions are
+  /// stored.
+  const uint32_t *MacroOffsets;
+
+  /// \brief Base macro ID for macros local to this module.
+  serialization::MacroID BaseMacroID;
+
+  /// \brief Remapping table for macro IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> MacroRemap;
+
   /// \brief The offset of the start of the set of defined macros.
   uint64_t MacroStartOffset;