Rework the (de-)serialization of macros, as stored in

MacroInfo*. Instead of simply dumping an offset into the current file,
give each macro definition a proper ID with all of the standard
modules-remapping facilities. Additionally, when a macro is modified
in a subsequent AST file (e.g., #undef'ing a macro loaded from another
module or from a precompiled header), provide a macro update record
rather than rewriting the entire macro definition. This gives us
greater consistency with the way we handle declarations, and ties
together macro definitions much more cleanly.

Note that we're still not actually deserializing macro history (we
never were), but it's far easy to do properly now.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@165560 91177308-0d34-0410-b5e6-96231b3b80d8

26 files changed, 474 insertions, 89 deletions

diff --git a/include/clang/AST/ASTConsumer.h b/include/clang/AST/ASTConsumer.h
index 8418b5a45a..a183eb059c 100644
--- a/include/clang/AST/ASTConsumer.h
+++ b/include/clang/AST/ASTConsumer.h
@@ -19,6 +19,7 @@ namespace clang {
   class CXXRecordDecl;
   class DeclGroupRef;
   class HandleTagDeclDefinition;
+  class PPMutationListener;
   class ASTMutationListener;
   class ASTDeserializationListener; // layering violation because void* is ugly
   class SemaConsumer; // layering violation required for safe SemaConsumer
@@ -111,6 +112,11 @@ public:
   /// it was actually used.
   virtual void HandleVTable(CXXRecordDecl *RD, bool DefinitionRequired) {}
 
+  /// \brief If the consumer is interested in preprocessor entities getting
+  /// modified after their initial creation, it should return a pointer to
+  /// a PPMutationListener here.
+  virtual PPMutationListener *GetPPMutationListener() { return 0; }
+
   /// \brief If the consumer is interested in entities getting modified after
   /// their initial creation, it should return a pointer to
   /// an ASTMutationListener here.
diff --git a/include/clang/AST/ASTMutationListener.h b/include/clang/AST/ASTMutationListener.h
index cb038a0a58..56d15260a5 100644
--- a/include/clang/AST/ASTMutationListener.h
+++ b/include/clang/AST/ASTMutationListener.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_CLANG_AST_ASTMUTATIONLISTENER_H
 #define LLVM_CLANG_AST_ASTMUTATIONLISTENER_H
 
+#include "clang/Basic/SourceLocation.h"
+
 namespace clang {
   class Decl;
   class DeclContext;
diff --git a/include/clang/Lex/PPMutationListener.h b/include/clang/Lex/PPMutationListener.h
new file mode 100644
index 0000000000..5319c66fa2
--- /dev/null
+++ b/include/clang/Lex/PPMutationListener.h
@@ -0,0 +1,43 @@
+//===--- PPMutationListener.h - Preprocessor Mutation Interface -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the PPMutationListener interface.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LEX_PPTMUTATIONLISTENER_H
+#define LLVM_CLANG_LEX_PPTMUTATIONLISTENER_H
+
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class MacroInfo;
+
+/// \brief A record that describes an update to a macro that was
+/// originally loaded to an AST file and has been modified within the
+/// current translation unit.
+struct MacroUpdate {
+  /// \brief The source location at which this macro was #undef'd.
+  SourceLocation UndefLoc;
+};
+
+/// \brief An abstract interface that should be implemented by
+/// listeners that want to be notified when a preprocessor entity gets
+/// modified after its initial creation.
+class PPMutationListener {
+public:
+  virtual ~PPMutationListener();
+
+  /// \brief A macro has been #undef'd.
+  virtual void UndefinedMacro(MacroInfo *MI) { }
+};
+
+} // end namespace clang
+
+#endif
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index 03ff02798b..12b99af6e0 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -18,6 +18,7 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/PTHLexer.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/PPMutationListener.h"
 #include "clang/Lex/TokenLexer.h"
 #include "clang/Lex/PTHManager.h"
 #include "clang/Basic/Builtins.h"
@@ -288,6 +289,11 @@ class Preprocessor : public RefCountedBase<Preprocessor> {
   /// encountered (e.g. a file is \#included, etc).
   PPCallbacks *Callbacks;
 
+  /// \brief Listener whose actions are invoked when an entity in the
+  /// preprocessor (e.g., a macro) that was loaded from an AST file is
+  /// later mutated.
+  PPMutationListener *Listener;
+
   struct MacroExpandsInfo {
     Token Tok;
     MacroInfo *MI;
@@ -480,6 +486,19 @@ public:
     Callbacks = C;
   }
 
+  /// \brief Attach an preprocessor mutation listener to the preprocessor.
+  ///
+  /// The preprocessor mutation listener provides the ability to track
+  /// modifications to the preprocessor entities committed after they were
+  /// initially created.
+  void setPPMutationListener(PPMutationListener *Listener) {
+    this->Listener = Listener;
+  }
+
+  /// \brief Retrieve a pointer to the preprocessor mutation listener
+  /// associated with this preprocessor, if any.
+  PPMutationListener *getPPMutationListener() const { return Listener; }
+
   /// \brief Given an identifier, return the MacroInfo it is \#defined to
   /// or null if it isn't \#define'd.
   MacroInfo *getMacroInfo(IdentifierInfo *II) const {
@@ -1339,6 +1358,8 @@ private:
   // Macro handling.
   void HandleDefineDirective(Token &Tok);
   void HandleUndefDirective(Token &Tok);
+  void UndefineMacro(IdentifierInfo *II, MacroInfo *MI,
+                     SourceLocation UndefLoc);
 
   // Conditional Inclusion.
   void HandleIfdefDirective(Token &Tok, bool isIfndef,
diff --git a/include/clang/Serialization/ASTBitCodes.h b/include/clang/Serialization/ASTBitCodes.h
index 57d896ee41..a0c62bb296 100644
--- a/include/clang/Serialization/ASTBitCodes.h
+++ b/include/clang/Serialization/ASTBitCodes.h
@@ -126,7 +126,13 @@ namespace clang {
 
     /// \brief The number of predefined identifier IDs.
     const unsigned int NUM_PREDEF_IDENT_IDS = 1;
+
+    /// \brief An ID number that refers to a macro in an AST file.
+    typedef uint32_t MacroID;
     
+    /// \brief The number of predefined macro IDs.
+    const unsigned int NUM_PREDEF_MACRO_IDS = 1;
+
     /// \brief An ID number that refers to an ObjC selector in an AST file.
     typedef uint32_t SelectorID;
 
@@ -472,7 +478,18 @@ namespace clang {
       ///
       /// This array can only be interpreted properly using the Objective-C
       /// categories map.
-      OBJC_CATEGORIES = 54
+      OBJC_CATEGORIES = 54,
+
+      /// \brief Record code for the table of offsets of each macro ID.
+      ///
+      /// The offset table contains offsets into the blob stored in
+      /// the preprocessor block. Each offset points to the corresponding
+      /// macro definition.
+      MACRO_OFFSET = 55,
+
+      /// \brief Record of updates for a macro that was modified after
+      /// being deserialized.
+      MACRO_UPDATES = 56
     };
 
     /// \brief Record types used within a source manager block.
diff --git a/include/clang/Serialization/ASTDeserializationListener.h b/include/clang/Serialization/ASTDeserializationListener.h
index ab0d313a5c..f24e39d8e1 100644
--- a/include/clang/Serialization/ASTDeserializationListener.h
+++ b/include/clang/Serialization/ASTDeserializationListener.h
@@ -23,6 +23,7 @@ class Decl;
 class ASTReader;
 class QualType;
 class MacroDefinition;
+class MacroInfo;
 class Module;
   
 class ASTDeserializationListener {
@@ -37,6 +38,8 @@ public:
   /// \brief An identifier was deserialized from the AST file.
   virtual void IdentifierRead(serialization::IdentID ID,
                               IdentifierInfo *II) { }
+  /// \brief A macro was read from the AST file.
+  virtual void MacroRead(serialization::MacroID ID, MacroInfo *MI) { }
   /// \brief A type was deserialized from the AST file. The ID here has the
   ///        qualifier bits already removed, and T is guaranteed to be locally
   ///        unqualified.
diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h
index 5875a5052b..a5fb396a50 100644
--- a/include/clang/Serialization/ASTReader.h
+++ b/include/clang/Serialization/ASTReader.h
@@ -24,6 +24,7 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/PPMutationListener.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
@@ -368,11 +369,33 @@ private:
   typedef ContinuousRangeMap<serialization::IdentID, ModuleFile *, 4>
     GlobalIdentifierMapType;
 
-  /// \brief Mapping from global identifer IDs to the module in which the
+  /// \brief Mapping from global identifier IDs to the module in which the
   /// identifier resides along with the offset that should be added to the
   /// global identifier ID to produce a local ID.
   GlobalIdentifierMapType GlobalIdentifierMap;
 
+  /// \brief A vector containing macros that have already been
+  /// loaded.
+  ///
+  /// If the pointer at index I is non-NULL, then it refers to the
+  /// MacroInfo for the identifier with ID=I+1 that has already
+  /// been loaded.
+  std::vector<MacroInfo *> MacrosLoaded;
+
+  typedef ContinuousRangeMap<serialization::MacroID, ModuleFile *, 4>
+    GlobalMacroMapType;
+
+  /// \brief Mapping from global macro IDs to the module in which the
+  /// macro resides along with the offset that should be added to the
+  /// global macro ID to produce a local ID.
+  GlobalMacroMapType GlobalMacroMap;
+
+  typedef llvm::DenseMap<serialization::MacroID, MacroUpdate> MacroUpdatesMap;
+
+  /// \brief Mapping from (global) macro IDs to the set of updates to be
+  /// performed to the corresponding macro.
+  MacroUpdatesMap MacroUpdates;
+
   /// \brief A vector containing submodules that have already been loaded.
   ///
   /// This vector is indexed by the Submodule ID (-1). NULL submodule entries
@@ -441,9 +464,8 @@ private:
   llvm::DenseMap<Selector, unsigned> SelectorGeneration;
 
   /// \brief Mapping from identifiers that represent macros whose definitions
-  /// have not yet been deserialized to the global offset where the macro
-  /// record resides.
-  llvm::DenseMap<IdentifierInfo *, uint64_t> UnreadMacroRecordOffsets;
+  /// have not yet been deserialized to the global ID of the macro.
+  llvm::DenseMap<IdentifierInfo *, serialization::MacroID> UnreadMacroIDs;
 
   typedef ContinuousRangeMap<unsigned, ModuleFile *, 4>
     GlobalPreprocessedEntityMapType;
@@ -1065,6 +1087,11 @@ public:
     return static_cast<unsigned>(IdentifiersLoaded.size());
   }
 
+  /// \brief Returns the number of macros found in the chain.
+  unsigned getTotalNumMacros() const {
+    return static_cast<unsigned>(MacrosLoaded.size());
+  }
+
   /// \brief Returns the number of types found in the chain.
   unsigned getTotalNumTypes() const {
     return static_cast<unsigned>(TypesLoaded.size());
@@ -1366,6 +1393,13 @@ public:
   serialization::IdentifierID getGlobalIdentifierID(ModuleFile &M,
                                                     unsigned LocalID);
 
+  /// \brief Retrieve the macro with the given ID.
+  MacroInfo *getMacro(serialization::MacroID ID);
+
+  /// \brief Retrieve the global macro ID corresponding to the given local
+  /// ID within the given module file.
+  serialization::MacroID getGlobalMacroID(ModuleFile &M, unsigned LocalID);
+
   /// \brief Read the source location entry with index ID.
   virtual bool ReadSLocEntry(int ID);
 
@@ -1521,14 +1555,11 @@ public:
   ///
   /// \param II The name of the macro.
   ///
-  /// \param F The module file from which the macro definition was deserialized.
-  ///
-  /// \param Offset The offset into the module file at which the macro 
-  /// definition is located.
+  /// \param ID The global macro ID.
   ///
   /// \param Visible Whether the macro should be made visible.
-  void setIdentifierIsMacro(IdentifierInfo *II, ModuleFile &F,
-                            uint64_t Offset, bool Visible);
+  void setIdentifierIsMacro(IdentifierInfo *II, serialization::MacroID ID,
+                            bool Visible);
 
   /// \brief Read the set of macros defined by this external macro source.
   virtual void ReadDefinedMacros();
@@ -1541,11 +1572,11 @@ public:
 
   /// \brief Note that this identifier is up-to-date.
   void markIdentifierUpToDate(IdentifierInfo *II);
-  
+
   /// \brief Read the macro definition corresponding to this iterator
   /// into the unread macro record offsets table.
   void LoadMacroDefinition(
-                     llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos);
+         llvm::DenseMap<IdentifierInfo *,serialization::MacroID>::iterator Pos);
 
   /// \brief Load all external visible decls in the given DeclContext.
   void completeVisibleDeclsMap(const DeclContext *DC);
diff --git a/include/clang/Serialization/ASTWriter.h b/include/clang/Serialization/ASTWriter.h
index d95b249a28..db5b8ca99e 100644
--- a/include/clang/Serialization/ASTWriter.h
+++ b/include/clang/Serialization/ASTWriter.h
@@ -18,6 +18,7 @@
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/ASTMutationListener.h"
+#include "clang/Lex/PPMutationListener.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Sema/SemaConsumer.h"
@@ -25,6 +26,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include <map>
@@ -51,6 +53,7 @@ class MemorizeStatCalls;
 class OpaqueValueExpr;
 class OpenCLOptions;
 class ASTReader;
+class MacroInfo;
 class Module;
 class PreprocessedEntity;
 class PreprocessingRecord;
@@ -70,6 +73,7 @@ namespace SrcMgr { class SLocEntry; }
 /// data structures. This bitstream can be de-serialized via an
 /// instance of the ASTReader class.
 class ASTWriter : public ASTDeserializationListener,
+                  public PPMutationListener,
                   public ASTMutationListener {
 public:
   typedef SmallVector<uint64_t, 64> RecordData;
@@ -214,6 +218,15 @@ private:
   /// IdentifierInfo.
   llvm::DenseMap<const IdentifierInfo *, serialization::IdentID> IdentifierIDs;
 
+  /// \brief The first ID number we can use for our own macros.
+  serialization::MacroID FirstMacroID;
+
+  /// \brief The identifier ID that will be assigned to the next new identifier.
+  serialization::MacroID NextMacroID;
+
+  /// \brief Map that provides the ID numbers of each macro.
+  llvm::DenseMap<MacroInfo *, serialization::MacroID> MacroIDs;
+
   /// @name FlushStmt Caches
   /// @{
 
@@ -249,17 +262,14 @@ private:
   /// table, indexed by the Selector ID (-1).
   std::vector<uint32_t> SelectorOffsets;
 
-  /// \brief Offsets of each of the macro identifiers into the
-  /// bitstream.
-  ///
-  /// For each identifier that is associated with a macro, this map
-  /// provides the offset into the bitstream where that macro is
-  /// defined.
-  llvm::DenseMap<const IdentifierInfo *, uint64_t> MacroOffsets;
-
   /// \brief The set of identifiers that had macro definitions at some point.
   std::vector<const IdentifierInfo *> DeserializedMacroNames;
 
+  typedef llvm::MapVector<MacroInfo *, MacroUpdate> MacroUpdatesMap;
+
+  /// \brief Updates to macro definitions that were loaded from an AST file.
+  MacroUpdatesMap MacroUpdates;
+
   /// \brief Mapping from macro definitions (as they occur in the preprocessing
   /// record) to the macro IDs.
   llvm::DenseMap<const MacroDefinition *, serialization::PreprocessedEntityID>
@@ -427,6 +437,7 @@ private:
   void WriteIdentifierTable(Preprocessor &PP, IdentifierResolver &IdResolver,
                             bool IsModule);
   void WriteAttributes(ArrayRef<const Attr*> Attrs, RecordDataImpl &Record);
+  void WriteMacroUpdates();
   void ResolveDeclUpdatesBlocks();
   void WriteDeclUpdatesBlocks();
   void WriteDeclReplacementsBlock();
@@ -500,6 +511,9 @@ public:
   /// \brief Emit a reference to an identifier.
   void AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Record);
 
+  /// \brief Emit a reference to a macro.
+  void addMacroRef(MacroInfo *MI, RecordDataImpl &Record);
+
   /// \brief Emit a Selector (which is a smart pointer reference).
   void AddSelectorRef(Selector, RecordDataImpl &Record);
 
@@ -517,15 +531,8 @@ public:
   /// \brief Get the unique number used to refer to the given identifier.
   serialization::IdentID getIdentifierRef(const IdentifierInfo *II);
 
-  /// \brief Retrieve the offset of the macro definition for the given
-  /// identifier.
-  ///
-  /// The identifier must refer to a macro.
-  uint64_t getMacroOffset(const IdentifierInfo *II) {
-    assert(MacroOffsets.find(II) != MacroOffsets.end() &&
-           "Identifier does not name a macro");
-    return MacroOffsets[II];
-  }
+  /// \brief Get the unique number used to refer to the given macro.
+  serialization::MacroID getMacroRef(MacroInfo *MI);
 
   /// \brief Emit a reference to a type.
   void AddTypeRef(QualType T, RecordDataImpl &Record);
@@ -688,13 +695,17 @@ public:
   // ASTDeserializationListener implementation
   void ReaderInitialized(ASTReader *Reader);
   void IdentifierRead(serialization::IdentID ID, IdentifierInfo *II);
+  void MacroRead(serialization::MacroID ID, MacroInfo *MI);
   void TypeRead(serialization::TypeIdx Idx, QualType T);
   void SelectorRead(serialization::SelectorID ID, Selector Sel);
   void MacroDefinitionRead(serialization::PreprocessedEntityID ID,
                            MacroDefinition *MD);
   void MacroVisible(IdentifierInfo *II);
   void ModuleRead(serialization::SubmoduleID ID, Module *Mod);
-                    
+
+  // PPMutationListener implementation.
+  virtual void UndefinedMacro(MacroInfo *MI);
+
   // ASTMutationListener implementation.
   virtual void CompletedTagDefinition(const TagDecl *D);
   virtual void AddedVisibleDecl(const DeclContext *DC, const Decl *D);
@@ -737,6 +748,7 @@ public:
   ~PCHGenerator();
   virtual void InitializeSema(Sema &S) { SemaPtr = &S; }
   virtual void HandleTranslationUnit(ASTContext &Ctx);
+  virtual PPMutationListener *GetPPMutationListener();
   virtual ASTMutationListener *GetASTMutationListener();
   virtual ASTDeserializationListener *GetASTDeserializationListener();
 };
diff --git a/include/clang/Serialization/Module.h b/include/clang/Serialization/Module.h
index 87a0e40e9e..ecf0ae1349 100644
--- a/include/clang/Serialization/Module.h
+++ b/include/clang/Serialization/Module.h
@@ -173,6 +173,22 @@ public:
   /// all of the macro definitions.
   llvm::BitstreamCursor MacroCursor;
 
+  /// \brief The number of macros in this AST file.
+  unsigned LocalNumMacros;
+
+  /// \brief Offsets of macros in the preprocessor block.
+  ///
+  /// This array is indexed by the macro ID (-1), and provides
+  /// the offset into the preprocessor block where macro definitions are
+  /// stored.
+  const uint32_t *MacroOffsets;
+
+  /// \brief Base macro ID for macros local to this module.
+  serialization::MacroID BaseMacroID;
+
+  /// \brief Remapping table for macro IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> MacroRemap;
+
   /// \brief The offset of the start of the set of defined macros.
   uint64_t MacroStartOffset;
 
diff --git a/lib/Frontend/ChainedIncludesSource.cpp b/lib/Frontend/ChainedIncludesSource.cpp
index dbb06bd23c..5f7ff74bc6 100644
--- a/lib/Frontend/ChainedIncludesSource.cpp
+++ b/lib/Frontend/ChainedIncludesSource.cpp
@@ -108,6 +108,8 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     OwningPtr<ASTConsumer> consumer;
     consumer.reset(new PCHGenerator(Clang->getPreprocessor(), "-", 0,
                                     /*isysroot=*/"", &OS));
+    Clang->getPreprocessor().setPPMutationListener(
+                                            consumer->GetPPMutationListener());
     Clang->getASTContext().setASTMutationListener(
                                             consumer->GetASTMutationListener());
     Clang->setASTConsumer(consumer.take());
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index bf46e549b5..24f38d428b 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -946,6 +946,8 @@ Module *CompilerInstance::loadModule(SourceLocation ImportLoc,
           getASTConsumer().GetASTDeserializationListener());
         getASTContext().setASTMutationListener(
           getASTConsumer().GetASTMutationListener());
+        getPreprocessor().setPPMutationListener(
+          getASTConsumer().GetPPMutationListener());
       }
       OwningPtr<ExternalASTSource> Source;
       Source.reset(ModuleManager);
diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp
index a4321e720e..ca8511247a 100644
--- a/lib/Frontend/FrontendAction.cpp
+++ b/lib/Frontend/FrontendAction.cpp
@@ -247,7 +247,9 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
       goto failure;
 
     CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener());
-
+    CI.getPreprocessor().setPPMutationListener(
+      Consumer->GetPPMutationListener());
+    
     if (!CI.getPreprocessorOpts().ChainedIncludes.empty()) {
       // Convert headers to PCH and chain them.
       OwningPtr<ExternalASTSource> source;
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 23cbef63ed..9314517ed3 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -1944,8 +1944,20 @@ void Preprocessor::HandleUndefDirective(Token &UndefTok) {
   if (MI->isWarnIfUnused())
     WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
 
-  MI->setUndefLoc(MacroNameTok.getLocation());
-  clearMacroInfo(MacroNameTok.getIdentifierInfo());
+  UndefineMacro(MacroNameTok.getIdentifierInfo(), MI,
+                MacroNameTok.getLocation());
+}
+
+void Preprocessor::UndefineMacro(IdentifierInfo *II, MacroInfo *MI,
+                                 SourceLocation UndefLoc) {
+  MI->setUndefLoc(UndefLoc);
+  if (MI->isFromAST()) {
+    MI->setChangedAfterLoad();
+    if (Listener)
+      Listener->UndefinedMacro(MI);
+  }
+
+  clearMacroInfo(II);
 }
 
 
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index b5bc9583f6..1f9c811f55 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -737,7 +737,7 @@ void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
     if (MacroInfo *CurrentMI = getMacroInfo(IdentInfo)) {
       if (CurrentMI->isWarnIfUnused())
         WarnUnusedMacroLocs.erase(CurrentMI->getDefinitionLoc());
-      CurrentMI->setUndefLoc(MessageLoc);
+      UndefineMacro(IdentInfo, CurrentMI, MessageLoc);
     }
 
     // Get the MacroInfo we want to reinstall.
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index a0fc265b6b..d9f206901d 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -49,6 +49,8 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
 
+PPMutationListener::~PPMutationListener() { }
+
 Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
                            const TargetInfo *target, SourceManager &SM,
                            HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
@@ -62,8 +64,8 @@ Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
     IncrementalProcessing(IncrProcessing), CodeComplete(0), 
     CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
     SkipMainFilePreamble(0, true), CurPPLexer(0), 
-    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 
-    Record(0), MIChainHead(0), MICache(0) 
+    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), Listener(0),
+    MacroArgCache(0), Record(0), MIChainHead(0), MICache(0) 
 {
   OwnsHeaderSearch = OwnsHeaders;
   
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index b7266ebbba..cdb9646758 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -573,7 +573,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   // definition.
   if (hadMacroDefinition) {
     // FIXME: Check for conflicts?
-    uint32_t Offset = ReadUnalignedLE32(d);
+    uint32_t LocalID = ReadUnalignedLE32(d);
     unsigned LocalSubmoduleID = ReadUnalignedLE32(d);
     
     // Determine whether this macro definition should be visible now, or
@@ -594,7 +594,8 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
       }
     }
 
-    Reader.setIdentifierIsMacro(II, F, Offset, Visible && hasMacroDefinition);
+    Reader.setIdentifierIsMacro(II, Reader.getGlobalMacroID(F, LocalID),
+                                Visible && hasMacroDefinition);
     DataLen -= 8;
   }
 
@@ -1314,10 +1315,19 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
         return;
       }
 
-      unsigned NextIndex = 1;
+      unsigned GlobalID = getGlobalMacroID(F, Record[1]);
+
+      // If this macro has already been loaded, don't do so again.
+      if (MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS])
+        return;
+
+      unsigned NextIndex = 2;
       SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex);
       MacroInfo *MI = PP.AllocateMacroInfo(Loc);
 
+      // Record this macro.
+      MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS] = MI;
+
       SourceLocation UndefLoc = ReadSourceLocation(F, Record, NextIndex);
       if (UndefLoc.isValid())
         MI->setUndefLoc(UndefLoc);
@@ -1345,6 +1355,20 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
                             PP.getPreprocessorAllocator());
       }
 
+      if (DeserializationListener)
+        DeserializationListener->MacroRead(GlobalID, MI);
+
+      // If an update record marked this as undefined, do so now.
+      MacroUpdatesMap::iterator Update = MacroUpdates.find(GlobalID);
+      if (Update != MacroUpdates.end()) {
+        if (MI->getUndefLoc().isInvalid()) {
+          MI->setUndefLoc(Update->second.UndefLoc);
+          if (PPMutationListener *Listener = PP.getPPMutationListener())
+            Listener->UndefinedMacro(MI);
+        }
+        MacroUpdates.erase(Update);
+      }
+
       // Finally, install the macro.
       PP.setMacroInfo(II, MI, /*LoadedFromAST=*/true);
 
@@ -1455,17 +1479,18 @@ HeaderFileInfoTrait::ReadData(const internal_key_type, const unsigned char *d,
   return HFI;
 }
 
-void ASTReader::setIdentifierIsMacro(IdentifierInfo *II, ModuleFile &F,
-                                     uint64_t LocalOffset, bool Visible) {
+void ASTReader::setIdentifierIsMacro(IdentifierInfo *II, MacroID ID,
+                                     bool Visible) {
   if (Visible) {
     // Note that this identifier has a macro definition.
     II->setHasMacroDefinition(true);
   } else {
     II->setHadMacroDefinition(true);
   }
-  
-  // Adjust the offset to a global offset.
-  UnreadMacroRecordOffsets[II] = F.GlobalBitOffset + LocalOffset;
+
+  // FIXME: This could end up overwriting a previously recording macro
+  // definition here, which is not cool at all.
+  UnreadMacroIDs[II] = ID;
 }
 
 void ASTReader::ReadDefinedMacros() {
@@ -1522,23 +1547,21 @@ void ASTReader::ReadDefinedMacros() {
   }
   
   // Drain the unread macro-record offsets map.
-  while (!UnreadMacroRecordOffsets.empty())
-    LoadMacroDefinition(UnreadMacroRecordOffsets.begin());
+  while (!UnreadMacroIDs.empty())
+    LoadMacroDefinition(UnreadMacroIDs.begin());
 }
 
 void ASTReader::LoadMacroDefinition(
-                    llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos) {
-  assert(Pos != UnreadMacroRecordOffsets.end() && "Unknown macro definition");
-  uint64_t Offset = Pos->second;
-  UnreadMacroRecordOffsets.erase(Pos);
-  
-  RecordLocation Loc = getLocalBitOffset(Offset);
-  ReadMacroRecord(*Loc.F, Loc.Offset);
+                    llvm::DenseMap<IdentifierInfo *, MacroID>::iterator Pos) {
+  assert(Pos != UnreadMacroIDs.end() && "Unknown macro definition");
+  uint64_t GlobalID = Pos->second;
+  UnreadMacroIDs.erase(Pos);
+  getMacro(GlobalID);
 }
 
 void ASTReader::LoadMacroDefinition(IdentifierInfo *II) {