[PCH/Modules] De/Serialize MacroInfos separately than MacroDirectives.

-Serialize the macro directives history into its own section
-Get rid of the macro updates section
-When de/serializing an identifier from a module, associate only one macro per
 submodule that defined+exported it.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177761 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 454 insertions, 279 deletions

diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index 73cd72f44c..c6dbfe1498 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -516,6 +516,8 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   Bits >>= 1;
   bool ExtensionToken = Bits & 0x01;
   Bits >>= 1;
+  bool hasSubmoduleMacros = Bits & 0x01;
+  Bits >>= 1;
   bool hadMacroDefinition = Bits & 0x01;
   Bits >>= 1;
 
@@ -554,13 +556,26 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   // If this identifier is a macro, deserialize the macro
   // definition.
   if (hadMacroDefinition) {
-    SmallVector<MacroID, 4> MacroIDs;
-    while (uint32_t LocalID = ReadUnalignedLE32(d)) {
-      MacroIDs.push_back(Reader.getGlobalMacroID(F, LocalID));
+    uint32_t MacroDirectivesOffset = ReadUnalignedLE32(d);
+    DataLen -= 4;
+    SmallVector<uint32_t, 8> LocalMacroIDs;
+    if (hasSubmoduleMacros) {
+      while (uint32_t LocalMacroID = ReadUnalignedLE32(d)) {
+        DataLen -= 4;
+        LocalMacroIDs.push_back(LocalMacroID);
+      }
       DataLen -= 4;
     }
-    DataLen -= 4;
-    Reader.setIdentifierIsMacro(II, MacroIDs);
+
+    if (F.Kind == MK_Module) {
+      for (SmallVectorImpl<uint32_t>::iterator
+             I = LocalMacroIDs.begin(), E = LocalMacroIDs.end(); I != E; ++I) {
+        MacroID MacID = Reader.getGlobalMacroID(F, *I);
+        Reader.addPendingMacroFromModule(II, &F, MacID, F.DirectImportLoc);
+      }
+    } else {
+      Reader.addPendingMacroFromPCH(II, &F, MacroDirectivesOffset);
+    }
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -1073,8 +1088,7 @@ bool ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, unsigned BlockID) {
   }
 }
 
-void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
-                                MacroDirective *Hint) {
+MacroInfo *ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
   BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
@@ -1086,24 +1100,6 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
   SmallVector<IdentifierInfo*, 16> MacroArgs;
   MacroInfo *Macro = 0;
 
-  // RAII object to add the loaded macro information once we're done
-  // adding tokens.
-  struct AddLoadedMacroInfoRAII {
-    Preprocessor &PP;
-    MacroDirective *Hint;
-    MacroDirective *MD;
-    IdentifierInfo *II;
-
-    AddLoadedMacroInfoRAII(Preprocessor &PP, MacroDirective *Hint)
-      : PP(PP), Hint(Hint), MD(), II() { }
-    ~AddLoadedMacroInfoRAII( ) {
-      if (MD) {
-        // Finally, install the macro.
-        PP.addLoadedMacroInfo(II, MD, Hint);
-      }
-    }
-  } AddLoadedMacroInfo(PP, Hint);
-
   while (true) {
     // Advance to the next record, but if we get to the end of the block, don't
     // pop it (removing all the abbreviations from the cursor) since we want to
@@ -1115,9 +1111,9 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
     case llvm::BitstreamEntry::SubBlock: // Handled for us already.
     case llvm::BitstreamEntry::Error:
       Error("malformed block record in AST file");
-      return;
+      return Macro;
     case llvm::BitstreamEntry::EndBlock:
-      return;
+      return Macro;
     case llvm::BitstreamEntry::Record:
       // The interesting case.
       break;
@@ -1128,47 +1124,24 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
     PreprocessorRecordTypes RecType =
       (PreprocessorRecordTypes)Stream.readRecord(Entry.ID, Record);
     switch (RecType) {
+    case PP_MACRO_DIRECTIVE_HISTORY:
+      return Macro;
+
     case PP_MACRO_OBJECT_LIKE:
     case PP_MACRO_FUNCTION_LIKE: {
       // If we already have a macro, that means that we've hit the end
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
-
-      IdentifierInfo *II = getLocalIdentifier(F, Record[0]);
-      if (II == 0) {
-        Error("macro must have a name in AST file");
-        return;
-      }
-
-      unsigned GlobalID = getGlobalMacroID(F, Record[1]);
+        return Macro;
 
-      // If this macro has already been loaded, don't do so again.
-      if (MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS])
-        return;
-
-      SubmoduleID GlobalSubmoduleID = getGlobalSubmoduleID(F, Record[2]);
-      unsigned NextIndex = 3;
+      unsigned NextIndex = 1; // Skip identifier ID.
+      SubmoduleID SubModID = getGlobalSubmoduleID(F, Record[NextIndex++]);
       SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex);
-      MacroInfo *MI = PP.AllocateDeserializedMacroInfo(Loc, GlobalSubmoduleID);
-      // FIXME: Location should be import location in case of module.
-      MacroDirective *MD = PP.AllocateMacroDirective(MI, Loc,
-                                                     /*isImported=*/true);
+      MacroInfo *MI = PP.AllocateDeserializedMacroInfo(Loc, SubModID);
       MI->setDefinitionEndLoc(ReadSourceLocation(F, Record, NextIndex));
-
-      // Record this macro.
-      MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS] = MD;
-
-      SourceLocation UndefLoc = ReadSourceLocation(F, Record, NextIndex);
-      if (UndefLoc.isValid())
-        MD->setUndefLoc(UndefLoc);
-
       MI->setIsUsed(Record[NextIndex++]);
 
-      bool IsPublic = Record[NextIndex++];
-      MD->setVisibility(IsPublic, ReadSourceLocation(F, Record, NextIndex));
-
       if (RecType == PP_MACRO_FUNCTION_LIKE) {
         // Decode function-like macro info.
         bool isC99VarArgs = Record[NextIndex++];
@@ -1188,61 +1161,6 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
                             PP.getPreprocessorAllocator());
       }
 
-      if (DeserializationListener)
-        DeserializationListener->MacroRead(GlobalID, MD);
-
-      // If an update record marked this as undefined, do so now.
-      // FIXME: Only if the submodule this update came from is visible?
-      MacroUpdatesMap::iterator Update = MacroUpdates.find(GlobalID);
-      if (Update != MacroUpdates.end()) {
-        if (MD->getUndefLoc().isInvalid()) {
-          for (unsigned I = 0, N = Update->second.size(); I != N; ++I) {
-            bool Hidden = false;
-            if (unsigned SubmoduleID = Update->second[I].first) {
-              if (Module *Owner = getSubmodule(SubmoduleID)) {
-                if (Owner->NameVisibility == Module::Hidden) {
-                  // Note that this #undef is hidden.
-                  Hidden = true;
-
-                  // Record this hiding for later.
-                  HiddenNamesMap[Owner].push_back(
-                    HiddenName(II, MD, Update->second[I].second.UndefLoc));
-                }
-              }
-            }
-
-            if (!Hidden) {
-              MD->setUndefLoc(Update->second[I].second.UndefLoc);
-              if (PPMutationListener *Listener = PP.getPPMutationListener())
-                Listener->UndefinedMacro(MD);
-              break;
-            }
-          }
-        }
-        MacroUpdates.erase(Update);
-      }
-
-      // Determine whether this macro definition is visible.
-      bool Hidden = !MD->isPublic();
-      if (!Hidden && GlobalSubmoduleID) {
-        if (Module *Owner = getSubmodule(GlobalSubmoduleID)) {
-          if (Owner->NameVisibility == Module::Hidden) {
-            // The owning module is not visible, and this macro definition
-            // should not be, either.
-            Hidden = true;
-
-            // Note that this macro definition was hidden because its owning
-            // module is not yet visible.
-            HiddenNamesMap[Owner].push_back(HiddenName(II, MD));
-          }
-        }
-      }
-      MD->setHidden(Hidden);
-
-      // Make sure we install the macro once we're done.
-      AddLoadedMacroInfo.MD = MD;
-      AddLoadedMacroInfo.II = II;
-
       // Remember that we saw this macro last so that we add the tokens that
       // form its body to it.
       Macro = MI;
@@ -1381,10 +1299,19 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
   return HFI;
 }
 
-void ASTReader::setIdentifierIsMacro(IdentifierInfo *II, ArrayRef<MacroID> IDs){
-  II->setHadMacroDefinition(true);
+void ASTReader::addPendingMacroFromModule(IdentifierInfo *II,
+                                          ModuleFile *M,
+                                          GlobalMacroID GMacID,
+                                          SourceLocation ImportLoc) {
+  assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard");
+  PendingMacroIDs[II].push_back(PendingMacroInfo(M, GMacID, ImportLoc));
+}
+
+void ASTReader::addPendingMacroFromPCH(IdentifierInfo *II,
+                                       ModuleFile *M,
+                                       uint64_t MacroDirectivesOffset) {
   assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard");
-  PendingMacroIDs[II].append(IDs.begin(), IDs.end());
+  PendingMacroIDs[II].push_back(PendingMacroInfo(M, MacroDirectivesOffset));
 }
 
 void ASTReader::ReadDefinedMacros() {
@@ -1526,6 +1453,119 @@ void ASTReader::markIdentifierUpToDate(IdentifierInfo *II) {
     IdentifierGeneration[II] = CurrentGeneration;
 }
 
+void ASTReader::resolvePendingMacro(IdentifierInfo *II,
+                                    const PendingMacroInfo &PMInfo) {
+  assert(II);
+
+  if (PMInfo.M->Kind != MK_Module) {
+    installPCHMacroDirectives(II, *PMInfo.M,
+                              PMInfo.PCHMacroData.MacroDirectivesOffset);
+    return;
+  }
+  
+  // Module Macro.
+
+  GlobalMacroID GMacID = PMInfo.ModuleMacroData.GMacID;
+  SourceLocation ImportLoc =
+      SourceLocation::getFromRawEncoding(PMInfo.ModuleMacroData.ImportLoc);
+
+  assert(GMacID);
+  // If this macro has already been loaded, don't do so again.
+  if (MacrosLoaded[GMacID - NUM_PREDEF_MACRO_IDS])
+    return;
+
+  MacroInfo *MI = getMacro(GMacID);
+  SubmoduleID SubModID = MI->getOwningModuleID();
+  MacroDirective *MD = PP.AllocateMacroDirective(MI, ImportLoc,
+                                                 /*isImported=*/true);
+
+  // Determine whether this macro definition is visible.
+  bool Hidden = false;
+  if (SubModID) {
+    if (Module *Owner = getSubmodule(SubModID)) {
+      if (Owner->NameVisibility == Module::Hidden) {
+        // The owning module is not visible, and this macro definition
+        // should not be, either.
+        Hidden = true;
+
+        // Note that this macro definition was hidden because its owning
+        // module is not yet visible.
+        HiddenNamesMap[Owner].push_back(HiddenName(II, MD));
+      }
+    }
+  }
+  MD->setHidden(Hidden);
+
+  if (!Hidden)
+    installImportedMacro(II, MD);
+}
+
+void ASTReader::installPCHMacroDirectives(IdentifierInfo *II,
+                                          ModuleFile &M, uint64_t Offset) {
+  assert(M.Kind != MK_Module);
+
+  BitstreamCursor &Cursor = M.MacroCursor;
+  SavedStreamPosition SavedPosition(Cursor);
+  Cursor.JumpToBit(Offset);
+
+  llvm::BitstreamEntry Entry =
+      Cursor.advance(BitstreamCursor::AF_DontPopBlockAtEnd);
+  if (Entry.Kind != llvm::BitstreamEntry::Record) {
+    Error("malformed block record in AST file");
+    return;
+  }
+
+  RecordData Record;
+  PreprocessorRecordTypes RecType =
+    (PreprocessorRecordTypes)Cursor.readRecord(Entry.ID, Record);
+  if (RecType != PP_MACRO_DIRECTIVE_HISTORY) {
+    Error("malformed block record in AST file");
+    return;
+  }
+
+  // Deserialize the macro directives history in reverse source-order.
+  MacroDirective *Latest = 0, *Earliest = 0;
+  unsigned Idx = 0, N = Record.size();
+  while (Idx < N) {
+    GlobalMacroID GMacID = getGlobalMacroID(M, Record[Idx++]);
+    MacroInfo *MI = getMacro(GMacID);
+    SourceLocation Loc = ReadSourceLocation(M, Record, Idx);
+    SourceLocation UndefLoc = ReadSourceLocation(M, Record, Idx);
+    SourceLocation VisibilityLoc = ReadSourceLocation(M, Record, Idx);
+    bool isImported = Record[Idx++];
+    bool isPublic = Record[Idx++];
+    bool isAmbiguous = Record[Idx++];
+
+    MacroDirective *MD = PP.AllocateMacroDirective(MI, Loc, isImported);
+    if (UndefLoc.isValid())
+      MD->setUndefLoc(UndefLoc);
+    if (VisibilityLoc.isValid())
+      MD->setVisibility(isPublic, VisibilityLoc);
+    MD->setAmbiguous(isAmbiguous);
+    MD->setIsFromPCH();
+
+    if (!Latest)
+      Latest = MD;
+    if (Earliest)
+      Earliest->setPrevious(MD);
+    Earliest = MD;
+  }
+
+  PP.setLoadedMacroDirective(II, Latest);
+}
+
+void ASTReader::installImportedMacro(IdentifierInfo *II, MacroDirective *MD) {
+  assert(II && MD);
+
+  MacroDirective *Prev = PP.getMacroDirective(II);
+  if (Prev && !Prev->getInfo()->isIdenticalTo(*MD->getInfo(), PP)) {
+    Prev->setAmbiguous(true);
+    MD->setAmbiguous(true);
+  }
+  
+  PP.setMacroDirective(II, MD);
+}
+
 InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
   // If this ID is bogus, just return an empty input file.
   if (ID == 0 || ID > F.InputFilesLoaded.size())
@@ -2629,18 +2669,8 @@ bool ASTReader::ReadASTBlock(ModuleFile &F) {
       break;
     }
 
-    case MACRO_UPDATES: {
-      for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
-        MacroID ID = getGlobalMacroID(F, Record[I++]);
-        if (I == N)
-          break;
-
-        SourceLocation UndefLoc = ReadSourceLocation(F, Record, I);
-        SubmoduleID SubmoduleID = getGlobalSubmoduleID(F, Record[I++]);;
-        MacroUpdate Update;
-        Update.UndefLoc = UndefLoc;
-        MacroUpdates[ID].push_back(std::make_pair(SubmoduleID, Update));
-      }
+    case MACRO_TABLE: {
+      // FIXME: Not used yet.
       break;
     }
     }
@@ -2695,7 +2725,7 @@ void ASTReader::makeNamesVisible(const HiddenNames &Names) {
       std::pair<IdentifierInfo *, MacroDirective *> Macro = Names[I].getMacro();
       Macro.second->setHidden(!Macro.second->isPublic());
       if (Macro.second->isDefined()) {
-        PP.makeLoadedMacroInfoVisible(Macro.first, Macro.second);
+        installImportedMacro(Macro.first, Macro.second);
       }
       break;
     }
@@ -5789,7 +5819,7 @@ void ASTReader::PrintStats() {
   unsigned NumMacrosLoaded
     = MacrosLoaded.size() - std::count(MacrosLoaded.begin(),
                                        MacrosLoaded.end(),
-                                       (MacroDirective *)0);
+                                       (MacroInfo *)0);
   unsigned NumSelectorsLoaded
     = SelectorsLoaded.size() - std::count(SelectorsLoaded.begin(),
                                           SelectorsLoaded.end(),
@@ -6398,7 +6428,7 @@ IdentifierID ASTReader::getGlobalIdentifierID(ModuleFile &M, unsigned LocalID) {
   return LocalID + I->second;
 }
 
-MacroDirective *ASTReader::getMacro(MacroID ID, MacroDirective *Hint) {
+MacroInfo *ASTReader::getMacro(MacroID ID) {
   if (ID == 0)
     return 0;
 
@@ -6414,7 +6444,11 @@ MacroDirective *ASTReader::getMacro(MacroID ID, MacroDirective *Hint) {
     assert(I != GlobalMacroMap.end() && "Corrupted global macro map");
     ModuleFile *M = I->second;
     unsigned Index = ID - M->BaseMacroID;
-    ReadMacroRecord(*M, M->MacroOffsets[Index], Hint);
+    MacrosLoaded[ID] = ReadMacroRecord(*M, M->MacroOffsets[Index]);
+    
+    if (DeserializationListener)
+      DeserializationListener->MacroRead(ID + NUM_PREDEF_MACRO_IDS,
+                                         MacrosLoaded[ID]);
   }
 
   return MacrosLoaded[ID];
@@ -7128,12 +7162,22 @@ void ASTReader::finishPendingActions() {
 
     // Load any pending macro definitions.
     for (unsigned I = 0; I != PendingMacroIDs.size(); ++I) {
-      // FIXME: std::move here
-      SmallVector<MacroID, 2> GlobalIDs = PendingMacroIDs.begin()[I].second;
-      MacroDirective *Hint = 0;
+      IdentifierInfo *II = PendingMacroIDs.begin()[I].first;
+      SmallVector<PendingMacroInfo, 2> GlobalIDs;
+      GlobalIDs.swap(PendingMacroIDs.begin()[I].second);
+      // Initialize the macro history from chained-PCHs ahead of module imports.
+      for (unsigned IDIdx = 0, NumIDs = GlobalIDs.size(); IDIdx !=  NumIDs;
+           ++IDIdx) {
+        const PendingMacroInfo &Info = GlobalIDs[IDIdx];
+        if (Info.M->Kind != MK_Module)
+          resolvePendingMacro(II, Info);
+      }
+      // Handle module imports.
       for (unsigned IDIdx = 0, NumIDs = GlobalIDs.size(); IDIdx !=  NumIDs;
            ++IDIdx) {
-        Hint = getMacro(GlobalIDs[IDIdx], Hint);
+        const PendingMacroInfo &Info = GlobalIDs[IDIdx];
+        if (Info.M->Kind == MK_Module)
+          resolvePendingMacro(II, Info);
       }
     }
     PendingMacroIDs.clear();
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index f0b12009fe..725896309e 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -835,7 +835,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(LOCAL_REDECLARATIONS);
   RECORD(OBJC_CATEGORIES);
   RECORD(MACRO_OFFSET);
-  RECORD(MACRO_UPDATES);
+  RECORD(MACRO_TABLE);
 
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
@@ -1787,11 +1787,47 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
 // Preprocessor Serialization
 //===----------------------------------------------------------------------===//
 
-static int compareMacroDefinitions(const void *XPtr, const void *YPtr) {
-  const std::pair<const IdentifierInfo *, MacroInfo *> &X =
-    *(const std::pair<const IdentifierInfo *, MacroInfo *>*)XPtr;
-  const std::pair<const IdentifierInfo *, MacroInfo *> &Y =
-    *(const std::pair<const IdentifierInfo *, MacroInfo *>*)YPtr;
+namespace {
+class ASTMacroTableTrait {
+public:
+  typedef IdentID key_type;
+  typedef key_type key_type_ref;
+
+  struct Data {
+    uint32_t MacroDirectivesOffset;
+  };
+
+  typedef Data data_type;
+  typedef const data_type &data_type_ref;
+
+  static unsigned ComputeHash(IdentID IdID) {
+    return llvm::hash_value(IdID);
+  }
+
+  std::pair<unsigned,unsigned>
+  static EmitKeyDataLength(raw_ostream& Out,
+                           key_type_ref Key, data_type_ref Data) {
+    unsigned KeyLen = 4; // IdentID.
+    unsigned DataLen = 4; // MacroDirectivesOffset.
+    return std::make_pair(KeyLen, DataLen);
+  }
+
+  static void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
+    clang::io::Emit32(Out, Key);
+  }
+
+  static void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
+                       unsigned) {
+    clang::io::Emit32(Out, Data.MacroDirectivesOffset);
+  }
+};
+} // end anonymous namespace
+
+static int compareMacroDirectives(const void *XPtr, const void *YPtr) {
+  const std::pair<const IdentifierInfo *, MacroDirective *> &X =
+    *(const std::pair<const IdentifierInfo *, MacroDirective *>*)XPtr;
+  const std::pair<const IdentifierInfo *, MacroDirective *> &Y =
+    *(const std::pair<const IdentifierInfo *, MacroDirective *>*)YPtr;
   return X.first->getName().compare(Y.first->getName());
 }
 
@@ -1837,24 +1873,68 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
     fprintf(stderr, "warning: precompiled header used __DATE__ or __TIME__.\n");
 
 
-  // Loop over all the macro definitions that are live at the end of the file,
+  // Loop over all the macro directives that are live at the end of the file,
   // emitting each to the PP section.
 
-  // Construct the list of macro definitions that need to be serialized.
+  // Construct the list of macro directives that need to be serialized.
   SmallVector<std::pair<const IdentifierInfo *, MacroDirective *>, 2>
-    MacrosToEmit;
-  for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0),
-                                    E = PP.macro_end(Chain == 0);
+    MacroDirectives;
+  for (Preprocessor::macro_iterator
+         I = PP.macro_begin(/*IncludeExternalMacros=*/false),
+         E = PP.macro_end(/*IncludeExternalMacros=*/false);
        I != E; ++I) {
-    if (!IsModule || I->second->isPublic()) {
-      MacrosToEmit.push_back(std::make_pair(I->first, I->second));
-    }
+    MacroDirectives.push_back(std::make_pair(I->first, I->second));
   }
 
   // Sort the set of macro definitions that need to be serialized by the
   // name of the macro, to provide a stable ordering.
-  llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(),
-                       &compareMacroDefinitions);
+  llvm::array_pod_sort(MacroDirectives.begin(), MacroDirectives.end(),
+                       &compareMacroDirectives);
+
+  OnDiskChainedHashTableGenerator<ASTMacroTableTrait> Generator;
+
+  // Emit the macro directives as a list and associate the offset with the
+  // identifier they belong to.
+  for (unsigned I = 0, N = MacroDirectives.size(); I != N; ++I) {
+    const IdentifierInfo *Name = MacroDirectives[I].first;
+    uint64_t MacroDirectiveOffset = Stream.GetCurrentBitNo();
+    MacroDirective *MD = MacroDirectives[I].second;
+
+    // If the macro or identifier need no updates, don't write the macro history
+    // for this one.
+    if (MD->isFromPCH() && !MD->hasChangedAfterLoad() &&
+        Name->isFromAST() && !Name->hasChangedSinceDeserialization())
+      continue;
+
+    // Emit the macro directives in reverse source order.
+    for (; MD; MD = MD->getPrevious()) {
+      if (shouldIgnoreMacro(MD, IsModule, PP))
+        continue;
+      MacroID InfoID = getMacroRef(MD->getInfo(), Name);
+      if (InfoID == 0)
+        continue;
+
+      Record.push_back(InfoID);
+      AddSourceLocation(MD->getLocation(), Record);
+      AddSourceLocation(MD->getUndefLoc(), Record);
+      AddSourceLocation(MD->getVisibilityLocation(), Record);
+      Record.push_back(MD->isImported());
+      Record.push_back(MD->isPublic());
+      Record.push_back(MD->isAmbiguous());
+    }
+    if (Record.empty())
+      continue;
+
+    Stream.EmitRecord(PP_MACRO_DIRECTIVE_HISTORY, Record);
+    Record.clear();
+
+    IdentMacroDirectivesOffsetMap[Name] = MacroDirectiveOffset;
+
+    IdentID NameID = getIdentifierRef(Name);
+    ASTMacroTableTrait::Data data;
+    data.MacroDirectivesOffset = MacroDirectiveOffset;
+    Generator.insert(NameID, data);
+  }
 
   /// \brief Offsets of each of the macros into the bitstream, indexed by
   /// the local macro ID
@@ -1864,101 +1944,107 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
   /// defined.
   std::vector<uint32_t> MacroOffsets;
 
-  for (unsigned I = 0, N = MacrosToEmit.size(); I != N; ++I) {
-    const IdentifierInfo *Name = MacrosToEmit[I].first;
+  for (unsigned I = 0, N = MacroInfosToEmit.size(); I != N; ++I) {
+    const IdentifierInfo *Name = MacroInfosToEmit[I].Name;
+    MacroInfo *MI = MacroInfosToEmit[I].MI;
+    MacroID ID = MacroInfosToEmit[I].ID;
 
-    for (MacroDirective *MD = MacrosToEmit[I].second; MD;
-         MD = MD->getPrevious()) {
-      if (shouldIgnoreMacro(MD, IsModule, PP))
-        continue;
-
-      MacroID ID = getMacroRef(MD);
-      if (!ID)
-        continue;
+    if (ID < FirstMacroID) {
+      assert(0 && "Loaded MacroInfo entered MacroInfosToEmit ?");
+      continue;
+    }
 
-      // Skip macros from a AST file if we're chaining.
-      if (Chain && MD->isImported() && !MD->hasChangedAfterLoad())
-        continue;
+    // Record the local offset of this macro.
+    unsigned Index = ID - FirstMacroID;
+    if (Index == MacroOffsets.size())
+      MacroOffsets.push_back(Stream.GetCurrentBitNo());
+    else {
+      if (Index > MacroOffsets.size())
+        MacroOffsets.resize(Index + 1);
 
-      if (ID < FirstMacroID) {
-        // This will have been dealt with via an update record.
-        assert(MacroUpdates.count(MD) > 0 && "Missing macro update");
-        continue;
-      }
+      MacroOffsets[Index] = Stream.GetCurrentBitNo();
+    }
 
-      // Record the local offset of this macro.
-      unsigned Index = ID - FirstMacroID;
-      if (Index == MacroOffsets.size())
-        MacroOffsets.push_back(Stream.GetCurrentBitNo());
-      else {
-        if (Index > MacroOffsets.size())
-          MacroOffsets.resize(Index + 1);
+    AddIdentifierRef(Name, Record);
+    Record.push_back(inferSubmoduleIDFromLocation(MI->getDefinitionLoc()));
+    AddSourceLocation(MI->getDefinitionLoc(), Record);
+    AddSourceLocation(MI->getDefinitionEndLoc(), Record);
+    Record.push_back(MI->isUsed());
+    unsigned Code;
+    if (MI->isObjectLike()) {
+      Code = PP_MACRO_OBJECT_LIKE;
+    } else {
+      Code = PP_MACRO_FUNCTION_LIKE;
 
-        MacroOffsets[Index] = Stream.GetCurrentBitNo();
-      }
+      Record.push_back(MI->isC99Varargs());
+      Record.push_back(MI->isGNUVarargs());
+      Record.push_back(MI->hasCommaPasting());
+      Record.push_back(MI->getNumArgs());
+      for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
+           I != E; ++I)
+        AddIdentifierRef(*I, Record);
+    }
 
-      AddIdentifierRef(Name, Record);
-      addMacroRef(MD, Record);
-      const MacroInfo *MI = MD->getInfo();
-      Record.push_back(inferSubmoduleIDFromLocation(MI->getDefinitionLoc()));
-      AddSourceLocation(MI->getDefinitionLoc(), Record);
-      AddSourceLocation(MI->getDefinitionEndLoc(), Record);
-      AddSourceLocation(MD->getUndefLoc(), Record);
-      Record.push_back(MI->isUsed());
-      Record.push_back(MD->isPublic());
-      AddSourceLocation(MD->getVisibilityLocation(), Record);
-      unsigned Code;
-      if (MI->isObjectLike()) {
-        Code = PP_MACRO_OBJECT_LIKE;
-      } else {
-        Code = PP_MACRO_FUNCTION_LIKE;
-
-        Record.push_back(MI->isC99Varargs());
-        Record.push_back(MI->isGNUVarargs());
-        Record.push_back(MI->hasCommaPasting());
-        Record.push_back(MI->getNumArgs());
-        for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
-             I != E; ++I)
-          AddIdentifierRef(*I, Record);
-      }
+    // If we have a detailed preprocessing record, record the macro definition
+    // ID that corresponds to this macro.
+    if (PPRec)
+      Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
 
-      // If we have a detailed preprocessing record, record the macro definition
-      // ID that corresponds to this macro.
-      if (PPRec)
-        Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
+    Stream.EmitRecord(Code, Record);
+    Record.clear();
 
-      Stream.EmitRecord(Code, Record);
+    // Emit the tokens array.
+    for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
+      // Note that we know that the preprocessor does not have any annotation
+      // tokens in it because they are created by the parser, and thus can't
+      // be in a macro definition.
+      const Token &Tok = MI->getReplacementToken(TokNo);
+
+      Record.push_back(Tok.getLocation().getRawEncoding());
+      Record.push_back(Tok.getLength());
+
+      // FIXME: When reading literal tokens, reconstruct the literal pointer
+      // if it is needed.
+      AddIdentifierRef(Tok.getIdentifierInfo(), Record);
+      // FIXME: Should translate token kind to a stable encoding.
+      Record.push_back(Tok.getKind());
+      // FIXME: Should translate token flags to a stable encoding.
+      Record.push_back(Tok.getFlags());
+
+      Stream.EmitRecord(PP_TOKEN, Record);
       Record.clear();
-
-      // Emit the tokens array.
-      for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
-        // Note that we know that the preprocessor does not have any annotation
-        // tokens in it because they are created by the parser, and thus can't
-        // be in a macro definition.
-        const Token &Tok = MI->getReplacementToken(TokNo);
-
-        Record.push_back(Tok.getLocation().getRawEncoding());
-        Record.push_back(Tok.getLength());
-
-        // FIXME: When reading literal tokens, reconstruct the literal pointer
-        // if it is needed.
-        AddIdentifierRef(Tok.getIdentifierInfo(), Record);
-        // FIXME: Should translate token kind to a stable encoding.
-        Record.push_back(Tok.getKind());
-        // FIXME: Should translate token flags to a stable encoding.
-        Record.push_back(Tok.getFlags());
-
-        Stream.EmitRecord(PP_TOKEN, Record);
-        Record.clear();
-      }
-      ++NumMacros;
     }
+    ++NumMacros;
   }
+
   Stream.ExitBlock();
 
-  // Write the offsets table for macro IDs.
+  // Create the on-disk hash table in a buffer.
+  SmallString<4096> MacroTable;
+  uint32_t BucketOffset;
+  {
+    llvm::raw_svector_ostream Out(MacroTable);
+    // Make sure that no bucket is at offset 0
+    clang::io::Emit32(Out, 0);
+    BucketOffset = Generator.Emit(Out);
+  }
+
+  // Write the macro table
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(MACRO_TABLE));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  unsigned MacroTableAbbrev = Stream.EmitAbbrev(Abbrev);
+
+  Record.push_back(MACRO_TABLE);
+  Record.push_back(BucketOffset);
+  Stream.EmitRecordWithBlob(MacroTableAbbrev, Record, MacroTable.str());
+  Record.clear();
+
+  // Write the offsets table for macro IDs.
+  using namespace llvm;
+  Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(MACRO_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macros
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID
@@ -2801,13 +2887,61 @@ class ASTIdentifierTableTrait {
     if (!II->hadMacroDefinition())
       return false;
 
-    if (Macro || (Macro = PP.getMacroDirectiveHistory(II)))
-      return !shouldIgnoreMacro(Macro, IsModule, PP) &&
-             (!IsModule || Macro->isPublic());
+    if (Macro || (Macro = PP.getMacroDirectiveHistory(II))) {
+      if (!IsModule)
+        return !shouldIgnoreMacro(Macro, IsModule, PP);
+      SubmoduleID ModID;
+      if (getFirstPublicSubmoduleMacro(Macro, ModID))
+        return true;
+    }
 
     return false;
   }
 
+  MacroDirective *getFirstPublicSubmoduleMacro(MacroDirective *MD,
+                                               SubmoduleID &ModID) {
+    if (shouldIgnoreMacro(MD, IsModule, PP))
+      return 0;
+    ModID = getSubmoduleID(MD);
+    if (ModID == 0)
+      return 0;
+    if (MD->isDefined() && MD->isPublic())
+      return MD;
+    return getNextPublicSubmoduleMacro(MD, ModID);
+  }
+
+  MacroDirective *getNextPublicSubmoduleMacro(MacroDirective *MD,
+                                              SubmoduleID &ModID) {
+    while (MD) {
+      MD = getNextSubmoduleMacro(MD, ModID);
+      if (MD && MD->isDefined() && MD->isPublic())
+        return MD;
+    }
+    return 0;
+  }
+
+  MacroDirective *getNextSubmoduleMacro(MacroDirective *CurrMD,
+                                        SubmoduleID &CurrModID) {
+    SubmoduleID OrigID = CurrModID;
+    while ((CurrMD = CurrMD->getPrevious())) {
+      if (shouldIgnoreMacro(CurrMD, IsModule, PP))
+        return 0;
+      CurrModID = getSubmoduleID(CurrMD);
+      if (CurrModID == 0)
+        return 0;
+      if (CurrModID != OrigID)
+        return CurrMD;
+    }
+    return 0;
+  }
+
+  SubmoduleID getSubmoduleID(MacroDirective *MD) {
+    MacroInfo *MI = MD->getInfo();
+    if (unsigned ID = MI->getOwningModuleID())
+      return ID;
+    return Writer.inferSubmoduleIDFromLocation(MI->getDefinitionLoc());
+  }
+
 public:
   typedef IdentifierInfo* key_type;
   typedef key_type  key_type_ref;
@@ -2832,14 +2966,15 @@ public:
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
       if (hadMacroDefinition(II, Macro)) {
-        for (MacroDirective *M = Macro; M; M = M->getPrevious()) {
-          if (shouldIgnoreMacro(M, IsModule, PP))
-            continue;
-          if (Writer.getMacroRef(M) != 0)
-            DataLen += 4;
+        DataLen += 4; // MacroDirectives offset.
+        if (IsModule) {
+          SubmoduleID ModID;
+          for (MacroDirective *MD = getFirstPublicSubmoduleMacro(Macro, ModID);
+                 MD; MD = getNextPublicSubmoduleMacro(MD, ModID)) {
+            DataLen += 4; // MacroInfo ID.
+          }
+          DataLen += 4;
         }
-
-        DataLen += 4;
       }
 
       for (IdentifierResolver::iterator D = IdResolver.begin(II),
@@ -2878,6 +3013,7 @@ public:
     Bits = 0;
     bool HadMacroDefinition = hadMacroDefinition(II, Macro);
     Bits = (Bits << 1) | unsigned(HadMacroDefinition);
+    Bits = (Bits << 1) | unsigned(IsModule);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
@@ -2885,15 +3021,18 @@ public:
     clang::io::Emit16(Out, Bits);
 
     if (HadMacroDefinition) {
-      // Write all of the macro IDs associated with this identifier.
-      for (MacroDirective *M = Macro; M; M = M->getPrevious()) {
-        if (shouldIgnoreMacro(M, IsModule, PP))
-          continue;
-        if (MacroID ID = Writer.getMacroRef(M))
-          clang::io::Emit32(Out, ID);
+      clang::io::Emit32(Out, Writer.getMacroDirectivesOffset(II));
+      if (IsModule) {
+        // Write the IDs of macros coming from different submodules.
+        SubmoduleID ModID;
+        for (MacroDirective *MD = getFirstPublicSubmoduleMacro(Macro, ModID);
+               MD; MD = getNextPublicSubmoduleMacro(MD, ModID)) {
+          MacroID InfoID = Writer.getMacroID(MD->getInfo());
+          assert(InfoID);
+          clang::io::Emit32(Out, InfoID);
+        }
+        clang::io::Emit32(Out, 0);
       }
-
-      clang::io::Emit32(Out, 0);
     }
 
     // Emit the declaration IDs in reverse order, because the
@@ -3968,7 +4107,6 @@ void ASTWriter::WriteASTCore(Sema &SemaRef,
     }
   }
 
-  WriteMacroUpdates();
   WriteDeclUpdatesBlocks();
   WriteDeclReplacementsBlock();
   WriteRedeclarations();
@@ -3985,21 +4123,6 @@ void ASTWriter::WriteASTCore(Sema &SemaRef,
   Stream.ExitBlock();
 }
 
-void ASTWriter::WriteMacroUpdates() {
-  if (MacroUpdates.empty())
-    return;
-
-  RecordData Record;
-  for (MacroUpdatesMap::iterator I = MacroUpdates.begin(),
-                                 E = MacroUpdates.end();
-       I != E; ++I) {
-    addMacroRef(I->first, Record);
-    AddSourceLocation(I->second.UndefLoc, Record);
-    Record.push_back(inferSubmoduleIDFromLocation(I->second.UndefLoc));
-  }
-  Stream.EmitRecord(MACRO_UPDATES, Record);
-}
-
 /// \brief Go through the declaration update blocks and resolve declaration
 /// pointers into declaration IDs.
 void ASTWriter::ResolveDeclUpdatesBlocks() {
@@ -4095,10 +4218,6 @@ void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Recor
   Record.push_back(getIdentifierRef(II));
 }
 
-void ASTWriter::addMacroRef(MacroDirective *MD, RecordDataImpl &Record) {
-  Record.push_back(getMacroRef(MD));
-}
-
 IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   if (II == 0)
     return 0;
@@ -4109,19 +4228,35 @@ IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   return ID;
 }
 
-MacroID ASTWriter::getMacroRef(MacroDirective *MD) {
+MacroID ASTWriter::getMacroRef(MacroInfo *MI, const IdentifierInfo *Name) {
   // Don't emit builtin macros like __LINE__ to the AST file unless they
   // have been redefined by the header (in which case they are not
   // isBuiltinMacro).
-  if (MD == 0 || MD->getInfo()->isBuiltinMacro())
+  if (MI == 0 || MI->isBuiltinMacro())
     return