<rdar://problem/13363214> Eliminate race condition between module rebuild and the global module index.

The global module index was querying the file manager for each of the
module files it knows about at load time, to prune out any out-of-date
information. The file manager would then cache the results of the
stat() falls used to find that module file.

Later, the same translation unit could end up trying to import one of the
module files that had previously been ignored by the module cache, but
after some other Clang instance rebuilt the module file to bring it
up-to-date. The stale stat() results in the file manager would
trigger a second rebuild of the already-up-to-date module, causing
failures down the line.

The global module index now lazily resolves its module file references
to actual AST reader module files only after the module file has been
loaded, eliminating the stat-caching race. Moreover, the AST reader
can communicate to its caller that a module file is missing (rather
than simply being out-of-date), allowing us to simplify the
module-loading logic and allowing the compiler to recover if a
dependent module file ends up getting deleted.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177367 91177308-0d34-0410-b5e6-96231b3b80d8

13 files changed, 491 insertions, 321 deletions

diff --git a/include/clang/Serialization/ASTReader.h b/include/clang/Serialization/ASTReader.h
index 2744865819..61a0cbebd7 100644
--- a/include/clang/Serialization/ASTReader.h
+++ b/include/clang/Serialization/ASTReader.h
@@ -48,6 +48,7 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <sys/stat.h>
 
 namespace llvm {
   class MemoryBuffer;
@@ -236,6 +237,8 @@ public:
     Success,
     /// \brief The AST file itself appears corrupted.
     Failure,
+    /// \brief The AST file was missing.
+    Missing,
     /// \brief The AST file is out-of-date relative to its input files,
     /// and needs to be regenerated.
     OutOfDate,
@@ -949,6 +952,7 @@ private:
   ASTReadResult ReadASTCore(StringRef FileName, ModuleKind Type,
                             SourceLocation ImportLoc, ModuleFile *ImportedBy,
                             SmallVectorImpl<ImportedModule> &Loaded,
+                            off_t ExpectedSize, time_t ExpectedModTime,
                             unsigned ClientLoadCapabilities);
   ASTReadResult ReadControlBlock(ModuleFile &F,
                                  SmallVectorImpl<ImportedModule> &Loaded,
@@ -1146,15 +1150,18 @@ public:
     /// \brief The client can't handle any AST loading failures.
     ARR_None = 0,
     /// \brief The client can handle an AST file that cannot load because it
+    /// is missing.
+    ARR_Missing = 0x1,
+    /// \brief The client can handle an AST file that cannot load because it
     /// is out-of-date relative to its input files.
-    ARR_OutOfDate = 0x1,
+    ARR_OutOfDate = 0x2,
     /// \brief The client can handle an AST file that cannot load because it
     /// was built with a different version of Clang.
-    ARR_VersionMismatch = 0x2,
+    ARR_VersionMismatch = 0x4,
     /// \brief The client can handle an AST file that cannot load because it's
     /// compiled configuration doesn't match that of the context it was
     /// loaded into.
-    ARR_ConfigurationMismatch = 0x4
+    ARR_ConfigurationMismatch = 0x8
   };
 
   /// \brief Load the AST file designated by the given file name.
diff --git a/include/clang/Serialization/GlobalModuleIndex.h b/include/clang/Serialization/GlobalModuleIndex.h
index ec8ad353e0..53f61e2d59 100644
--- a/include/clang/Serialization/GlobalModuleIndex.h
+++ b/include/clang/Serialization/GlobalModuleIndex.h
@@ -34,9 +34,44 @@ class DirectoryEntry;
 class FileEntry;
 class FileManager;
 
+namespace serialization {
+  class ModuleFile;
+};
+
 using llvm::SmallVector;
 using llvm::SmallVectorImpl;
 using llvm::StringRef;
+using serialization::ModuleFile;
+
+/// \brief Abstract class that resolves a module file name to a ModuleFile
+/// pointer, which is used to uniquely describe a module file.
+class ModuleFileNameResolver {
+public:
+  virtual ~ModuleFileNameResolver();
+
+  /// \brief Attempt to resolve the given module file name to a specific,
+  /// already-loaded module.
+  ///
+  /// \param FileName The name of the module file.
+  ///
+  /// \param ExpectedSize The size that the module file is expected to have.
+  /// If the actual size differs, the resolver should return \c true.
+  ///
+  /// \param ExpectedModTime The modification time that the module file is
+  /// expected to have. If the actual modification time differs, the resolver
+  /// should return \c true.
+  ///
+  /// \param File Will be set to the module file if there is one, or null
+  /// otherwise.
+  ///
+  /// \returns True if a module file exists but does not meet the size/
+  /// modification time criteria, false if the module file is available or has
+  /// not yet been loaded.
+  virtual bool resolveModuleFileName(StringRef FileName,
+                                     off_t ExpectedSize,
+                                     time_t ExpectedModTime,
+                                     ModuleFile *&File) = 0;
+};
 
 /// \brief A global index for a set of module files, providing information about
 /// the identifiers within those module files.
@@ -54,6 +89,9 @@ class GlobalModuleIndex {
   /// as the global module index is live.
   llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
 
+  /// \brief The module file name resolver.
+  ModuleFileNameResolver *Resolver;
+
   /// \brief The hash table.
   ///
   /// This pointer actually points to a IdentifierIndexTable object,
@@ -63,25 +101,39 @@ class GlobalModuleIndex {
 
   /// \brief Information about a given module file.
   struct ModuleInfo {
-    ModuleInfo() : File() { }
+    ModuleInfo() : File(), Size(), ModTime() { }
+
+    /// \brief The module file, if it is known.
+    ModuleFile *File;
+
+    /// \brief The module file name.
+    std::string FileName;
+
+    /// \brief Size of the module file at the time the global index was built.
+    off_t Size;
 
-    /// \brief The module file entry.
-    const FileEntry *File;
+    /// \brief Modification time of the module file at the time the global
+    /// index was built.
+    time_t ModTime;
 
-    /// \brief The module files on which this module directly depends.
-    llvm::SmallVector<const FileEntry *, 4> Dependencies;
+    /// \brief The module IDs on which this module directly depends.
+    /// FIXME: We don't really need a vector here.
+    llvm::SmallVector<unsigned, 4> Dependencies;
+
+    /// \brief The module IDs that directly depend on this module.
+    llvm::SmallVector<unsigned, 4> ImportedBy;
   };
 
   /// \brief A mapping from module IDs to information about each module.
   ///
   /// This vector may have gaps, if module files have been removed or have
   /// been updated since the index was built. A gap is indicated by an empty
-  /// \c File pointer.
+  /// file name.
   llvm::SmallVector<ModuleInfo, 16> Modules;
 
-  /// \brief Lazily-populated mapping from module file entries to their
+  /// \brief Lazily-populated mapping from module files to their
   /// corresponding index into the \c Modules vector.
-  llvm::DenseMap<const FileEntry *, unsigned> ModulesByFile;
+  llvm::DenseMap<ModuleFile *, unsigned> ModulesByFile;
 
   /// \brief The number of identifier lookups we performed.
   unsigned NumIdentifierLookups;
@@ -91,7 +143,7 @@ class GlobalModuleIndex {
   unsigned NumIdentifierLookupHits;
 
   /// \brief Internal constructor. Use \c readIndex() to read an index.
-  explicit GlobalModuleIndex(FileManager &FileMgr, llvm::MemoryBuffer *Buffer,
+  explicit GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
                              llvm::BitstreamCursor Cursor);
 
   GlobalModuleIndex(const GlobalModuleIndex &) LLVM_DELETED_FUNCTION;
@@ -115,29 +167,27 @@ public:
 
   /// \brief Read a global index file for the given directory.
   ///
-  /// \param FileMgr The file manager to use for reading files.
-  ///
   /// \param Path The path to the specific module cache where the module files
   /// for the intended configuration reside.
   ///
   /// \returns A pair containing the global module index (if it exists) and
   /// the error code.
   static std::pair<GlobalModuleIndex *, ErrorCode>
-  readIndex(FileManager &FileMgr, StringRef Path);
+  readIndex(StringRef Path);
 
   /// \brief Retrieve the set of modules that have up-to-date indexes.
   ///
   /// \param ModuleFiles Will be populated with the set of module files that
   /// have been indexed.
-  void getKnownModules(SmallVectorImpl<const FileEntry *> &ModuleFiles);
+  void getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles);
 
   /// \brief Retrieve the set of module files on which the given module file
   /// directly depends.
-  void getModuleDependencies(const FileEntry *ModuleFile,
-                             SmallVectorImpl<const FileEntry *> &Dependencies);
+  void getModuleDependencies(ModuleFile *File,
+                             SmallVectorImpl<ModuleFile *> &Dependencies);
 
   /// \brief A set of module files in which we found a result.
-  typedef llvm::SmallPtrSet<const FileEntry *, 4> HitSet;
+  typedef llvm::SmallPtrSet<ModuleFile *, 4> HitSet;
   
   /// \brief Look for all of the module files with information about the given
   /// identifier, e.g., a global function, variable, or type with that name.
@@ -150,6 +200,20 @@ public:
   /// \returns true if the identifier is known to the index, false otherwise.
   bool lookupIdentifier(StringRef Name, HitSet &Hits);
 
+  /// \brief Set the module file name resolver.
+  void setResolver(ModuleFileNameResolver *Resolver) {
+    this->Resolver = Resolver;
+  }
+
+  /// \brief Note that additional modules have been loaded, which invalidates
+  /// the module file -> module cache.
+  void noteAdditionalModulesLoaded() {
+    ModulesByFile.clear();
+  }
+
+  /// \brief Resolve the module file for the module with the given ID.
+  ModuleFile *resolveModuleFile(unsigned ID);
+
   /// \brief Print statistics to standard error.
   void printStats();
 
diff --git a/include/clang/Serialization/ModuleManager.h b/include/clang/Serialization/ModuleManager.h
index 60e3b40a7a..9b58b75ebb 100644
--- a/include/clang/Serialization/ModuleManager.h
+++ b/include/clang/Serialization/ModuleManager.h
@@ -16,17 +16,18 @@
 #define LLVM_CLANG_SERIALIZATION_MODULE_MANAGER_H
 
 #include "clang/Basic/FileManager.h"
+#include "clang/Serialization/GlobalModuleIndex.h"
 #include "clang/Serialization/Module.h"
 #include "llvm/ADT/DenseMap.h"
 
 namespace clang { 
 
-class GlobalModuleIndex;
+class ModuleMap;
 
 namespace serialization {
   
 /// \brief Manages the set of modules loaded by an AST reader.
-class ModuleManager {
+class ModuleManager : public ModuleFileNameResolver {
   /// \brief The chain of AST files. The first entry is the one named by the
   /// user, the last one is the one that doesn't depend on anything further.
   SmallVector<ModuleFile *, 2> Chain;
@@ -143,6 +144,19 @@ public:
   
   /// \brief Number of modules loaded
   unsigned size() const { return Chain.size(); }
+
+  /// \brief The result of attempting to add a new module.
+  enum AddModuleResult {
+    /// \brief The module file had already been loaded.
+    AlreadyLoaded,
+    /// \brief The module file was just loaded in response to this call.
+    NewlyLoaded,
+    /// \brief The module file is missing.
+    Missing,
+    /// \brief The module file is out-of-date.
+    OutOfDate
+  };
+
   /// \brief Attempts to create a new module and add it to the list of known
   /// modules.
   ///
@@ -157,18 +171,30 @@ public:
   ///
   /// \param Generation The generation in which this module was loaded.
   ///
+  /// \param ExpectedSize The expected size of the module file, used for
+  /// validation. This will be zero if unknown.
+  ///
+  /// \param ExpectedModTime The expected modification time of the module
+  /// file, used for validation. This will be zero if unknown.
+  ///
+  /// \param Module A pointer to the module file if the module was successfully
+  /// loaded.
+  ///
   /// \param ErrorStr Will be set to a non-empty string if any errors occurred
   /// while trying to load the module.
   ///
   /// \return A pointer to the module that corresponds to this file name,
-  /// and a boolean indicating whether the module was newly added.
-  std::pair<ModuleFile *, bool> 
-  addModule(StringRef FileName, ModuleKind Type, SourceLocation ImportLoc,
-            ModuleFile *ImportedBy, unsigned Generation,
-            std::string &ErrorStr);
+  /// and a value indicating whether the module was loaded.
+  AddModuleResult addModule(StringRef FileName, ModuleKind Type,
+                            SourceLocation ImportLoc,
+                            ModuleFile *ImportedBy, unsigned Generation,
+                            off_t ExpectedSize, time_t ExpectedModTime,
+                            ModuleFile *&Module,
+                            std::string &ErrorStr);
 
   /// \brief Remove the given set of modules.
-  void removeModules(ModuleIterator first, ModuleIterator last);
+  void removeModules(ModuleIterator first, ModuleIterator last,
+                     ModuleMap *modMap);
 
   /// \brief Add an in-memory buffer the list of known buffers
   void addInMemoryBuffer(StringRef FileName, llvm::MemoryBuffer *Buffer);
@@ -200,7 +226,7 @@ public:
   /// Any module that is known to both the global module index and the module
   /// manager that is *not* in this set can be skipped.
   void visit(bool (*Visitor)(ModuleFile &M, void *UserData), void *UserData,
-             llvm::SmallPtrSet<const FileEntry *, 4> *ModuleFilesHit = 0);
+             llvm::SmallPtrSet<ModuleFile *, 4> *ModuleFilesHit = 0);
   
   /// \brief Visit each of the modules with a depth-first traversal.
   ///
@@ -221,7 +247,34 @@ public:
   void visitDepthFirst(bool (*Visitor)(ModuleFile &M, bool Preorder, 
                                        void *UserData), 
                        void *UserData);
-  
+
+  /// \brief Attempt to resolve the given module file name to a file entry.
+  ///
+  /// \param FileName The name of the module file.
+  ///
+  /// \param ExpectedSize The size that the module file is expected to have.
+  /// If the actual size differs, the resolver should return \c true.
+  ///
+  /// \param ExpectedModTime The modification time that the module file is
+  /// expected to have. If the actual modification time differs, the resolver
+  /// should return \c true.
+  ///
+  /// \param File Will be set to the file if there is one, or null
+  /// otherwise.
+  ///
+  /// \returns True if a file exists but does not meet the size/
+  /// modification time criteria, false if the file is either available and
+  /// suitable, or is missing.
+  bool lookupModuleFile(StringRef FileName,
+                        off_t ExpectedSize,
+                        time_t ExpectedModTime,
+                        const FileEntry *&File);
+
+  virtual bool resolveModuleFileName(StringRef FileName,
+                                     off_t ExpectedSize,
+                                     time_t ExpectedModTime,
+                                     ModuleFile *&File);
+
   /// \brief View the graphviz representation of the module graph.
   void viewGraph();
 };
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 4a956ff3ab..c1115aedbf 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -801,6 +801,7 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
     break;
 
   case ASTReader::Failure:
+  case ASTReader::Missing:
   case ASTReader::OutOfDate:
   case ASTReader::VersionMismatch:
   case ASTReader::ConfigurationMismatch:
diff --git a/lib/Frontend/ChainedIncludesSource.cpp b/lib/Frontend/ChainedIncludesSource.cpp
index f414f93812..3b31bf6362 100644
--- a/lib/Frontend/ChainedIncludesSource.cpp
+++ b/lib/Frontend/ChainedIncludesSource.cpp
@@ -47,6 +47,7 @@ static ASTReader *createASTReader(CompilerInstance &CI,
     return Reader.take();
 
   case ASTReader::Failure:
+  case ASTReader::Missing:
   case ASTReader::OutOfDate:
   case ASTReader::VersionMismatch:
   case ASTReader::ConfigurationMismatch:
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index ce6572aa1b..d5c1a27ec8 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -338,6 +338,7 @@ CompilerInstance::createPCHExternalASTSource(StringRef Path,
     // Unrecoverable failure: don't even try to process the input file.
     break;
 
+  case ASTReader::Missing:
   case ASTReader::OutOfDate:
   case ASTReader::VersionMismatch:
   case ASTReader::ConfigurationMismatch:
@@ -945,88 +946,6 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
     } else
       ModuleFileName = PP->getHeaderSearchInfo().getModuleFileName(ModuleName);
 
-    if (ModuleFileName.empty()) {
-      getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
-        << ModuleName
-        << SourceRange(ImportLoc, ModuleNameLoc);
-      LastModuleImportLoc = ImportLoc;
-      LastModuleImportResult = ModuleLoadResult();
-      return LastModuleImportResult;
-    }
-    
-    const FileEntry *ModuleFile
-      = getFileManager().getFile(ModuleFileName, /*OpenFile=*/false,
-                                 /*CacheFailure=*/false);
-    bool BuildingModule = false;
-    if (!ModuleFile && Module) {
-      // The module is not cached, but we have a module map from which we can
-      // build the module.
-
-      // Check whether there is a cycle in the module graph.
-      ModuleBuildStack Path = getSourceManager().getModuleBuildStack();
-      ModuleBuildStack::iterator Pos = Path.begin(), PosEnd = Path.end();
-      for (; Pos != PosEnd; ++Pos) {
-        if (Pos->first == ModuleName)
-          break;
-      }
-
-      if (Pos != PosEnd) {
-        SmallString<256> CyclePath;
-        for (; Pos != PosEnd; ++Pos) {
-          CyclePath += Pos->first;
-          CyclePath += " -> ";
-        }
-        CyclePath += ModuleName;
-
-        getDiagnostics().Report(ModuleNameLoc, diag::err_module_cycle)
-          << ModuleName << CyclePath;
-        return ModuleLoadResult();
-      }
-
-      // Check whether we have already attempted to build this module (but
-      // failed).
-      if (getPreprocessorOpts().FailedModules &&
-          getPreprocessorOpts().FailedModules->hasAlreadyFailed(ModuleName)) {
-        getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_built)
-          << ModuleName
-          << SourceRange(ImportLoc, ModuleNameLoc);
-        ModuleBuildFailed = true;
-        return ModuleLoadResult();
-      }
-
-      BuildingModule = true;
-      compileModule(*this, ModuleNameLoc, Module, ModuleFileName);
-      ModuleFile = FileMgr->getFile(ModuleFileName, /*OpenFile=*/false,
-                                    /*CacheFailure=*/false);
-
-      if (!ModuleFile && getPreprocessorOpts().FailedModules)
-        getPreprocessorOpts().FailedModules->addFailed(ModuleName);
-    }
-
-    if (!ModuleFile) {
-      getDiagnostics().Report(ModuleNameLoc,
-                              BuildingModule? diag::err_module_not_built
-                                            : diag::err_module_not_found)
-        << ModuleName
-        << SourceRange(ImportLoc, ModuleNameLoc);
-      ModuleBuildFailed = true;
-      return ModuleLoadResult();
-    }
-
-    // If there is already a module file associated with this module, make sure
-    // it is the same as the module file we're looking for. Otherwise, we
-    // have two module files for the same module.
-    if (const FileEntry *CurModuleFile = Module? Module->getASTFile() : 0) {
-      if (CurModuleFile != ModuleFile) {
-        getDiagnostics().Report(ModuleNameLoc, diag::err_module_file_conflict)
-          << ModuleName
-          << CurModuleFile->getName()
-          << ModuleFile->getName();
-        ModuleBuildFailed = true;
-        return ModuleLoadResult();
-      }
-    }
-
     // If we don't already have an ASTReader, create one now.
     if (!ModuleManager) {
       if (!hasASTContext())
@@ -1056,21 +975,53 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         ModuleManager->StartTranslationUnit(&getASTConsumer());
     }
 
-    // Try to load the module we found.
-    unsigned ARRFlags = ASTReader::ARR_None;
-    if (Module)
-      ARRFlags |= ASTReader::ARR_OutOfDate;
-    switch (ModuleManager->ReadAST(ModuleFile->getName(),
-                                   serialization::MK_Module, ImportLoc,
-                                   ARRFlags)) {
+    // Try to load the module file.
+    unsigned ARRFlags = ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing;
+    switch (ModuleManager->ReadAST(ModuleFileName, serialization::MK_Module,
+                                   ImportLoc, ARRFlags)) {
     case ASTReader::Success:
       break;
 
     case ASTReader::OutOfDate: {
-      // The module file is out-of-date. Rebuild it.
-      getFileManager().invalidateCache(ModuleFile);
+      // The module file is out-of-date. Remove it, then rebuild it.
       bool Existed;
       llvm::sys::fs::remove(ModuleFileName, Existed);
+    }
+    // Fall through to build the module again.
+
+    case ASTReader::Missing: {
+      // The module file is (now) missing. Build it.
+
+      // If we don't have a module, we don't know how to build the module file.
+      // Complain and return.
+      if (!Module) {
+        getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
+          << ModuleName
+          << SourceRange(ImportLoc, ModuleNameLoc);
+        ModuleBuildFailed = true;
+        return ModuleLoadResult();
+      }
+
+      // Check whether there is a cycle in the module graph.
+      ModuleBuildStack ModPath = getSourceManager().getModuleBuildStack();
+      ModuleBuildStack::iterator Pos = ModPath.begin(), PosEnd = ModPath.end();
+      for (; Pos != PosEnd; ++Pos) {
+        if (Pos->first == ModuleName)
+          break;
+      }
+
+      if (Pos != PosEnd) {
+        SmallString<256> CyclePath;
+        for (; Pos != PosEnd; ++Pos) {
+          CyclePath += Pos->first;
+          CyclePath += " -> ";
+        }
+        CyclePath += ModuleName;
+
+        getDiagnostics().Report(ModuleNameLoc, diag::err_module_cycle)
+          << ModuleName << CyclePath;
+        return ModuleLoadResult();
+      }
 
       // Check whether we have already attempted to build this module (but
       // failed).
@@ -1083,15 +1034,23 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
         return ModuleLoadResult();
       }
 
+      // Try to compile the module.
       compileModule(*this, ModuleNameLoc, Module, ModuleFileName);
 
-      // Try loading the module again.
-      ModuleFile = FileMgr->getFile(ModuleFileName, /*OpenFile=*/false,
-                                    /*CacheFailure=*/false);
-      if (!ModuleFile ||
-          ModuleManager->ReadAST(ModuleFileName,
+      // Try to read the module file, now that we've compiled it.
+      ASTReader::ASTReadResult ReadResult
+        = ModuleManager->ReadAST(ModuleFileName,
                                  serialization::MK_Module, ImportLoc,
-                                 ASTReader::ARR_None) != ASTReader::Success) {
+                                 ASTReader::ARR_Missing);
+      if (ReadResult != ASTReader::Success) {
+        if (ReadResult == ASTReader::Missing) {
+          getDiagnostics().Report(ModuleNameLoc,
+                                  Module? diag::err_module_not_built
+                                        : diag::err_module_not_found)
+            << ModuleName
+            << SourceRange(ImportLoc, ModuleNameLoc);
+        }
+
         if (getPreprocessorOpts().FailedModules)
           getPreprocessorOpts().FailedModules->addFailed(ModuleName);
         KnownModules[Path[0].first] = 0;
@@ -1125,10 +1084,6 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
                  .findModule((Path[0].first->getName()));
     }
 
-    if (Module) {
-      Module->setASTFile(ModuleFile);
-    }
-    
     // Cache the result of this top-level module lookup for later.
     Known = KnownModules.insert(std::make_pair(Path[0].first, Module)).first;
   }
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index cc9b5d506a..6530bae509 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -1616,8 +1616,10 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
          || StoredTime != File->getModificationTime()
 #endif
          )) {
-      if (Complain)
+      if (Complain) {
         Error(diag::err_fe_pch_file_modified, Filename, F.FileName);
+      }
+
       IsOutOfDate = true;
     }
 
@@ -1774,6 +1776,8 @@ ASTReader::ReadControlBlock(ModuleFile &F,
         // location info are setup.
         SourceLocation ImportLoc =
             SourceLocation::getFromRawEncoding(Record[Idx++]);
+        off_t StoredSize = (off_t)Record[Idx++];
+        time_t StoredModTime = (time_t)Record[Idx++];
         unsigned Length = Record[Idx++];
         SmallString<128> ImportedFile(Record.begin() + Idx,
                                       Record.begin() + Idx + Length);
@@ -1781,9 +1785,11 @@ ASTReader::ReadControlBlock(ModuleFile &F,
 
         // Load the AST file.
         switch(ReadASTCore(ImportedFile, ImportedKind, ImportLoc, &F, Loaded,
+                           StoredSize, StoredModTime,
                            ClientLoadCapabilities)) {
         case Failure: return Failure;
           // If we have to ignore the dependency, we'll have to ignore this too.
+        case Missing: return Missing;
         case OutOfDate: return OutOfDate;
         case VersionMismatch: return VersionMismatch;
         case ConfigurationMismatch: return ConfigurationMismatch;
@@ -2774,7 +2780,7 @@ bool ASTReader::loadGlobalIndex() {
   StringRef ModuleCachePath
     = getPreprocessor().getHeaderSearchInfo().getModuleCachePath();
   std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode> Result
-    = GlobalModuleIndex::readIndex(FileMgr, ModuleCachePath);
+    = GlobalModuleIndex::readIndex(ModuleCachePath);
   if (!Result.first)
     return true;
 
@@ -2799,13 +2805,18 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
   SmallVector<ImportedModule, 4> Loaded;
   switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type, ImportLoc,
                                                 /*ImportedBy=*/0, Loaded,
+                                                0, 0,
                                                 ClientLoadCapabilities)) {
   case Failure:
+  case Missing:
   case OutOfDate:
   case VersionMismatch:
   case ConfigurationMismatch:
   case HadErrors:
-    ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, ModuleMgr.end());
+    ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, ModuleMgr.end(),
+                            Context.getLangOpts().Modules
+                              ? &PP.getHeaderSearchInfo().getModuleMap()
+                              : 0);
 
     // If we find that any modules are unusable, the global index is going
     // to be out-of-date. Just remove it.
@@ -2923,27 +2934,54 @@ ASTReader::ReadASTCore(StringRef FileName,
                        SourceLocation ImportLoc,
                        ModuleFile *ImportedBy,
                        SmallVectorImpl<ImportedModule> &Loaded,
+                       off_t ExpectedSize, time_t ExpectedModTime,
                        unsigned ClientLoadCapabilities) {
   ModuleFile *M;
-  bool NewModule;
   std::string ErrorStr;
-  llvm::tie(M, NewModule) = ModuleMgr.addModule(FileName, Type, ImportLoc,
-                                                ImportedBy, CurrentGeneration,
-                                                ErrorStr);
-
-  if (!M) {
-    // We couldn't load the module.
-    std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
-      + ErrorStr;
-    Error(Msg);
-    return Failure;
-  }
+  ModuleManager::AddModuleResult AddResult
+    = ModuleMgr.addModule(FileName, Type, ImportLoc, ImportedBy,
+                          CurrentGeneration, ExpectedSize, ExpectedModTime,
+                          M, ErrorStr);
 
-  if (!NewModule) {
-    // We've already loaded this module.
+  switch (AddResult) {
+  case ModuleManager::AlreadyLoaded:
     return Success;
+
+  case ModuleManager::NewlyLoaded:
+    // Load module file below.
+    break;
+
+  case ModuleManager::Missing:
+    // The module file was missing; if the client handle handle, that, return
+    // it.
+    if (ClientLoadCapabilities & ARR_Missing)
+      return Missing;
+
+    // Otherwise, return an error.
+    {
+      std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+                      + ErrorStr;
+      Error(Msg);
+    }
+    return Failure;
+
+  case ModuleManager::OutOfDate:
+    // We couldn't load the module file because it is out-of-date. If the
+    // client can handle out-of-date, return it.
+    if (ClientLoadCapabilities & ARR_OutOfDate)
+      return OutOfDate;
+
+    // Otherwise, return an error.
+    {
+      std::string Msg = "Unable to load module \"" + FileName.str() + "\": "
+                      + ErrorStr;
+      Error(Msg);
+    }
+    return Failure;
   }
 
+  assert(M && "Missing module file");
+
   // FIXME: This seems rather a hack. Should CurrentDir be part of the
   // module?
   if (FileName != "-") {
@@ -2997,6 +3035,7 @@ ASTReader::ReadASTCore(StringRef FileName,
         break;
 
       case Failure: return Failure;
+      case Missing: return Missing;
       case OutOfDate: return OutOfDate;
       case VersionMismatch: return VersionMismatch;
       case ConfigurationMismatch: return ConfigurationMismatch;
@@ -3467,18 +3506,22 @@ bool ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
         return true;
       }
 
-      if (const FileEntry *CurFile = CurrentModule->getASTFile()) {
-        if (CurFile != F.File) {
-          if (!Diags.isDiagnosticInFlight()) {
-            Diag(diag::err_module_file_conflict)
-              << CurrentModule->getTopLevelModuleName()
-              << CurFile->getName()
-              << F.File->getName();
+      if (!ParentModule) {
+        if (const FileEntry *CurFile = CurrentModule->getASTFile()) {
+          if (CurFile != F.File) {
+            if (!Diags.isDiagnosticInFlight()) {
+              Diag(diag::err_module_file_conflict)
+                << CurrentModule->getTopLevelModuleName()
+                << CurFile->getName()
+                << F.File->getName();
+            }
+            return true;
           }
-          return true;
         }
+
+        CurrentModule->setASTFile(F.File);
       }
-      CurrentModule->setASTFile(F.File);
+      
       CurrentModule->IsFromModuleFile = true;
       CurrentModule->IsSystem = IsSystem || CurrentModule->IsSystem;
       CurrentModule->InferSubmodules = InferSubmodules;
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index 716b1fbb30..724c8cf825 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -1034,6 +1034,8 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
 
       Record.push_back((unsigned)(*M)->Kind); // FIXME: Stable encoding
       AddSourceLocation((*M)->ImportLoc, Record);
+      Record.push_back((*M)->File->getSize());
+      Record.push_back((*M)->File->getModificationTime());
       // FIXME: This writes the absolute path for AST files we depend on.
       const std::string &FileName = (*M)->FileName;
       Record.push_back(FileName.size());
diff --git a/lib/Serialization/GlobalModuleIndex.cpp b/lib/Serialization/GlobalModuleIndex.cpp
index 7d34f85fd8..8b5851dbe5 100644
--- a/lib/Serialization/GlobalModuleIndex.cpp
+++ b/lib/Serialization/GlobalModuleIndex.cpp
@@ -57,6 +57,8 @@ static const char * const IndexFileName = "modules.idx";
 /// \brief The global index file version.
 static const unsigned CurrentVersion = 1;
 
+ModuleFileNameResolver::~ModuleFileNameResolver() { }
+
 //----------------------------------------------------------------------------//
 // Global module index reader.
 //----------------------------------------------------------------------------//
@@ -115,29 +117,16 @@ public:
 
 typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;
 
-/// \brief Module information as it was loaded from the index file.
-struct LoadedModuleInfo {
-  const FileEntry *File;
-  SmallVector<unsigned, 2> Dependencies;
-  SmallVector<unsigned, 2> ImportedBy;
-};
-
 }
 
-GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
-                                     llvm::MemoryBuffer *Buffer,
+GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
                                      llvm::BitstreamCursor Cursor)
-  : Buffer(Buffer), IdentifierIndex(),
+  : Buffer(Buffer), Resolver(), IdentifierIndex(),
     NumIdentifierLookups(), NumIdentifierLookupHits()
 {
-  typedef llvm::DenseMap<unsigned, LoadedModuleInfo> LoadedModulesMap;
-  LoadedModulesMap LoadedModules;
-  
   // Read the global index.
-  unsigned LargestID = 0;
   bool InGlobalIndexBlock = false;
   bool Done = false;
-  bool AnyOutOfDate = false;
   while (!Done) {
     llvm::BitstreamEntry Entry = Cursor.advance();
 
@@ -185,41 +174,33 @@ GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr,
     case MODULE: {
       unsigned Idx = 0;
       unsigned ID = Record[Idx++];
-      if (ID > LargestID)
-        LargestID = ID;
-      
-      off_t Size = Record[Idx++];
-      ti