Introduce a flag in SourceManager to treat non-system source files

as "volatile", meaning there's a high enough chance that they may
change while we are trying to use them.

This flag is only enabled by libclang.
Currently "volatile" source files will be stat'ed immediately
before opening them, because the file size stat info
may not be accurate since when we got it (e.g. from the PCH).
This avoids crashes when trying to reference mmap'ed memory
from a file whose size is not what we expect.

Note that there's still a window for a racing issue to occur
but the window for it should be way smaller than before.
We can consider later on to avoid mmap completely on such files.

rdar://11612916

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@160074 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 94 insertions, 32 deletions

diff --git a/include/clang/Basic/FileManager.h b/include/clang/Basic/FileManager.h
index ea58e2d76a..930523f4a7 100644
--- a/include/clang/Basic/FileManager.h
+++ b/include/clang/Basic/FileManager.h
@@ -221,7 +221,8 @@ public:
   /// \brief Open the specified file as a MemoryBuffer, returning a new
   /// MemoryBuffer if successful, otherwise returning null.
   llvm::MemoryBuffer *getBufferForFile(const FileEntry *Entry,
-                                       std::string *ErrorStr = 0);
+                                       std::string *ErrorStr = 0,
+                                       bool isVolatile = false);
   llvm::MemoryBuffer *getBufferForFile(StringRef Filename,
                                        std::string *ErrorStr = 0);
 
diff --git a/include/clang/Basic/SourceManager.h b/include/clang/Basic/SourceManager.h
index 415b017b73..082cbb1fb1 100644
--- a/include/clang/Basic/SourceManager.h
+++ b/include/clang/Basic/SourceManager.h
@@ -128,14 +128,20 @@ namespace SrcMgr {
     /// When true, the original entry may be a virtual file that does not
     /// exist.
     unsigned BufferOverridden : 1;
+
+    /// \brief True if this content cache was initially created for a source
+    /// file considered as a system one.
+    unsigned IsSystemFile : 1;
     
     ContentCache(const FileEntry *Ent = 0)
       : Buffer(0, false), OrigEntry(Ent), ContentsEntry(Ent),
-        SourceLineCache(0), NumLines(0), BufferOverridden(false) {}
+        SourceLineCache(0), NumLines(0), BufferOverridden(false),
+        IsSystemFile(false) {}
     
     ContentCache(const FileEntry *Ent, const FileEntry *contentEnt)
       : Buffer(0, false), OrigEntry(Ent), ContentsEntry(contentEnt),
-        SourceLineCache(0), NumLines(0), BufferOverridden(false) {}
+        SourceLineCache(0), NumLines(0), BufferOverridden(false),
+        IsSystemFile(false) {}
     
     ~ContentCache();
     
@@ -143,7 +149,8 @@ namespace SrcMgr {
     /// a non-NULL Buffer or SourceLineCache.  Ownership of allocated memory
     /// is not transferred, so this is a logical error.
     ContentCache(const ContentCache &RHS)
-      : Buffer(0, false), SourceLineCache(0), BufferOverridden(false)
+      : Buffer(0, false), SourceLineCache(0), BufferOverridden(false),
+        IsSystemFile(false)
     {
       OrigEntry = RHS.OrigEntry;
       ContentsEntry = RHS.ContentsEntry;
@@ -534,6 +541,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// files, should report the original file name. Defaults to true.
   bool OverridenFilesKeepOriginalName;
 
+  /// \brief True if non-system source files should be treated as volatile
+  /// (likely to change while trying to use them). Defaults to false.
+  bool UserFilesAreVolatile;
+
   struct OverriddenFilesInfoTy {
     /// \brief Files that have been overriden with the contents from another
     /// file.
@@ -639,7 +650,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   explicit SourceManager(const SourceManager&);
   void operator=(const SourceManager&);
 public:
-  SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr);
+  SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
+                bool UserFilesAreVolatile = false);
   ~SourceManager();
 
   void clearIDTables();
@@ -654,6 +666,10 @@ public:
     OverridenFilesKeepOriginalName = value;
   }
 
+  /// \brief True if non-system source files should be treated as volatile
+  /// (likely to change while trying to use them).
+  bool userFilesAreVolatile() const { return UserFilesAreVolatile; }
+
   /// \brief Create the FileID for a memory buffer that will represent the
   /// FileID for the main source.
   ///
@@ -706,7 +722,9 @@ public:
   FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos,
                       SrcMgr::CharacteristicKind FileCharacter,
                       int LoadedID = 0, unsigned LoadedOffset = 0) {
-    const SrcMgr::ContentCache *IR = getOrCreateContentCache(SourceFile);
+    const SrcMgr::ContentCache *
+      IR = getOrCreateContentCache(SourceFile,
+                              /*isSystemFile=*/FileCharacter != SrcMgr::C_User);
     assert(IR && "getOrCreateContentCache() cannot return NULL");
     return createFileID(IR, IncludePos, FileCharacter, LoadedID, LoadedOffset);
   }
@@ -1519,7 +1537,8 @@ private:
                       int LoadedID, unsigned LoadedOffset);
 
   const SrcMgr::ContentCache *
-    getOrCreateContentCache(const FileEntry *SourceFile);
+    getOrCreateContentCache(const FileEntry *SourceFile,
+                            bool isSystemFile = false);
 
   /// \brief Create a new ContentCache for the specified  memory buffer.
   const SrcMgr::ContentCache*
diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h
index afaf60e89a..8dd863e304 100644
--- a/include/clang/Frontend/ASTUnit.h
+++ b/include/clang/Frontend/ASTUnit.h
@@ -254,6 +254,10 @@ private:
   /// \brief Whether to include brief documentation within the set of code
   /// completions cached.
   bool IncludeBriefCommentsInCodeCompletion : 1;
+
+  /// \brief True if non-system source files should be treated as volatile
+  /// (likely to change while trying to use them).
+  bool UserFilesAreVolatile : 1;
  
   /// \brief The language options used when we load an AST file.
   LangOptions ASTFileLangOpts;
@@ -619,7 +623,8 @@ public:
   /// \brief Create a ASTUnit. Gets ownership of the passed CompilerInvocation. 
   static ASTUnit *create(CompilerInvocation *CI,
                          IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
-                         bool CaptureDiagnostics = false);
+                         bool CaptureDiagnostics,
+                         bool UserFilesAreVolatile);
 
   /// \brief Create a ASTUnit from an AST file.
   ///
@@ -636,7 +641,8 @@ public:
                                   RemappedFile *RemappedFiles = 0,
                                   unsigned NumRemappedFiles = 0,
                                   bool CaptureDiagnostics = false,
-                                  bool AllowPCHWithCompilerErrors = false);
+                                  bool AllowPCHWithCompilerErrors = false,
+                                  bool UserFilesAreVolatile = false);
 
 private:
   /// \brief Helper function for \c LoadFromCompilerInvocation() and
@@ -687,6 +693,7 @@ public:
                                              bool PrecompilePreamble = false,
                                        bool CacheCodeCompletionResults = false,
                               bool IncludeBriefCommentsInCodeCompletion = false,
+                                       bool UserFilesAreVolatile = false,
                                        OwningPtr<ASTUnit> *ErrAST = 0);
 
   /// LoadFromCompilerInvocation - Create an ASTUnit from a source file, via a
@@ -707,7 +714,8 @@ public:
                                              bool PrecompilePreamble = false,
                                       TranslationUnitKind TUKind = TU_Complete,
                                        bool CacheCodeCompletionResults = false,
-                            bool IncludeBriefCommentsInCodeCompletion = false);
+                            bool IncludeBriefCommentsInCodeCompletion = false,
+                                             bool UserFilesAreVolatile = false);
 
   /// LoadFromCommandLine - Create an ASTUnit from a vector of command line
   /// arguments, which must specify exactly one source file.
@@ -742,6 +750,7 @@ public:
                             bool IncludeBriefCommentsInCodeCompletion = false,
                                       bool AllowPCHWithCompilerErrors = false,
                                       bool SkipFunctionBodies = false,
+                                      bool UserFilesAreVolatile = false,
                                       OwningPtr<ASTUnit> *ErrAST = 0);
   
   /// \brief Reparse the source files using the same command-line options that
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
index ddf8934b62..b921f3e0d9 100644
--- a/lib/Basic/FileManager.cpp
+++ b/lib/Basic/FileManager.cpp
@@ -488,15 +488,21 @@ void FileManager::FixupRelativePath(SmallVectorImpl<char> &path) const {
 }
 
 llvm::MemoryBuffer *FileManager::
-getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
+getBufferForFile(const FileEntry *Entry, std::string *ErrorStr,
+                 bool isVolatile) {
   OwningPtr<llvm::MemoryBuffer> Result;
   llvm::error_code ec;
 
+  uint64_t FileSize = Entry->getSize();
+  // If there's a high enough chance that the file have changed since we
+  // got its size, force a stat before opening it.
+  if (isVolatile)
+    FileSize = -1;
+
   const char *Filename = Entry->getName();
   // If the file is already open, use the open file descriptor.
   if (Entry->FD != -1) {
-    ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result,
-                                         Entry->getSize());
+    ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, FileSize);
     if (ErrorStr)
       *ErrorStr = ec.message();
 
@@ -508,7 +514,7 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
   // Otherwise, open the file.
 
   if (FileSystemOpts.WorkingDir.empty()) {
-    ec = llvm::MemoryBuffer::getFile(Filename, Result, Entry->getSize());
+    ec = llvm::MemoryBuffer::getFile(Filename, Result, FileSize);
     if (ec && ErrorStr)
       *ErrorStr = ec.message();
     return Result.take();
@@ -516,7 +522,7 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
 
   SmallString<128> FilePath(Entry->getName());
   FixupRelativePath(FilePath);
-  ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, Entry->getSize());
+  ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, FileSize);
   if (ec && ErrorStr)
     *ErrorStr = ec.message();
   return Result.take();
diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp
index 5dc100a07a..9ec2474299 100644
--- a/lib/Basic/SourceManager.cpp
+++ b/lib/Basic/SourceManager.cpp
@@ -97,7 +97,10 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(DiagnosticsEngine &Diag,
   }    
 
   std::string ErrorStr;
-  Buffer.setPointer(SM.getFileManager().getBufferForFile(ContentsEntry, &ErrorStr));
+  bool isVolatile = SM.userFilesAreVolatile() && !IsSystemFile;
+  Buffer.setPointer(SM.getFileManager().getBufferForFile(ContentsEntry,
+                                                         &ErrorStr,
+                                                         isVolatile));
 
   // If we were unable to open the file, then we are in an inconsistent
   // situation where the content cache referenced a file which no longer
@@ -367,8 +370,10 @@ LineTableInfo &SourceManager::getLineTable() {
 // Private 'Create' methods.
 //===----------------------------------------------------------------------===//
 
-SourceManager::SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr)
+SourceManager::SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
+                             bool UserFilesAreVolatile)
   : Diag(Diag), FileMgr(FileMgr), OverridenFilesKeepOriginalName(true),
+    UserFilesAreVolatile(UserFilesAreVolatile),
     ExternalSLocEntries(0), LineTable(0), NumLinearScans(0),
     NumBinaryProbes(0), FakeBufferForRecovery(0),
     FakeContentCacheForRecovery(0) {
@@ -426,7 +431,8 @@ void SourceManager::clearIDTables() {
 /// getOrCreateContentCache - Create or return a cached ContentCache for the
 /// specified file.
 const ContentCache *
-SourceManager::getOrCreateContentCache(const FileEntry *FileEnt) {
+SourceManager::getOrCreateContentCache(const FileEntry *FileEnt,
+                                       bool isSystemFile) {
   assert(FileEnt && "Didn't specify a file entry to use?");
 
   // Do we already have information about this file?
@@ -455,6 +461,8 @@ SourceManager::getOrCreateContentCache(const FileEntry *FileEnt) {
     new (Entry) ContentCache(FileEnt);
   }
 
+  Entry->IsSystemFile = isSystemFile;
+
   return Entry;
 }
 
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 7be25e96b4..ea2f773c56 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -215,7 +215,7 @@ ASTUnit::ASTUnit(bool _MainFileIsAST)
     PreambleRebuildCounter(0), SavedMainFileBuffer(0), PreambleBuffer(0),
     NumWarningsInPreamble(0),
     ShouldCacheCodeCompletionResults(false),
-    IncludeBriefCommentsInCodeCompletion(false),
+    IncludeBriefCommentsInCodeCompletion(false), UserFilesAreVolatile(false),
     CompletionCacheTopLevelHashValue(0),
     PreambleTopLevelHashValue(0),
     CurrentTopLevelHashValue(0),
@@ -658,7 +658,8 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
                                   RemappedFile *RemappedFiles,
                                   unsigned NumRemappedFiles,
                                   bool CaptureDiagnostics,
-                                  bool AllowPCHWithCompilerErrors) {
+                                  bool AllowPCHWithCompilerErrors,
+                                  bool UserFilesAreVolatile) {
   OwningPtr<ASTUnit> AST(new ASTUnit(true));
 
   // Recover resources if we crash before exiting this method.
@@ -674,8 +675,10 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->Diagnostics = Diags;
   AST->FileMgr = new FileManager(FileSystemOpts);
+  AST->UserFilesAreVolatile = UserFilesAreVolatile;
   AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
-                                     AST->getFileManager());
+                                     AST->getFileManager(),
+                                     UserFilesAreVolatile);
   AST->HeaderInfo.reset(new HeaderSearch(AST->getFileManager(),
                                          AST->getDiagnostics(),
                                          AST->ASTFileLangOpts,
@@ -1077,7 +1080,8 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   LangOpts = &Clang->getLangOpts();
   FileSystemOpts = Clang->getFileSystemOpts();
   FileMgr = new FileManager(FileSystemOpts);
-  SourceMgr = new SourceManager(getDiagnostics(), *FileMgr);
+  SourceMgr = new SourceManager(getDiagnostics(), *FileMgr,
+                                UserFilesAreVolatile);
   TheSema.reset();
   Ctx = 0;
   PP = 0;
@@ -1665,7 +1669,8 @@ StringRef ASTUnit::getMainFileName() const {
 
 ASTUnit *ASTUnit::create(CompilerInvocation *CI,
                          IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
-                         bool CaptureDiagnostics) {
+                         bool CaptureDiagnostics,
+                         bool UserFilesAreVolatile) {
   OwningPtr<ASTUnit> AST;
   AST.reset(new ASTUnit(false));
   ConfigureDiags(Diags, 0, 0, *AST, CaptureDiagnostics);
@@ -1673,7 +1678,9 @@ ASTUnit *ASTUnit::create(CompilerInvocation *CI,
   AST->Invocation = CI;
   AST->FileSystemOpts = CI->getFileSystemOpts();
   AST->FileMgr = new FileManager(AST->FileSystemOpts);
-  AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr);
+  AST->UserFilesAreVolatile = UserFilesAreVolatile;
+  AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
+                                     UserFilesAreVolatile);
 
   return AST.take();
 }
@@ -1689,6 +1696,7 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
                                              bool PrecompilePreamble,
                                              bool CacheCodeCompletionResults,
                                     bool IncludeBriefCommentsInCodeCompletion,
+                                             bool UserFilesAreVolatile,
                                              OwningPtr<ASTUnit> *ErrAST) {
   assert(CI && "A CompilerInvocation is required");
 
@@ -1696,7 +1704,7 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
   ASTUnit *AST = Unit;
   if (!AST) {
     // Create the AST unit.
-    OwnAST.reset(create(CI, Diags, CaptureDiagnostics));
+    OwnAST.reset(create(CI, Diags, CaptureDiagnostics, UserFilesAreVolatile));
     AST = OwnAST.get();
   }
   
@@ -1859,7 +1867,8 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
                                              bool PrecompilePreamble,
                                              TranslationUnitKind TUKind,
                                              bool CacheCodeCompletionResults,
-                                    bool IncludeBriefCommentsInCodeCompletion) {
+                                    bool IncludeBriefCommentsInCodeCompletion,
+                                             bool UserFilesAreVolatile) {
   // Create the AST unit.
   OwningPtr<ASTUnit> AST;
   AST.reset(new ASTUnit(false));
@@ -1872,6 +1881,7 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   AST->IncludeBriefCommentsInCodeCompletion
     = IncludeBriefCommentsInCodeCompletion;
   AST->Invocation = CI;
+  AST->UserFilesAreVolatile = UserFilesAreVolatile;
   
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
@@ -1898,6 +1908,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                       bool IncludeBriefCommentsInCodeCompletion,
                                       bool AllowPCHWithCompilerErrors,
                                       bool SkipFunctionBodies,
+                                      bool UserFilesAreVolatile,
                                       OwningPtr<ASTUnit> *ErrAST) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
@@ -1957,6 +1968,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
   AST->IncludeBriefCommentsInCodeCompletion
     = IncludeBriefCommentsInCodeCompletion;
+  AST->UserFilesAreVolatile = UserFilesAreVolatile;
   AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
   AST->StoredDiagnostics.swap(StoredDiagnostics);
   AST->Invocation = CI;
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index 46bd55e83e..4073e32c2c 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -1127,8 +1127,9 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
       // This is the module's main file.
       IncludeLoc = getImportLocation(F);
     }
-    FileID FID = SourceMgr.createFileID(File, IncludeLoc,
-                                        (SrcMgr::CharacteristicKind)Record[2],
+    SrcMgr::CharacteristicKind
+      FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
+    FileID FID = SourceMgr.createFileID(File, IncludeLoc, FileCharacter,
                                         ID, BaseOffset + Record[0]);
     SrcMgr::FileInfo &FileInfo =
           const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile());
@@ -1145,7 +1146,8 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
     }
     
     const SrcMgr::ContentCache *ContentCache
-      = SourceMgr.getOrCreateContentCache(File);
+      = SourceMgr.getOrCreateContentCache(File,
+                              /*isSystemFile=*/FileCharacter != SrcMgr::C_User);
     if (OverriddenBuffer && !ContentCache->BufferOverridden &&
         ContentCache->ContentsEntry == ContentCache->OrigEntry) {
       unsigned Code = SLocEntryCursor.ReadCode();
diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp
index 2233a1125e..de56b19931 100644
--- a/tools/libclang/CIndex.cpp
+++ b/tools/libclang/CIndex.cpp
@@ -2464,7 +2464,8 @@ CXTranslationUnit clang_createTranslationUnit(CXIndex CIdx,
                                   CXXIdx->getOnlyLocalDecls(),
                                   0, 0,
                                   /*CaptureDiagnostics=*/true,
-                                  /*AllowPCHWithCompilerErrors=*/true);
+                                  /*AllowPCHWithCompilerErrors=*/true,
+                                  /*UserFilesAreVolatile=*/true);
   return MakeCXTranslationUnit(CXXIdx, TU);
 }
 
@@ -2612,6 +2613,7 @@ static void clang_parseTranslationUnit_Impl(void *UserData) {
                                  IncludeBriefCommentsInCodeCompletion,
                                  /*AllowPCHWithCompilerErrors=*/true,
                                  SkipFunctionBodies,
+                                 /*UserFilesAreVolatile=*/true,
                                  &ErrUnit));
 
   if (NumErrors != Diags->getClient()->getNumErrors()) {
diff --git a/tools/libclang/Indexing.cpp b/tools/libclang/Indexing.cpp
index 6b681340f1..8fe9c36556 100644
--- a/tools/libclang/Indexing.cpp
+++ b/tools/libclang/Indexing.cpp
@@ -353,7 +353,8 @@ static void clang_indexSourceFile_Impl(void *UserData) {
     CInvok->getDiagnosticOpts().IgnoreWarnings = true;
 
   ASTUnit *Unit = ASTUnit::create(CInvok.getPtr(), Diags,
-                                  /*CaptureDiagnostics=*/true);
+                                  /*CaptureDiagnostics=*/true,
+                                  /*UserFilesAreVolatile=*/true);
   OwningPtr<CXTUOwner> CXTU(new CXTUOwner(MakeCXTranslationUnit(CXXIdx, Unit)));
 
   // Recover resources if we crash before exiting this method.
@@ -396,7 +397,9 @@ static void clang_indexSourceFile_Impl(void *UserData) {
                                                        OnlyLocalDecls,
                                                     /*CaptureDiagnostics=*/true,
                                                        PrecompilePreamble,
-                                                    CacheCodeCompletionResults);
+                                                    CacheCodeCompletionResults,
+                                 /*IncludeBriefCommentsInCodeCompletion=*/false,
+                                                 /*UserFilesAreVolatile=*/true);
   if (DiagTrap.hasErrorOccurred() && CXXIdx->getDisplayDiagnostics())
     printDiagsToStderr(Unit);