<rdar://problem/13434605> Periodically prune the module cache so that it does not grow forever.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@177918 91177308-0d34-0410-b5e6-96231b3b80d8

7 files changed, 187 insertions, 2 deletions

diff --git a/docs/Modules.rst b/docs/Modules.rst
index ba55892cde..b892e382fc 100644
--- a/docs/Modules.rst
+++ b/docs/Modules.rst
@@ -174,6 +174,12 @@ Command-line parameters
 ``-fmodules-ignore-macro=macroname``
   Instruct modules to ignore the named macro when selecting an appropriate module variant. Use this for macros defined on the command line that don't affect how modules are built, to improve sharing of compiled module files.
 
+``-fmodules-prune-interval=seconds``
+  Specify the minimum delay (in seconds) between attempts to prune the module cache. Module cache pruning attempts to clear out old, unused module files so that the module cache itself does not grow without bound. The default delay is large (604,800 seconds, or 7 days) because this is an expensive operation. Set this value to 0 to turn off pruning.
+
+``-fmodules-prune-after=seconds``
+  Specify the minimum time (in seconds) for which a file in the module cache must be unused (according to access time) before module pruning will remove it. The default delay is large (2,678,400 seconds, or 31 days) to avoid excessive module rebuilding.
+
 Module Map Language
 ===================
 
diff --git a/include/clang/Driver/Options.td b/include/clang/Driver/Options.td
index a6ede0f450..22415ca39d 100644
--- a/include/clang/Driver/Options.td
+++ b/include/clang/Driver/Options.td
@@ -499,6 +499,12 @@ def fdelayed_template_parsing : Flag<["-"], "fdelayed-template-parsing">, Group<
 def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
   Flags<[NoForward,CC1Option]>, MetaVarName<"<directory>">,
   HelpText<"Specify the module cache path">;
+def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i_Group>,
+  Flags<[CC1Option]>, MetaVarName<"<seconds>">,
+  HelpText<"Specify the interval (in seconds) between attempts to prune the module cache">;
+def fmodules_prune_after : Joined<["-"], "fmodules-prune-after=">, Group<i_Group>,
+  Flags<[CC1Option]>, MetaVarName<"<seconds>">,
+  HelpText<"Specify the interval (in seconds) after which a module file will be considered unused">;
 def fmodules : Flag <["-"], "fmodules">, Group<f_Group>, Flags<[NoForward,CC1Option]>,
   HelpText<"Enable the 'modules' language feature">;
 def fmodules_autolink : Flag <["-"], "fmodules-autolink">, Group<f_Group>, Flags<[NoForward,CC1Option]>,
diff --git a/include/clang/Lex/HeaderSearchOptions.h b/include/clang/Lex/HeaderSearchOptions.h
index c45884360d..afce5ba18b 100644
--- a/include/clang/Lex/HeaderSearchOptions.h
+++ b/include/clang/Lex/HeaderSearchOptions.h
@@ -95,6 +95,24 @@ public:
   /// Note: Only used for testing!
   unsigned DisableModuleHash : 1;
 
+  /// \brief The interval (in seconds) between pruning operations.
+  ///
+  /// This operation is expensive, because it requires Clang to walk through
+  /// the directory structure of the module cache, stat()'ing and removing
+  /// files.
+  ///
+  /// The default value is large, e.g., the operation runs once a week.
+  unsigned ModuleCachePruneInterval;
+
+  /// \brief The time (in seconds) after which an unused module file will be
+  /// considered unused and will, therefore, be pruned.
+  ///
+  /// When the module cache is pruned, any module file that has not been
+  /// accessed in this many seconds will be removed. The default value is
+  /// large, e.g., a month, to avoid forcing infrequently-used modules to be
+  /// regenerated often.
+  unsigned ModuleCachePruneAfter;
+
   /// \brief The set of macro names that should be ignored for the purposes
   /// of computing the module hash.
   llvm::SetVector<std::string> ModulesIgnoreMacros;
@@ -116,7 +134,10 @@ public:
 
 public:
   HeaderSearchOptions(StringRef _Sysroot = "/")
-    : Sysroot(_Sysroot), DisableModuleHash(0), UseBuiltinIncludes(true),
+    : Sysroot(_Sysroot), DisableModuleHash(0),
+      ModuleCachePruneInterval(7*24*60*60),
+      ModuleCachePruneAfter(31*24*60*60),
+      UseBuiltinIncludes(true),
       UseStandardSystemIncludes(true), UseStandardCXXIncludes(true),
       UseLibcxx(false), Verbose(false) {}
 
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index bff8848db0..c7e47ad111 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -2827,6 +2827,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Pass through all -fmodules-ignore-macro arguments.
   Args.AddAllArgs(CmdArgs, options::OPT_fmodules_ignore_macro);
+  Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_interval);
+  Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_after);
 
   // -fmodules-autolink (on by default when modules is enabled) automatically
   // links against libraries for imported modules.  This requires the
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index f83c8704f6..2a35ca0794 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -44,6 +44,8 @@
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
+#include <sys/stat.h>
+#include <sys/time.h>
 
 using namespace clang;
 
@@ -996,6 +998,97 @@ static void checkConfigMacro(Preprocessor &PP, StringRef ConfigMacro,
     << false;
 }
 
+/// \brief Write a new timestamp file with the given path.
+static void writeTimestampFile(StringRef TimestampFile) {
+  std::string ErrorInfo;
+  llvm::raw_fd_ostream Out(TimestampFile.str().c_str(), ErrorInfo,
+                           llvm::raw_fd_ostream::F_Binary);
+}
+
+/// \brief Prune the module cache of modules that haven't been accessed in
+/// a long time.
+static void pruneModuleCache(const HeaderSearchOptions &HSOpts) {
+  struct stat StatBuf;
+  llvm::SmallString<128> TimestampFile;
+  TimestampFile = HSOpts.ModuleCachePath;
+  llvm::sys::path::append(TimestampFile, "modules.timestamp");
+
+  // Try to stat() the timestamp file.
+  if (::stat(TimestampFile.c_str(), &StatBuf)) {
+    // If the timestamp file wasn't there, create one now.
+    if (errno == ENOENT) {
+      writeTimestampFile(TimestampFile);
+    }
+    return;
+  }
+
+  // Check whether the time stamp is older than our pruning interval.
+  // If not, do nothing.
+  time_t TimeStampModTime = StatBuf.st_mtime;
+  time_t CurrentTime = time(0);
+  if (CurrentTime - TimeStampModTime <= HSOpts.ModuleCachePruneInterval) {
+    return;
+  }
+
+  // Write a new timestamp file so that nobody else attempts to prune.
+  // There is a benign race condition here, if two Clang instances happen to
+  // notice at the same time that the timestamp is out-of-date.
+  writeTimestampFile(TimestampFile);
+
+  // Walk the entire module cache, looking for unused module files and module
+  // indices.
+  llvm::error_code EC;
+  SmallString<128> ModuleCachePathNative;
+  llvm::sys::path::native(HSOpts.ModuleCachePath, ModuleCachePathNative);
+  for (llvm::sys::fs::directory_iterator
+         Dir(ModuleCachePathNative.str(), EC), DirEnd;
+       Dir != DirEnd && !EC; Dir.increment(EC)) {
+    // If we don't have a directory, there's nothing to look into.
+    bool IsDirectory;
+    if (llvm::sys::fs::is_directory(Dir->path(), IsDirectory) || !IsDirectory)
+      continue;
+
+    // Walk all of the files within this directory.
+    bool RemovedAllFiles = true;
+    for (llvm::sys::fs::directory_iterator File(Dir->path(), EC), FileEnd;
+         File != FileEnd && !EC; File.increment(EC)) {
+      // We only care about module and global module index files.
+      if (llvm::sys::path::extension(File->path()) != ".pcm" &&
+          llvm::sys::path::filename(File->path()) != "modules.idx") {
+        RemovedAllFiles = false;
+        continue;
+      }
+
+      // Look at this file. If we can't stat it, there's nothing interesting
+      // there.
+      if (::stat(File->path().c_str(), &StatBuf)) {
+        RemovedAllFiles = false;
+        continue;
+      }
+
+      // If the file has been used recently enough, leave it there.
+      time_t FileAccessTime = StatBuf.st_atime;
+      if (CurrentTime - FileAccessTime <= HSOpts.ModuleCachePruneAfter) {
+        RemovedAllFiles = false;;
+        continue;
+      }
+
+      // Remove the file.
+      bool Existed;
+      if (llvm::sys::fs::remove(File->path(), Existed) || !Existed) {
+        RemovedAllFiles = false;
+      }
+    }
+
+    // If we removed all of the files in the directory, remove the directory
+    // itself.
+    if (RemovedAllFiles) {
+      bool Existed;
+      llvm::sys::fs::remove(Dir->path(), Existed);
+    }
+  }
+}
+
 ModuleLoadResult
 CompilerInstance::loadModule(SourceLocation ImportLoc,
                              ModuleIdPath Path,
@@ -1042,6 +1135,14 @@ CompilerInstance::loadModule(SourceLocation ImportLoc,
       if (!hasASTContext())
         createASTContext();
 
+      // If we're not recursively building a module, check whether we
+      // need to prune the module cache.
+      if (getSourceManager().getModuleBuildStack().empty() &&
+          getHeaderSearchOpts().ModuleCachePruneInterval > 0 &&
+          getHeaderSearchOpts().ModuleCachePruneAfter > 0) {
+        pruneModuleCache(getHeaderSearchOpts());
+      }
+
       std::string Sysroot = getHeaderSearchOpts().Sysroot;
       const PreprocessorOptions &PPOpts = getPreprocessorOpts();
       ModuleManager = new ASTReader(getPreprocessor(), *Context,
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 301a082cbd..2d945b5b04 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -836,7 +836,10 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   Opts.ResourceDir = Args.getLastArgValue(OPT_resource_dir);
   Opts.ModuleCachePath = Args.getLastArgValue(OPT_fmodules_cache_path);
   Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
-
+  Opts.ModuleCachePruneInterval
+    = Args.getLastArgIntValue(OPT_fmodules_prune_interval, 7*24*60*60);
+  Opts.ModuleCachePruneAfter
+    = Args.getLastArgIntValue(OPT_fmodules_prune_after, 31*24*60*60);
   for (arg_iterator it = Args.filtered_begin(OPT_fmodules_ignore_macro),
        ie = Args.filtered_end(); it != ie; ++it) {
     StringRef MacroDef = (*it)->getValue();
diff --git a/test/Modules/prune.m b/test/Modules/prune.m
new file mode 100644
index 0000000000..2e2c2eee77
--- /dev/null
+++ b/test/Modules/prune.m
@@ -0,0 +1,46 @@
+// Test the automatic pruning of module cache entries.
+#ifdef IMPORT_DEPENDS_ON_MODULE
+@import DependsOnModule;
+#else
+@import Module;
+#endif
+
+// We need 'touch' and 'find' for this test to work.
+// REQUIRES: shell
+
+// Clear out the module cache
+// RUN: rm -rf %t
+// Run Clang twice so we end up creating the timestamp file (the second time).
+// RUN: %clang_cc1 -DIMPORT_DEPENDS_ON_MODULE -fmodules-ignore-macro=DIMPORT_DEPENDS_ON_MODULE -fmodules -F %S/Inputs -fmodules-cache-path=%t %s -verify
+// RUN: %clang_cc1 -DIMPORT_DEPENDS_ON_MODULE -fmodules-ignore-macro=DIMPORT_DEPENDS_ON_MODULE -fmodules -F %S/Inputs -fmodules-cache-path=%t %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set the timestamp back more than two days. We should try to prune,
+// but nothing gets pruned because the module files are new enough.
+// RUN: touch -m -a -A -481200 %t/modules.timestamp 
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set the DependsOnModule access time back more than four days.
+// This shouldn't prune anything, because the timestamp has been updated, so
+// the pruning mechanism won't fire.
+// RUN: touch -a -A -961200 `find /Volumes/Data/dgregor/Projects/llvm-build-xcode/tools/clang/test/Modules/Output/prune.m.tmp -name DependsOnModule.pcm`
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | grep DependsOnModule.pcm
+
+// Set both timestamp and DependsOnModule.pcm back beyond the cutoff.
+// This should trigger pruning, which will remove DependsOnModule but not Module.
+// RUN: touch -m -a -A -481200 %t/modules.timestamp 
+// RUN: touch -a -A -961200 `find /Volumes/Data/dgregor/Projects/llvm-build-xcode/tools/clang/test/Modules/Output/prune.m.tmp -name DependsOnModule.pcm`
+// RUN: %clang_cc1 -fmodules -F %S/Inputs -fmodules-cache-path=%t -fmodules -fmodules-prune-interval=172800 -fmodules-prune-after=345600 %s -verify
+// RUN: ls %t | grep modules.timestamp
+// RUN: ls -R %t | grep ^Module.pcm
+// RUN: ls -R %t | not grep DependsOnModule.pcm
+
+// expected-no-diagnostics