[clang][scan-deps] Add option to disable caching stat failures

While the source code isn't supposed to change during a build, in some
environments it does. This adds an option that disables caching of
stat failures, meaning that source files can be added to the build
during scanning.

This adds a `-no-cache-negative-stats` option to clang-scan-deps to
enable this behavior. There are no tests for clang-scan-deps as
there's no reliable way to do so from it. A unit test has been added
that modifies the filesystem between scans to test it.

Internally this uses an environment variable.

`CLANG_SCAN_CACHE_NEGATIVE_STATS`: If this is set or set to 1 negative
stat caching is enabled. If it's 0 then it's disabled.

Negative stat caching is disabled by default.

Author: Bigcheese

clang/include/clang-c/Dependencies.h

@@ -161,6 +161,17 @@ CINDEX_DEPRECATED CINDEX_LINKAGE void
 clang_experimental_DependencyScannerServiceOptions_setActionCache(
     CXDependencyScannerServiceOptions Opts, CXCASActionCache Cache);
 
+/**
+ * Set if negative stat caching is enabled or disabled. If this is not called
+ * the default value is used.
+ *
+ * The default is false unless the environment variable
+ * `CLANG_SCAN_CACHE_NEGATIVE_STATS` is set to empty or "1".
+ */
+CINDEX_LINKAGE void
+clang_experimental_DependencyScannerServiceOptions_setCacheNegativeStats(
+    CXDependencyScannerServiceOptions Opts, bool CacheNegativeStats);
+
 /**
  * Create a \c CXDependencyScannerService object.
  * Must be disposed with \c

clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h

@@ -38,9 +38,8 @@ class DependencyScanningCASFilesystem
 public:
   static const char ID;
 
-  DependencyScanningCASFilesystem(
-      IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> WorkerFS,
-      llvm::cas::ActionCache &Cache);
+  DependencyScanningCASFilesystem(DependencyScanningService &Service,
+      IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> WorkerFS);
 
   ~DependencyScanningCASFilesystem();
 
@@ -109,6 +108,8 @@ class DependencyScanningCASFilesystem
 
   llvm::cas::ObjectStore &CAS;
   llvm::cas::ActionCache &Cache;
+  /// The service associated with this VFS.
+  DependencyScanningService &Service;
   std::optional<llvm::cas::ObjectRef> ClangFullVersionID;
   std::optional<llvm::cas::ObjectRef> DepDirectivesID;
   std::optional<llvm::cas::ObjectRef> EmptyBlobID;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

@@ -25,6 +25,8 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
 
+class DependencyScanningService;
+
 using DependencyDirectivesTy =
     SmallVector<dependency_directives_scan::Directive, 20>;
 
@@ -387,7 +389,7 @@ class DependencyScanningWorkerFilesystem
   static const char ID;
 
   DependencyScanningWorkerFilesystem(
-      DependencyScanningFilesystemSharedCache &SharedCache,
+      DependencyScanningService &Service,
       IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
@@ -476,10 +478,7 @@ class DependencyScanningWorkerFilesystem
   /// Returns entry associated with the unique ID in the shared cache or nullptr
   /// if none is found.
   const CachedFileSystemEntry *
-  findSharedEntryByUID(llvm::vfs::Status Stat) const {
-    return SharedCache.getShardForUID(Stat.getUniqueID())
-        .findEntryByUID(Stat.getUniqueID());
-  }
+  findSharedEntryByUID(llvm::vfs::Status Stat) const;
 
   /// Associates the given entry with the filename in the local cache and
   /// returns it.
@@ -493,20 +492,14 @@ class DependencyScanningWorkerFilesystem
   /// some. Otherwise, constructs new one with the given error code, associates
   /// it with the filename and returns the result.
   const CachedFileSystemEntry &
-  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
-    return SharedCache.getShardForFilename(Filename)
-        .getOrEmplaceEntryForFilename(Filename, EC);
-  }
+  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC);
 
   /// Returns entry associated with the filename in the shared cache if there is
   /// some. Otherwise, associates the given entry with the filename and returns
   /// it.
   const CachedFileSystemEntry &
   getOrInsertSharedEntryForFilename(StringRef Filename,
-                                    const CachedFileSystemEntry &Entry) {
-    return SharedCache.getShardForFilename(Filename)
-        .getOrInsertEntryForFilename(Filename, Entry);
-  }
+                                    const CachedFileSystemEntry &Entry);
 
   void printImpl(raw_ostream &OS, PrintType Type,
                  unsigned IndentLevel) const override {
@@ -519,8 +512,9 @@ class DependencyScanningWorkerFilesystem
   /// VFS.
   bool shouldBypass(StringRef Path) const;
 
-  /// The global cache shared between worker threads.
-  DependencyScanningFilesystemSharedCache &SharedCache;
+  /// The service associated with this VFS.
+  DependencyScanningService &Service;
+
   /// The local cache is used by the worker thread to cache file system queries
   /// locally instead of querying the global cache every time.
   DependencyScanningFilesystemLocalCache LocalCache;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -97,6 +97,12 @@ enum class ScanningOptimizations {
 
 #undef DSS_LAST_BITMASK_ENUM
 
+bool shouldCacheNegativeStatsDefault();
+
+/// \return true if failed stats for files with this name should be cached,
+///         false otherwise.
+bool shouldCacheNegativeStatsForPath(StringRef Path);
+
 /// The dependency scanning service contains shared configuration and state that
 /// is used by the individual dependency scanning workers.
 class DependencyScanningService {
@@ -109,7 +115,8 @@ class DependencyScanningService {
       ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
       bool EagerLoadModules = false, bool TraceVFS = false,
       std::time_t BuildSessionTimestamp =
-          llvm::sys::toTimeT(std::chrono::system_clock::now()));
+          llvm::sys::toTimeT(std::chrono::system_clock::now()),
+      bool CacheNegativeStats = shouldCacheNegativeStatsDefault());
 
   ScanningMode getMode() const { return Mode; }
 
@@ -121,6 +128,8 @@ class DependencyScanningService {
 
   bool shouldTraceVFS() const { return TraceVFS; }
 
+  bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     assert(!SharedFS && "Expected not to have a CASFS");
     assert(SharedCache && "Expected a shared cache");
@@ -152,6 +161,7 @@ class DependencyScanningService {
   const bool EagerLoadModules;
   /// Whether to trace VFS accesses.
   const bool TraceVFS;
+  const bool CacheNegativeStats;
   /// Shared CachingOnDiskFileSystem. Set to nullptr to not use CAS dependency
   /// scanning.
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS;

clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Basic/Version.h"
 #include "clang/Lex/DependencyDirectivesScanner.h"
 #include "llvm/CAS/ActionCache.h"
@@ -35,10 +36,10 @@ static void reportAsFatalIfError(llvm::Error E) {
 using llvm::Error;
 
 DependencyScanningCASFilesystem::DependencyScanningCASFilesystem(
-    IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> WorkerFS,
-    llvm::cas::ActionCache &Cache)
-    : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) {
-}
+    DependencyScanningService &Service,
+    IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> WorkerFS)
+    : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()),
+      Cache(*Service.getCache()), Service(Service) {}
 
 const char DependencyScanningCASFilesystem::ID = 0;
 DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default;
@@ -201,13 +202,6 @@ DependencyScanningCASFilesystem::getOriginal(cas::CASID InputDataID) {
   return Blob.takeError();
 }
 
-static bool shouldCacheStatFailures(StringRef Filename) {
-  StringRef Ext = llvm::sys::path::extension(Filename);
-  if (Ext.empty())
-    return false; // This may be the module cache directory.
-  return true;
-}
-
 llvm::cas::CachingOnDiskFileSystem &
 DependencyScanningCASFilesystem::getCachingFS() {
   return static_cast<llvm::cas::CachingOnDiskFileSystem &>(*FS);
@@ -233,7 +227,8 @@ DependencyScanningCASFilesystem::lookupPath(const Twine &Path) {
   llvm::ErrorOr<llvm::vfs::Status> MaybeStatus =
       getCachingFS().statusAndFileID(PathRef, FileID);
   if (!MaybeStatus) {
-    if (shouldCacheStatFailures(PathRef))
+    if (Service.shouldCacheNegativeStats() &&
+        shouldCacheNegativeStatsForPath(PathRef))
       Entries[PathRef].EC = MaybeStatus.getError();
     return LookupPathResult{nullptr, MaybeStatus.getError()};
   }

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/CAS/CASFileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Threading.h"
@@ -255,19 +256,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
 }
 
 DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
-    DependencyScanningFilesystemSharedCache &SharedCache,
+    DependencyScanningService &Service,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
     : llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
                         llvm::vfs::ProxyFileSystem>(std::move(FS)),
-      SharedCache(SharedCache),
-      WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
+      Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
   updateWorkingDirForCacheLookup();
 }
 
 const CachedFileSystemEntry &
 DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
     TentativeEntry TEntry) {
-  auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
+  auto &Shard =
+      Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID());
   return Shard.getOrEmplaceEntryForUID(
       TEntry.Status.getUniqueID(), std::move(TEntry.Status),
       std::move(TEntry.Contents), std::move(TEntry.CASContents));
@@ -278,18 +279,46 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
     StringRef Filename) {
   if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
     return Entry;
-  auto &Shard = SharedCache.getShardForFilename(Filename);
+  auto &Shard = Service.getSharedCache().getShardForFilename(Filename);
   if (const auto *Entry = Shard.findEntryByFilename(Filename))
     return &LocalCache.insertEntryForFilename(Filename, *Entry);
   return nullptr;
 }
 
+const CachedFileSystemEntry *
+DependencyScanningWorkerFilesystem::findSharedEntryByUID(
+    llvm::vfs::Status Stat) const {
+  return Service.getSharedCache()
+      .getShardForUID(Stat.getUniqueID())
+      .findEntryByUID(Stat.getUniqueID());
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename(
+    StringRef Filename, std::error_code EC) {
+  return Service.getSharedCache()
+      .getShardForFilename(Filename)
+      .getOrEmplaceEntryForFilename(Filename, EC);
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename(
+    StringRef Filename, const CachedFileSystemEntry &Entry) {
+  return Service.getSharedCache()
+      .getShardForFilename(Filename)
+      .getOrInsertEntryForFilename(Filename, Entry);
+}
+
 llvm::ErrorOr<const CachedFileSystemEntry &>
 DependencyScanningWorkerFilesystem::computeAndStoreResult(
     StringRef OriginalFilename, StringRef FilenameForLookup) {
   llvm::ErrorOr<llvm::vfs::Status> Stat =
       getUnderlyingFS().status(OriginalFilename);
   if (!Stat) {
+    if (!Service.shouldCacheNegativeStats() ||
+        !shouldCacheNegativeStatsForPath(OriginalFilename))
+      return Stat.getError();
+
     const auto &Entry =
         getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
     return insertLocalEntryForFilename(FilenameForLookup, Entry);
@@ -478,7 +507,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
     return HandleCachedRealPath(*RealPath);
 
   // If we have the result in the shared cache, cache it locally.
-  auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
+  auto &Shard =
+      Service.getSharedCache().getShardForFilename(*FilenameForLookup);
   if (const auto *ShardRealPath =
           Shard.findRealPathByFilename(*FilenameForLookup)) {
     const auto &RealPath = LocalCache.insertRealPathForFilename(

clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp

@@ -11,21 +11,49 @@
 #include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Process.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
+bool clang::tooling::dependencies::shouldCacheNegativeStatsDefault() {
+  if (std::optional<std::string> MaybeNegStats =
+          llvm::sys::Process::GetEnv("CLANG_SCAN_CACHE_NEGATIVE_STATS")) {
+    if (MaybeNegStats->empty())
+      return true;
+    return llvm::StringSwitch<bool>(*MaybeNegStats)
+        .Case("1", true)
+        .Case("0", false)
+        .Default(false);
+  }
+  return false;
+}
+
+// rdar://148027982
+// rdar://127079541
+// Negative caching directories can cause build failures due to incorrectly
+// configured projects.
+bool dependencies::shouldCacheNegativeStatsForPath(StringRef Path) {
+  StringRef Ext = llvm::sys::path::extension(Path);
+  if (Ext.empty())
+    return false;
+  if (Ext == ".framework")
+    return false;
+  return true;
+}
+
 DependencyScanningService::DependencyScanningService(
     ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts,
     std::shared_ptr<llvm::cas::ObjectStore> CAS,
     std::shared_ptr<llvm::cas::ActionCache> Cache,
     IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS,
     ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
-    std::time_t BuildSessionTimestamp)
+    std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
     : Mode(Mode), Format(Format), CASOpts(std::move(CASOpts)),
       CAS(std::move(CAS)), Cache(std::move(Cache)), OptimizeArgs(OptimizeArgs),
       EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
+      CacheNegativeStats(CacheNegativeStats),
       SharedFS(std::move(SharedFS)),
       BuildSessionTimestamp(BuildSessionTimestamp) {
   if (!this->SharedFS)

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -34,15 +34,15 @@ DependencyScanningWorker::DependencyScanningWorker(
 
   if (Service.useCASFS()) {
     CacheFS = Service.getSharedFS().createProxyFS();
-    DepCASFS = new DependencyScanningCASFilesystem(CacheFS, *Service.getCache());
+    DepCASFS = new DependencyScanningCASFilesystem(Service, CacheFS);
     BaseFS = DepCASFS;
     return;
   }
 
   switch (Service.getMode()) {
   case ScanningMode::DependencyDirectivesScan:
     DepFS = llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>(
-        Service.getSharedCache(), FS);
+        Service, FS);
     BaseFS = DepFS;
     break;
   case ScanningMode::CanonicalPreprocessing:

clang/tools/clang-scan-deps/ClangScanDeps.cpp

@@ -92,6 +92,7 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make;
 static ScanningOptimizations OptimizeArgs;
 static std::string ModuleFilesDir;
 static bool EagerLoadModules;
+static bool CacheNegativeStats;
 static unsigned NumThreads = 0;
 static std::string CompilationDB;
 static std::optional<std::string> ModuleNames;
@@ -212,6 +213,8 @@ static void ParseArgs(int argc, char **argv) {
 
   EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
 
+  CacheNegativeStats = Args.hasArg(OPT_cache_negative_stats);
+
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
     StringRef S{A->getValue()};
     if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -1541,9 +1544,10 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
     });
   };
 
-  DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs,
-                                    EagerLoadModules,
-      /*TraceVFS=*/Verbose);
+  DependencyScanningService Service(
+      ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules,
+      /*TraceVFS=*/Verbose,
+      llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats);
 
   llvm::Timer T;
   T.startTimer();

clang/tools/clang-scan-deps/Opts.td

@@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir",
 
 def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">;
 def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">;
+def cache_negative_stats : F<"cache-negative-stats", "Cache stat failures">;
 
 def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">;