[DependencyScan] Correct setup clang VFS for dependency scanning

Currently, dependency scanner is not reporting the redirecting files
that are baked inside swift-frontend for platform support. This causes
dependency scanner returns virtual path for those files, and
swift-driver/build-system will not be able to correct validate the files
on incremental build, causing incremental build to be almost clean
builds.

This behavior issue is caused by the dependency scanning file system
layer inside clang dependency scanner that caches stats. If the
redirecting files are created underneath the layer, the real path is
lost. This fixes the issue by moving the redirecting files above the
caching layer using `-ivfsoverlay` option.

In addition to that, this commit also unifies how clang importer and
clang dependency scanner initiate the VFS, making the logic much
simpler.

Author: cachemeifyoucan

include/swift/ClangImporter/ClangImporter.h

@@ -148,6 +148,18 @@ typedef llvm::PointerUnion<const clang::Decl *, const clang::MacroInfo *,
                            const clang::Type *, const clang::Token *>
     ImportDiagnosticTarget;
 
+/// Addition file mapping information for ClangImporter.
+struct ClangInvocationFileMapping {
+  /// Mapping from a file name to an existing file path.
+  SmallVector<std::pair<std::string, std::string>, 2> redirectedFiles;
+
+  /// MemoryBuffer that represents the file name and content to overload.
+  SmallVector<std::pair<std::string, std::string>, 2> overridenFiles;
+
+  bool requiresBuiltinHeadersInSystemModules = false;
+  bool needSystemVFSOverlay = false;
+};
+
 /// Class that imports Clang modules into Swift, mapping directly
 /// from Clang ASTs over to Swift ASTs.
 class ClangImporter final : public ClangModuleLoader {
@@ -166,7 +178,7 @@ class ClangImporter final : public ClangModuleLoader {
 private:
   Implementation &Impl;
 
-  bool requiresBuiltinHeadersInSystemModules = false;
+  ClangInvocationFileMapping clangFileMapping;
 
   ClangImporter(ASTContext &ctx, DependencyTracker *tracker,
                 DWARFImporterDelegate *dwarfImporterDelegate);
@@ -205,6 +217,14 @@ class ClangImporter final : public ClangModuleLoader {
          DWARFImporterDelegate *dwarfImporterDelegate = nullptr,
          bool ignoreFileMapping = false);
 
+  static std::string getClangSystemOverlayFile(const SearchPathOptions &Opts);
+
+  static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+  computeClangImporterFileSystem(
+      const ASTContext &ctx, const ClangInvocationFileMapping &fileMapping,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseFS,
+      bool suppressDiagnostics = false);
+
   std::vector<std::string>
   getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget = false);
 
@@ -531,6 +551,11 @@ class ClangImporter final : public ClangModuleLoader {
 
   std::string getClangModuleHash() const;
 
+  /// Get clang file mapping.
+  const ClangInvocationFileMapping &getClangFileMapping() const {
+    return clangFileMapping;
+  }
+
   /// Get clang import creation cc1 args for swift explicit module build.
   std::vector<std::string> getSwiftExplicitModuleDirectCC1Args() const;
 
@@ -883,56 +908,17 @@ std::optional<ResultConvention>
 getCxxRefConventionWithAttrs(const clang::Decl *decl);
 } // namespace importer
 
-struct ClangInvocationFileMapping {
-  /// Mapping from a file name to an existing file path.
-  SmallVector<std::pair<std::string, std::string>, 2> redirectedFiles;
-
-  /// Mapping from a file name to a string of characters that represents the
-  /// contents of the file.
-  SmallVector<std::pair<std::string, std::string>, 1> overridenFiles;
-
-  bool requiresBuiltinHeadersInSystemModules;
-};
-
-class ClangInvocationFileMappingContext {
-public:
-  const LangOptions &LangOpts;
-  SearchPathOptions &SearchPathOpts;
-  ClangImporterOptions &ClangImporterOpts;
-  const CASOptions &CASOpts;
-  DiagnosticEngine &Diags;
-
-  ClangInvocationFileMappingContext(
-    const LangOptions &LangOpts, SearchPathOptions &SearchPathOpts,
-    ClangImporterOptions &ClangImporterOpts, const CASOptions &CASOpts,
-    DiagnosticEngine &Diags)
-    : LangOpts(LangOpts), SearchPathOpts(SearchPathOpts),
-      ClangImporterOpts(ClangImporterOpts), CASOpts(CASOpts),
-      Diags(Diags) {}
-
-  ClangInvocationFileMappingContext(const swift::ASTContext &Ctx);
-};
-
 /// On Linux, some platform libraries (glibc, libstdc++) are not modularized.
 /// We inject modulemaps for those libraries into their include directories
 /// to allow using them from Swift.
 ///
 /// `suppressDiagnostic` prevents us from emitting warning messages when we
 /// are unable to find headers.
 ClangInvocationFileMapping getClangInvocationFileMapping(
-    const ClangInvocationFileMappingContext &ctx,
+    const ASTContext &ctx,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs = nullptr,
     bool suppressDiagnostic = false);
 
-/// Apply the given file mapping to the specified 'fileSystem', used
-/// primarily to inject modulemaps on platforms with non-modularized
-/// platform libraries.
-ClangInvocationFileMapping applyClangInvocationMapping(
-    const ClangInvocationFileMappingContext &ctx,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseVFS,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &fileSystem,
-    bool suppressDiagnostics = false);
-
 /// Information used to compute the access level of inherited C++ members.
 class ClangInheritanceInfo {
   /// The cumulative inheritance access specifier, that is used to compute the

lib/ClangImporter/ClangImporter.cpp

@@ -828,10 +828,9 @@ getEmbedBitcodeInvocationArguments(std::vector<std::string> &invocationArgStrs,
   });
 }
 
-void
-importer::addCommonInvocationArguments(
-    std::vector<std::string> &invocationArgStrs,
-    ASTContext &ctx, bool requiresBuiltinHeadersInSystemModules,
+void importer::addCommonInvocationArguments(
+    std::vector<std::string> &invocationArgStrs, ASTContext &ctx,
+    bool requiresBuiltinHeadersInSystemModules, bool needSystemVFSOverlay,
     bool ignoreClangTarget) {
   using ImporterImpl = ClangImporter::Implementation;
   llvm::Triple triple = ctx.LangOpts.Target;
@@ -1003,6 +1002,12 @@ importer::addCommonInvocationArguments(
     invocationArgStrs.push_back("-Xclang");
     invocationArgStrs.push_back("-fbuiltin-headers-in-system-modules");
   }
+
+  if (needSystemVFSOverlay) {
+    invocationArgStrs.push_back("-ivfsoverlay");
+    invocationArgStrs.push_back(
+        ClangImporter::getClangSystemOverlayFile(ctx.SearchPathOpts));
+  }
 }
 
 bool ClangImporter::canReadPCH(StringRef PCHFilename) {
@@ -1151,6 +1156,85 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions,
   return PCHFilename.value();
 }
 
+std::string
+ClangImporter::getClangSystemOverlayFile(const SearchPathOptions &Opts) {
+  llvm::SmallString<256> overlayPath(Opts.RuntimeResourcePath);
+  llvm::sys::path::append(overlayPath,
+                          Implementation::clangSystemVFSOverlayName);
+  return overlayPath.str().str();
+}
+
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+ClangImporter::computeClangImporterFileSystem(
+    const ASTContext &ctx, const ClangInvocationFileMapping &fileMapping,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseFS,
+    bool suppressDiagnostics) {
+  // Configure ClangImporter file system. There are two situations:
+  // * If caching is used, thus file system is immutable, the one immutable file
+  //   system is shared between swift frontend and ClangImporter.
+  // * Otherwise, ClangImporter file system is configure from scratch from
+  //   VFS in SourceMgr using ivfsoverlay options.
+  if (ctx.CASOpts.HasImmutableFileSystem)
+    return baseFS;
+
+  auto importerOpts = ctx.ClangImporterOpts;
+  auto fileSystem = baseFS;
+  std::unique_ptr<llvm::MemoryBuffer> redirectYAMLFile;
+  if (!fileMapping.redirectedFiles.empty()) {
+    if (importerOpts.DumpClangDiagnostics) {
+      llvm::errs() << "clang importer redirected file mappings:\n";
+      for (const auto &mapping : fileMapping.redirectedFiles) {
+        llvm::errs() << "   mapping real file '" << mapping.second
+                     << "' to virtual file '" << mapping.first << "'\n";
+      }
+      llvm::errs() << "\n";
+    }
+    // Create a vfs overlay map for all redirects.
+    llvm::vfs::YAMLVFSWriter vfsWriter;
+    vfsWriter.setUseExternalNames(true);
+    for (const auto &mapping : fileMapping.redirectedFiles)
+      vfsWriter.addFileMapping(mapping.first, mapping.second);
+
+    std::string vfsYAML;
+    llvm::raw_string_ostream os(vfsYAML);
+    vfsWriter.write(os);
+
+    redirectYAMLFile = llvm::MemoryBuffer::getMemBufferCopy(
+        vfsYAML, getClangSystemOverlayFile(ctx.SearchPathOpts));
+  }
+
+  if (!fileMapping.overridenFiles.empty() || redirectYAMLFile) {
+    auto overridenVFS =
+        llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+    for (auto &file : fileMapping.overridenFiles) {
+      if (importerOpts.DumpClangDiagnostics) {
+        llvm::errs() << "clang importer overriding file '" << file.first
+                     << "' with the following contents:\n";
+        llvm::errs() << file.second << "\n";
+      }
+      // Note MemoryBuffer is guaranteeed to be null-terminated.
+      overridenVFS->addFile(file.first, 0,
+                            llvm::MemoryBuffer::getMemBufferCopy(file.second));
+    }
+    if (redirectYAMLFile) {
+      if (importerOpts.DumpClangDiagnostics) {
+        llvm::errs() << "clang importer overriding file for redirects'"
+                     << redirectYAMLFile->getBufferIdentifier()
+                     << "' with the following contents:\n";
+        llvm::errs() << redirectYAMLFile->getBuffer() << "\n";
+      }
+      std::string yamlFile = redirectYAMLFile->getBufferIdentifier().str();
+      overridenVFS->addFile(yamlFile, 0, std::move(redirectYAMLFile));
+    }
+    auto overlayVFS =
+        llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(fileSystem);
+    overlayVFS->pushOverlay(std::move(overridenVFS));
+    fileSystem = std::move(overlayVFS);
+  }
+
+  return fileSystem;
+}
+
 std::vector<std::string>
 ClangImporter::getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget) {
   assert(!ctx.ClangImporterOpts.DirectClangCC1ModuleBuild &&
@@ -1168,8 +1252,10 @@ ClangImporter::getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget)
     getEmbedBitcodeInvocationArguments(invocationArgStrs, ctx);
     break;
   }
-  addCommonInvocationArguments(invocationArgStrs, ctx,
-      requiresBuiltinHeadersInSystemModules, ignoreClangTarget);
+  addCommonInvocationArguments(
+      invocationArgStrs, ctx,
+      clangFileMapping.requiresBuiltinHeadersInSystemModules,
+      clangFileMapping.needSystemVFSOverlay, ignoreClangTarget);
   return invocationArgStrs;
 }
 
@@ -1235,6 +1321,10 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
         CI->getTargetOpts().DarwinTargetVariantTriple = ctx.LangOpts.TargetVariant->str();
     }
 
+    if (clangFileMapping.needSystemVFSOverlay)
+      CI->getHeaderSearchOpts().AddVFSOverlayFile(
+          getClangSystemOverlayFile(ctx.SearchPathOpts));
+
     // Forward the index store path. That information is not passed to scanner
     // and it is cached invariant so we don't want to re-scan if that changed.
     CI->getFrontendOpts().IndexStorePath = ctx.ClangImporterOpts.IndexStorePath;
@@ -1348,18 +1438,16 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
     }
   }
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
-      ctx.SourceMgr.getFileSystem();
-
-  ClangInvocationFileMapping fileMapping =
-    applyClangInvocationMapping(ctx, nullptr, VFS, ignoreFileMapping);
-
-  importer->requiresBuiltinHeadersInSystemModules =
-      fileMapping.requiresBuiltinHeadersInSystemModules;
+  importer->clangFileMapping = getClangInvocationFileMapping(
+      ctx, ctx.SourceMgr.getFileSystem(), ignoreFileMapping);
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs =
+      computeClangImporterFileSystem(ctx, importer->clangFileMapping,
+                                     ctx.SourceMgr.getFileSystem(),
+                                     ignoreFileMapping);
 
   // Create a new Clang compiler invocation.
   {
-    if (auto ClangArgs = importer->getClangCC1Arguments(ctx, VFS))
+    if (auto ClangArgs = importer->getClangCC1Arguments(ctx, vfs))
       importer->Impl.ClangArgs = *ClangArgs;
     else
       return nullptr;
@@ -1373,7 +1461,7 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
       llvm::errs() << "'\n";
     }
     importer->Impl.Invocation = createClangInvocation(
-        importer.get(), importerOpts, VFS, importer->Impl.ClangArgs);
+        importer.get(), importerOpts, vfs, importer->Impl.ClangArgs);
     if (!importer->Impl.Invocation)
       return nullptr;
   }
@@ -1424,7 +1512,7 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
   auto actualDiagClient = std::make_unique<ClangDiagnosticConsumer>(
       importer->Impl, instance.getDiagnosticOpts(),
       importerOpts.DumpClangDiagnostics);
-  instance.createVirtualFileSystem(std::move(VFS), actualDiagClient.get());
+  instance.createVirtualFileSystem(std::move(vfs), actualDiagClient.get());
   instance.createFileManager();
   instance.createDiagnostics(actualDiagClient.release());