[CAS] swift dependency scanning using CAS for compiler caching (#66366)

Teach swift dependency scanner to use CAS to capture the full dependencies for a build and construct build commands with immutable inputs from CAS. This allows swift compilation caching using CAS.
2025-12-14 20:36:38 +01:00 · 2023-06-12 10:55:53 -07:00
parent 2db4a038c3
commit b1f99b8e93
34 changed files with 2671 additions and 298 deletions
--- a/include/swift/AST/ModuleDependencies.h
+++ b/include/swift/AST/ModuleDependencies.h
@@ -23,9 +23,16 @@
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/CASProvidingFileSystem.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Mutex.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <vector>
 #include <unordered_map>
@@ -99,8 +106,10 @@ public:
  const ModuleDependencyKind dependencyKind;

  ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
+                                  StringRef moduleCacheKey = "",
                                  bool resolved = false)
-      : dependencyKind(dependencyKind), resolved(resolved) { }
+      : dependencyKind(dependencyKind), moduleCacheKey(moduleCacheKey.str()),
+        resolved(resolved) {}

  virtual ModuleDependencyInfoStorageBase *clone() const = 0;

@@ -117,12 +126,20 @@ public:
  /// The set of modules on which this module depends, resolved
  /// to Module IDs, qualified by module kind: Swift, Clang, etc.
  std::vector<ModuleDependencyID> resolvedModuleDependencies;
+
+  /// The cache key for the produced module.
+  std::string moduleCacheKey;
+
  bool resolved;
 };

 struct CommonSwiftTextualModuleDependencyDetails {
-  CommonSwiftTextualModuleDependencyDetails(ArrayRef<StringRef> extraPCMArgs)
-     : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()) {}
+  CommonSwiftTextualModuleDependencyDetails(
+      ArrayRef<StringRef> extraPCMArgs, ArrayRef<StringRef> buildCommandLine,
+      const std::string &CASFileSystemRootID)
+      : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
+        buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
+        CASFileSystemRootID(CASFileSystemRootID) {}

  /// To build a PCM to be used by this Swift module, we need to append these
  /// arguments to the generic PCM build arguments reported from the dependency
@@ -141,6 +158,16 @@ struct CommonSwiftTextualModuleDependencyDetails {
  /// Dependencies comprised of Swift overlay modules of direct and
  /// transitive Clang dependencies.
  std::vector<ModuleDependencyID> swiftOverlayDependencies;
+
+  /// The Swift frontend invocation arguments to build the Swift module from the
+  /// interface.
+  std::vector<std::string> buildCommandLine;
+
+  /// CASID for the Root of CASFS. Empty if CAS is not used.
+  std::string CASFileSystemRootID;
+
+  /// CASID for the Root of bridgingHeaderClangIncludeTree. Empty if not used.
+  std::string CASBridgingHeaderIncludeTreeRootID;
 };

 /// Describes the dependencies of a Swift module described by an Swift interface file.
@@ -158,10 +185,6 @@ public:
  /// Potentially ready-to-use compiled modules for the interface file.
  const std::vector<std::string> compiledModuleCandidates;

-  /// The Swift frontend invocation arguments to build the Swift module from the
-  /// interface.
-  std::vector<std::string> buildCommandLine;
-
  /// The hash value that will be used for the generated module
  const std::string contextHash;

@@ -172,22 +195,20 @@ public:
  CommonSwiftTextualModuleDependencyDetails textualModuleDetails;

  SwiftInterfaceModuleDependenciesStorage(
-      const std::string moduleOutputPath,
-      const std::string swiftInterfaceFile,
+      const std::string &moduleOutputPath,
+      const std::string &swiftInterfaceFile,
      ArrayRef<std::string> compiledModuleCandidates,
-      ArrayRef<StringRef> buildCommandLine,
-      ArrayRef<StringRef> extraPCMArgs,
-      StringRef contextHash,
-      bool isFramework
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface),
-      moduleOutputPath(moduleOutputPath),
-      swiftInterfaceFile(swiftInterfaceFile),
-      compiledModuleCandidates(compiledModuleCandidates.begin(),
-                               compiledModuleCandidates.end()),
-      buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
-      contextHash(contextHash), isFramework(isFramework),
-      textualModuleDetails(extraPCMArgs)
-      {}
+      ArrayRef<StringRef> buildCommandLine, ArrayRef<StringRef> extraPCMArgs,
+      StringRef contextHash, bool isFramework, const std::string &RootID,
+      const std::string &moduleCacheKey)
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface,
+                                        moduleCacheKey),
+        moduleOutputPath(moduleOutputPath),
+        swiftInterfaceFile(swiftInterfaceFile),
+        compiledModuleCandidates(compiledModuleCandidates.begin(),
+                                 compiledModuleCandidates.end()),
+        contextHash(contextHash), isFramework(isFramework),
+        textualModuleDetails(extraPCMArgs, buildCommandLine, RootID) {}

  ModuleDependencyInfoStorageBase *clone() const override {
    return new SwiftInterfaceModuleDependenciesStorage(*this);
@@ -198,7 +219,7 @@ public:
  }

  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
-    buildCommandLine = newCommandLine;
+    textualModuleDetails.buildCommandLine = newCommandLine;
  }
 };

@@ -218,11 +239,18 @@ public:
  /// Collection of module imports that were detected to be `@Testable`
  llvm::StringSet<> testableImports;

+  /// The Swift frontend invocation arguments to build bridging header.
+  std::vector<std::string> bridgingHeaderBuildCommandLine;
+
  SwiftSourceModuleDependenciesStorage(
-    ArrayRef<StringRef> extraPCMArgs
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
-      textualModuleDetails(extraPCMArgs),
-      testableImports(llvm::StringSet<>()) {}
+      const std::string &RootID, ArrayRef<StringRef> buildCommandLine,
+      ArrayRef<StringRef> bridgingHeaderBuildCommandLine,
+      ArrayRef<StringRef> extraPCMArgs)
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
+        textualModuleDetails(extraPCMArgs, buildCommandLine, RootID),
+        testableImports(llvm::StringSet<>()),
+        bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(),
+                                       bridgingHeaderBuildCommandLine.end()) {}

  ModuleDependencyInfoStorageBase *clone() const override {
    return new SwiftSourceModuleDependenciesStorage(*this);
@@ -232,6 +260,15 @@ public:
    return base->dependencyKind == ModuleDependencyKind::SwiftSource;
  }

+  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
+    textualModuleDetails.buildCommandLine = newCommandLine;
+  }
+
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    bridgingHeaderBuildCommandLine = newCommandLine;
+  }
+
  void addTestableImport(ImportPath::Module module) {
    testableImports.insert(module.front().Item.str());
  }
@@ -245,12 +282,12 @@ public:
  SwiftBinaryModuleDependencyStorage(const std::string &compiledModulePath,
                                     const std::string &moduleDocPath,
                                     const std::string &sourceInfoPath,
-                                     const bool isFramework)
-      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary),
-        compiledModulePath(compiledModulePath),
-        moduleDocPath(moduleDocPath),
-        sourceInfoPath(sourceInfoPath),
-        isFramework(isFramework) {}
+                                     const bool isFramework,
+                                     const std::string &moduleCacheKey)
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary,
+                                        moduleCacheKey),
+        compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
+        sourceInfoPath(sourceInfoPath), isFramework(isFramework) {}

  ModuleDependencyInfoStorageBase *clone() const override {
    return new SwiftBinaryModuleDependencyStorage(*this);
@@ -288,7 +325,7 @@ public:
  const std::string contextHash;

  /// Partial (Clang) command line that can be used to build this module.
-  const std::vector<std::string> nonPathCommandLine;
+  std::vector<std::string> buildCommandLine;

  /// The file dependencies
  const std::vector<std::string> fileDependencies;
@@ -297,20 +334,28 @@ public:
  /// as found by the scanning action that discovered it
  const std::vector<std::string> capturedPCMArgs;

-  ClangModuleDependencyStorage(
-      const std::string &pcmOutputPath,
-      const std::string &moduleMapFile,
-      const std::string &contextHash,
-      const std::vector<std::string> &nonPathCommandLine,
-      const std::vector<std::string> &fileDependencies,
-      const std::vector<std::string> &capturedPCMArgs
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang),
-      pcmOutputPath(pcmOutputPath),
-      moduleMapFile(moduleMapFile),
-      contextHash(contextHash),
-      nonPathCommandLine(nonPathCommandLine),
-      fileDependencies(fileDependencies),
-      capturedPCMArgs(capturedPCMArgs) {}
+  /// CASID for the Root of CASFS. Empty if CAS is not used.
+  std::string CASFileSystemRootID;
+
+  /// CASID for the Root of ClangIncludeTree. Empty if not used.
+  std::string CASClangIncludeTreeRootID;
+
+  ClangModuleDependencyStorage(const std::string &pcmOutputPath,
+                               const std::string &moduleMapFile,
+                               const std::string &contextHash,
+                               const std::vector<std::string> &buildCommandLine,
+                               const std::vector<std::string> &fileDependencies,
+                               const std::vector<std::string> &capturedPCMArgs,
+                               const std::string &CASFileSystemRootID,
+                               const std::string &clangIncludeTreeRoot,
+                               const std::string &moduleCacheKey)
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang,
+                                        moduleCacheKey),
+        pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile),
+        contextHash(contextHash), buildCommandLine(buildCommandLine),
+        fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs),
+        CASFileSystemRootID(CASFileSystemRootID),
+        CASClangIncludeTreeRootID(clangIncludeTreeRoot) {}

  ModuleDependencyInfoStorageBase *clone() const override {
    return new ClangModuleDependencyStorage(*this);
@@ -319,6 +364,10 @@ public:
  static bool classof(const ModuleDependencyInfoStorageBase *base) {
    return base->dependencyKind == ModuleDependencyKind::Clang;
  }
+
+  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
+    buildCommandLine = newCommandLine;
+  }
 };

 /// Describes an placeholder Swift module dependency module stub.
@@ -381,35 +430,44 @@ public:

  /// Describe the module dependencies for a Swift module that can be
  /// built from a Swift interface file (\c .swiftinterface).
-  static ModuleDependencyInfo forSwiftInterfaceModule(
-      const std::string &moduleOutputPath,
-      const std::string &swiftInterfaceFile,
-      ArrayRef<std::string> compiledCandidates,
-      ArrayRef<StringRef> buildCommands,
-      ArrayRef<StringRef> extraPCMArgs,
-      StringRef contextHash,
-      bool isFramework) {
+  static ModuleDependencyInfo
+  forSwiftInterfaceModule(const std::string &moduleOutputPath,
+                          const std::string &swiftInterfaceFile,
+                          ArrayRef<std::string> compiledCandidates,
+                          ArrayRef<StringRef> buildCommands,
+                          ArrayRef<StringRef> extraPCMArgs,
+                          StringRef contextHash, bool isFramework,
+                          const std::string &CASFileSystemRootID,
+                          const std::string &moduleCacheKey) {
    return ModuleDependencyInfo(
        std::make_unique<SwiftInterfaceModuleDependenciesStorage>(
-          moduleOutputPath, swiftInterfaceFile, compiledCandidates, buildCommands,
-          extraPCMArgs, contextHash, isFramework));
+            moduleOutputPath, swiftInterfaceFile, compiledCandidates,
+            buildCommands, extraPCMArgs, contextHash, isFramework,
+            CASFileSystemRootID, moduleCacheKey));
  }

  /// Describe the module dependencies for a serialized or parsed Swift module.
-  static ModuleDependencyInfo forSwiftBinaryModule(
-      const std::string &compiledModulePath,
-      const std::string &moduleDocPath,
-      const std::string &sourceInfoPath,
-      bool isFramework) {
+  static ModuleDependencyInfo
+  forSwiftBinaryModule(const std::string &compiledModulePath,
+                       const std::string &moduleDocPath,
+                       const std::string &sourceInfoPath, bool isFramework,
+                       const std::string &moduleCacheKey) {
    return ModuleDependencyInfo(
        std::make_unique<SwiftBinaryModuleDependencyStorage>(
-          compiledModulePath, moduleDocPath, sourceInfoPath, isFramework));
+            compiledModulePath, moduleDocPath, sourceInfoPath, isFramework,
+            moduleCacheKey));
  }

  /// Describe the main Swift module.
-  static ModuleDependencyInfo forSwiftSourceModule(ArrayRef<StringRef> extraPCMArgs) {
+  static ModuleDependencyInfo
+  forSwiftSourceModule(const std::string &CASFileSystemRootID,
+                       ArrayRef<StringRef> buildCommands,
+                       ArrayRef<StringRef> bridgingHeaderBuildCommands,
+                       ArrayRef<StringRef> extraPCMArgs) {
    return ModuleDependencyInfo(
-        std::make_unique<SwiftSourceModuleDependenciesStorage>(extraPCMArgs));
+        std::make_unique<SwiftSourceModuleDependenciesStorage>(
+            CASFileSystemRootID, buildCommands, bridgingHeaderBuildCommands,
+            extraPCMArgs));
  }

  /// Describe the module dependencies for a Clang module that can be
@@ -420,11 +478,15 @@ public:
      const std::string &contextHash,
      const std::vector<std::string> &nonPathCommandLine,
      const std::vector<std::string> &fileDependencies,
-      const std::vector<std::string> &capturedPCMArgs) {
+      const std::vector<std::string> &capturedPCMArgs,
+      const std::string &CASFileSystemRootID,
+      const std::string &clangIncludeTreeRoot,
+      const std::string &moduleCacheKey) {
    return ModuleDependencyInfo(
        std::make_unique<ClangModuleDependencyStorage>(
          pcmOutputPath, moduleMapFile, contextHash,
-          nonPathCommandLine, fileDependencies, capturedPCMArgs));
+          nonPathCommandLine, fileDependencies, capturedPCMArgs,
+          CASFileSystemRootID, clangIncludeTreeRoot,  moduleCacheKey));
  }

  /// Describe a placeholder dependency swift module.
@@ -453,6 +515,15 @@ public:
    return storage->resolvedModuleDependencies;
  }

+  std::string getModuleCacheKey() const {
+    assert(storage->resolved);
+    return storage->moduleCacheKey;
+  }
+
+  void updateModuleCacheKey(const std::string &key) {
+    storage->moduleCacheKey = key;
+  }
+
  /// Resolve a dependency's set of `imports` with qualified Module IDs
  void resolveDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
    assert(!storage->resolved && "Resolving an already-resolved dependency");
@@ -474,9 +545,41 @@ public:
    textualModuleDetails->swiftOverlayDependencies.assign(dependencyIDs.begin(), dependencyIDs.end());
  }

+  std::vector<std::string> getCommandline() const {
+    if (auto *detail = getAsClangModule())
+      return detail->buildCommandLine;
+    else if (auto *detail = getAsSwiftInterfaceModule())
+      return detail->textualModuleDetails.buildCommandLine;
+    else if (auto *detail = getAsSwiftSourceModule())
+      return detail->textualModuleDetails.buildCommandLine;
+    return {};
+  }
+
  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
-    assert(isSwiftInterfaceModule() && "Can only update command line on Swift interface dependency");
-    cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())->updateCommandLine(newCommandLine);
+    if (isSwiftInterfaceModule())
+      return cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    else if (isSwiftSourceModule())
+      return cast<SwiftSourceModuleDependenciesStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    else if (isClangModule())
+      return cast<ClangModuleDependencyStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    llvm_unreachable("Unexpected type");
+  }
+
+  std::vector<std::string> getBridgingHeaderCommandline() const {
+    if (auto *detail = getAsSwiftSourceModule())
+      return detail->bridgingHeaderBuildCommandLine;
+    return {};
+  }
+
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    if (isSwiftSourceModule())
+      return cast<SwiftSourceModuleDependenciesStorage>(storage.get())
+          ->updateBridgingHeaderCommandLine(newCommandLine);
+    llvm_unreachable("Unexpected type");
  }

  bool isResolved() const {
@@ -568,6 +671,18 @@ public:
  /// Get the bridging header.
  Optional<std::string> getBridgingHeader() const;

+  /// Get CAS Filesystem RootID.
+  Optional<std::string> getCASFSRootID() const;
+
+  /// Get Clang Include Tree ID.
+  Optional<std::string> getClangIncludeTree() const;
+
+  /// Get bridging header Include Tree ID.
+  Optional<std::string> getBridgingHeaderIncludeTree() const;
+
+  /// Get module output path.
+  std::string getModuleOutputPath() const;
+
  /// Add a bridging header to a Swift module's dependencies.
  void addBridgingHeader(StringRef bridgingHeader);

@@ -581,6 +696,9 @@ public:
  void addBridgingModuleDependency(StringRef module,
                                   llvm::StringSet<> &alreadyAddedModules);

+  /// Add bridging header include tree.
+  void addBridgingHeaderIncludeTree(StringRef ID);
+
  /// Collect a map from a secondary module name to a list of cross-import
  /// overlays, when this current module serves as the primary module.
  llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
@@ -597,6 +715,23 @@ using ModuleDependenciesKindRefMap =
                       llvm::StringMap<const ModuleDependencyInfo *>,
                       ModuleDependencyKindHash>;

+// MARK: SwiftDependencyTracker
+/// Track swift dependency
+class SwiftDependencyTracker {
+public:
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+                         const std::vector<std::string> &CommonFiles)
+      : FS(FS.createProxyFS()), Files(CommonFiles) {}
+
+  void startTracking();
+  void trackFile(const Twine &path) { (void)FS->status(path); }
+  llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
+
+private:
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  const std::vector<std::string> &Files;
+};
+
 // MARK: SwiftDependencyScanningService
 /// A carrier of state shared among possibly multiple invocations of the dependency
 /// scanner. Acts as a global cache of discovered module dependencies and
@@ -620,7 +755,21 @@ class SwiftDependencyScanningService {
  };

  /// The persistent Clang dependency scanner service
-  clang::tooling::dependencies::DependencyScanningService ClangScanningService;
+  Optional<clang::tooling::dependencies::DependencyScanningService>
+      ClangScanningService;
+
+  /// CachingOnDiskFileSystem for dependency tracking.
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
+
+  /// If use clang include tree.
+  bool UseClangIncludeTree = false;
+
+  /// CAS ObjectStore Instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+
+  /// The common dependencies that is needed for every swift compiler instance.
+  std::vector<std::string> CommonDependencyFiles;
+
  /// The global file system cache.
  Optional<
      clang::tooling::dependencies::DependencyScanningFilesystemSharedCache>
@@ -671,9 +820,37 @@ public:
    return getCacheForScanningContextHash(scanningContextHash)->alreadySeenClangModules;
  }

+  bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
+
+  llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
+    assert(CacheFS && "Expect CachingOnDiskFileSystem");
+    return *CacheFS;
+  }
+
+  Optional<SwiftDependencyTracker> createSwiftDependencyTracker() const {
+    if (!CacheFS)
+      return None;
+
+    return SwiftDependencyTracker(*CacheFS, CommonDependencyFiles);
+  }
+
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
+    if (usingCachingFS())
+      return CacheFS->createProxyFS();
+
+    if (UseClangIncludeTree)
+      return llvm::cas::createCASProvidingFileSystem(
+          CAS, llvm::vfs::createPhysicalFileSystem());
+
+    return llvm::vfs::createPhysicalFileSystem();
+  }
+
  /// Wrap the filesystem on the specified `CompilerInstance` with a
  /// caching `DependencyScanningWorkerFilesystem`
  void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);
+
+  /// Setup caching service.
+  void setupCachingDependencyScanningService(CompilerInstance &Instance);
 private:
  /// Enforce clients not being allowed to query this cache directly, it must be
  /// wrapped in an instance of `ModuleDependenciesCache`.
@@ -769,6 +946,9 @@ public:
  clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
    return clangScanningTool;
  }
+  SwiftDependencyScanningService &getScanService() {
+    return globalScanningService;
+  }
  llvm::StringSet<>& getAlreadySeenClangModules() {
    return globalScanningService.getAlreadySeenClangModules(scannerContextHash);
  }