[CAS] swift dependency scanning using CAS for compiler caching (#66366)

Teach swift dependency scanner to use CAS to capture the full dependencies for a build and construct build commands with immutable inputs from CAS.

This allows swift compilation caching using CAS.

include/swift-c/DependencyScan/DependencyScan.h

@@ -25,7 +25,7 @@
 /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions.
 /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 3
+#define SWIFTSCAN_VERSION_MINOR 4
 
 SWIFTSCAN_BEGIN_DECLS
 
@@ -139,6 +139,10 @@ SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_textual_detail_get_command_line(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_set_t *
+swiftscan_swift_textual_detail_get_bridging_pch_command_line(
+    swiftscan_module_details_t details);
+
 SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_textual_detail_get_extra_pcm_args(
     swiftscan_module_details_t details);
@@ -154,6 +158,14 @@ SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_swift_textual_detail_get_swift_overlay_dependencies(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_swift_textual_detail_get_cas_fs_root_id(
+    swiftscan_module_details_t details);
+
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_swift_textual_detail_get_module_cache_key(
+    swiftscan_module_details_t details);
+
 //=== Swift Binary Module Details query APIs ------------------------------===//
 
 SWIFTSCAN_PUBLIC swiftscan_string_ref_t
@@ -172,6 +184,10 @@ SWIFTSCAN_PUBLIC bool
 swiftscan_swift_binary_detail_get_is_framework(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_swift_binary_detail_get_module_cache_key(
+    swiftscan_module_details_t details);
+
 //=== Swift Placeholder Module Details query APIs -------------------------===//
 
 SWIFTSCAN_PUBLIC swiftscan_string_ref_t
@@ -200,6 +216,12 @@ swiftscan_clang_detail_get_command_line(swiftscan_module_details_t details);
 SWIFTSCAN_PUBLIC swiftscan_string_set_t *
 swiftscan_clang_detail_get_captured_pcm_args(swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_clang_detail_get_cas_fs_root_id(swiftscan_module_details_t details);
+
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_clang_detail_get_module_cache_key(swiftscan_module_details_t details);
+
 //=== Batch Scan Input Functions ------------------------------------------===//
 
 /// Create an \c swiftscan_batch_scan_input_t instance.
@@ -402,6 +424,40 @@ swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner);
 /// An entry point to invoke the compiler via a library call.
 SWIFTSCAN_PUBLIC int invoke_swift_compiler(int argc, const char **argv);
 
+//=== Scanner CAS Operations ----------------------------------------------===//
+
+/// Opaque container for a CAS instance that includes both ObjectStore and
+/// ActionCache.
+typedef struct swiftscan_cas_s *swiftscan_cas_t;
+
+/// Enum types for output types for cache key computation.
+/// TODO: complete the list.
+typedef enum {
+  SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
+  SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIAVEINTERFACE = 3,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
+  SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
+} swiftscan_output_kind_t;
+
+/// Create a \c cas instance that points to path.
+SWIFTSCAN_PUBLIC swiftscan_cas_t swiftscan_cas_create(const char *path);
+
+/// Dispose the \c cas instance.
+SWIFTSCAN_PUBLIC void swiftscan_cas_dispose(swiftscan_cas_t cas);
+
+/// Store content into CAS. Return \c CASID as string.
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t swiftscan_cas_store(swiftscan_cas_t cas,
+                                                            uint8_t *data,
+                                                            unsigned size);
+
+/// Compute \c CacheKey for output of \c kind from the compiler invocation \c
+/// argc and \c argv with \c input. Return \c CacheKey as string.
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv,
+                            const char *input, swiftscan_output_kind_t kind);
+
 //===----------------------------------------------------------------------===//
 
 SWIFTSCAN_END_DECLS

include/swift/AST/ModuleDependencies.h

@@ -23,9 +23,16 @@
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/CASProvidingFileSystem.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Mutex.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <vector>
 #include <unordered_map>
@@ -99,8 +106,10 @@ public:
   const ModuleDependencyKind dependencyKind;
 
   ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
+                                  StringRef moduleCacheKey = "",
                                   bool resolved = false)
-      : dependencyKind(dependencyKind), resolved(resolved) { }
+      : dependencyKind(dependencyKind), moduleCacheKey(moduleCacheKey.str()),
+        resolved(resolved) {}
 
   virtual ModuleDependencyInfoStorageBase *clone() const = 0;
 
@@ -117,12 +126,20 @@ public:
   /// The set of modules on which this module depends, resolved
   /// to Module IDs, qualified by module kind: Swift, Clang, etc.
   std::vector<ModuleDependencyID> resolvedModuleDependencies;
+
+  /// The cache key for the produced module.
+  std::string moduleCacheKey;
+
   bool resolved;
 };
 
 struct CommonSwiftTextualModuleDependencyDetails {
-  CommonSwiftTextualModuleDependencyDetails(ArrayRef<StringRef> extraPCMArgs)
+  CommonSwiftTextualModuleDependencyDetails(
-     : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()) {}
+      ArrayRef<StringRef> extraPCMArgs, ArrayRef<StringRef> buildCommandLine,
+      const std::string &CASFileSystemRootID)
+      : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
+        buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
+        CASFileSystemRootID(CASFileSystemRootID) {}
 
   /// To build a PCM to be used by this Swift module, we need to append these
   /// arguments to the generic PCM build arguments reported from the dependency
@@ -141,6 +158,16 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// Dependencies comprised of Swift overlay modules of direct and
   /// transitive Clang dependencies.
   std::vector<ModuleDependencyID> swiftOverlayDependencies;
+
+  /// The Swift frontend invocation arguments to build the Swift module from the
+  /// interface.
+  std::vector<std::string> buildCommandLine;
+
+  /// CASID for the Root of CASFS. Empty if CAS is not used.
+  std::string CASFileSystemRootID;
+
+  /// CASID for the Root of bridgingHeaderClangIncludeTree. Empty if not used.
+  std::string CASBridgingHeaderIncludeTreeRootID;
 };
 
 /// Describes the dependencies of a Swift module described by an Swift interface file.
@@ -158,10 +185,6 @@ public:
   /// Potentially ready-to-use compiled modules for the interface file.
   const std::vector<std::string> compiledModuleCandidates;
 
-  /// The Swift frontend invocation arguments to build the Swift module from the
-  /// interface.
-  std::vector<std::string> buildCommandLine;
-
   /// The hash value that will be used for the generated module
   const std::string contextHash;
 
@@ -172,22 +195,20 @@ public:
   CommonSwiftTextualModuleDependencyDetails textualModuleDetails;
 
   SwiftInterfaceModuleDependenciesStorage(
-      const std::string moduleOutputPath,
+      const std::string &moduleOutputPath,
-      const std::string swiftInterfaceFile,
+      const std::string &swiftInterfaceFile,
       ArrayRef<std::string> compiledModuleCandidates,
-      ArrayRef<StringRef> buildCommandLine,
+      ArrayRef<StringRef> buildCommandLine, ArrayRef<StringRef> extraPCMArgs,
-      ArrayRef<StringRef> extraPCMArgs,
+      StringRef contextHash, bool isFramework, const std::string &RootID,
-      StringRef contextHash,
+      const std::string &moduleCacheKey)
-      bool isFramework
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface,
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftInterface),
+                                        moduleCacheKey),
-      moduleOutputPath(moduleOutputPath),
+        moduleOutputPath(moduleOutputPath),
-      swiftInterfaceFile(swiftInterfaceFile),
+        swiftInterfaceFile(swiftInterfaceFile),
-      compiledModuleCandidates(compiledModuleCandidates.begin(),
+        compiledModuleCandidates(compiledModuleCandidates.begin(),
-                               compiledModuleCandidates.end()),
+                                 compiledModuleCandidates.end()),
-      buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
+        contextHash(contextHash), isFramework(isFramework),
-      contextHash(contextHash), isFramework(isFramework),
+        textualModuleDetails(extraPCMArgs, buildCommandLine, RootID) {}
-      textualModuleDetails(extraPCMArgs)
-      {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new SwiftInterfaceModuleDependenciesStorage(*this);
@@ -198,7 +219,7 @@ public:
   }
 
   void updateCommandLine(const std::vector<std::string> &newCommandLine) {
-    buildCommandLine = newCommandLine;
+    textualModuleDetails.buildCommandLine = newCommandLine;
   }
 };
 
@@ -218,11 +239,18 @@ public:
   /// Collection of module imports that were detected to be `@Testable`
   llvm::StringSet<> testableImports;
 
+  /// The Swift frontend invocation arguments to build bridging header.
+  std::vector<std::string> bridgingHeaderBuildCommandLine;
+
   SwiftSourceModuleDependenciesStorage(
-    ArrayRef<StringRef> extraPCMArgs
+      const std::string &RootID, ArrayRef<StringRef> buildCommandLine,
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
+      ArrayRef<StringRef> bridgingHeaderBuildCommandLine,
-      textualModuleDetails(extraPCMArgs),
+      ArrayRef<StringRef> extraPCMArgs)
-      testableImports(llvm::StringSet<>()) {}
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftSource),
+        textualModuleDetails(extraPCMArgs, buildCommandLine, RootID),
+        testableImports(llvm::StringSet<>()),
+        bridgingHeaderBuildCommandLine(bridgingHeaderBuildCommandLine.begin(),
+                                       bridgingHeaderBuildCommandLine.end()) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new SwiftSourceModuleDependenciesStorage(*this);
@@ -232,6 +260,15 @@ public:
     return base->dependencyKind == ModuleDependencyKind::SwiftSource;
   }
 
+  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
+    textualModuleDetails.buildCommandLine = newCommandLine;
+  }
+
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    bridgingHeaderBuildCommandLine = newCommandLine;
+  }
+
   void addTestableImport(ImportPath::Module module) {
     testableImports.insert(module.front().Item.str());
   }
@@ -245,12 +282,12 @@ public:
   SwiftBinaryModuleDependencyStorage(const std::string &compiledModulePath,
                                      const std::string &moduleDocPath,
                                      const std::string &sourceInfoPath,
-                                     const bool isFramework)
+                                     const bool isFramework,
-      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary),
+                                     const std::string &moduleCacheKey)
-        compiledModulePath(compiledModulePath),
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary,
-        moduleDocPath(moduleDocPath),
+                                        moduleCacheKey),
-        sourceInfoPath(sourceInfoPath),
+        compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
-        isFramework(isFramework) {}
+        sourceInfoPath(sourceInfoPath), isFramework(isFramework) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new SwiftBinaryModuleDependencyStorage(*this);
@@ -288,7 +325,7 @@ public:
   const std::string contextHash;
 
   /// Partial (Clang) command line that can be used to build this module.
-  const std::vector<std::string> nonPathCommandLine;
+  std::vector<std::string> buildCommandLine;
 
   /// The file dependencies
   const std::vector<std::string> fileDependencies;
@@ -297,20 +334,28 @@ public:
   /// as found by the scanning action that discovered it
   const std::vector<std::string> capturedPCMArgs;
 
-  ClangModuleDependencyStorage(
+  /// CASID for the Root of CASFS. Empty if CAS is not used.
-      const std::string &pcmOutputPath,
+  std::string CASFileSystemRootID;
-      const std::string &moduleMapFile,
+
-      const std::string &contextHash,
+  /// CASID for the Root of ClangIncludeTree. Empty if not used.
-      const std::vector<std::string> &nonPathCommandLine,
+  std::string CASClangIncludeTreeRootID;
-      const std::vector<std::string> &fileDependencies,
+
-      const std::vector<std::string> &capturedPCMArgs
+  ClangModuleDependencyStorage(const std::string &pcmOutputPath,
-  ) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang),
+                               const std::string &moduleMapFile,
-      pcmOutputPath(pcmOutputPath),
+                               const std::string &contextHash,
-      moduleMapFile(moduleMapFile),
+                               const std::vector<std::string> &buildCommandLine,
-      contextHash(contextHash),
+                               const std::vector<std::string> &fileDependencies,
-      nonPathCommandLine(nonPathCommandLine),
+                               const std::vector<std::string> &capturedPCMArgs,
-      fileDependencies(fileDependencies),
+                               const std::string &CASFileSystemRootID,
-      capturedPCMArgs(capturedPCMArgs) {}
+                               const std::string &clangIncludeTreeRoot,
+                               const std::string &moduleCacheKey)
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang,
+                                        moduleCacheKey),
+        pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile),
+        contextHash(contextHash), buildCommandLine(buildCommandLine),
+        fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs),
+        CASFileSystemRootID(CASFileSystemRootID),
+        CASClangIncludeTreeRootID(clangIncludeTreeRoot) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new ClangModuleDependencyStorage(*this);
@@ -319,6 +364,10 @@ public:
   static bool classof(const ModuleDependencyInfoStorageBase *base) {
     return base->dependencyKind == ModuleDependencyKind::Clang;
   }
+
+  void updateCommandLine(const std::vector<std::string> &newCommandLine) {
+    buildCommandLine = newCommandLine;
+  }
 };
 
 /// Describes an placeholder Swift module dependency module stub.
@@ -381,35 +430,44 @@ public:
 
   /// Describe the module dependencies for a Swift module that can be
   /// built from a Swift interface file (\c .swiftinterface).
-  static ModuleDependencyInfo forSwiftInterfaceModule(
+  static ModuleDependencyInfo
-      const std::string &moduleOutputPath,
+  forSwiftInterfaceModule(const std::string &moduleOutputPath,
-      const std::string &swiftInterfaceFile,
+                          const std::string &swiftInterfaceFile,
-      ArrayRef<std::string> compiledCandidates,
+                          ArrayRef<std::string> compiledCandidates,
-      ArrayRef<StringRef> buildCommands,
+                          ArrayRef<StringRef> buildCommands,
-      ArrayRef<StringRef> extraPCMArgs,
+                          ArrayRef<StringRef> extraPCMArgs,
-      StringRef contextHash,
+                          StringRef contextHash, bool isFramework,
-      bool isFramework) {
+                          const std::string &CASFileSystemRootID,
+                          const std::string &moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftInterfaceModuleDependenciesStorage>(
-          moduleOutputPath, swiftInterfaceFile, compiledCandidates, buildCommands,
+            moduleOutputPath, swiftInterfaceFile, compiledCandidates,
-          extraPCMArgs, contextHash, isFramework));
+            buildCommands, extraPCMArgs, contextHash, isFramework,
+            CASFileSystemRootID, moduleCacheKey));
   }
 
   /// Describe the module dependencies for a serialized or parsed Swift module.
-  static ModuleDependencyInfo forSwiftBinaryModule(
+  static ModuleDependencyInfo
-      const std::string &compiledModulePath,
+  forSwiftBinaryModule(const std::string &compiledModulePath,
-      const std::string &moduleDocPath,
+                       const std::string &moduleDocPath,
-      const std::string &sourceInfoPath,
+                       const std::string &sourceInfoPath, bool isFramework,
-      bool isFramework) {
+                       const std::string &moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftBinaryModuleDependencyStorage>(
-          compiledModulePath, moduleDocPath, sourceInfoPath, isFramework));
+            compiledModulePath, moduleDocPath, sourceInfoPath, isFramework,
+            moduleCacheKey));
   }
 
   /// Describe the main Swift module.
-  static ModuleDependencyInfo forSwiftSourceModule(ArrayRef<StringRef> extraPCMArgs) {
+  static ModuleDependencyInfo
+  forSwiftSourceModule(const std::string &CASFileSystemRootID,
+                       ArrayRef<StringRef> buildCommands,
+                       ArrayRef<StringRef> bridgingHeaderBuildCommands,
+                       ArrayRef<StringRef> extraPCMArgs) {
     return ModuleDependencyInfo(
-        std::make_unique<SwiftSourceModuleDependenciesStorage>(extraPCMArgs));
+        std::make_unique<SwiftSourceModuleDependenciesStorage>(
+            CASFileSystemRootID, buildCommands, bridgingHeaderBuildCommands,
+            extraPCMArgs));
   }
 
   /// Describe the module dependencies for a Clang module that can be
@@ -420,11 +478,15 @@ public:
       const std::string &contextHash,
       const std::vector<std::string> &nonPathCommandLine,
       const std::vector<std::string> &fileDependencies,
-      const std::vector<std::string> &capturedPCMArgs) {
+      const std::vector<std::string> &capturedPCMArgs,
+      const std::string &CASFileSystemRootID,
+      const std::string &clangIncludeTreeRoot,
+      const std::string &moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<ClangModuleDependencyStorage>(
           pcmOutputPath, moduleMapFile, contextHash,
-          nonPathCommandLine, fileDependencies, capturedPCMArgs));
+          nonPathCommandLine, fileDependencies, capturedPCMArgs,
+          CASFileSystemRootID, clangIncludeTreeRoot,  moduleCacheKey));
   }
 
   /// Describe a placeholder dependency swift module.
@@ -453,6 +515,15 @@ public:
     return storage->resolvedModuleDependencies;
   }
 
+  std::string getModuleCacheKey() const {
+    assert(storage->resolved);
+    return storage->moduleCacheKey;
+  }
+
+  void updateModuleCacheKey(const std::string &key) {
+    storage->moduleCacheKey = key;
+  }
+
   /// Resolve a dependency's set of `imports` with qualified Module IDs
   void resolveDependencies(const ArrayRef<ModuleDependencyID> dependencyIDs) {
     assert(!storage->resolved && "Resolving an already-resolved dependency");
@@ -474,9 +545,41 @@ public:
     textualModuleDetails->swiftOverlayDependencies.assign(dependencyIDs.begin(), dependencyIDs.end());
   }
 
+  std::vector<std::string> getCommandline() const {
+    if (auto *detail = getAsClangModule())
+      return detail->buildCommandLine;
+    else if (auto *detail = getAsSwiftInterfaceModule())
+      return detail->textualModuleDetails.buildCommandLine;
+    else if (auto *detail = getAsSwiftSourceModule())
+      return detail->textualModuleDetails.buildCommandLine;
+    return {};
+  }
+
   void updateCommandLine(const std::vector<std::string> &newCommandLine) {
-    assert(isSwiftInterfaceModule() && "Can only update command line on Swift interface dependency");
+    if (isSwiftInterfaceModule())
-    cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())->updateCommandLine(newCommandLine);
+      return cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    else if (isSwiftSourceModule())
+      return cast<SwiftSourceModuleDependenciesStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    else if (isClangModule())
+      return cast<ClangModuleDependencyStorage>(storage.get())
+          ->updateCommandLine(newCommandLine);
+    llvm_unreachable("Unexpected type");
+  }
+
+  std::vector<std::string> getBridgingHeaderCommandline() const {
+    if (auto *detail = getAsSwiftSourceModule())
+      return detail->bridgingHeaderBuildCommandLine;
+    return {};
+  }
+
+  void updateBridgingHeaderCommandLine(
+      const std::vector<std::string> &newCommandLine) {
+    if (isSwiftSourceModule())
+      return cast<SwiftSourceModuleDependenciesStorage>(storage.get())
+          ->updateBridgingHeaderCommandLine(newCommandLine);
+    llvm_unreachable("Unexpected type");
   }
 
   bool isResolved() const {
@@ -568,6 +671,18 @@ public:
   /// Get the bridging header.
   Optional<std::string> getBridgingHeader() const;
 
+  /// Get CAS Filesystem RootID.
+  Optional<std::string> getCASFSRootID() const;
+
+  /// Get Clang Include Tree ID.
+  Optional<std::string> getClangIncludeTree() const;
+
+  /// Get bridging header Include Tree ID.
+  Optional<std::string> getBridgingHeaderIncludeTree() const;
+
+  /// Get module output path.
+  std::string getModuleOutputPath() const;
+
   /// Add a bridging header to a Swift module's dependencies.
   void addBridgingHeader(StringRef bridgingHeader);
 
@@ -581,6 +696,9 @@ public:
   void addBridgingModuleDependency(StringRef module,
                                    llvm::StringSet<> &alreadyAddedModules);
 
+  /// Add bridging header include tree.
+  void addBridgingHeaderIncludeTree(StringRef ID);
+
   /// Collect a map from a secondary module name to a list of cross-import
   /// overlays, when this current module serves as the primary module.
   llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
@@ -597,6 +715,23 @@ using ModuleDependenciesKindRefMap =
                        llvm::StringMap<const ModuleDependencyInfo *>,
                        ModuleDependencyKindHash>;
 
+// MARK: SwiftDependencyTracker
+/// Track swift dependency
+class SwiftDependencyTracker {
+public:
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+                         const std::vector<std::string> &CommonFiles)
+      : FS(FS.createProxyFS()), Files(CommonFiles) {}
+
+  void startTracking();
+  void trackFile(const Twine &path) { (void)FS->status(path); }
+  llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
+
+private:
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  const std::vector<std::string> &Files;
+};
+
 // MARK: SwiftDependencyScanningService
 /// A carrier of state shared among possibly multiple invocations of the dependency
 /// scanner. Acts as a global cache of discovered module dependencies and
@@ -620,7 +755,21 @@ class SwiftDependencyScanningService {
   };
 
   /// The persistent Clang dependency scanner service
-  clang::tooling::dependencies::DependencyScanningService ClangScanningService;
+  Optional<clang::tooling::dependencies::DependencyScanningService>
+      ClangScanningService;
+
+  /// CachingOnDiskFileSystem for dependency tracking.
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
+
+  /// If use clang include tree.
+  bool UseClangIncludeTree = false;
+
+  /// CAS ObjectStore Instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+
+  /// The common dependencies that is needed for every swift compiler instance.
+  std::vector<std::string> CommonDependencyFiles;
+
   /// The global file system cache.
   Optional<
       clang::tooling::dependencies::DependencyScanningFilesystemSharedCache>
@@ -671,9 +820,37 @@ public:
     return getCacheForScanningContextHash(scanningContextHash)->alreadySeenClangModules;
   }
 
+  bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
+
+  llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
+    assert(CacheFS && "Expect CachingOnDiskFileSystem");
+    return *CacheFS;
+  }
+
+  Optional<SwiftDependencyTracker> createSwiftDependencyTracker() const {
+    if (!CacheFS)
+      return None;
+
+    return SwiftDependencyTracker(*CacheFS, CommonDependencyFiles);
+  }
+
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
+    if (usingCachingFS())
+      return CacheFS->createProxyFS();
+
+    if (UseClangIncludeTree)
+      return llvm::cas::createCASProvidingFileSystem(
+          CAS, llvm::vfs::createPhysicalFileSystem());
+
+    return llvm::vfs::createPhysicalFileSystem();
+  }
+
   /// Wrap the filesystem on the specified `CompilerInstance` with a
   /// caching `DependencyScanningWorkerFilesystem`
   void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);
+
+  /// Setup caching service.
+  void setupCachingDependencyScanningService(CompilerInstance &Instance);
 private:
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
@@ -769,6 +946,9 @@ public:
   clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
     return clangScanningTool;
   }
+  SwiftDependencyScanningService &getScanService() {
+    return globalScanningService;
+  }
   llvm::StringSet<>& getAlreadySeenClangModules() {
     return globalScanningService.getAlreadySeenClangModules(scannerContextHash);
   }

include/swift/Basic/LangOptions.h

@@ -822,6 +822,9 @@ namespace swift {
     /// clang CASOptions.
     std::string CASPath;
 
+    /// Cache key for imported bridging header.
+    std::string BridgingHeaderPCHCacheKey;
+
     /// Disable validating the persistent PCH.
     bool PCHDisableValidation = false;
 
@@ -891,6 +894,9 @@ namespace swift {
     /// built and provided to the compiler invocation.
     bool DisableImplicitClangModules = false;
 
+    /// Enable ClangIncludeTree for explicit module builds.
+    bool UseClangIncludeTree = false;
+
     /// Return a hash code of any components from these options that should
     /// contribute to a Swift Bridging PCH hash.
     llvm::hash_code getPCHHashComponents() const {

include/swift/ClangImporter/ClangImporter.h

@@ -54,6 +54,7 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
   struct ModuleDeps;
+  struct TranslationUnitDeps;
   using ModuleDepsGraph = std::vector<ModuleDeps>;
 }
 }
@@ -431,6 +432,10 @@ public:
       ModuleDependenciesCache &cache,
       const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies);
 
+  void recordBridgingHeaderOptions(
+      ModuleDependencyInfo &MDI,
+      const clang::tooling::dependencies::TranslationUnitDeps &deps);
+
   Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate,

include/swift/DependencyScan/DependencyScanImpl.h

@@ -86,6 +86,9 @@ typedef struct {
   /// Options to the compile command required to build this module interface
   swiftscan_string_set_t *command_line;
 
+  /// Options to the compile command required to build bridging header.
+  swiftscan_string_set_t *bridging_pch_command_line;
+
   /// To build a PCM to be used by this Swift module, we need to append these
   /// arguments to the generic PCM build arguments reported from the dependency
   /// graph.
@@ -96,6 +99,15 @@ typedef struct {
 
   /// A flag to indicate whether or not this module is a framework.
   bool is_framework;
+
+  /// The CASID for CASFileSystemRoot
+  swiftscan_string_ref_t cas_fs_root_id;
+
+  /// The CASID for bridging header include tree
+  swiftscan_string_ref_t bridging_header_include_tree;
+
+  /// ModuleCacheKey
+  swiftscan_string_ref_t module_cache_key;
 } swiftscan_swift_textual_details_t;
 
 /// Swift modules with only a binary module file.
@@ -111,6 +123,9 @@ typedef struct {
 
   /// A flag to indicate whether or not this module is a framework.
   bool is_framework;
+
+  /// ModuleCacheKey
+  swiftscan_string_ref_t module_cache_key;
 } swiftscan_swift_binary_details_t;
 
 /// Swift placeholder modules carry additional details that specify their
@@ -139,6 +154,15 @@ typedef struct {
 
   /// The swift-specific PCM arguments captured by this dependencies object
   swiftscan_string_set_t *captured_pcm_args;
+
+  /// The CASID for CASFileSystemRoot
+  swiftscan_string_ref_t cas_fs_root_id;
+
+  /// The CASID for CASFileSystemRoot
+  swiftscan_string_ref_t clang_include_tree;
+
+  /// ModuleCacheKey
+  swiftscan_string_ref_t module_cache_key;
 } swiftscan_clang_details_t;
 
 struct swiftscan_module_details_s {

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -140,7 +140,10 @@ using SwiftInterfaceModuleDetailsLayout =
                    FileIDArrayIDField,                  // sourceFiles
                    FileIDArrayIDField,                  // bridgingSourceFiles
                    FileIDArrayIDField,                  // bridgingModuleDependencies
-                   DependencyIDArrayIDField             // swiftOverlayDependencies
+                   DependencyIDArrayIDField,            // swiftOverlayDependencies
+                   IdentifierIDField,                   // CASFileSystemRootID
+                   IdentifierIDField,                   // bridgingHeaderIncludeTree
+                   IdentifierIDField                    // moduleCacheKey
                    >;
 
 using SwiftSourceModuleDetailsLayout =
@@ -150,7 +153,11 @@ using SwiftSourceModuleDetailsLayout =
                    FileIDArrayIDField,               // sourceFiles
                    FileIDArrayIDField,               // bridgingSourceFiles
                    FileIDArrayIDField,               // bridgingModuleDependencies
-                   DependencyIDArrayIDField          // swiftOverlayDependencies
+                   DependencyIDArrayIDField,         // swiftOverlayDependencies
+                   IdentifierIDField,                // CASFileSystemRootID
+                   IdentifierIDField,                // bridgingHeaderIncludeTree
+                   FlagIDArrayIDField,               // buildCommandLine
+                   FlagIDArrayIDField                // bridgingHeaderBuildCommandLine
                    >;
 
 using SwiftBinaryModuleDetailsLayout =
@@ -158,7 +165,8 @@ using SwiftBinaryModuleDetailsLayout =
                    FileIDField,                      // compiledModulePath
                    FileIDField,                      // moduleDocPath
                    FileIDField,                      // moduleSourceInfoPath
-                   IsFrameworkField                  // isFramework
+                   IsFrameworkField,                 // isFramework
+                   IdentifierIDField                 // moduleCacheKey
                    >;
 
 using SwiftPlaceholderModuleDetailsLayout =
@@ -175,7 +183,10 @@ using ClangModuleDetailsLayout =
                    ContextHashIDField,        // contextHash
                    FlagIDArrayIDField,        // commandLine
                    FileIDArrayIDField,        // fileDependencies
-                   FlagIDArrayIDField         // capturedPCMArgs
+                   FlagIDArrayIDField,        // capturedPCMArgs
+                   IdentifierIDField,         // CASFileSystemRootID
+                   IdentifierIDField,         // clangIncludeTreeRoot
+                   IdentifierIDField          // moduleCacheKey
                    >;
 } // namespace graph_block
 

include/swift/Frontend/CachingUtils.h

@@ -19,6 +19,7 @@
 #include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/CAS/CASReference.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <memory>
 
@@ -54,6 +55,32 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
                                       StringRef CorrespondingInput,
                                       file_types::ID OutputKind);
 
+llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
+createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
+                    ArrayRef<std::string> IncludeTreeRoots);
+
+namespace cas {
+/// Helper class to manage CAS/Caching from libSwiftScan C APIs.
+class CachingTool {
+public:
+  // Create the tool with a list of arguments from compiler invocation.
+  CachingTool(llvm::StringRef Path);
+
+  // Compute the CASID for PCH output from invocation.
+  std::string computeCacheKey(llvm::ArrayRef<const char *> Args,
+                              StringRef InputPath, file_types::ID OutputKind);
+
+  // Store content into CAS.
+  std::string storeContent(llvm::StringRef Content);
+
+  // Check if the tool is correctly initialized.
+  bool isValid() const { return CAS && Cache; }
+
+private:
+  std::unique_ptr<llvm::cas::ObjectStore> CAS;
+  std::unique_ptr<llvm::cas::ActionCache> Cache;
+};
+} // namespace cas
 }
 
 #endif

include/swift/Frontend/FrontendOptions.h

@@ -129,7 +129,10 @@ public:
   std::string CASPath;
 
   /// CASFS Root.
-  std::string CASFSRootID;
+  std::vector<std::string> CASFSRootIDs;
+
+  /// Clang Include Trees.
+  std::vector<std::string> ClangIncludeTrees;
 
   /// Number of retry opening an input file if the previous opening returns
   /// bad file descriptor error.

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -111,9 +111,17 @@
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/ModuleInterfaceSupport.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/YAMLTraits.h"
 
+namespace llvm {
+namespace cas {
+class ObjectStore;
+class ActionCache;
+} // namespace cas
+} // namespace llvm
+
 namespace clang {
 class CompilerInstance;
 }
@@ -178,6 +186,55 @@ public:
   ~ExplicitSwiftModuleLoader();
 };
 
+class ExplicitCASModuleLoader : public SerializedModuleLoaderBase {
+  explicit ExplicitCASModuleLoader(ASTContext &ctx, llvm::cas::ObjectStore &CAS,
+                                   llvm::cas::ActionCache &cache,
+                                   DependencyTracker *tracker,
+                                   ModuleLoadingMode loadMode,
+                                   bool IgnoreSwiftSourceInfoFile);
+
+  bool findModule(ImportPath::Element moduleID,
+                  SmallVectorImpl<char> *moduleInterfacePath,
+                  SmallVectorImpl<char> *moduleInterfaceSourcePath,
+                  std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
+                  std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
+                  std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
+                  bool skipBuildingInterface, bool isTestableDependencyLookup,
+                  bool &isFramework, bool &isSystemModule) override;
+
+  std::error_code findModuleFilesInDirectory(
+      ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
+      SmallVectorImpl<char> *ModuleInterfacePath,
+      SmallVectorImpl<char> *ModuleInterfaceSourcePath,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
+
+  bool canImportModule(ImportPath::Module named,
+                       ModuleVersionInfo *versionInfo,
+                       bool isTestableDependencyLookup = false) override;
+
+  struct Implementation;
+  Implementation &Impl;
+
+public:
+  static std::unique_ptr<ExplicitCASModuleLoader>
+  create(ASTContext &ctx, llvm::cas::ObjectStore &CAS,
+         llvm::cas::ActionCache &cache, DependencyTracker *tracker,
+         ModuleLoadingMode loadMode, StringRef ExplicitSwiftModuleMap,
+         const std::vector<std::pair<std::string, std::string>>
+             &ExplicitSwiftModuleInputs,
+         bool IgnoreSwiftSourceInfoFile);
+
+  /// Append visible module names to \p names. Note that names are possibly
+  /// duplicated, and not guaranteed to be ordered in any way.
+  void collectVisibleTopLevelModuleNames(
+      SmallVectorImpl<Identifier> &names) const override;
+
+  ~ExplicitCASModuleLoader();
+};
 
 // Explicitly-specified Swift module inputs
 struct ExplicitSwiftModuleInputInfo {
@@ -185,12 +242,14 @@ struct ExplicitSwiftModuleInputInfo {
                                llvm::Optional<std::string> moduleDocPath,
                                llvm::Optional<std::string> moduleSourceInfoPath,
                                bool isFramework = false,
-                               bool isSystem = false)
+                               bool isSystem = false,
+                               llvm::Optional<std::string> moduleCacheKey = None)
     : modulePath(modulePath),
       moduleDocPath(moduleDocPath),
       moduleSourceInfoPath(moduleSourceInfoPath),
       isFramework(isFramework),
-      isSystem(isSystem) {}
+      isSystem(isSystem),
+      moduleCacheKey(moduleCacheKey) {}
   // Path of the .swiftmodule file.
   std::string modulePath;
   // Path of the .swiftmoduledoc file.
@@ -201,6 +260,8 @@ struct ExplicitSwiftModuleInputInfo {
   bool isFramework = false;
   // A flag that indicates whether this module is a system module
   bool isSystem = false;
+  // The cache key for clang module.
+  llvm::Optional<std::string> moduleCacheKey;
 };
 
 // Explicitly-specified Clang module inputs
@@ -208,11 +269,13 @@ struct ExplicitClangModuleInputInfo {
   ExplicitClangModuleInputInfo(std::string moduleMapPath,
                                std::string modulePath,
                                bool isFramework = false,
-                               bool isSystem = false)
+                               bool isSystem = false,
+                               llvm::Optional<std::string> moduleCacheKey = None)
     : moduleMapPath(moduleMapPath),
       modulePath(modulePath),
       isFramework(isFramework),
-      isSystem(isSystem) {}
+      isSystem(isSystem),
+      moduleCacheKey(moduleCacheKey) {}
   // Path of the Clang module map file.
   std::string moduleMapPath;
   // Path of a compiled Clang explicit module file (pcm).
@@ -221,6 +284,8 @@ struct ExplicitClangModuleInputInfo {
   bool isFramework = false;
   // A flag that indicates whether this module is a system module
   bool isSystem = false;
+  // The cache key for clang module.
+  llvm::Optional<std::string> moduleCacheKey;
 };
 
 /// Parser of explicit module maps passed into the compiler.
@@ -233,6 +298,8 @@ struct ExplicitClangModuleInputInfo {
 //      "isFramework": false,
 //      "clangModuleMapPath": "A/module.modulemap",
 //      "clangModulePath": "A.pcm",
+//      "moduleCacheKey": "llvmcas://<hash>",
+//      "clangModuleCacheKey": "llvmcas://<hash>",
 //    },
 //    {
 //      "moduleName": "B",
@@ -242,6 +309,8 @@ struct ExplicitClangModuleInputInfo {
 //      "isFramework": false,
 //      "clangModuleMapPath": "B/module.modulemap",
 //      "clangModulePath": "B.pcm",
+//      "moduleCacheKey": "llvmcas://<hash>",
+//      "clangModuleCacheKey": "llvmcas://<hash>",
 //    }
 //  ]
 class ExplicitModuleMapParser {
@@ -249,21 +318,14 @@ public:
   ExplicitModuleMapParser(llvm::BumpPtrAllocator &Allocator) : Saver(Allocator) {}
 
   std::error_code
-  parseSwiftExplicitModuleMap(llvm::StringRef fileName,
+  parseSwiftExplicitModuleMap(llvm::MemoryBufferRef BufferRef,
                               llvm::StringMap<ExplicitSwiftModuleInputInfo> &swiftModuleMap,
                               llvm::StringMap<ExplicitClangModuleInputInfo> &clangModuleMap) {
     using namespace llvm::yaml;
-    // Load the input file.
-    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
-        llvm::MemoryBuffer::getFile(fileName);
-    if (!fileBufOrErr) {
-      return std::make_error_code(std::errc::no_such_file_or_directory);
-    }
-    StringRef Buffer = fileBufOrErr->get()->getBuffer();
     // Use a new source manager instead of the one from ASTContext because we
     // don't want the JSON file to be persistent.
     llvm::SourceMgr SM;
-    Stream Stream(llvm::MemoryBufferRef(Buffer, fileName), SM);
+    Stream Stream(BufferRef, SM);
     for (auto DI = Stream.begin(); DI != Stream.end(); ++DI) {
       assert(DI != Stream.end() && "Failed to read a document");
       if (auto *MN = dyn_cast_or_null<SequenceNode>(DI->getRoot())) {
@@ -305,7 +367,8 @@ private:
       return true;
     StringRef moduleName;
     llvm::Optional<std::string> swiftModulePath, swiftModuleDocPath,
-                                swiftModuleSourceInfoPath;
+                                swiftModuleSourceInfoPath, swiftModuleCacheKey,
+                                clangModuleCacheKey;
     std::string clangModuleMapPath = "", clangModulePath = "";
     bool isFramework = false, isSystem = false;
     for (auto &entry : *mapNode) {
@@ -327,6 +390,10 @@ private:
         clangModuleMapPath = val.str();
       } else if (key == "clangModulePath") {
         clangModulePath = val.str();
+      } else if (key == "moduleCacheKey") {
+        swiftModuleCacheKey = val.str();
+      } else if (key == "clangModuleCacheKey") {
+        clangModuleCacheKey = val.str();
       } else {
         // Being forgiving for future fields.
         continue;
@@ -343,7 +410,8 @@ private:
                                          swiftModuleDocPath,
                                          swiftModuleSourceInfoPath,
                                          isFramework,
-                                         isSystem);
+                                         isSystem,
+                                         swiftModuleCacheKey);
       swiftModuleMap.try_emplace(moduleName, std::move(entry));
     } else {
       assert((!clangModuleMapPath.empty() ||
@@ -352,7 +420,8 @@ private:
       ExplicitClangModuleInputInfo entry(clangModuleMapPath,
                                          clangModulePath,
                                          isFramework,
-                                         isSystem);
+                                         isSystem,
+                                         clangModuleCacheKey);
       clangModuleMap.try_emplace(moduleName, std::move(entry));
     }
 

include/swift/Option/Options.td

@@ -1829,6 +1829,15 @@ def allow_unstable_cache_key_for_testing: Flag<["-"], "allow-unstable-cache-key-
   Flags<[FrontendOption, HelpHidden, NoDriverOption]>,
   HelpText<"Allow compilation caching with unstable inputs for testing purpose">;
 
+def bridging_header_pch_key : Separate<["-"], "bridging-header-pch-key">,
+  Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
+  HelpText<"Cache Key for bridging header pch">;
+
+def clang_include_tree: Flag<["-"], "clang-include-tree">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Use clang include tree">;
+
+
 // END ONLY SUPPORTED IN NEW DRIVER
 
 def enable_cas: Flag<["-"], "enable-cas">,
@@ -1839,6 +1848,10 @@ def cas_fs: Separate<["-"], "cas-fs">,
   Flags<[FrontendOption, NoDriverOption]>,
   HelpText<"Root CASID for CAS FileSystem">, MetaVarName<"<cas-id>">;
 
+def clang_include_tree_root: Separate<["-"], "clang-include-tree-root">,
+  Flags<[FrontendOption, NoDriverOption]>,
+  HelpText<"Clang Include Tree CASID">, MetaVarName<"<cas-id>">;
+
 def load_plugin_library:
   Separate<["-"], "load-plugin-library">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,

include/swift/Serialization/ModuleDependencyScanner.h

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/AST/ASTContext.h"
+#include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
 
@@ -41,18 +42,22 @@ namespace swift {
 
       /// Location where pre-built moduels are to be built into.
       std::string moduleCachePath;
+
+      Optional<SwiftDependencyTracker> dependencyTracker;
     public:
       Optional<ModuleDependencyInfo> dependencies;
 
       ModuleDependencyScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
                               Identifier moduleName,
                               InterfaceSubContextDelegate &astDelegate,
-                              ScannerKind kind = MDS_plain)
+                              ScannerKind kind = MDS_plain,
+                              Optional<SwiftDependencyTracker> tracker = None)
           : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                        /*IgnoreSwiftSourceInfoFile=*/true),
             kind(kind), moduleName(moduleName), astDelegate(astDelegate),
             moduleCachePath(getModuleCachePathFromClang(
-                ctx.getClangModuleLoader()->getClangInstance())) {}
+                ctx.getClangModuleLoader()->getClangInstance())),
+            dependencyTracker(tracker) {}
 
       std::error_code findModuleFilesInDirectory(
           ImportPath::Element ModuleID,
@@ -87,9 +92,16 @@ namespace swift {
       void parsePlaceholderModuleMap(StringRef fileName) {
         ExplicitModuleMapParser parser(Allocator);
         llvm::StringMap<ExplicitClangModuleInputInfo> ClangDependencyModuleMap;
-        auto result =
+        llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
-          parser.parseSwiftExplicitModuleMap(fileName, PlaceholderDependencyModuleMap,
+            llvm::MemoryBuffer::getFile(fileName);
-                                             ClangDependencyModuleMap);
+        if (!fileBufOrErr) {
+          Ctx.Diags.diagnose(SourceLoc(),
+                             diag::explicit_swift_module_map_missing, fileName);
+          return;
+        }
+        auto result = parser.parseSwiftExplicitModuleMap(
+            (*fileBufOrErr)->getMemBufferRef(), PlaceholderDependencyModuleMap,
+            ClangDependencyModuleMap);
         if (result == std::errc::invalid_argument) {
           Ctx.Diags.diagnose(SourceLoc(),
                              diag::placeholder_dependency_module_map_corrupted,
@@ -109,9 +121,10 @@ namespace swift {
       PlaceholderSwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
                                     Identifier moduleName,
                                     StringRef PlaceholderDependencyModuleMap,
-                                    InterfaceSubContextDelegate &astDelegate)
+                                    InterfaceSubContextDelegate &astDelegate,
+                                    Optional<SwiftDependencyTracker> tracker = None)
           : ModuleDependencyScanner(ctx, LoadMode, moduleName, astDelegate,
-                                    MDS_placeholder) {
+                                    MDS_placeholder, tracker) {
 
         // FIXME: Find a better place for this map to live, to avoid
         // doing the parsing on every module.

lib/AST/ModuleDependencies.cpp

@@ -15,8 +15,14 @@
 //===----------------------------------------------------------------------===//
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/AST/Decl.h"
+#include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/SourceFile.h"
 #include "swift/Frontend/Frontend.h"
+#include "llvm/CAS/CASProvidingFileSystem.h"
+#include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include <system_error>
 using namespace swift;
 
 ModuleDependencyInfoStorageBase::~ModuleDependencyInfoStorageBase() {}
@@ -175,6 +181,108 @@ Optional<std::string> ModuleDependencyInfo::getBridgingHeader() const {
   }
 }
 
+Optional<std::string> ModuleDependencyInfo::getCASFSRootID() const {
+  std::string Root;
+  switch (getKind()) {
+  case swift::ModuleDependencyKind::SwiftInterface: {
+    auto swiftInterfaceStorage =
+        cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
+    Root = swiftInterfaceStorage->textualModuleDetails.CASFileSystemRootID;
+    break;
+  }
+  case swift::ModuleDependencyKind::SwiftSource: {
+    auto swiftSourceStorage =
+        cast<SwiftSourceModuleDependenciesStorage>(storage.get());
+    Root = swiftSourceStorage->textualModuleDetails.CASFileSystemRootID;
+    break;
+  }
+  case swift::ModuleDependencyKind::Clang: {
+    auto clangModuleStorage = cast<ClangModuleDependencyStorage>(storage.get());
+    Root = clangModuleStorage->CASFileSystemRootID;
+    break;
+  }
+  default:
+    return None;
+  }
+  if (Root.empty())
+    return None;
+
+  return Root;
+}
+
+Optional<std::string> ModuleDependencyInfo::getClangIncludeTree() const {
+  std::string Root;
+  switch (getKind()) {
+  case swift::ModuleDependencyKind::Clang: {
+    auto clangModuleStorage = cast<ClangModuleDependencyStorage>(storage.get());
+    Root = clangModuleStorage->CASClangIncludeTreeRootID;
+    break;
+  }
+  default:
+    return None;
+  }
+  if (Root.empty())
+    return None;
+
+  return Root;
+}
+
+Optional<std::string>
+ModuleDependencyInfo::getBridgingHeaderIncludeTree() const {
+  std::string Root;
+  switch (getKind()) {
+  case swift::ModuleDependencyKind::SwiftInterface: {
+    auto swiftInterfaceStorage =
+        cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
+    Root = swiftInterfaceStorage->textualModuleDetails
+               .CASBridgingHeaderIncludeTreeRootID;
+    break;
+  }
+  case swift::ModuleDependencyKind::SwiftSource: {
+    auto swiftSourceStorage =
+        cast<SwiftSourceModuleDependenciesStorage>(storage.get());
+    Root = swiftSourceStorage->textualModuleDetails
+               .CASBridgingHeaderIncludeTreeRootID;
+    break;
+  }
+  default:
+    return None;
+  }
+  if (Root.empty())
+    return None;
+
+  return Root;
+}
+
+std::string ModuleDependencyInfo::getModuleOutputPath() const {
+  switch (getKind()) {
+  case swift::ModuleDependencyKind::SwiftInterface: {
+    auto swiftInterfaceStorage =
+        cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
+    return swiftInterfaceStorage->moduleOutputPath;
+  }
+  case swift::ModuleDependencyKind::SwiftSource: {
+    return "<swiftmodule>";
+  }
+  case swift::ModuleDependencyKind::Clang: {
+    auto clangModuleStorage = cast<ClangModuleDependencyStorage>(storage.get());
+    return clangModuleStorage->pcmOutputPath;
+  }
+  case swift::ModuleDependencyKind::SwiftBinary: {
+    auto swiftBinaryStorage =
+        cast<SwiftBinaryModuleDependencyStorage>(storage.get());
+    return swiftBinaryStorage->compiledModulePath;
+  }
+  case swift::ModuleDependencyKind::SwiftPlaceholder: {
+    auto swiftPlaceholderStorage =
+        cast<SwiftPlaceholderModuleDependencyStorage>(storage.get());
+    return swiftPlaceholderStorage->compiledModulePath;
+  }
+  default:
+    llvm_unreachable("Unexpected dependency kind");
+  }
+}
+
 void ModuleDependencyInfo::addBridgingHeader(StringRef bridgingHeader) {
   switch (getKind()) {
   case swift::ModuleDependencyKind::SwiftInterface: {
@@ -217,6 +325,27 @@ void ModuleDependencyInfo::addBridgingSourceFile(StringRef bridgingSourceFile) {
   }
 }
 
+void ModuleDependencyInfo::addBridgingHeaderIncludeTree(StringRef ID) {
+  switch (getKind()) {
+  case swift::ModuleDependencyKind::SwiftInterface: {
+    auto swiftInterfaceStorage =
+        cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
+    swiftInterfaceStorage->textualModuleDetails
+        .CASBridgingHeaderIncludeTreeRootID = ID.str();
+    break;
+  }
+  case swift::ModuleDependencyKind::SwiftSource: {
+    auto swiftSourceStorage =
+        cast<SwiftSourceModuleDependenciesStorage>(storage.get());
+    swiftSourceStorage->textualModuleDetails
+        .CASBridgingHeaderIncludeTreeRootID = ID.str();
+    break;
+  }
+  default:
+    llvm_unreachable("Unexpected dependency kind");
+  }
+}
+
 void ModuleDependencyInfo::addSourceFile(StringRef sourceFile) {
   switch (getKind()) {
   case swift::ModuleDependencyKind::SwiftSource: {
@@ -253,25 +382,97 @@ void ModuleDependencyInfo::addBridgingModuleDependency(
   }
 }
 
-SwiftDependencyScanningService::SwiftDependencyScanningService()
+SwiftDependencyScanningService::SwiftDependencyScanningService() {
-  : ClangScanningService(clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
+  ClangScanningService.emplace(
-                         clang::tooling::dependencies::ScanningOutputFormat::Full,
+      clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
-                         clang::CASOptions(),
+      clang::tooling::dependencies::ScanningOutputFormat::FullTree,
-                         /* CAS (llvm::cas::ObjectStore) */ nullptr,
+      clang::CASOptions(),
-                         /* Cache (llvm::cas::ActionCache) */ nullptr,
+      /* CAS (llvm::cas::ObjectStore) */ nullptr,
-                         /* SharedFS */ nullptr,
+      /* Cache (llvm::cas::ActionCache) */ nullptr,
-                         /* OptimizeArgs */ true) {
+      /* SharedFS */ nullptr,
-    SharedFilesystemCache.emplace();
+      /* OptimizeArgs */ true);
+  SharedFilesystemCache.emplace();
 }
 
-void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance) {
+void SwiftDependencyTracker::startTracking() {
- auto existingFS = Instance.getSourceMgr().getFileSystem();
+  FS->trackNewAccesses();
- llvm::IntrusiveRefCntPtr<
+
-     clang::tooling::dependencies::DependencyScanningWorkerFilesystem>
+  for (auto &file : Files)
-     depFS =
+    (void)FS->status(file);
-         new clang::tooling::dependencies::DependencyScanningWorkerFilesystem(
+}
-             getSharedFilesystemCache(), existingFS);
+
- Instance.getSourceMgr().setFileSystem(depFS);
+llvm::Expected<llvm::cas::ObjectProxy>
+SwiftDependencyTracker::createTreeFromDependencies() {
+  return FS->createTreeFromNewAccesses();
+}
+
+void SwiftDependencyScanningService::overlaySharedFilesystemCacheForCompilation(
+    CompilerInstance &Instance) {
+  auto existingFS = Instance.getSourceMgr().getFileSystem();
+  llvm::IntrusiveRefCntPtr<
+      clang::tooling::dependencies::DependencyScanningWorkerFilesystem>
+      depFS =
+          new clang::tooling::dependencies::DependencyScanningWorkerFilesystem(
+              getSharedFilesystemCache(), existingFS);
+  Instance.getSourceMgr().setFileSystem(depFS);
+}
+
+void SwiftDependencyScanningService::setupCachingDependencyScanningService(
+    CompilerInstance &Instance) {
+  if (!Instance.getInvocation().getFrontendOptions().EnableCAS)
+    return;
+
+  // Setup CAS.
+  CAS = Instance.getSharedCASInstance();
+
+  // Add SDKSetting file.
+  SmallString<256> SDKSettingPath;
+  llvm::sys::path::append(
+      SDKSettingPath,
+      Instance.getInvocation().getSearchPathOptions().getSDKPath(),
+      "SDKSettings.json");
+  CommonDependencyFiles.emplace_back(SDKSettingPath.data(),
+                                     SDKSettingPath.size());
+
+  // Add Legacy layout file (maybe just hard code instead of searching).
+  for (auto RuntimeLibPath :
+       Instance.getInvocation().getSearchPathOptions().RuntimeLibraryPaths) {
+    auto &FS = Instance.getFileSystem();
+    std::error_code EC;
+    for (auto F = FS.dir_begin(RuntimeLibPath, EC);
+         !EC && F != llvm::vfs::directory_iterator(); F.increment(EC)) {
+      if (F->path().endswith(".yaml"))
+        CommonDependencyFiles.emplace_back(F->path().str());
+    }
+  }
+
+  auto CachingFS =
+      llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
+  if (!CachingFS) {
+    Instance.getDiags().diagnose(SourceLoc(), diag::error_create_cas,
+                                 "CachingOnDiskFS",
+                                 toString(CachingFS.takeError()));
+    return;
+  }
+  CacheFS = std::move(*CachingFS);
+
+  clang::CASOptions CASOpts;
+  CASOpts.CASPath = Instance.getInvocation().getFrontendOptions().CASPath;
+  CASOpts.ensurePersistentCAS();
+
+  UseClangIncludeTree =
+      Instance.getInvocation().getClangImporterOptions().UseClangIncludeTree;
+  const clang::tooling::dependencies::ScanningOutputFormat ClangScanningFormat =
+      UseClangIncludeTree
+          ? clang::tooling::dependencies::ScanningOutputFormat::FullIncludeTree
+          : clang::tooling::dependencies::ScanningOutputFormat::FullTree;
+
+  ClangScanningService.emplace(
+      clang::tooling::dependencies::ScanningMode::DependencyDirectivesScan,
+      ClangScanningFormat, CASOpts, Instance.getSharedCASInstance(),
+      Instance.getSharedCacheInstance(),
+      UseClangIncludeTree ? nullptr : CacheFS,
+      /* ReuseFileManager */ false, /* OptimizeArgs */ false);
 }
 
 SwiftDependencyScanningService::ContextSpecificGlobalCacheState *
@@ -385,12 +586,12 @@ ModuleDependenciesCache::getDependencyReferencesMap(
 
 ModuleDependenciesCache::ModuleDependenciesCache(
     SwiftDependencyScanningService &globalScanningService,
-    std::string mainScanModuleName,
+    std::string mainScanModuleName, std::string scannerContextHash)
-    std::string scannerContextHash)
     : globalScanningService(globalScanningService),
       mainScanModuleName(mainScanModuleName),
       scannerContextHash(scannerContextHash),
-      clangScanningTool(globalScanningService.ClangScanningService) {
+      clangScanningTool(*globalScanningService.ClangScanningService,
+                        globalScanningService.getClangScanningFS()) {
   globalScanningService.configureForContextHash(scannerContextHash);
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind) {
@@ -399,8 +600,7 @@ ModuleDependenciesCache::ModuleDependenciesCache(
   }
 }
 
-Optional<const ModuleDependencyInfo*>
+Optional<const ModuleDependencyInfo *> ModuleDependenciesCache::findDependency(
-ModuleDependenciesCache::findDependency(
     StringRef moduleName, Optional<ModuleDependencyKind> kind) const {
   auto optionalDep = globalScanningService.findDependency(moduleName, kind,
                                                           scannerContextHash);

lib/AST/ModuleLoader.cpp

@@ -94,7 +94,9 @@ static bool findOverlayFilesInDirectory(ASTContext &ctx, StringRef path,
     callback(file);
   }
 
-  if (error && error != std::errc::no_such_file_or_directory) {
+  // A CAS file list returns operation not permitted on directory iterations.
+  if (error && error != std::errc::no_such_file_or_directory &&
+      error != std::errc::operation_not_permitted) {
     ctx.Diags.diagnose(diagLoc, diag::cannot_list_swiftcrossimport_dir,
                        moduleName, error.message(), path);
   }

lib/ClangImporter/ClangImporter.cpp

@@ -708,6 +708,15 @@ importer::getNormalInvocationArguments(
   invocationArgStrs.push_back((llvm::Twine(searchPathOpts.RuntimeResourcePath) +
                                llvm::sys::path::get_separator() +
                                "apinotes").str());
+
+  if (!importerOpts.CASPath.empty()) {
+    invocationArgStrs.push_back("-Xclang");
+    invocationArgStrs.push_back("-fcas-path");
+    invocationArgStrs.push_back("-Xclang");
+    invocationArgStrs.push_back(importerOpts.CASPath);
+    invocationArgStrs.push_back("-Xclang");
+    invocationArgStrs.push_back("-fno-pch-timestamp");
+  }
 }
 
 static void
@@ -1721,7 +1730,8 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() {
 
   auto &FrontendOpts = invocation->getFrontendOpts();
   FrontendOpts.DisableFree = false;
-  FrontendOpts.Inputs.clear();
+  if (FrontendOpts.CASIncludeTreeID.empty())
+    FrontendOpts.Inputs.clear();
 
   auto clonedInstance = std::make_unique<clang::CompilerInstance>(
     Impl.Instance->getPCHContainerOperations(),
@@ -1752,7 +1762,8 @@ bool ClangImporter::emitBridgingPCH(
   auto inputFile = clang::FrontendInputFile(headerPath, language);
 
   auto &FrontendOpts = invocation.getFrontendOpts();
-  FrontendOpts.Inputs = {inputFile};
+  if (invocation.getFrontendOpts().CASIncludeTreeID.empty())
+    FrontendOpts.Inputs = {inputFile};
   FrontendOpts.OutputFile = outputPCHPath.str();
   FrontendOpts.ProgramAction = clang::frontend::GeneratePCH;
 
@@ -1784,7 +1795,8 @@ bool ClangImporter::runPreprocessor(
   auto inputFile = clang::FrontendInputFile(inputPath, language);
 
   auto &FrontendOpts = invocation.getFrontendOpts();
-  FrontendOpts.Inputs = {inputFile};
+  if (invocation.getFrontendOpts().CASIncludeTreeID.empty())
+    FrontendOpts.Inputs = {inputFile};
   FrontendOpts.OutputFile = outputPath.str();
   FrontendOpts.ProgramAction = clang::frontend::PrintPreprocessedInput;
 
@@ -1807,11 +1819,13 @@ bool ClangImporter::emitPrecompiledModule(
   auto language = getLanguageFromOptions(LangOpts);
 
   auto &FrontendOpts = invocation.getFrontendOpts();
-  auto inputFile = clang::FrontendInputFile(
+  if (invocation.getFrontendOpts().CASIncludeTreeID.empty()) {
-      moduleMapPath, clang::InputKind(
+    auto inputFile = clang::FrontendInputFile(
-          language, clang::InputKind::ModuleMap, false),
+        moduleMapPath,
-      FrontendOpts.IsSystemModule);
+        clang::InputKind(language, clang::InputKind::ModuleMap, false),
-  FrontendOpts.Inputs = {inputFile};
+        FrontendOpts.IsSystemModule);
+    FrontendOpts.Inputs = {inputFile};
+  }
   FrontendOpts.OriginalModuleMap = moduleMapPath.str();
   FrontendOpts.OutputFile = outputPath.str();
   FrontendOpts.ProgramAction = clang::frontend::GenerateModule;
@@ -1839,7 +1853,8 @@ bool ClangImporter::dumpPrecompiledModule(
           clang::Language::Unknown, clang::InputKind::Precompiled, false));
 
   auto &FrontendOpts = invocation.getFrontendOpts();
-  FrontendOpts.Inputs = {inputFile};
+  if (invocation.getFrontendOpts().CASIncludeTreeID.empty())
+    FrontendOpts.Inputs = {inputFile};
   FrontendOpts.OutputFile = outputPath.str();
 
   auto action = std::make_unique<clang::DumpModuleInfoAction>();

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -18,11 +18,17 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/ClangImporter/ClangImporter.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 
 using namespace swift;
 
@@ -175,18 +181,86 @@ void ClangImporter::recordModuleDependencies(
     // Swift frontend option for input file path (Foo.modulemap).
     swiftArgs.push_back(clangModuleDep.ClangModuleMapFile);
 
+    // Handle VFSOverlay.
+    if (!ctx.SearchPathOpts.VFSOverlayFiles.empty()) {
+      for (auto &overlay : ctx.SearchPathOpts.VFSOverlayFiles) {
+        swiftArgs.push_back("-vfsoverlay");
+        swiftArgs.push_back(overlay);
+      }
+    }
+
     // Add args reported by the scanner.
-    llvm::for_each(clangModuleDep.BuildArguments, addClangArg);
+
+    // Round-trip clang args to canonicalize and clear the options that swift
+    // compiler doesn't need.
+    clang::CompilerInvocation depsInvocation;
+    clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(),
+                                        new clang::DiagnosticOptions(),
+                                        new clang::IgnoringDiagConsumer());
+
+    llvm::SmallVector<const char*> clangArgs;
+    llvm::for_each(clangModuleDep.BuildArguments, [&](const std::string &Arg) {
+      clangArgs.push_back(Arg.c_str());
+    });
+
+    bool success = clang::CompilerInvocation::CreateFromArgs(
+        depsInvocation, clangArgs, clangDiags);
+    (void)success;
+    assert(success && "clang option from dep scanner round trip failed");
+
+    // Clear the cache key for module. The module key is computed from clang
+    // invocation, not swift invocation.
+    depsInvocation.getFrontendOpts().ModuleCacheKeys.clear();
+
+    // FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
+    // from clang scanner and pass to swift.
+    for (auto overlay : depsInvocation.getHeaderSearchOpts().VFSOverlayFiles) {
+      if (llvm::is_contained(ctx.SearchPathOpts.VFSOverlayFiles, overlay))
+        continue;
+      swiftArgs.push_back("-vfsoverlay");
+      swiftArgs.push_back(overlay);
+    }
+
+    llvm::BumpPtrAllocator allocator;
+    llvm::StringSaver saver(allocator);
+    clangArgs.clear();
+    depsInvocation.generateCC1CommandLine(
+        clangArgs,
+        [&saver](const llvm::Twine &T) { return saver.save(T).data(); });
+
+    llvm::for_each(clangArgs, addClangArg);
+
+    // CASFileSystemRootID.
+    std::string RootID = clangModuleDep.CASFileSystemRootID
+                             ? clangModuleDep.CASFileSystemRootID->toString()
+                             : "";
+
+    std::string IncludeTree =
+        clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
+
+    if (!RootID.empty() || !IncludeTree.empty()) {
+      swiftArgs.push_back("-enable-cas");
+      swiftArgs.push_back("-cas-path");
+      swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
+    }
+
+    if (!RootID.empty()) {
+      swiftArgs.push_back("-cas-fs");
+      swiftArgs.push_back(RootID);
+    }
+
+    if (!IncludeTree.empty()) {
+      swiftArgs.push_back("-clang-include-tree");
+      swiftArgs.push_back("-clang-include-tree-root");
+      swiftArgs.push_back(IncludeTree);
+    }
 
     // Module-level dependencies.
     llvm::StringSet<> alreadyAddedModules;
     auto dependencies = ModuleDependencyInfo::forClangModule(
-        pcmPath,
+        pcmPath, clangModuleDep.ClangModuleMapFile,
-        clangModuleDep.ClangModuleMapFile,
+        clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs,
-        clangModuleDep.ID.ContextHash,
+        RootID, IncludeTree, /*module-cache-key*/ "");
-        swiftArgs,
-        fileDeps,
-        capturedPCMArgs);
     for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
       dependencies.addModuleImport(moduleName.ModuleName, &alreadyAddedModules);
       // It is safe to assume that all dependencies of a Clang module are Clang modules.
@@ -200,6 +274,85 @@ void ClangImporter::recordModuleDependencies(
   }
 }
 
+void ClangImporter::recordBridgingHeaderOptions(
+    ModuleDependencyInfo &MDI,
+    const clang::tooling::dependencies::TranslationUnitDeps &deps) {
+  auto &ctx = Impl.SwiftContext;
+
+  std::vector<std::string> swiftArgs;
+  auto addClangArg = [&](Twine arg) {
+    swiftArgs.push_back("-Xcc");
+    swiftArgs.push_back(arg.str());
+  };
+
+  // We are using Swift frontend mode.
+  swiftArgs.push_back("-frontend");
+
+  // Swift frontend action: -emit-pcm
+  swiftArgs.push_back("-emit-pch");
+
+  // We pass the entire argument list via -Xcc, so the invocation should
+  // use extra clang options alone.
+  swiftArgs.push_back("-only-use-extra-clang-opts");
+
+  // Ensure that the resulting PCM build invocation uses Clang frontend
+  // directly
+  swiftArgs.push_back("-direct-clang-cc1-module-build");
+
+  // Add args reported by the scanner.
+
+  // Round-trip clang args to canonicalize and clear the options that swift
+  // compiler doesn't need.
+  clang::CompilerInvocation depsInvocation;
+  clang::DiagnosticsEngine clangDiags(new clang::DiagnosticIDs(),
+                                      new clang::DiagnosticOptions(),
+                                      new clang::IgnoringDiagConsumer());
+
+  llvm::SmallVector<const char *> clangArgs;
+  llvm::for_each(deps.Commands[0].Arguments, [&](const std::string &Arg) {
+    clangArgs.push_back(Arg.c_str());
+  });
+
+  bool success = clang::CompilerInvocation::CreateFromArgs(
+      depsInvocation, clangArgs, clangDiags);
+  (void)success;
+  assert(success && "clang option from dep scanner round trip failed");
+
+  // Clear the cache key for module. The module key is computed from clang
+  // invocation, not swift invocation.
+  depsInvocation.getFrontendOpts().ProgramAction =
+      clang::frontend::ActionKind::GeneratePCH;
+  depsInvocation.getFrontendOpts().ModuleCacheKeys.clear();
+  depsInvocation.getFrontendOpts().OutputFile = "";
+
+  llvm::BumpPtrAllocator allocator;
+  llvm::StringSaver saver(allocator);
+  clangArgs.clear();
+  depsInvocation.generateCC1CommandLine(
+      clangArgs,
+      [&saver](const llvm::Twine &T) { return saver.save(T).data(); });
+
+  llvm::for_each(clangArgs, addClangArg);
+
+  if (!ctx.ClangImporterOpts.CASPath.empty()) {
+    swiftArgs.push_back("-enable-cas");
+    swiftArgs.push_back("-cas-path");
+    swiftArgs.push_back(ctx.ClangImporterOpts.CASPath);
+  }
+
+  if (auto Tree = deps.IncludeTreeID) {
+    swiftArgs.push_back("-clang-include-tree");
+    swiftArgs.push_back("-clang-include-tree-root");
+    swiftArgs.push_back(*Tree);
+  }
+  if (auto CASFS = deps.CASFileSystemRootID) {
+    swiftArgs.push_back("-cas-fs");
+    swiftArgs.push_back(*CASFS);
+  }
+
+  MDI.updateBridgingHeaderCommandLine(swiftArgs);
+}
+
 // The Swift compiler does not have a concept of a working directory.
 // It is instead handled by the Swift driver by resolving relative paths
 // according to the driver's notion of a working directory. On the other hand,
@@ -337,6 +490,11 @@ bool ClangImporter::addBridgingHeaderDependencies(
                                              alreadyAddedModules);
   }
 
+  if (auto TreeID = clangModuleDependencies->IncludeTreeID)
+    targetModule.addBridgingHeaderIncludeTree(*TreeID);
+
+  recordBridgingHeaderOptions(targetModule, *clangModuleDependencies);
+
   // Update the cache with the new information for the module.
   cache.updateDependency({moduleName.str(), moduleKind}, targetModule);
 

lib/DependencyScan/DependencyScanningTool.cpp

@@ -236,9 +236,6 @@ DependencyScanningTool::initCompilerInstanceForScan(
   auto Instance = std::make_unique<CompilerInstance>();
   Instance->addDiagnosticConsumer(&CDC);
 
-  // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
-  ScanningService->overlaySharedFilesystemCacheForCompilation(*Instance);
-
   // Basic error checking on the arguments
   if (CommandArgs.empty()) {
     Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args);
@@ -280,6 +277,10 @@ DependencyScanningTool::initCompilerInstanceForScan(
   if (Instance->setup(Invocation, InstanceSetupError)) {
     return std::make_error_code(std::errc::not_supported);
   }
+
+  // Setup the caching service after the instance finishes setup.
+  ScanningService->setupCachingDependencyScanningService(*Instance);
+
   (void)Instance->getMainModule();
 
   return Instance;

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -235,17 +235,19 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
         llvm::report_fatal_error(
             "Unexpected SWIFT_TEXTUAL_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
-      unsigned outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID,
+      unsigned outputPathFileID, interfaceFileID,
-          buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID,
+          compiledModuleCandidatesArrayID, buildCommandLineArrayID,
-          isFramework, bridgingHeaderFileID, sourceFilesArrayID,
+          extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID,
-          bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID,
+          sourceFilesArrayID, bridgingSourceFilesArrayID,
-          overlayDependencyIDArrayID;
+          bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID,
+          CASFileSystemRootID, bridgingHeaderIncludeTreeID, moduleCacheKeyID;
       SwiftInterfaceModuleDetailsLayout::readRecord(
-          Scratch, outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID,
+          Scratch, outputPathFileID, interfaceFileID,
-          buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID,
+          compiledModuleCandidatesArrayID, buildCommandLineArrayID,
-          isFramework, bridgingHeaderFileID, sourceFilesArrayID,
+          extraPCMArgsArrayID, contextHashID, isFramework, bridgingHeaderFileID,
-          bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID,
+          sourceFilesArrayID, bridgingSourceFilesArrayID,
-          overlayDependencyIDArrayID);
+          bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID,
+          CASFileSystemRootID, bridgingHeaderIncludeTreeID, moduleCacheKeyID);
 
       auto outputModulePath = getIdentifier(outputPathFileID);
       if (!outputModulePath)
@@ -278,11 +280,18 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       for (auto &arg : *extraPCMArgs)
         extraPCMRefs.push_back(arg);
 
+      auto rootFileSystemID = getIdentifier(CASFileSystemRootID);
+      if (!rootFileSystemID)
+        llvm::report_fatal_error("Bad CASFileSystem RootID");
+      auto moduleCacheKey = getIdentifier(moduleCacheKeyID);
+      if (!moduleCacheKeyID)
+        llvm::report_fatal_error("Bad moduleCacheKey");
+
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftInterfaceModule(
-          outputModulePath.value(),
+          outputModulePath.value(), optionalSwiftInterfaceFile.value(),
-          optionalSwiftInterfaceFile.value(), *compiledModuleCandidates,
+          *compiledModuleCandidates, buildCommandRefs, extraPCMRefs,
-          buildCommandRefs, extraPCMRefs, *contextHash, isFramework);
+          *contextHash, isFramework, *rootFileSystemID, *moduleCacheKey);
 
       // Add imports of this module
       for (const auto &moduleName : *currentModuleImports)
@@ -328,6 +337,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
         llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies");
       moduleDep.setOverlayDependencies(overlayModuleDependencyIDs.value());
 
+      // Add bridging header include tree
+      auto bridgingHeaderIncludeTree =
+          getIdentifier(bridgingHeaderIncludeTreeID);
+      if (!bridgingHeaderIncludeTree)
+        llvm::report_fatal_error("Bad bridging header include tree");
+      if (!bridgingHeaderIncludeTree->empty())
+        moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree);
+
       cache.recordDependency(currentModuleName, std::move(moduleDep),
                              getContextHash());
       hasCurrentModule = false;
@@ -345,11 +362,15 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
             "SWIFT_SOURCE_MODULE_DETAILS_NODE record");
       unsigned extraPCMArgsArrayID, bridgingHeaderFileID, sourceFilesArrayID,
           bridgingSourceFilesArrayID, bridgingModuleDependenciesArrayID,
-          overlayDependencyIDArrayID;
+          overlayDependencyIDArrayID, CASFileSystemRootID,
+          bridgingHeaderIncludeTreeID, buildCommandLineArrayID,
+          bridgingHeaderBuildCommandLineArrayID;
       SwiftSourceModuleDetailsLayout::readRecord(
           Scratch, extraPCMArgsArrayID, bridgingHeaderFileID,
           sourceFilesArrayID, bridgingSourceFilesArrayID,
-          bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID);
+          bridgingModuleDependenciesArrayID, overlayDependencyIDArrayID,
+          CASFileSystemRootID, bridgingHeaderIncludeTreeID,
+          buildCommandLineArrayID, bridgingHeaderBuildCommandLineArrayID);
 
       auto extraPCMArgs = getStringArray(extraPCMArgsArrayID);
       if (!extraPCMArgs)
@@ -358,8 +379,27 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       for (auto &arg : *extraPCMArgs)
         extraPCMRefs.push_back(arg);
 
+      auto rootFileSystemID = getIdentifier(CASFileSystemRootID);
+      if (!rootFileSystemID)
+        llvm::report_fatal_error("Bad CASFileSystem RootID");
+      auto commandLine = getStringArray(buildCommandLineArrayID);
+      if (!commandLine)
+        llvm::report_fatal_error("Bad command line");
+      std::vector<StringRef> buildCommandRefs;
+      for (auto &arg : *commandLine)
+        buildCommandRefs.push_back(arg);
+      std::vector<StringRef> bridgingHeaderBuildCommandRefs;
+      auto bridgingHeaderCommandLine =
+          getStringArray(bridgingHeaderBuildCommandLineArrayID);
+      if (!bridgingHeaderCommandLine)
+        llvm::report_fatal_error("Bad bridging header command line");
+      for (auto &arg : *bridgingHeaderCommandLine)
+        bridgingHeaderBuildCommandRefs.push_back(arg);
+
       // Form the dependencies storage object
-      auto moduleDep = ModuleDependencyInfo::forSwiftSourceModule(extraPCMRefs);
+      auto moduleDep = ModuleDependencyInfo::forSwiftSourceModule(
+          *rootFileSystemID, buildCommandRefs, bridgingHeaderBuildCommandRefs,
+          extraPCMRefs);
 
       // Add dependencies of this module
       for (const auto &moduleName : *currentModuleImports)
@@ -402,6 +442,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
         llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies");
       moduleDep.setOverlayDependencies(overlayModuleDependencyIDs.value());
 
+      // Add bridging header include tree
+      auto bridgingHeaderIncludeTree =
+          getIdentifier(bridgingHeaderIncludeTreeID);
+      if (!bridgingHeaderIncludeTree)
+        llvm::report_fatal_error("Bad bridging header include tree");
+      if (!bridgingHeaderIncludeTree->empty())
+        moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree);
+
       cache.recordDependency(currentModuleName, std::move(moduleDep),
                              getContextHash());
       hasCurrentModule = false;
@@ -414,10 +462,10 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
             "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
       unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
-          isFramework;
+          isFramework, moduleCacheKeyID;
       SwiftBinaryModuleDetailsLayout::readRecord(
           Scratch, compiledModulePathID, moduleDocPathID,
-          moduleSourceInfoPathID, isFramework);
+          moduleSourceInfoPathID, isFramework, moduleCacheKeyID);
 
       auto compiledModulePath = getIdentifier(compiledModulePathID);
       if (!compiledModulePath)
@@ -428,11 +476,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       auto moduleSourceInfoPath = getIdentifier(moduleSourceInfoPathID);
       if (!moduleSourceInfoPath)
         llvm::report_fatal_error("Bad module source info path");
+      auto moduleCacheKey = getIdentifier(moduleCacheKeyID);
+      if (!moduleCacheKeyID)
+        llvm::report_fatal_error("Bad moduleCacheKey");
 
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
           *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
-          isFramework);
+          isFramework, *moduleCacheKey);
       // Add dependencies of this module
       for (const auto &moduleName : *currentModuleImports)
         moduleDep.addModuleImport(moduleName);
@@ -481,11 +532,15 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
         llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
       unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID,
-               fileDependenciesArrayID, capturedPCMArgsArrayID;
+               fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID,
+               clangIncludeTreeRootID, moduleCacheKeyID;
       ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID,
                                            contextHashID, commandLineArrayID,
                                            fileDependenciesArrayID,
-                                           capturedPCMArgsArrayID);
+                                           capturedPCMArgsArrayID,
+                                           CASFileSystemRootID,
+                                           clangIncludeTreeRootID,
+                                           moduleCacheKeyID);
       auto pcmOutputPath = getIdentifier(pcmOutputPathID);
       if (!pcmOutputPath)
         llvm::report_fatal_error("Bad pcm output path");
@@ -504,11 +559,21 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       auto capturedPCMArgs = getStringArray(capturedPCMArgsArrayID);
       if (!capturedPCMArgs)
         llvm::report_fatal_error("Bad captured PCM Args");
+      auto rootFileSystemID = getIdentifier(CASFileSystemRootID);
+      if (!rootFileSystemID)
+        llvm::report_fatal_error("Bad CASFileSystem RootID");
+      auto clangIncludeTreeRoot = getIdentifier(clangIncludeTreeRootID);
+      if (!clangIncludeTreeRoot)
+        llvm::report_fatal_error("Bad clang include tree ID");
+      auto moduleCacheKey = getIdentifier(moduleCacheKeyID);
+      if (!moduleCacheKeyID)
+        llvm::report_fatal_error("Bad moduleCacheKey");
 
       // Form the dependencies storage object
-      auto moduleDep = ModuleDependencyInfo::forClangModule(*pcmOutputPath,
+      auto moduleDep = ModuleDependencyInfo::forClangModule(
-          *moduleMapPath, *contextHash, *commandLineArgs, *fileDependencies,
+          *pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs,
-          *capturedPCMArgs);
+          *fileDependencies, *capturedPCMArgs, *rootFileSystemID,
+          *clangIncludeTreeRoot, *moduleCacheKey);
 
       // Add dependencies of this module
       for (const auto &moduleName : *currentModuleImports)
@@ -647,6 +712,7 @@ enum ModuleIdentifierArrayKind : uint8_t {
   BridgingSourceFiles,
   BridgingModuleDependencies,
   SwiftOverlayDependencyIDs,
+  BridgingHeaderBuildCommandLine,
   NonPathCommandLine,
   FileDependencies,
   CapturedPCMArgs,
@@ -852,18 +918,23 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
             : 0;
     SwiftInterfaceModuleDetailsLayout::emitRecord(
         Out, ScratchRecord, AbbrCodes[SwiftInterfaceModuleDetailsLayout::Code],
-        outputModulePathFileId,
+        outputModulePathFileId, swiftInterfaceFileId,
-        swiftInterfaceFileId,
+        getArrayID(moduleID,
-        getArrayID(moduleID, ModuleIdentifierArrayKind::CompiledModuleCandidates),
+                   ModuleIdentifierArrayKind::CompiledModuleCandidates),
         getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
         getArrayID(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs),
-        getIdentifier(swiftTextDeps->contextHash),
+        getIdentifier(swiftTextDeps->contextHash), swiftTextDeps->isFramework,
-        swiftTextDeps->isFramework,
         bridgingHeaderFileId,
         getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
         getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies),
+        getArrayID(moduleID,
-        getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs));
+                   ModuleIdentifierArrayKind::BridgingModuleDependencies),
+        getArrayID(moduleID,
+                   ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs),
+        getIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID),
+        getIdentifier(swiftTextDeps->textualModuleDetails
+                          .CASBridgingHeaderIncludeTreeRootID),
+        getIdentifier(swiftTextDeps->moduleCacheKey));
     break;
   }
   case swift::ModuleDependencyKind::SwiftSource: {
@@ -882,8 +953,17 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
         bridgingHeaderFileId,
         getArrayID(moduleID, ModuleIdentifierArrayKind::SourceFiles),
         getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies),
+        getArrayID(moduleID,
-        getArrayID(moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs));
+                   ModuleIdentifierArrayKind::BridgingModuleDependencies),
+        getArrayID(moduleID,
+                   ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs),
+        getIdentifier(
+            swiftSourceDeps->textualModuleDetails.CASFileSystemRootID),
+        getIdentifier(swiftSourceDeps->textualModuleDetails
+                          .CASBridgingHeaderIncludeTreeRootID),
+        getArrayID(moduleID, ModuleIdentifierArrayKind::BuildCommandLine),
+        getArrayID(moduleID,
+                   ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine));
     break;
   }
   case swift::ModuleDependencyKind::SwiftBinary: {
@@ -895,7 +975,8 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
         getIdentifier(swiftBinDeps->compiledModulePath),
         getIdentifier(swiftBinDeps->moduleDocPath),
         getIdentifier(swiftBinDeps->sourceInfoPath),
-        swiftBinDeps->isFramework);
+        swiftBinDeps->isFramework,
+        getIdentifier(swiftBinDeps->moduleCacheKey));
 
     break;
   }
@@ -922,7 +1003,10 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
         getIdentifier(clangDeps->contextHash),
         getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine),
         getArrayID(moduleID, ModuleIdentifierArrayKind::FileDependencies),
-        getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs));
+        getArrayID(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs),
+        getIdentifier(clangDeps->CASFileSystemRootID),
+        getIdentifier(clangDeps->CASClangIncludeTreeRootID),
+        getIdentifier(clangDeps->moduleCacheKey));
 
     break;
   }
@@ -1035,26 +1119,30 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftTextDeps->moduleOutputPath);
         addIdentifier(swiftTextDeps->swiftInterfaceFile);
         addStringArray(moduleID,
-                       ModuleIdentifierArrayKind::CompiledModuleCandidates,
+                 ModuleIdentifierArrayKind::CompiledModuleCandidates,
-                       swiftTextDeps->compiledModuleCandidates);
+                 swiftTextDeps->compiledModuleCandidates);
         addStringArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine,
-                       swiftTextDeps->buildCommandLine);
+                 swiftTextDeps->textualModuleDetails.buildCommandLine);
         addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs,
-                       swiftTextDeps->textualModuleDetails.extraPCMArgs);
+                 swiftTextDeps->textualModuleDetails.extraPCMArgs);
         addIdentifier(swiftTextDeps->contextHash);
         if (swiftTextDeps->textualModuleDetails.bridgingHeaderFile.has_value())
-          addIdentifier(
+          addIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderFile
-              swiftTextDeps->textualModuleDetails.bridgingHeaderFile.value());
+                        .value());
         addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles,
-                       std::vector<std::string>());
+                 std::vector<std::string>());
         addStringArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles,
-                       swiftTextDeps->textualModuleDetails.bridgingSourceFiles);
+                 swiftTextDeps->textualModuleDetails.bridgingSourceFiles);
         addStringArray(
-            moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies,
+                 moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies,
-            swiftTextDeps->textualModuleDetails.bridgingModuleDependencies);
+                 swiftTextDeps->textualModuleDetails.bridgingModuleDependencies);
         addDependencyIDArray(
             moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs,
             swiftTextDeps->textualModuleDetails.swiftOverlayDependencies);
+        addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID);
+        addIdentifier(swiftTextDeps->textualModuleDetails
+                          .CASBridgingHeaderIncludeTreeRootID);
+        addIdentifier(swiftTextDeps->moduleCacheKey);
         break;
       }
       case swift::ModuleDependencyKind::SwiftBinary: {
@@ -1063,6 +1151,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->compiledModulePath);
         addIdentifier(swiftBinDeps->moduleDocPath);
         addIdentifier(swiftBinDeps->sourceInfoPath);
+        addIdentifier(swiftBinDeps->moduleCacheKey);
         break;
       }
       case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1093,6 +1182,14 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addDependencyIDArray(
             moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs,
             swiftSourceDeps->textualModuleDetails.swiftOverlayDependencies);
+        addStringArray(
+            moduleID, ModuleIdentifierArrayKind::BuildCommandLine,
+            swiftSourceDeps->textualModuleDetails.buildCommandLine);
+        addStringArray(
+            moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine,
+            swiftSourceDeps->bridgingHeaderBuildCommandLine);
+        addIdentifier(
+            swiftSourceDeps->textualModuleDetails.CASFileSystemRootID);
         break;
       }
       case swift::ModuleDependencyKind::Clang: {
@@ -1102,11 +1199,14 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(clangDeps->moduleMapFile);
         addIdentifier(clangDeps->contextHash);
         addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine,
-                       clangDeps->nonPathCommandLine);
+                       clangDeps->buildCommandLine);
         addStringArray(moduleID, ModuleIdentifierArrayKind::FileDependencies,
                        clangDeps->fileDependencies);
         addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs,
                        clangDeps->capturedPCMArgs);
+        addIdentifier(clangDeps->CASFileSystemRootID);
+        addIdentifier(clangDeps->CASClangIncludeTreeRootID);
+        addIdentifier(clangDeps->moduleCacheKey);
         break;
       }
       default:

lib/DependencyScan/ScanDependencies.cpp

@@ -12,14 +12,12 @@
 
 #include "swift/Basic/PrettyStackTrace.h"
 
-#include "swift/DependencyScan/ScanDependencies.h"
-#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h"
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticEngine.h"
+#include "swift/AST/DiagnosticsDriver.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/DiagnosticsSema.h"
-#include "swift/AST/DiagnosticsDriver.h"
 #include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/ModuleDependencies.h"
@@ -30,28 +28,38 @@
 #include "swift/Basic/STLExtras.h"
 #include "swift/ClangImporter/ClangImporter.h"
 #include "swift/DependencyScan/DependencyScanImpl.h"
+#include "swift/DependencyScan/ScanDependencies.h"
+#include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h"
 #include "swift/DependencyScan/StringUtils.h"
+#include "swift/Frontend/CachingUtils.h"
+#include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/FrontendOptions.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Strings.h"
 #include "clang/Basic/Module.h"
-#include "llvm/ADT/SetVector.h"
+#include "clang/Frontend/CompileJobCacheResult.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
-#include <set>
-#include <string>
-#include <sstream>
 #include <algorithm>
+#include <set>
+#include <sstream>
+#include <string>
 
 using namespace swift;
 using namespace swift::dependencies;
@@ -249,17 +257,56 @@ computeTransitiveClosureOfExplicitDependencies(
   return result;
 }
 
-static void
+static llvm::Expected<llvm::cas::ObjectRef>
-resolveExplicitModuleInputs(ModuleDependencyID moduleID,
+updateModuleCacheKey(ModuleDependencyInfo &depInfo,
-                            const ModuleDependencyInfo &resolvingDepInfo,
+                     llvm::cas::ObjectStore &CAS) {
-                            const std::set<ModuleDependencyID> &dependencies,
+  auto commandLine = depInfo.getCommandline();
-                            ModuleDependenciesCache &cache) {
+  std::vector<const char *> Args;
-  auto resolvingInterfaceDepDetails =
+  if (commandLine.size() > 1)
-      resolvingDepInfo.getAsSwiftInterfaceModule();
+    for (auto &c : ArrayRef<std::string>(commandLine).drop_front(1))
-  assert(resolvingInterfaceDepDetails &&
+      Args.push_back(c.c_str());
-         "Expected Swift Interface dependency.");
 
-  auto commandLine = resolvingInterfaceDepDetails->buildCommandLine;
+  auto base = createCompileJobBaseCacheKey(CAS, Args);
+  if (!base)
+    return base.takeError();
+
+  StringRef InputPath;
+  file_types::ID OutputType = file_types::ID::TY_INVALID;
+  if (auto *dep = depInfo.getAsClangModule()) {
+    OutputType = file_types::ID::TY_ClangModuleFile;
+    InputPath = dep->moduleMapFile;
+  } else if (auto *dep = depInfo.getAsSwiftInterfaceModule()) {
+    OutputType = file_types::ID::TY_SwiftModuleFile;
+    InputPath = dep->swiftInterfaceFile;
+  } else
+    llvm_unreachable("Unhandled dependency kind");
+
+  auto key =
+      createCompileJobCacheKeyForOutput(CAS, *base, InputPath, OutputType);
+  if (!key)
+    return key.takeError();
+
+  depInfo.updateModuleCacheKey(CAS.getID(*key).toString());
+  return *key;
+}
+
+static llvm::Error resolveExplicitModuleInputs(
+    ModuleDependencyID moduleID, const ModuleDependencyInfo &resolvingDepInfo,
+    const std::set<ModuleDependencyID> &dependencies,
+    ModuleDependenciesCache &cache, CompilerInstance &instance) {
+  // Only need to resolve dependency for following dependencies.
+  if (moduleID.second == ModuleDependencyKind::SwiftPlaceholder)
+    return llvm::Error::success();
+
+  std::vector<std::string> rootIDs;
+  if (auto ID = resolvingDepInfo.getCASFSRootID())
+    rootIDs.push_back(*ID);
+
+  std::vector<std::string> includeTrees;
+  if (auto ID = resolvingDepInfo.getClangIncludeTree())
+    includeTrees.push_back(*ID);
+
+  std::vector<std::string> commandLine = resolvingDepInfo.getCommandline();
   for (const auto &depModuleID : dependencies) {
     const auto optionalDepInfo =
         cache.findDependency(depModuleID.first, depModuleID.second);
@@ -269,14 +316,20 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID,
     case swift::ModuleDependencyKind::SwiftInterface: {
       auto interfaceDepDetails = depInfo->getAsSwiftInterfaceModule();
       assert(interfaceDepDetails && "Expected Swift Interface dependency.");
+      auto &path = interfaceDepDetails->moduleCacheKey.empty()
+                       ? interfaceDepDetails->moduleOutputPath
+                       : interfaceDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.first + "=" +
-                            interfaceDepDetails->moduleOutputPath);
+                            path);
     } break;
     case swift::ModuleDependencyKind::SwiftBinary: {
       auto binaryDepDetails = depInfo->getAsSwiftBinaryModule();
       assert(binaryDepDetails && "Expected Swift Binary Module dependency.");
+      auto &path = binaryDepDetails->moduleCacheKey.empty()
+                       ? binaryDepDetails->compiledModulePath
+                       : binaryDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.first + "=" +
-                            binaryDepDetails->compiledModulePath);
+                            path);
     } break;
     case swift::ModuleDependencyKind::SwiftPlaceholder: {
       auto placeholderDetails = depInfo->getAsPlaceholderDependencyModule();
@@ -287,13 +340,65 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID,
     case swift::ModuleDependencyKind::Clang: {
       auto clangDepDetails = depInfo->getAsClangModule();
       assert(clangDepDetails && "Expected Clang Module dependency.");
-      commandLine.push_back("-Xcc");
+      if (!resolvingDepInfo.isClangModule()) {
-      commandLine.push_back("-fmodule-file=" + depModuleID.first + "=" +
+        commandLine.push_back("-Xcc");
-                            clangDepDetails->pcmOutputPath);
+        commandLine.push_back("-fmodule-file=" + depModuleID.first + "=" +
-      commandLine.push_back("-Xcc");
+                              clangDepDetails->pcmOutputPath);
-      commandLine.push_back("-fmodule-map-file=" +
+        if (!instance.getInvocation()
-                            clangDepDetails->moduleMapFile);
+                 .getClangImporterOptions()
+                 .UseClangIncludeTree) {
+          commandLine.push_back("-Xcc");
+          commandLine.push_back("-fmodule-map-file=" +
+                                clangDepDetails->moduleMapFile);
+        }
+      }
+      if (!clangDepDetails->moduleCacheKey.empty()) {
+        auto appendXclang = [&]() {
+          if (!resolvingDepInfo.isClangModule()) {
+            // clang module build using cc1 arg so this is not needed.
+            commandLine.push_back("-Xcc");
+            commandLine.push_back("-Xclang");
+          }
+          commandLine.push_back("-Xcc");
+        };
+        appendXclang();
+        commandLine.push_back("-fmodule-file-cache-key");
+        appendXclang();
+        commandLine.push_back(clangDepDetails->pcmOutputPath);
+        appendXclang();
+        commandLine.push_back(clangDepDetails->moduleCacheKey);
+      }
+
+      // Only need to merge the CASFS from clang importer.
+      if (auto ID = depInfo->getCASFSRootID())
+        rootIDs.push_back(*ID);
+      if (auto ID = depInfo->getClangIncludeTree())
+        includeTrees.push_back(*ID);
     } break;
+    case swift::ModuleDependencyKind::SwiftSource: {
+      auto sourceDepDetails = depInfo->getAsSwiftSourceModule();
+      assert(sourceDepDetails && "Expected source dependency");
+      if (sourceDepDetails->textualModuleDetails
+              .CASBridgingHeaderIncludeTreeRootID.empty()) {
+        if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles
+                 .empty()) {
+          if (auto tracker =
+                  cache.getScanService().createSwiftDependencyTracker()) {
+            tracker->startTracking();
+            for (auto &file :
+                 sourceDepDetails->textualModuleDetails.bridgingSourceFiles)
+              tracker->trackFile(file);
+            auto bridgeRoot = tracker->createTreeFromDependencies();
+            if (!bridgeRoot)
+              return bridgeRoot.takeError();
+            rootIDs.push_back(bridgeRoot->getID().toString());
+          }
+        }
+      } else
+        includeTrees.push_back(sourceDepDetails->textualModuleDetails
+                                   .CASBridgingHeaderIncludeTreeRootID);
+      break;
+    }
     default:
       llvm_unreachable("Unhandled dependency kind.");
     }
@@ -301,8 +406,88 @@ resolveExplicitModuleInputs(ModuleDependencyID moduleID,
 
   // Update the dependency in the cache with the modified command-line.
   auto dependencyInfoCopy = resolvingDepInfo;
-  dependencyInfoCopy.updateCommandLine(commandLine);
+  if (resolvingDepInfo.isSwiftInterfaceModule() ||
+      resolvingDepInfo.isClangModule())
+    dependencyInfoCopy.updateCommandLine(commandLine);
+
+  // Handle CAS options.
+  if (instance.getInvocation().getFrontendOptions().EnableCAS) {
+    // Merge CASFS from clang dependency.
+    auto &CASFS = cache.getScanService().getSharedCachingFS();
+    auto &CAS = CASFS.getCAS();
+
+    // Update build command line.
+    if (resolvingDepInfo.isSwiftInterfaceModule() ||
+        resolvingDepInfo.isSwiftSourceModule()) {
+      // Update with casfs option.
+      std::vector<std::string> newCommandLine =
+          dependencyInfoCopy.getCommandline();
+      for (auto rootID : rootIDs) {
+        newCommandLine.push_back("-cas-fs");
+        newCommandLine.push_back(rootID);
+      }
+
+      for (auto tree : includeTrees) {
+        newCommandLine.push_back("-clang-include-tree-root");
+        newCommandLine.push_back(tree);
+      }
+      dependencyInfoCopy.updateCommandLine(newCommandLine);
+    }
+
+    if (auto *sourceDep = resolvingDepInfo.getAsSwiftSourceModule()) {
+      std::vector<std::string> newCommandLine =
+          dependencyInfoCopy.getBridgingHeaderCommandline();
+      for (auto bridgingDep :
+           sourceDep->textualModuleDetails.bridgingModuleDependencies) {
+        auto dep =
+            cache.findDependency(bridgingDep, ModuleDependencyKind::Clang);
+        assert(dep && "unknown clang dependency");
+        auto *clangDep = (*dep)->getAsClangModule();
+        assert(clangDep && "wrong module dependency kind");
+        if (!clangDep->moduleCacheKey.empty()) {
+          newCommandLine.push_back("-Xcc");
+          newCommandLine.push_back("-fmodule-file-cache-key");
+          newCommandLine.push_back("-Xcc");
+          newCommandLine.push_back(clangDep->pcmOutputPath);
+          newCommandLine.push_back("-Xcc");
+          newCommandLine.push_back(clangDep->moduleCacheKey);
+        }
+        dependencyInfoCopy.updateBridgingHeaderCommandLine(newCommandLine);
+      }
+    }
+
+    if (resolvingDepInfo.isClangModule() ||
+        resolvingDepInfo.isSwiftInterfaceModule()) {
+      // Compute and update module cache key.
+      auto Key = updateModuleCacheKey(dependencyInfoCopy, CAS);
+      if (!Key)
+        return Key.takeError();
+    }
+
+    // For binary module, we need to make sure the lookup key is setup here in
+    // action cache. We just use the CASID of the binary module itself as key.
+    if (auto *binaryDep = dependencyInfoCopy.getAsSwiftBinaryModule()) {
+      auto Ref =
+          CASFS.getObjectRefForFileContent(binaryDep->compiledModulePath);
+      if (!Ref)
+        return llvm::errorCodeToError(Ref.getError());
+      assert(*Ref && "Binary module should be loaded into CASFS already");
+      dependencyInfoCopy.updateModuleCacheKey(CAS.getID(**Ref).toString());
+
+      clang::cas::CompileJobCacheResult::Builder Builder;
+      Builder.addOutput(
+          clang::cas::CompileJobCacheResult::OutputKind::MainOutput, **Ref);
+      auto Result = Builder.build(CAS);
+      if (!Result)
+        return Result.takeError();
+      if (auto E = instance.getActionCache().put(CAS.getID(**Ref),
+                                                 CAS.getID(*Result)))
+        return E;
+    }
+  }
   cache.updateDependency(moduleID, dependencyInfoCopy);
+
+  return llvm::Error::success();
 }
 
 /// Resolve the direct dependencies of the given module.
@@ -363,11 +548,11 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID moduleI
     // A record of all of the Clang modules referenced from this Swift module.
     std::vector<std::string> allClangModules;
     llvm::StringSet<> knownModules;
+    auto clangImporter =
+        static_cast<ClangImporter *>(ctx.getClangModuleLoader());
 
     // If the Swift module has a bridging header, add those dependencies.
     if (knownDependencies->getBridgingHeader()) {
-      auto clangImporter =
-          static_cast<ClangImporter *>(ctx.getClangModuleLoader());
       if (!clangImporter->addBridgingHeaderDependencies(moduleID.first,
                                                         moduleID.second, cache)) {
         // Grab the updated module dependencies.
@@ -477,7 +662,8 @@ static void discoverCrossImportOverlayDependencies(
   // Construct a dummy main to resolve the newly discovered cross import
   // overlays.
   StringRef dummyMainName = "DummyMainModuleForResolvingCrossImportOverlays";
-  auto dummyMainDependencies = ModuleDependencyInfo::forSwiftSourceModule({});
+  auto dummyMainDependencies =
+      ModuleDependencyInfo::forSwiftSourceModule({}, {}, {}, {});
   std::for_each(newOverlays.begin(), newOverlays.end(),
                 [&](Identifier modName) {
                   dummyMainDependencies.addModuleImport(modName.str());
@@ -858,6 +1044,16 @@ static void writeJSON(llvm::raw_ostream &out,
       bool commaAfterFramework =
           swiftTextualDeps->extra_pcm_args->count != 0 || hasBridgingHeaderPath;
 
+      if (swiftTextualDeps->cas_fs_root_id.length != 0) {
+        writeJSONSingleField(out, "casFSRootID",
+                             swiftTextualDeps->cas_fs_root_id, 5,
+                             /*trailingComma=*/true);
+      }
+      if (swiftTextualDeps->module_cache_key.length != 0) {
+        writeJSONSingleField(out, "moduleCacheKey",
+                             swiftTextualDeps->module_cache_key, 5,
+                             /*trailingComma=*/true);
+      }
       writeJSONSingleField(out, "isFramework", swiftTextualDeps->is_framework,
                            5, commaAfterFramework);
       if (swiftTextualDeps->extra_pcm_args->count != 0) {
@@ -886,16 +1082,36 @@ static void writeJSON(llvm::raw_ostream &out,
         writeJSONSingleField(out, "sourceFiles",
                              swiftTextualDeps->bridging_source_files, 6,
                              /*trailingComma=*/true);
+        if (swiftTextualDeps->bridging_header_include_tree.length != 0) {
+          writeJSONSingleField(out, "includeTree",
+                               swiftTextualDeps->bridging_header_include_tree,
+                               6, /*trailingComma=*/true);
+        }
         writeJSONSingleField(out, "moduleDependencies",
                              swiftTextualDeps->bridging_module_dependencies, 6,
-                             /*trailingComma=*/false);
+                             /*trailingComma=*/true);
+        out.indent(6 * 2);
+        out << "\"commandLine\": [\n";
+        for (int i = 0,
+                 count = swiftTextualDeps->bridging_pch_command_line->count;
+             i < count; ++i) {
+          const auto &arg = get_C_string(
+              swiftTextualDeps->bridging_pch_command_line->strings[i]);
+          out.indent(7 * 2);
+          out << "\"" << quote(arg) << "\"";
+          if (i != count - 1)
+            out << ",";
+          out << "\n";
+        }
+        out.indent(6 * 2);
+        out << "]\n";
         out.indent(5 * 2);
         out << (hasOverlayDependencies ? "},\n" : "}\n");
       }
       if (hasOverlayDependencies) {
         writeDependencies(out, swiftTextualDeps->swift_overlay_module_dependencies,
                           "swiftOverlayDependencies", 5,
-                          /*trailingComma=*/true);
+                          /*trailingComma=*/false);
       }
     } else if (swiftPlaceholderDeps) {
       out << "\"swiftPlaceholder\": {\n";
@@ -942,6 +1158,11 @@ static void writeJSON(llvm::raw_ostream &out,
                              swiftBinaryDeps->module_source_info_path,
                              /*indentLevel=*/5,
                              /*trailingComma=*/true);
+      if (swiftBinaryDeps->module_cache_key.length != 0) {
+        writeJSONSingleField(out, "moduleCacheKey",
+                             swiftBinaryDeps->module_cache_key, 5,
+                             /*trailingComma=*/true);
+      }
       writeJSONSingleField(out, "isFramework", swiftBinaryDeps->is_framework,
                            5, /*trailingComma=*/false);
     } else {
@@ -959,9 +1180,22 @@ static void writeJSON(llvm::raw_ostream &out,
       writeJSONSingleField(out, "commandLine", clangDeps->command_line, 5,
                            /*trailingComma=*/true);
 
+      if (clangDeps->cas_fs_root_id.length != 0)
+        writeJSONSingleField(out, "casFSRootID", clangDeps->cas_fs_root_id, 5,
+                             /*trailingComma=*/true);
+      if (clangDeps->clang_include_tree.length != 0)
+        writeJSONSingleField(out, "clangIncludeTree",
+                             clangDeps->clang_include_tree, 5,
+                             /*trailingComma=*/true);
+      if (clangDeps->module_cache_key.length != 0)
+        writeJSONSingleField(out, "moduleCacheKey", clangDeps->module_cache_key,
+                             5,
+                             /*trailingComma=*/true);
+
       // Captured PCM arguments.
       writeJSONSingleField(out, "capturedPCMArgs", clangDeps->captured_pcm_args, 5,
                            /*trailingComma=*/false, /*nested=*/true);
+
     }
 
     out.indent(4 * 2);
@@ -1104,13 +1338,21 @@ generateFullDependencyGraph(CompilerInstance &instance,
             moduleInterfacePath,
             create_set(swiftTextualDeps->compiledModuleCandidates),
             bridgingHeaderPath,
-            create_set(swiftTextualDeps->textualModuleDetails.bridgingSourceFiles),
+            create_set(
-            create_set(swiftTextualDeps->textualModuleDetails.bridgingModuleDependencies),
+                swiftTextualDeps->textualModuleDetails.bridgingSourceFiles),
+            create_set(swiftTextualDeps->textualModuleDetails
+                           .bridgingModuleDependencies),
             create_set(bridgedOverlayDependencyNames),
-            create_set(swiftTextualDeps->buildCommandLine),
+            create_set(swiftTextualDeps->textualModuleDetails.buildCommandLine),
+            /*bridgingHeaderBuildCommand*/ create_set({}),
             create_set(swiftTextualDeps->textualModuleDetails.extraPCMArgs),
             create_clone(swiftTextualDeps->contextHash.c_str()),
-            swiftTextualDeps->isFramework};
+            swiftTextualDeps->isFramework,
+            create_clone(swiftTextualDeps->textualModuleDetails
+                             .CASFileSystemRootID.c_str()),
+            create_clone(swiftTextualDeps->textualModuleDetails
+                             .CASBridgingHeaderIncludeTreeRootID.c_str()),
+            create_clone(swiftTextualDeps->moduleCacheKey.c_str())};
       } else if (swiftSourceDeps) {
         swiftscan_string_ref_t moduleInterfacePath = create_null();
         swiftscan_string_ref_t bridgingHeaderPath =
@@ -1128,13 +1370,23 @@ generateFullDependencyGraph(CompilerInstance &instance,
             moduleInterfacePath,
             create_empty_set(),
             bridgingHeaderPath,
-            create_set(swiftSourceDeps->textualModuleDetails.bridgingSourceFiles),
+            create_set(
-            create_set(swiftSourceDeps->textualModuleDetails.bridgingModuleDependencies),
+                swiftSourceDeps->textualModuleDetails.bridgingSourceFiles),
+            create_set(swiftSourceDeps->textualModuleDetails
+                           .bridgingModuleDependencies),
             create_set(bridgedOverlayDependencyNames),
-            create_empty_set(),
+            create_set(swiftSourceDeps->textualModuleDetails.buildCommandLine),
+            create_set(swiftSourceDeps->bridgingHeaderBuildCommandLine),
             create_set(swiftSourceDeps->textualModuleDetails.extraPCMArgs),
-            /*contextHash*/create_null(),
+            /*contextHash*/ create_null(),
-            /*isFramework*/false};
+            /*isFramework*/ false,
+            /*CASFS*/
+            create_clone(swiftSourceDeps->textualModuleDetails
+                             .CASFileSystemRootID.c_str()),
+            /*IncludeTree*/
+            create_clone(swiftSourceDeps->textualModuleDetails
+                             .CASBridgingHeaderIncludeTreeRootID.c_str()),
+            /*CacheKey*/ create_clone("")};
       } else if (swiftPlaceholderDeps) {
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER;
         details->swift_placeholder_details = {
@@ -1147,16 +1399,19 @@ generateFullDependencyGraph(CompilerInstance &instance,
             create_clone(swiftBinaryDeps->compiledModulePath.c_str()),
             create_clone(swiftBinaryDeps->moduleDocPath.c_str()),
             create_clone(swiftBinaryDeps->sourceInfoPath.c_str()),
-            swiftBinaryDeps->isFramework};
+            swiftBinaryDeps->isFramework,
+            create_clone(swiftBinaryDeps->moduleCacheKey.c_str())};
       } else {
         // Clang module details
         details->kind = SWIFTSCAN_DEPENDENCY_INFO_CLANG;
         details->clang_details = {
             create_clone(clangDeps->moduleMapFile.c_str()),
             create_clone(clangDeps->contextHash.c_str()),
-            create_set(clangDeps->nonPathCommandLine),
+            create_set(clangDeps->buildCommandLine),
-            create_set(clangDeps->capturedPCMArgs)
+            create_set(clangDeps->capturedPCMArgs),
-        };
+            create_clone(clangDeps->CASFileSystemRootID.c_str()),
+            create_clone(clangDeps->CASClangIncludeTreeRootID.c_str()),
+            create_clone(clangDeps->moduleCacheKey.c_str())};
       }
       return details;
     };
@@ -1353,8 +1608,9 @@ forEachBatchEntry(CompilerInstance &invocationInstance,
   return false;
 }
 
-static ModuleDependencyInfo
+static ModuleDependencyInfo identifyMainModuleDependencies(
-identifyMainModuleDependencies(CompilerInstance &instance) {
+    CompilerInstance &instance,
+    Optional<SwiftDependencyTracker> tracker = None) {
   ModuleDecl *mainModule = instance.getMainModule();
   // Main module file name.
   auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
@@ -1375,7 +1631,22 @@ identifyMainModuleDependencies(CompilerInstance &instance) {
     ExtraPCMArgs.insert(ExtraPCMArgs.begin(),
                         {"-Xcc", "-target", "-Xcc",
                          instance.getASTContext().LangOpts.Target.str()});
-  auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(ExtraPCMArgs);
+
+  std::string rootID;
+  if (tracker) {
+    tracker->startTracking();
+    for (auto fileUnit : mainModule->getFiles()) {
+      auto sf = dyn_cast<SourceFile>(fileUnit);
+      if (!sf)
+        continue;
+      tracker->trackFile(sf->getFilename());
+    }
+    auto root = cantFail(tracker->createTreeFromDependencies());
+    rootID = root.getID().toString();
+  }
+
+  auto mainDependencies =
+      ModuleDependencyInfo::forSwiftSourceModule(rootID, {}, {}, ExtraPCMArgs);
 
   // Compute Implicit dependencies of the main module
   {
@@ -1475,6 +1746,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) {
     deserializeDependencyCache(instance, service);
   // Wrap the filesystem with a caching `DependencyScanningWorkerFilesystem`
   service.overlaySharedFilesystemCacheForCompilation(instance);
+  service.setupCachingDependencyScanningService(instance);
   ModuleDependenciesCache cache(service,
                                 instance.getMainModule()->getNameStr().str(),
                                 instance.getInvocation().getModuleScanningHash());
@@ -1542,6 +1814,7 @@ bool swift::dependencies::batchScanDependencies(
 
   SwiftDependencyScanningService singleUseService;
   singleUseService.overlaySharedFilesystemCacheForCompilation(instance);
+  singleUseService.setupCachingDependencyScanningService(instance);
   ModuleDependenciesCache cache(singleUseService,
                                 instance.getMainModule()->getNameStr().str(),
                                 instance.getInvocation().getModuleScanningHash());
@@ -1630,7 +1903,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
                                        ModuleDependenciesCache &cache) {
   ModuleDecl *mainModule = instance.getMainModule();
   // First, identify the dependencies of the main module
-  auto mainDependencies = identifyMainModuleDependencies(instance);
+  auto mainDependencies = identifyMainModuleDependencies(
+      instance, cache.getScanService().createSwiftDependencyTracker());
   auto &ctx = instance.getASTContext();
 
   // Add the main module.
@@ -1702,17 +1976,18 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   auto moduleTransitiveClosures =
       computeTransitiveClosureOfExplicitDependencies(topoSortedModuleList,
                                                      cache);
-  for (const auto &dependencyClosure : moduleTransitiveClosures) {
+  for (const auto &modID : llvm::reverse(topoSortedModuleList)) {
-    auto &modID = dependencyClosure.first;
+    auto dependencyClosure = moduleTransitiveClosures[modID];
     // For main module or binary modules, no command-line to resolve.
     // For Clang modules, their dependencies are resolved by the clang Scanner
     // itself for us.
-    if (modID.second != ModuleDependencyKind::SwiftInterface)
-      continue;
     auto optionalDeps = cache.findDependency(modID.first, modID.second);
     assert(optionalDeps.has_value());
     auto deps = optionalDeps.value();
-    resolveExplicitModuleInputs(modID, *deps, dependencyClosure.second, cache);
+    if (auto E = resolveExplicitModuleInputs(modID, *deps, dependencyClosure,
+                                             cache, instance))
+      instance.getDiags().diagnose(SourceLoc(), diag::error_cas,
+                                   toString(std::move(E)));
   }
 
   auto dependencyGraph = generateFullDependencyGraph(

lib/Frontend/ArgsToFrontendOptionsConverter.cpp

@@ -354,8 +354,11 @@ bool ArgsToFrontendOptionsConverter::convert(
   Opts.EnableCAS = Args.hasArg(OPT_enable_cas);
   Opts.CASPath =
       Args.getLastArgValue(OPT_cas_path, llvm::cas::getDefaultOnDiskCASPath());
-  Opts.CASFSRootID = Args.getLastArgValue(OPT_cas_fs);
+  Opts.CASFSRootIDs = Args.getAllArgValues(OPT_cas_fs);
-  if (Opts.EnableCAS && Opts.CASFSRootID.empty() &&
+  Opts.ClangIncludeTrees = Args.getAllArgValues(OPT_clang_include_tree_root);
+
+  if (Opts.EnableCAS && Opts.CASFSRootIDs.empty() &&
+      Opts.ClangIncludeTrees.empty() &&
       FrontendOptions::supportCompilationCaching(Opts.RequestedAction)) {
     if (!Args.hasArg(OPT_allow_unstable_cache_key_for_testing)) {
         Diags.diagnose(SourceLoc(), diag::error_caching_no_cas_fs);

lib/Frontend/CachingUtils.cpp

@@ -15,13 +15,19 @@
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/FileTypes.h"
 #include "swift/Frontend/CompileJobCacheKey.h"
+#include "clang/CAS/IncludeTree.h"
 #include "clang/Frontend/CompileJobCacheResult.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
+#include "llvm/CAS/CASFileSystem.h"
+#include "llvm/CAS/HierarchicalTreeBuilder.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/CAS/TreeEntry.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackends.h"
 #include "llvm/Support/VirtualOutputFile.h"
 #include <memory>
@@ -342,4 +348,121 @@ Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
   return Error::success();
 }
 
+static llvm::Error createCASObjectNotFoundError(const llvm::cas::CASID &ID) {
+  return createStringError(llvm::inconvertibleErrorCode(),
+                           "CASID missing from Object Store " + ID.toString());
+}
+
+static Expected<ObjectRef> mergeCASFileSystem(ObjectStore &CAS,
+                                              ArrayRef<std::string> FSRoots) {
+  llvm::cas::HierarchicalTreeBuilder Builder;
+  for (auto &Root : FSRoots) {
+    auto ID = CAS.parseID(Root);
+    if (!ID)
+      return ID.takeError();
+
+    auto Ref = CAS.getReference(*ID);
+    if (!Ref)
+      return createCASObjectNotFoundError(*ID);
+    Builder.pushTreeContent(*Ref, "");
+  }
+
+  auto NewRoot = Builder.create(CAS);
+  if (!NewRoot)
+    return NewRoot.takeError();
+
+  return NewRoot->getRef();
+}
+
+Expected<IntrusiveRefCntPtr<vfs::FileSystem>>
+createCASFileSystem(ObjectStore &CAS, ArrayRef<std::string> FSRoots,
+                    ArrayRef<std::string> IncludeTrees) {
+  assert(!FSRoots.empty() || !IncludeTrees.empty() && "no root ID provided");
+  if (FSRoots.size() == 1 && IncludeTrees.empty()) {
+    auto ID = CAS.parseID(FSRoots.front());
+    if (!ID)
+      return ID.takeError();
+    auto Ref = CAS.getReference(*ID);
+    if (!Ref)
+      return createCASObjectNotFoundError(*ID);
+  }
+
+  auto NewRoot = mergeCASFileSystem(CAS, FSRoots);
+  if (!NewRoot)
+    return NewRoot.takeError();
+
+  auto FS = createCASFileSystem(CAS, CAS.getID(*NewRoot));
+  if (!FS)
+    return FS.takeError();
+
+  auto CASFS = makeIntrusiveRefCnt<vfs::OverlayFileSystem>(std::move(*FS));
+  // Push all Include File System onto overlay.
+  for (auto &Tree : IncludeTrees) {
+    auto ID = CAS.parseID(Tree);
+    if (!ID)
+      return ID.takeError();
+
+    auto Ref = CAS.getReference(*ID);
+    if (!Ref)
+      return createCASObjectNotFoundError(*ID);
+    auto IT = clang::cas::IncludeTreeRoot::get(CAS, *Ref);
+    if (!IT)
+      return IT.takeError();
+
+    auto ITFS = clang::cas::createIncludeTreeFileSystem(*IT);
+    if (!ITFS)
+      return ITFS.takeError();
+    CASFS->pushOverlay(std::move(*ITFS));
+  }
+
+  return CASFS;
+}
+
+namespace cas {
+
+CachingTool::CachingTool(StringRef Path) {
+  auto DB = llvm::cas::createOnDiskUnifiedCASDatabases(Path);
+  if (!DB) {
+    llvm::errs() << "Failed to create CAS at " << Path << ": "
+                 << toString(DB.takeError()) << "\n";
+    return;
+  }
+
+  CAS = std::move(DB->first);
+  Cache = std::move(DB->second);
+}
+
+std::string CachingTool::computeCacheKey(ArrayRef<const char *> Args,
+                                         StringRef InputPath,
+                                         file_types::ID OutputKind) {
+  auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args);
+  if (!BaseKey) {
+    llvm::errs() << "Failed to create cache key: "
+                 << toString(BaseKey.takeError()) << "\n";
+    return "";
+  }
+
+  auto Key =
+      createCompileJobCacheKeyForOutput(*CAS, *BaseKey, InputPath, OutputKind);
+  if (!Key) {
+    llvm::errs() << "Failed to create cache key: " << toString(Key.takeError())
+                 << "\n";
+    return "";
+  }
+
+  return CAS->getID(*Key).toString();
+}
+
+std::string CachingTool::storeContent(StringRef Content) {
+  auto Result = CAS->storeFromString({}, Content);
+  if (!Result) {
+    llvm::errs() << "Failed to store to CAS: " << toString(Result.takeError())
+                 << "\n";
+    return "";
+  }
+
+  return CAS->getID(*Result).toString();
+}
+
+} // namespace cas
 } // namespace swift

lib/Frontend/CompilerInvocation.cpp

@@ -1438,12 +1438,15 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
       Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val);
   }
 
-  if (!workingDirectory.empty()) {
+  if (FrontendOpts.CASFSRootIDs.empty() &&
-    // Provide a working directory to Clang as well if there are any -Xcc
+      FrontendOpts.ClangIncludeTrees.empty()) {
-    // options, in case some of them are search-related. But do it at the
+    if (!workingDirectory.empty()) {
-    // beginning, so that an explicit -Xcc -working-directory will win.
+      // Provide a working directory to Clang as well if there are any -Xcc
-    Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(),
+      // options, in case some of them are search-related. But do it at the
-                          {"-working-directory", workingDirectory.str()});
+      // beginning, so that an explicit -Xcc -working-directory will win.
+      Opts.ExtraArgs.insert(Opts.ExtraArgs.begin(),
+                            {"-working-directory", workingDirectory.str()});
+    }
   }
 
   Opts.DumpClangDiagnostics |= Args.hasArg(OPT_dump_clang_diagnostics);
@@ -1463,6 +1466,8 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
 
   if (auto *A = Args.getLastArg(OPT_import_objc_header))
     Opts.BridgingHeader = A->getValue();
+  Opts.BridgingHeaderPCHCacheKey =
+      Args.getLastArgValue(OPT_bridging_header_pch_key);
   Opts.DisableSwiftBridgeAttr |= Args.hasArg(OPT_disable_swift_bridge_attr);
 
   Opts.DisableOverlayModules |= Args.hasArg(OPT_emit_imported_modules);
@@ -1473,6 +1478,7 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
 
   Opts.ExtraArgsOnly |= Args.hasArg(OPT_extra_clang_options_only);
   Opts.DirectClangCC1ModuleBuild |= Args.hasArg(OPT_direct_clang_cc1_module_build);
+  Opts.UseClangIncludeTree |= Args.hasArg(OPT_clang_include_tree);
 
   if (const Arg *A = Args.getLastArg(OPT_pch_output_dir)) {
     Opts.PrecompiledHeaderOutputDir = A->getValue();

lib/Frontend/Frontend.cpp

@@ -439,22 +439,6 @@ bool CompilerInstance::setupCASIfNeeded(ArrayRef<const char *> Args) {
   ResultCache = std::move(MaybeCache->second);
 
   // create baseline key.
-  llvm::Optional<llvm::cas::ObjectRef> FSRef;
-  if (!Opts.CASFSRootID.empty()) {
-    auto CASFSID = CAS->parseID(Opts.CASFSRootID);
-    if (!CASFSID) {
-      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
-                           toString(CASFSID.takeError()));
-      return true;
-    }
-    FSRef = CAS->getReference(*CASFSID);
-    if (!FSRef) {
-      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
-                           "-cas-fs value does not exist in CAS");
-      return true;
-    }
-  }
-
   auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args);
   if (!BaseKey) {
     Diagnostics.diagnose(SourceLoc(), diag::error_cas,
@@ -561,25 +545,39 @@ bool CompilerInstance::setup(const CompilerInvocation &Invoke,
 
 bool CompilerInstance::setUpVirtualFileSystemOverlays() {
   if (Invocation.getFrontendOptions().EnableCAS &&
-      !Invocation.getFrontendOptions().CASFSRootID.empty()) {
+      (!Invocation.getFrontendOptions().CASFSRootIDs.empty() ||
+       !Invocation.getFrontendOptions().ClangIncludeTrees.empty())) {
     // Set up CASFS as BaseFS.
-    auto RootID = CAS->parseID(Invocation.getFrontendOptions().CASFSRootID);
+    const auto &Opts = getInvocation().getFrontendOptions();
-    if (!RootID) {
+    auto FS =
-      Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id,
+        createCASFileSystem(*CAS, Opts.CASFSRootIDs, Opts.ClangIncludeTrees);
-                           Invocation.getFrontendOptions().CASFSRootID,
-                           toString(RootID.takeError()));
-      return true;
-    }
-    auto FS = llvm::cas::createCASFileSystem(*CAS, *RootID);
     if (!FS) {
-      Diagnostics.diagnose(SourceLoc(), diag::error_invalid_cas_id,
+      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
-                           Invocation.getFrontendOptions().CASFSRootID,
                            toString(FS.takeError()));
       return true;
     }
     SourceMgr.setFileSystem(std::move(*FS));
   }
 
+  // If we have a bridging header cache key, try load it now and overlay it.
+  if (!Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey.empty() &&
+      Invocation.getFrontendOptions().EnableCAS) {
+    auto loadedBridgingBuffer = loadCachedCompileResultFromCacheKey(
+        getObjectStore(), getActionCache(), Diagnostics,
+        Invocation.getClangImporterOptions().BridgingHeaderPCHCacheKey,
+        Invocation.getClangImporterOptions().BridgingHeader);
+    if (loadedBridgingBuffer) {
+      llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS =
+          new llvm::vfs::InMemoryFileSystem();
+      PCHFS->addFile(Invocation.getClangImporterOptions().BridgingHeader, 0,
+                     std::move(loadedBridgingBuffer));
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayVFS =
+          new llvm::vfs::OverlayFileSystem(SourceMgr.getFileSystem());
+      OverlayVFS->pushOverlay(PCHFS);
+      SourceMgr.setFileSystem(std::move(OverlayVFS));
+    }
+  }
+
   auto ExpectedOverlay =
       Invocation.getSearchPathOptions().makeOverlayFileSystem(
           SourceMgr.getFileSystem());
@@ -701,17 +699,24 @@ bool CompilerInstance::setUpModuleLoaders() {
   // If using `-explicit-swift-module-map-file`, create the explicit loader
   // before creating `ClangImporter` because the entries in the map influence
   // the Clang flags. The loader is added to the context below.
-  std::unique_ptr<ExplicitSwiftModuleLoader> ESML = nullptr;
+  std::unique_ptr<SerializedModuleLoaderBase> ESML = nullptr;
   bool ExplicitModuleBuild =
       Invocation.getFrontendOptions().DisableImplicitModules;
   if (ExplicitModuleBuild ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleMap.empty() ||
       !Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs.empty()) {
-    ESML = ExplicitSwiftModuleLoader::create(
+    if (Invocation.getFrontendOptions().EnableCAS)
-        *Context, getDependencyTracker(), MLM,
+      ESML = ExplicitCASModuleLoader::create(
-        Invocation.getSearchPathOptions().ExplicitSwiftModuleMap,
+          *Context, getObjectStore(), getActionCache(), getDependencyTracker(),
-        Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs,
+          MLM, Invocation.getSearchPathOptions().ExplicitSwiftModuleMap,
-        IgnoreSourceInfoFile);
+          Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs,
+          IgnoreSourceInfoFile);
+    else
+      ESML = ExplicitSwiftModuleLoader::create(
+          *Context, getDependencyTracker(), MLM,
+          Invocation.getSearchPathOptions().ExplicitSwiftModuleMap,
+          Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs,
+          IgnoreSourceInfoFile);
   }
 
   // Wire up the Clang importer. If the user has specified an SDK, use it.
@@ -839,6 +844,12 @@ std::string CompilerInstance::getBridgingHeaderPath() const {
 }
 
 bool CompilerInstance::setUpInputs() {
+  // There is no input file when building PCM using ClangIncludeTree.
+  if (Invocation.getFrontendOptions().RequestedAction ==
+          FrontendOptions::ActionType::EmitPCM &&
+      Invocation.getClangImporterOptions().UseClangIncludeTree)
+    return false;
+
   // Adds to InputSourceCodeBufferIDs, so may need to happen before the
   // per-input setup.
   const Optional<unsigned> ideInspectionTargetBufferID =

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -20,6 +20,7 @@
 #include "swift/AST/FileSystem.h"
 #include "swift/AST/Module.h"
 #include "swift/Basic/Platform.h"
+#include "swift/Frontend/CachingUtils.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Frontend/ModuleInterfaceSupport.h"
 #include "swift/Parse/ParseVersion.h"
@@ -28,6 +29,7 @@
 #include "swift/Serialization/Validation.h"
 #include "swift/Strings.h"
 #include "clang/Basic/Module.h"
+#include "clang/Frontend/CompileJobCacheResult.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
@@ -36,9 +38,12 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include "llvm/Support/YAMLParser.h"
@@ -1572,6 +1577,20 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
     GenericArgs.push_back("-clang-build-session-file");
     GenericArgs.push_back(clangImporterOpts.BuildSessionFilePath);
   }
+
+  if (!clangImporterOpts.CASPath.empty()) {
+    genericSubInvocation.getClangImporterOptions().CASPath =
+        clangImporterOpts.CASPath;
+    GenericArgs.push_back("-enable-cas");
+    GenericArgs.push_back("-cas-path");
+    GenericArgs.push_back(clangImporterOpts.CASPath);
+  }
+
+  if (clangImporterOpts.UseClangIncludeTree) {
+    genericSubInvocation.getClangImporterOptions().UseClangIncludeTree =
+        clangImporterOpts.UseClangIncludeTree;
+    GenericArgs.push_back("-clang-include-tree");
+  }
 }
 
 bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
@@ -1661,6 +1680,7 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
   // required by sourcekitd.
   subClangImporterOpts.DetailedPreprocessingRecord =
     clangImporterOpts.DetailedPreprocessingRecord;
+  subClangImporterOpts.CASPath = clangImporterOpts.CASPath;
 
   // If the compiler has been asked to be strict with ensuring downstream dependencies
   // get the parent invocation's context, or this is an Explicit build, inherit the
@@ -1941,15 +1961,22 @@ struct ExplicitSwiftModuleLoader::Implementation {
   void parseSwiftExplicitModuleMap(StringRef fileName) {
     ExplicitModuleMapParser parser(Allocator);
     llvm::StringMap<ExplicitClangModuleInputInfo> ExplicitClangModuleMap;
-    auto result =
+    // Load the input file.
-        parser.parseSwiftExplicitModuleMap(fileName, ExplicitModuleMap,
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
-                                           ExplicitClangModuleMap);
+        llvm::MemoryBuffer::getFile(fileName);
-    if (result == std::errc::invalid_argument)
+    if (!fileBufOrErr) {
-      Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted,
-                         fileName);
-    else if (result == std::errc::no_such_file_or_directory)
       Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing,
                          fileName);
+      return;
+    }
+
+    auto hasError = parser.parseSwiftExplicitModuleMap(
+        (*fileBufOrErr)->getMemBufferRef(), ExplicitModuleMap,
+        ExplicitClangModuleMap);
+
+    if (hasError)
+      Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted,
+                         fileName);
 
     // A single module map can define multiple modules; keep track of the ones
     // we've seen so that we don't generate duplicate flags.
@@ -2154,3 +2181,333 @@ ExplicitSwiftModuleLoader::create(ASTContext &ctx,
 
   return result;
 }
+
+struct ExplicitCASModuleLoader::Implementation {
+  ASTContext &Ctx;
+  llvm::BumpPtrAllocator Allocator;
+  llvm::cas::ObjectStore &CAS;
+  llvm::cas::ActionCache &Cache;
+
+  llvm::StringMap<ExplicitSwiftModuleInputInfo> ExplicitModuleMap;
+
+  Implementation(ASTContext &Ctx, llvm::cas::ObjectStore &CAS,
+                 llvm::cas::ActionCache &Cache)
+      : Ctx(Ctx), CAS(CAS), Cache(Cache) {}
+
+  llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> loadBuffer(StringRef ID) {
+    auto key = CAS.parseID(ID);
+    if (!key)
+      return key.takeError();
+
+    auto ref = CAS.getReference(*key);
+    if (!ref)
+      return nullptr;
+
+    auto loaded = CAS.getProxy(*ref);
+    if (!loaded)
+      return loaded.takeError();
+
+    return loaded->getMemoryBuffer();
+  }
+
+  // Same as the regular explicit module map but must be loaded from
+  // CAS, instead of a file that is not tracked by the dependency.
+  void parseSwiftExplicitModuleMap(StringRef ID) {
+    ExplicitModuleMapParser parser(Allocator);
+    llvm::StringMap<ExplicitClangModuleInputInfo> ExplicitClangModuleMap;
+    auto buf = loadBuffer(ID);
+    if (!buf) {
+      Ctx.Diags.diagnose(SourceLoc(), diag::error_cas,
+                         toString(buf.takeError()));
+      return;
+    }
+    if (!*buf) {
+      Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing,
+                         ID);
+      return;
+    }
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
+        llvm::MemoryBuffer::getFile(ID);
+
+    auto hasError = parser.parseSwiftExplicitModuleMap(
+        (*buf)->getMemBufferRef(), ExplicitModuleMap, ExplicitClangModuleMap);
+
+    if (hasError)
+      Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted,
+                         ID);
+
+    std::set<std::string> moduleMapsSeen;
+    std::vector<std::string> &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs;
+    for (auto &entry : ExplicitClangModuleMap) {
+      const auto &moduleMapPath = entry.getValue().moduleMapPath;
+      if (!moduleMapPath.empty() &&
+          !Ctx.ClangImporterOpts.UseClangIncludeTree &&
+          moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) {
+        moduleMapsSeen.insert(moduleMapPath);
+        extraClangArgs.push_back(
+            (Twine("-fmodule-map-file=") + moduleMapPath).str());
+      }
+
+      const auto &modulePath = entry.getValue().modulePath;
+      if (!modulePath.empty()) {
+        extraClangArgs.push_back(
+            (Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath)
+                .str());
+      }
+      auto cachePath = entry.getValue().moduleCacheKey;
+      if (cachePath) {
+        extraClangArgs.push_back("-Xclang");
+        extraClangArgs.push_back("-fmodule-file-cache-key");
+        extraClangArgs.push_back("-Xclang");
+        extraClangArgs.push_back(modulePath);
+        extraClangArgs.push_back("-Xclang");
+        extraClangArgs.push_back(*cachePath);
+      }
+    }
+  }
+
+  void addCommandLineExplicitInputs(
+      const std::vector<std::pair<std::string, std::string>>
+          &commandLineExplicitInputs) {
+    for (const auto &moduleInput : commandLineExplicitInputs) {
+      ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {});
+      ExplicitModuleMap.try_emplace(moduleInput.first, std::move(entry));
+    }
+  }
+
+  llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+  loadFileBuffer(StringRef ID, StringRef Name) {
+    auto key = CAS.parseID(ID);
+    if (!key)
+      return key.takeError();
+
+    auto moduleLookup = Cache.get(*key);
+    if (!moduleLookup)
+      return moduleLookup.takeError();
+    if (!*moduleLookup)
+      return nullptr;
+
+    auto moduleRef = CAS.getReference(**moduleLookup);
+    if (!moduleRef)
+      return nullptr;
+
+    clang::cas::CompileJobResultSchema schema(CAS);
+    auto result = schema.load(*moduleRef);
+    if (!result)
+      return result.takeError();
+    auto output = result->getOutput(
+        clang::cas::CompileJobCacheResult::OutputKind::MainOutput);
+    if (!output)
+      return nullptr;
+
+    auto buf = CAS.getProxy(output->Object);
+    if (!buf)
+      return buf.takeError();
+
+    return buf->getMemoryBuffer(Name);
+  }
+
+  llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
+  loadModuleFromPath(StringRef Path, DiagnosticEngine &Diags) {
+    for (auto &Deps : ExplicitModuleMap) {
+      if (Deps.second.modulePath == Path) {
+        if (!Deps.second.moduleCacheKey)
+          return nullptr;
+        return loadCachedCompileResultFromCacheKey(
+            CAS, Cache, Diags, *Deps.second.moduleCacheKey, Path);
+      }
+    }
+    return nullptr;
+  }
+};
+
+ExplicitCASModuleLoader::ExplicitCASModuleLoader(ASTContext &ctx,
+                                                 llvm::cas::ObjectStore &CAS,
+                                                 llvm::cas::ActionCache &cache,
+                                                 DependencyTracker *tracker,
+                                                 ModuleLoadingMode loadMode,
+                                                 bool IgnoreSwiftSourceInfoFile)
+    : SerializedModuleLoaderBase(ctx, tracker, loadMode,
+                                 IgnoreSwiftSourceInfoFile),
+      Impl(*new Implementation(ctx, CAS, cache)) {}
+
+ExplicitCASModuleLoader::~ExplicitCASModuleLoader() { delete &Impl; }
+
+bool ExplicitCASModuleLoader::findModule(
+    ImportPath::Element ModuleID, SmallVectorImpl<char> *ModuleInterfacePath,
+    SmallVectorImpl<char> *ModuleInterfaceSourcePath,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
+    bool skipBuildingInterface, bool isTestableDependencyLookup,
+    bool &IsFramework, bool &IsSystemModule) {
+  // Find a module with an actual, physical name on disk, in case
+  // -module-alias is used (otherwise same).
+  //
+  // For example, if '-module-alias Foo=Bar' is passed in to the frontend, and
+  // an input file has 'import Foo', a module called Bar (real name) should be
+  // searched.
+  StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str();
+
+  auto it = Impl.ExplicitModuleMap.find(moduleName);
+  // If no explicit module path is given matches the name, return with an
+  // error code.
+  if (it == Impl.ExplicitModuleMap.end()) {
+    return false;
+  }
+  auto &moduleInfo = it->getValue();
+
+  // Set IsFramework bit according to the moduleInfo
+  IsFramework = moduleInfo.isFramework;
+  IsSystemModule = moduleInfo.isSystem;
+
+  // Fallback check for module cache key passed on command-line as module path.
+  std::string moduleCASID = moduleInfo.moduleCacheKey
+                                ? *moduleInfo.moduleCacheKey
+                                : moduleInfo.modulePath;
+
+  // FIXME: the loaded module buffer doesn't set an identifier so it
+  // is not tracked in dependency tracker, which doesn't handle modules
+  // that are not located on disk.
+  auto moduleBuf = loadCachedCompileResultFromCacheKey(Impl.CAS, Impl.Cache,
+                                                       Ctx.Diags, moduleCASID);
+  if (!moduleBuf) {
+    // We cannot read the module content, diagnose.
+    Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
+                       moduleInfo.modulePath);
+    return false;
+  }
+
+  const bool isForwardingModule =
+      !serialization::isSerializedAST(moduleBuf->getBuffer());
+  // If the module is a forwarding module, read the actual content from the path
+  // encoded in the forwarding module as the actual module content.
+  if (isForwardingModule) {
+    auto forwardingModule = ForwardingModule::load(*moduleBuf.get());
+    if (forwardingModule) {
+      // Look through ExplicitModuleMap for paths.
+      // TODO: need to have dependency scanner reports forwarded module as
+      // dependency for this compilation and ingested into CAS.
+      auto moduleOrErr = Impl.loadModuleFromPath(
+          forwardingModule->underlyingModulePath, Ctx.Diags);
+      if (!moduleOrErr) {
+        llvm::consumeError(moduleOrErr.takeError());
+        Ctx.Diags.diagnose(SourceLoc(),
+                           diag::error_opening_explicit_module_file,
+                           moduleInfo.modulePath);
+        return false;
+      }
+      moduleBuf = std::move(*moduleOrErr);
+      if (!moduleBuf) {
+        // We cannot read the module content, diagnose.
+        Ctx.Diags.diagnose(SourceLoc(),
+                           diag::error_opening_explicit_module_file,
+                           moduleInfo.modulePath);
+        return false;
+      }
+    } else {
+      // We cannot read the module content, diagnose.
+      Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
+                         moduleInfo.modulePath);
+      return false;
+    }
+  }
+  assert(moduleBuf);
+  // Move the opened module buffer to the caller.
+  *ModuleBuffer = std::move(moduleBuf);
+
+  // TODO: support .swiftdoc file and .swiftsourceinfo file
+  return true;
+}
+
+std::error_code ExplicitCASModuleLoader::findModuleFilesInDirectory(
+    ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
+    SmallVectorImpl<char> *ModuleInterfacePath,
+    SmallVectorImpl<char> *ModuleInterfaceSourcePath,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
+    bool skipBuildingInterface, bool IsFramework,
+    bool IsTestableDependencyLookup) {
+  llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
+  return std::make_error_code(std::errc::not_supported);
+}
+
+bool ExplicitCASModuleLoader::canImportModule(
+    ImportPath::Module path, ModuleVersionInfo *versionInfo,
+    bool isTestableDependencyLookup) {
+  // FIXME: Swift submodules?
+  if (path.hasSubmodule())
+    return false;
+  ImportPath::Element mID = path.front();
+  // Look up the module with the real name (physical name on disk);
+  // in case `-module-alias` is used, the name appearing in source files
+  // and the real module name are different. For example, '-module-alias
+  // Foo=Bar' maps Foo appearing in source files, e.g. 'import Foo', to the real
+  // module name Bar (on-disk name), which should be searched for loading.
+  StringRef moduleName = Ctx.getRealModuleName(mID.Item).str();
+  auto it = Impl.ExplicitModuleMap.find(moduleName);
+  // If no provided explicit module matches the name, then it cannot be
+  // imported.
+  if (it == Impl.ExplicitModuleMap.end()) {
+    return false;
+  }
+
+  // If the caller doesn't want version info we're done.
+  if (!versionInfo)
+    return true;
+
+  // Open .swiftmodule file and read out the version
+  std::string moduleCASID = it->second.moduleCacheKey
+                                ? *it->second.moduleCacheKey
+                                : it->second.modulePath;
+  auto moduleBuf = Impl.loadFileBuffer(moduleCASID, it->second.modulePath);
+  if (!moduleBuf) {
+    Ctx.Diags.diagnose(SourceLoc(), diag::error_cas,
+                       toString(moduleBuf.takeError()));
+    return false;
+  }
+  if (!*moduleBuf) {
+    Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
+                       it->second.modulePath);
+    return false;
+  }
+  auto metaData = serialization::validateSerializedAST(
+      (*moduleBuf)->getBuffer(), Ctx.SILOpts.EnableOSSAModules,
+      Ctx.LangOpts.SDKName, !Ctx.LangOpts.DebuggerSupport);
+  versionInfo->setVersion(metaData.userModuleVersion,
+                          ModuleVersionSourceKind::SwiftBinaryModule);
+  return true;
+}
+
+void ExplicitCASModuleLoader::collectVisibleTopLevelModuleNames(
+    SmallVectorImpl<Identifier> &names) const {
+  for (auto &entry : Impl.ExplicitModuleMap) {
+    names.push_back(Ctx.getIdentifier(entry.getKey()));
+  }
+}
+
+std::unique_ptr<ExplicitCASModuleLoader> ExplicitCASModuleLoader::create(
+    ASTContext &ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &cache,
+    DependencyTracker *tracker, ModuleLoadingMode loadMode,
+    StringRef ExplicitSwiftModuleMap,
+    const std::vector<std::pair<std::string, std::string>>
+        &ExplicitSwiftModuleInputs,
+    bool IgnoreSwiftSourceInfoFile) {
+  auto result =
+      std::unique_ptr<ExplicitCASModuleLoader>(new ExplicitCASModuleLoader(
+          ctx, CAS, cache, tracker, loadMode, IgnoreSwiftSourceInfoFile));
+  auto &Impl = result->Impl;
+  // If the explicit module map is given, try parse it.
+  if (!ExplicitSwiftModuleMap.empty()) {
+    // Parse a JSON file to collect explicitly built modules.
+    Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap);
+  }
+  // If some modules are provided with explicit
+  // '-swift-module-file' options, add those as well.
+  if (!ExplicitSwiftModuleInputs.empty()) {
+    Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs);
+  }
+
+  return result;
+}

lib/Serialization/ModuleDependencyScanner.cpp

@@ -22,6 +22,10 @@
 #include "swift/Serialization/SerializedModuleLoader.h"
 #include "swift/Serialization/ModuleDependencyScanner.h"
 #include "swift/Subsystems.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include "ModuleFileSharedCore.h"
 
 #include <algorithm>
@@ -151,11 +155,6 @@ ErrorOr<ModuleDependencyInfo> ModuleDependencyScanner::scanInterfaceFile(
     Args.push_back("-o");
     Args.push_back(outputPathBase.str().str());
 
-    std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
-    Result = ModuleDependencyInfo::forSwiftInterfaceModule(
-        outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs, PCMArgs,
-        Hash, isFramework);
-
     // Open the interface file.
     auto &fs = *Ctx.SourceMgr.getFileSystem();
     auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath);
@@ -172,6 +171,21 @@ ErrorOr<ModuleDependencyInfo> ModuleDependencyScanner::scanInterfaceFile(
         *moduleDecl, SourceFileKind::Interface, bufferID, parsingOpts);
     moduleDecl->addAuxiliaryFile(*sourceFile);
 
+    std::string RootID;
+    if (dependencyTracker) {
+      dependencyTracker->startTracking();
+      dependencyTracker->trackFile(moduleInterfacePath);
+      auto RootOrError = dependencyTracker->createTreeFromDependencies();
+      if (!RootOrError)
+        return llvm::errorToErrorCode(RootOrError.takeError());
+      RootID = RootOrError->getID().toString();
+    }
+
+    std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
+    Result = ModuleDependencyInfo::forSwiftInterfaceModule(
+        outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs,
+        PCMArgs, Hash, isFramework, RootID, /*module-cache-key*/ "");
+
     // Walk the source file to find the import declarations.
     llvm::StringSet<> alreadyAddedModules;
     Result->addModuleImport(*sourceFile, alreadyAddedModules);
@@ -255,9 +269,10 @@ Optional<const ModuleDependencyInfo*> SerializedModuleLoaderBase::getModuleDepen
   // FIXME: submodules?
   scanners.push_back(std::make_unique<PlaceholderSwiftModuleScanner>(
       Ctx, LoadMode, moduleId, Ctx.SearchPathOpts.PlaceholderDependencyModuleMap,
-      delegate));
+      delegate, cache.getScanService().createSwiftDependencyTracker()));
   scanners.push_back(std::make_unique<ModuleDependencyScanner>(
-      Ctx, LoadMode, moduleId, delegate));
+      Ctx, LoadMode, moduleId, delegate, ModuleDependencyScanner::MDS_plain,
+      cache.getScanService().createSwiftDependencyTracker()));
 
   // Check whether there is a module with this name that we can import.
   assert(isa<PlaceholderSwiftModuleScanner>(scanners[0].get()) &&

lib/Serialization/SerializedModuleLoader.cpp

@@ -442,7 +442,8 @@ SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework) {
   const std::string sourceInfoPath;
   // Map the set of dependencies over to the "module dependencies".
   auto dependencies = ModuleDependencyInfo::forSwiftBinaryModule(
-      modulePath.str(), moduleDocPath, sourceInfoPath, isFramework);
+      modulePath.str(), moduleDocPath, sourceInfoPath, isFramework,
+      /*module-cache-key*/ "");
   // Some transitive dependencies of binary modules are not required to be
   // imported during normal builds.
   // TODO: This is worth revisiting for debugger purposes where

test/CAS/Inputs/SwiftDepsExtractor.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#
+# Usage: SwiftDepsExtractor.py file.json ModuleName Key
+
+import json
+import sys
+
+input_json = sys.argv[1]
+module_name = sys.argv[2]
+key = sys.argv[3]
+
+mode = 'swift'
+
+if module_name.startswith('clang:'):
+    mode = 'clang'
+    module_name = module_name[6:]
+elif module_name.startswith('swiftPrebuiltExternal:'):
+    mode = 'swiftPrebuiltExternal'
+    module_name = module_name[22:]
+
+with open(input_json, 'r') as file:
+    deps = json.load(file)
+    module_names = deps['modules'][::2]
+    module_details = deps['modules'][1::2]
+    for name, detail in zip(module_names, module_details):
+        if name.get(mode, '') != module_name:
+            continue
+
+        if key in detail.keys():
+            print(detail[key])
+            break
+
+        print(detail['details'][mode][key])
+        break

test/CAS/binary_module_deps.swift

@@ -0,0 +1,41 @@
+// REQUIRES: objc_interop
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/cas
+
+// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %S/../ScanDependencies/Inputs/Swift/E.swiftinterface -o %t/E.swiftmodule -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -swift-version 4
+// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %S/../ScanDependencies/Inputs/Swift/A.swiftinterface -o %t/A.swiftmodule -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -swift-version 4
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %t -swift-version 4 -enable-cas -cas-path %t/cas
+// RUN: %FileCheck %s -DTEMP=%t < %t/deps.json
+
+/// Test binary module key: binary module key is the CASID of itself.
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:A moduleCacheKey > %t/A.key.casid
+// RUN: llvm-cas --cas %t/cas --cat-blob @%t/A.key.casid > %t/Loaded.swiftmodule
+// RUN: diff %t/A.swiftmodule %t/Loaded.swiftmodule
+
+import A
+import E
+
+
+/// Main module
+// CHECK-LABEL: "swift": "deps"
+// CHECK: "directDependencies": [
+// CHECK: "swiftPrebuiltExternal": "A"
+// CHECK: "swiftPrebuiltExternal": "E"
+// CHECK: "details": {
+// CHECK: "casFSRootID"
+
+/// E.swiftmodule
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}E.swiftmodule",
+// CHECK: "details": {
+// CHECK: "swiftPrebuiltExternal": {
+// CHECK: "compiledModulePath":
+// CHECK: "moduleCacheKey":
+
+/// A.swiftmodule
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A.swiftmodule",
+// CHECK: "details": {
+// CHECK: "swiftPrebuiltExternal": {
+// CHECK: "compiledModulePath":
+// CHECK: "moduleCacheKey":

test/CAS/cas-explicit-module-map.swift

@@ -0,0 +1,129 @@
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/cas
+// RUN: split-file %s %t
+// RUN: %target-swift-emit-pcm -module-cache-path %t/clang-module-cache -module-name SwiftShims %swift-lib-dir/swift/shims/module.modulemap -o %t/SwiftShims.pcm
+// RUN: %target-swift-emit-pcm -module-cache-path %t/clang-module-cache -module-name _SwiftConcurrencyShims %swift-lib-dir/swift/shims/module.modulemap -o %t/_SwiftConcurrencyShims.pcm
+// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/A.swift -o %t/A.swiftmodule -swift-version 5
+// RUN: %target-swift-frontend -emit-module -module-cache-path %t/clang-module-cache %t/B.swift -o %t/B.swiftmodule -I %t -swift-version 5
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %t/Test.swift -o %t/deps.json -I %t -swift-version 5 -enable-cas -cas-path %t/cas
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:A moduleCacheKey | tr -d '\n' > %t/A.key
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json swiftPrebuiltExternal:B moduleCacheKey | tr -d '\n' > %t/B.key
+
+/// Prepare the cas objects that can be used to construct CompileJobResultSchema object.
+// RUN: llvm-cas --cas %t/cas --get-cache-result @%t/A.key > %t/A.result
+// RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/A.result | tail -n 1 > %t/schema.casid
+// RUN: llvm-cas --cas %t/cas --cat-blob @%t/A.result > %t/kind.blob
+
+/// Make keys for module loads. The result casid construction is tied with the actual structure of CompilerJobResultSchema.
+// RUN: llvm-cas --cas %t/cas --make-blob --data %stdlib_module | tr -d '\n' > %t/Swift.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Swift.key @%t/schema.casid  > %t/Swift.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Swift.key @%t/Swift.casid
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %ononesupport_module | tr -d '\n' > %t/ONone.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/ONone.key @%t/schema.casid > %t/ONone.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/ONone.key @%t/ONone.casid
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %concurrency_module | tr -d '\n' > %t/Concurrency.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Concurrency.key @%t/schema.casid > %t/Concurrency.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Concurrency.key @%t/Concurrency.casid
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %string_processing_module | tr -d '\n' > %t/String.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/String.key @%t/schema.casid > %t/String.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/String.key @%t/String.casid
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/SwiftShims.pcm | tr -d '\n' > %t/Shims.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/Shims.key @%t/schema.casid > %t/Shims.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/Shims.key @%t/Shims.casid
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/_SwiftConcurrencyShims.pcm | tr -d '\n' > %t/ConcurrencyShims.key
+// RUN: llvm-cas --cas %t/cas --make-node --data %t/kind.blob @%t/ConcurrencyShims.key @%t/schema.casid > %t/ConcurrencyShims.casid
+// RUN: llvm-cas --cas %t/cas --put-cache-key @%t/ConcurrencyShims.key @%t/ConcurrencyShims.casid
+
+// RUN: echo "[{" > %/t/map.json
+// RUN: echo "\"moduleName\": \"A\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"A.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/A.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"B\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"B.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/B.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"Swift\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"Swift.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/Swift.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"SwiftOnoneSupport\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"SwiftOnoneSupport.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/ONone.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"_Concurrency\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"_Concurrency.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/Concurrency.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"_StringProcessing\"," >> %/t/map.json
+// RUN: echo "\"modulePath\": \"_StringProcessing.swiftmodule\"," >> %/t/map.json
+// RUN: echo -n "\"moduleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/String.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"SwiftShims\"," >> %/t/map.json
+// RUN: echo "\"clangModulePath\": \"SwiftShims.pcm\"," >> %/t/map.json
+// RUN: echo -n "\"clangModuleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/Shims.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}," >> %/t/map.json
+// RUN: echo "{" >> %/t/map.json
+// RUN: echo "\"moduleName\": \"_SwiftConcurrencyShims\"," >> %/t/map.json
+// RUN: echo "\"clangModulePath\": \"_SwiftConcurrency.pcm\"," >> %/t/map.json
+// RUN: echo -n "\"clangModuleCacheKey\": \"" >> %/t/map.json
+// RUN: cat %t/ConcurrencyShims.key >> %/t/map.json
+// RUN: echo "\"," >> %/t/map.json
+// RUN: echo "\"isFramework\": false" >> %/t/map.json
+// RUN: echo "}]" >> %/t/map.json
+
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %target-swift-frontend -emit-module -emit-module-path %t/Foo.swiftmodule -disable-implicit-swift-modules -module-cache-path %t.module-cache -explicit-swift-module-map-file @%t/map.casid -Rmodule-loading -Xcc -Rmodule-import %s -enable-cas -cas-path %t/cas -allow-unstable-cache-key-for-testing 2>&1 | %FileCheck %s
+
+// CHECK-DAG: loaded module 'A'
+// CHECK-DAG: loaded module 'B'
+// CHECK-DAG: loaded module 'Swift'
+// CHECK-DAG: loaded module '_StringProcessing'
+// CHECK-DAG: loaded module '_Concurrency'
+// CHECK-DAG: loaded module 'SwiftOnoneSupport'
+
+//--- A.swift
+func test() {}
+
+//--- B.swift
+import A
+func myTest() {}
+
+//--- Test.swift
+import B
+
+

test/CAS/cas_fs.swift

@@ -8,7 +8,6 @@
 
 // RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/source.casid
 // RUN: not %target-swift-frontend -typecheck -enable-cas -cas-fs @%t/source.casid -cas-path %t/cas %s 2>&1 | %FileCheck %s --check-prefix NO-RESOURCES
-// NO-RESOURCES: error: unable to set working directory
 // NO-RESOURCES: error: unable to load standard library
 
 /// Ingest the resource directory to satisfy the file system requirement. Also switch CWD to resource dir.

test/CAS/module_deps.swift

@@ -0,0 +1,195 @@
+// REQUIRES: objc_interop
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/cas
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas
+// Check the contents of the JSON output
+// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json
+
+// Check the contents of the JSON output
+// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json
+
+// Check the make-style dependencies file
+// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d
+
+// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas
+// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json
+
+// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples.
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -clang-target %target-cpu-apple-macosx10.14 -enable-cas -cas-path %t/cas
+// Check the contents of the JSON output
+// RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json
+
+/// check cas-fs content
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json E casFSRootID > %t/E_fs.casid
+// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/E_fs.casid | %FileCheck %s -check-prefix FS_ROOT_E
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F casFSRootID > %t/F_fs.casid
+// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/F_fs.casid | %FileCheck %s -check-prefix FS_ROOT_F
+
+// FS_ROOT_E-DAG: E.swiftinterface
+// FS_ROOT_E-DAG: SDKSettings.json
+
+// FS_ROOT_F: CHeaders/A.h
+// FS_ROOT_F: CHeaders/B.h
+// FS_ROOT_F: CHeaders/C.h
+// FS_ROOT_F: CHeaders/D.h
+// FS_ROOT_F: CHeaders/F.h
+// FS_ROOT_F: CHeaders/G.h
+// FS_ROOT_F: CHeaders/H.h
+// FS_ROOT_F: CHeaders/I.h
+// FS_ROOT_F: CHeaders/X.h
+// FS_ROOT_F: CHeaders/module.modulemap
+
+import C
+import E
+import G
+import SubE
+
+// CHECK: "mainModuleName": "deps"
+
+/// --------Main module
+// CHECK-LABEL: "modulePath": "deps.swiftmodule",
+// CHECK-NEXT: sourceFiles
+// CHECK-NEXT: module_deps.swift
+// CHECK-NEXT: ],
+// CHECK-NEXT: "directDependencies": [
+// CHECK-DAG:     "swift": "A"
+// CHECK-DAG:     "clang": "C"
+// CHECK-DAG:     "swift": "E"
+// CHECK-DAG:     "swift": "F"
+// CHECK-DAG:     "swift": "G"
+// CHECK-DAG:     "swift": "SubE"
+// CHECK-DAG:     "swift": "Swift"
+// CHECK-DAG:     "swift": "SwiftOnoneSupport"
+// CHECK-DAG:     "swift": "_Concurrency"
+// CHECK-DAG:     "swift": "_cross_import_E"
+// CHECK: ],
+
+// CHECK:      "extraPcmArgs": [
+// CHECK-NEXT:    "-Xcc",
+// CHECK-NEXT:    "-target",
+// CHECK-NEXT:    "-Xcc",
+// CHECK:         "-fapinotes-swift-version=4"
+// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from"
+// CHECK: "bridgingHeader":
+// CHECK-NEXT: "path":
+// CHECK-SAME: Bridging.h
+
+// CHECK-NEXT: "sourceFiles":
+// CHECK-NEXT: Bridging.h
+// CHECK-NEXT: BridgingOther.h
+
+// CHECK: "moduleDependencies": [
+// CHECK-NEXT: "F"
+// CHECK-NEXT: ]
+
+/// --------Swift module A
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule",
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-DAG:   "clang": "A"
+// CHECK-DAG:   "swift": "Swift"
+// CHECK-NEXT: },
+// CHECK: "details":
+// CHECK: "moduleCacheKey":
+
+/// --------Swift module F
+// CHECK:      "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule",
+// CHECK-NEXT: "sourceFiles": [
+// CHECK-NEXT: ],
+// CHECK-NEXT: "directDependencies": [
+// CHECK-NEXT:   {
+// CHECK-DAG:     "clang": "F"
+// CHECK-DAG:     "swift": "Swift"
+// CHECK-DAG:     "swift": "SwiftOnoneSupport"
+// CHECK-NEXT:   }
+// CHECK-NEXT: ],
+// CHECK: "details":
+// CHECK: "moduleCacheKey":
+
+/// --------Swift module G
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule"
+// CHECK: "directDependencies"
+// CHECK-NEXT: {
+// CHECK-DAG:   "clang": "G"
+// CHECK-DAG:   "swift": "Swift"
+// CHECK-DAG:   "swift": "SwiftOnoneSupport"
+// CHECK: ],
+// CHECK-NEXT: "details": {
+
+// CHECK: "contextHash": "{{.*}}",
+// CHECK: "commandLine": [
+// CHECK: "-compile-module-from-interface"
+// CHECK: "-target"
+// CHECK: "-module-name"
+// CHECK: "G"
+// CHECK: "-swift-version"
+// CHECK: "5"
+// CHECK: ],
+// CHECK_NO_CLANG_TARGET: "extraPcmArgs": [
+// CHECK_NO_CLANG_TARGET-NEXT:   "-Xcc",
+// CHECK_NO_CLANG_TARGET-NEXT:   "-target",
+// CHECK_CLANG_TARGET: "extraPcmArgs": [
+// CHECK_CLANG_TARGET-NEXT:   "-Xcc",
+// CHECK_CLANG_TARGET-NEXT:   "-fapinotes-swift-version={{.*}}"
+// CHECK_CLANG_TARGET-NEXT:   ]
+
+/// --------Swift module E
+// CHECK: "swift": "E"
+// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule"
+// CHECK: "directDependencies"
+// CHECK-NEXT: {
+// CHECK-NEXT: "swift": "Swift"
+
+// CHECK: "moduleInterfacePath"
+// CHECK-SAME: E.swiftinterface
+
+/// --------Swift module Swift
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule",
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang": "SwiftShims"
+
+/// --------Clang module SwiftShims
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm",
+// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]",
+// CHECK: "-o"
+// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm
+// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path"
+
+/// --------Clang module C
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm",
+
+// CHECK: "sourceFiles": [
+// CHECK-DAG: module.modulemap
+// CHECK-DAG: C.h
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang": "B"
+
+// CHECK: "moduleMapPath"
+// CHECK-SAME: module.modulemap
+
+// CHECK: "contextHash"
+// CHECK-SAME: "{{.*}}"
+
+/// --------Clang module B
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm",
+// CHECK: "contextHash": "[[B_CONTEXT:.*]]",
+// CHECK: "-o"
+// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm
+
+// Check make-style dependencies
+// CHECK-MAKE-DEPS: module_deps.swift
+// CHECK-MAKE-DEPS-SAME: A.swiftinterface
+// CHECK-MAKE-DEPS-SAME: G.swiftinterface
+// CHECK-MAKE-DEPS-SAME: B.h
+// CHECK-MAKE-DEPS-SAME: F.h
+// CHECK-MAKE-DEPS-SAME: Bridging.h
+// CHECK-MAKE-DEPS-SAME: BridgingOther.h
+// CHECK-MAKE-DEPS-SAME: module.modulemap

test/CAS/module_deps_include_tree.swift

@@ -0,0 +1,190 @@
+// REQUIRES: objc_interop
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/cas
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas -clang-include-tree
+// Check the contents of the JSON output
+// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json
+
+// Check the contents of the JSON output
+// RUN: %FileCheck %s -check-prefix CHECK -check-prefix CHECK-NO-SEARCH-PATHS < %t/deps.json
+
+// Check the make-style dependencies file
+// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d
+
+// RUN: %target-swift-frontend -scan-dependencies -test-dependency-scan-cache-serialization -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cas -cas-path %t/cas -clang-include-tree
+// RUN: %FileCheck -check-prefix CHECK -check-prefix CHECK_NO_CLANG_TARGET %s < %t/deps.json
+
+// Ensure that scanning with `-clang-target` makes sure that Swift modules' respective PCM-dependency-build-argument sets do not contain target triples.
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps_clang_target.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -clang-target %target-cpu-apple-macosx10.14 -enable-cas -cas-path %t/cas -clang-include-tree
+// Check the contents of the JSON output
+// RUN: %FileCheck -check-prefix CHECK_CLANG_TARGET %s < %t/deps_clang_target.json
+
+/// check cas-fs content
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json E casFSRootID > %t/E_fs.casid
+// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/E_fs.casid | %FileCheck %s -check-prefix FS_ROOT_E
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:F clangIncludeTree > %t/F_tree.casid
+// RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/F_tree.casid | %FileCheck %s -check-prefix INCLUDE_TREE_F
+
+// FS_ROOT_E-DAG: E.swiftinterface
+// FS_ROOT_E-DAG: SDKSettings.json
+
+// INCLUDE_TREE_F: <module-includes>
+// INCLUDE_TREE_F: <built-in>
+// INCLUDE_TREE_F: Files:
+// INCLUDE_TREE_F-NEXT: CHeaders/F.h
+
+import C
+import E
+import G
+import SubE
+
+// CHECK: "mainModuleName": "deps"
+
+/// --------Main module
+// CHECK-LABEL: "modulePath": "deps.swiftmodule",
+// CHECK-NEXT: sourceFiles
+// CHECK-NEXT: module_deps_include_tree.swift
+// CHECK-NEXT: ],
+// CHECK-NEXT: "directDependencies": [
+// CHECK-DAG:     "swift": "A"
+// CHECK-DAG:     "clang": "C"
+// CHECK-DAG:     "swift": "E"
+// CHECK-DAG:     "swift": "F"
+// CHECK-DAG:     "swift": "G"
+// CHECK-DAG:     "swift": "SubE"
+// CHECK-DAG:     "swift": "Swift"
+// CHECK-DAG:     "swift": "SwiftOnoneSupport"
+// CHECK-DAG:     "swift": "_Concurrency"
+// CHECK-DAG:     "swift": "_cross_import_E"
+// CHECK: ],
+
+// CHECK:      "extraPcmArgs": [
+// CHECK-NEXT:    "-Xcc",
+// CHECK-NEXT:    "-target",
+// CHECK-NEXT:    "-Xcc",
+// CHECK:         "-fapinotes-swift-version=4"
+// CHECK-NOT: "error: cannot open Swift placeholder dependency module map from"
+// CHECK: "bridgingHeader":
+// CHECK-NEXT: "path":
+// CHECK-SAME: Bridging.h
+
+// CHECK-NEXT: "sourceFiles":
+// CHECK-NEXT: Bridging.h
+// CHECK-NEXT: BridgingOther.h
+
+// CHECK: "moduleDependencies": [
+// CHECK-NEXT: "F"
+// CHECK-NEXT: ]
+
+/// --------Swift module A
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}A-{{.*}}.swiftmodule",
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-DAG:   "clang": "A"
+// CHECK-DAG:   "swift": "Swift"
+// CHECK-NEXT: },
+// CHECK: "details":
+// CHECK: "moduleCacheKey":
+
+/// --------Swift module F
+// CHECK:      "modulePath": "{{.*}}{{/|\\}}F-{{.*}}.swiftmodule",
+// CHECK-NEXT: "sourceFiles": [
+// CHECK-NEXT: ],
+// CHECK-NEXT: "directDependencies": [
+// CHECK-NEXT:   {
+// CHECK-DAG:     "clang": "F"
+// CHECK-DAG:     "swift": "Swift"
+// CHECK-DAG:     "swift": "SwiftOnoneSupport"
+// CHECK-NEXT:   }
+// CHECK-NEXT: ],
+// CHECK: "details":
+// CHECK: "moduleCacheKey":
+
+/// --------Swift module G
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}G-{{.*}}.swiftmodule"
+// CHECK: "directDependencies"
+// CHECK-NEXT: {
+// CHECK-DAG:   "clang": "G"
+// CHECK-DAG:   "swift": "Swift"
+// CHECK-DAG:   "swift": "SwiftOnoneSupport"
+// CHECK: ],
+// CHECK-NEXT: "details": {
+
+// CHECK: "contextHash": "{{.*}}",
+// CHECK: "commandLine": [
+// CHECK: "-compile-module-from-interface"
+// CHECK: "-target"
+// CHECK: "-module-name"
+// CHECK: "G"
+// CHECK: "-swift-version"
+// CHECK: "5"
+// CHECK: ],
+// CHECK_NO_CLANG_TARGET: "extraPcmArgs": [
+// CHECK_NO_CLANG_TARGET-NEXT:   "-Xcc",
+// CHECK_NO_CLANG_TARGET-NEXT:   "-target",
+// CHECK_CLANG_TARGET: "extraPcmArgs": [
+// CHECK_CLANG_TARGET-NEXT:   "-Xcc",
+// CHECK_CLANG_TARGET-NEXT:   "-fapinotes-swift-version={{.*}}"
+// CHECK_CLANG_TARGET-NEXT:   ]
+
+/// --------Swift module E
+// CHECK: "swift": "E"
+// CHECK-LABEL: modulePath": "{{.*}}{{/|\\}}E-{{.*}}.swiftmodule"
+// CHECK: "directDependencies"
+// CHECK-NEXT: {
+// CHECK-NEXT: "swift": "Swift"
+
+// CHECK: "moduleInterfacePath"
+// CHECK-SAME: E.swiftinterface
+
+/// --------Swift module Swift
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}Swift-{{.*}}.swiftmodule",
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang": "SwiftShims"
+
+/// --------Clang module SwiftShims
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}SwiftShims-{{.*}}.pcm",
+// CHECK: "contextHash": "[[SHIMS_CONTEXT:.*]]",
+// CHECK: "-o"
+// CHECK-NEXT: SwiftShims-{{.*}}[[SHIMS_CONTEXT]].pcm
+// CHECK-NO-SEARCH-PATHS-NOT: "-prebuilt-module-cache-path"
+
+/// --------Clang module C
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}C-{{.*}}.pcm",
+
+// CHECK: "sourceFiles": [
+// CHECK-DAG: module.modulemap
+// CHECK-DAG: C.h
+
+// CHECK: directDependencies
+// CHECK-NEXT: {
+// CHECK-NEXT: "clang": "B"
+
+// CHECK: "moduleMapPath"
+// CHECK-SAME: module.modulemap
+
+// CHECK: "contextHash"
+// CHECK-SAME: "{{.*}}"
+
+/// --------Clang module B
+// CHECK-LABEL: "modulePath": "{{.*}}{{/|\\}}B-{{.*}}.pcm",
+// CHECK: "contextHash": "[[B_CONTEXT:.*]]",
+// CHECK: "-o"
+// CHECK-NEXT: B-{{.*}}[[B_CONTEXT]].pcm
+
+// Check make-style dependencies
+// CHECK-MAKE-DEPS: module_deps_include_tree.swift
+// CHECK-MAKE-DEPS-SAME: A.swiftinterface
+// CHECK-MAKE-DEPS-SAME: G.swiftinterface
+// CHECK-MAKE-DEPS-SAME: B.h
+// CHECK-MAKE-DEPS-SAME: F.h
+// CHECK-MAKE-DEPS-SAME: Bridging.h
+// CHECK-MAKE-DEPS-SAME: BridgingOther.h
+// CHECK-MAKE-DEPS-SAME: module.modulemap

test/CMakeLists.txt

@@ -71,6 +71,7 @@ function(get_test_dependencies SDK result_var_name)
       c-arcmt-test
       c-index-test
       clang
+      clang-cas-test
       count
       dsymutil
       FileCheck

tools/libSwiftScan/libSwiftScan.cpp

@@ -20,11 +20,15 @@
 #include "swift/DependencyScan/DependencyScanImpl.h"
 #include "swift/DependencyScan/DependencyScanningTool.h"
 #include "swift/DependencyScan/StringUtils.h"
+#include "swift/Frontend/CachingUtils.h"
 #include "swift/Option/Options.h"
+#include "llvm/CAS/ObjectStore.h"
 
 using namespace swift::dependencies;
+using namespace swift::cas;
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningTool, swiftscan_scanner_t)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CachingTool, swiftscan_cas_t)
 
 //=== Private Cleanup Functions -------------------------------------------===//
 
@@ -48,6 +52,12 @@ void swiftscan_dependency_info_details_dispose(
     swiftscan_string_set_dispose(
         details_impl->swift_textual_details.extra_pcm_args);
     swiftscan_string_dispose(details_impl->swift_textual_details.context_hash);
+    swiftscan_string_dispose(
+        details_impl->swift_textual_details.cas_fs_root_id);
+    swiftscan_string_dispose(
+        details_impl->swift_textual_details.bridging_header_include_tree);
+    swiftscan_string_dispose(
+        details_impl->swift_textual_details.module_cache_key);
     break;
   case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY:
     swiftscan_string_dispose(
@@ -56,6 +66,8 @@ void swiftscan_dependency_info_details_dispose(
         details_impl->swift_binary_details.module_doc_path);
     swiftscan_string_dispose(
         details_impl->swift_binary_details.module_source_info_path);
+    swiftscan_string_dispose(
+        details_impl->swift_binary_details.module_cache_key);
     break;
   case SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER:
     swiftscan_string_dispose(
@@ -70,6 +82,8 @@ void swiftscan_dependency_info_details_dispose(
     swiftscan_string_dispose(details_impl->clang_details.context_hash);
     swiftscan_string_set_dispose(details_impl->clang_details.command_line);
     swiftscan_string_set_dispose(details_impl->clang_details.captured_pcm_args);
+    swiftscan_string_dispose(details_impl->clang_details.cas_fs_root_id);
+    swiftscan_string_dispose(details_impl->clang_details.module_cache_key);
     break;
   }
   delete details_impl;
@@ -276,6 +290,12 @@ swiftscan_string_set_t *swiftscan_swift_textual_detail_get_command_line(
   return details->swift_textual_details.command_line;
 }
 
+swiftscan_string_set_t *
+swiftscan_swift_textual_detail_get_bridging_pch_command_line(
+    swiftscan_module_details_t details) {
+  return details->swift_textual_details.bridging_pch_command_line;
+}
+
 swiftscan_string_set_t *swiftscan_swift_textual_detail_get_extra_pcm_args(
     swiftscan_module_details_t details) {
   return details->swift_textual_details.extra_pcm_args;
@@ -296,6 +316,16 @@ swiftscan_string_set_t *swiftscan_swift_textual_detail_get_swift_overlay_depende
   return details->swift_textual_details.swift_overlay_module_dependencies;
 }
 
+swiftscan_string_ref_t swiftscan_swift_textual_detail_get_cas_fs_root_id(
+    swiftscan_module_details_t details) {
+  return details->swift_textual_details.cas_fs_root_id;
+}
+
+swiftscan_string_ref_t swiftscan_swift_textual_detail_get_module_cache_key(
+    swiftscan_module_details_t details) {
+  return details->swift_textual_details.module_cache_key;
+}
+
 //=== Swift Binary Module Details query APIs ------------------------------===//
 
 swiftscan_string_ref_t swiftscan_swift_binary_detail_get_compiled_module_path(
@@ -319,6 +349,12 @@ bool swiftscan_swift_binary_detail_get_is_framework(
   return details->swift_binary_details.is_framework;
 }
 
+swiftscan_string_ref_t swiftscan_swift_binary_detail_get_module_cache_key(
+    swiftscan_module_details_t details) {
+  return details->swift_binary_details.module_cache_key;
+}
+
+
 //=== Swift Placeholder Module Details query APIs -------------------------===//
 
 swiftscan_string_ref_t
@@ -360,6 +396,16 @@ swiftscan_clang_detail_get_captured_pcm_args(swiftscan_module_details_t details)
   return details->clang_details.captured_pcm_args;
 }
 
+swiftscan_string_ref_t
+swiftscan_clang_detail_get_cas_fs_root_id(swiftscan_module_details_t details) {
+  return details->clang_details.cas_fs_root_id;
+}
+
+swiftscan_string_ref_t swiftscan_clang_detail_get_module_cache_key(
+    swiftscan_module_details_t details) {
+  return details->clang_details.module_cache_key;
+}
+
 //=== Batch Scan Input Functions ------------------------------------------===//
 
 swiftscan_batch_scan_input_t *swiftscan_batch_scan_input_create() {
@@ -617,6 +663,60 @@ swiftscan_diagnostics_set_dispose(swiftscan_diagnostic_set_t* diagnostics){
   delete diagnostics;
 }
 
+//=== CAS Functions ----------------------------------------------------------//
+
+swiftscan_cas_t swiftscan_cas_create(const char *path) {
+  std::string CASPath(path);
+  if (CASPath.empty())
+    CASPath = llvm::cas::getDefaultOnDiskCASPath();
+
+  CachingTool *tool = new CachingTool(CASPath);
+  if (!tool->isValid()) {
+    delete tool;
+    return nullptr;
+  }
+  return wrap(tool);
+}
+
+void swiftscan_cas_dispose(swiftscan_cas_t cas) { delete unwrap(cas); }
+
+swiftscan_string_ref_t
+swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data, unsigned size) {
+  llvm::StringRef StrContent((char*)data, size);
+  auto ID = unwrap(cas)->storeContent(StrContent);
+  return swift::c_string_utils::create_clone(ID.c_str());
+}
+
+static swift::file_types::ID
+getFileTypeFromScanOutputKind(swiftscan_output_kind_t kind) {
+  switch (kind) {
+  case SWIFTSCAN_OUTPUT_TYPE_OBJECT:
+    return swift::file_types::ID::TY_Object;
+  case SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE:
+    return swift::file_types::ID::TY_SwiftModuleFile;
+  case SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE:
+    return swift::file_types::ID::TY_SwiftModuleInterfaceFile;
+  case SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIAVEINTERFACE:
+    return swift::file_types::ID::TY_PrivateSwiftModuleInterfaceFile;
+  case SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE:
+    return swift::file_types::ID::TY_ClangModuleFile;
+  case SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH:
+    return swift::file_types::ID::TY_PCH;
+  }
+}
+
+swiftscan_string_ref_t
+swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv,
+                            const char *input, swiftscan_output_kind_t kind) {
+  std::vector<const char *> Compilation;
+  for (int i = 0; i < argc; ++i)
+    Compilation.push_back(argv[i]);
+
+  auto ID = unwrap(cas)->computeCacheKey(Compilation, input,
+                                         getFileTypeFromScanOutputKind(kind));
+  return swift::c_string_utils::create_clone(ID.c_str());
+}
+
 //=== Experimental Compiler Invocation Functions ------------------------===//
 
 int invoke_swift_compiler(int argc, const char **argv) {

tools/libSwiftScan/libSwiftScan.exports

@@ -12,14 +12,18 @@ swiftscan_swift_textual_detail_get_bridging_header_path
 swiftscan_swift_textual_detail_get_bridging_source_files
 swiftscan_swift_textual_detail_get_bridging_module_dependencies
 swiftscan_swift_textual_detail_get_command_line
+swiftscan_swift_textual_detail_get_bridging_pch_command_line
 swiftscan_swift_textual_detail_get_extra_pcm_args
 swiftscan_swift_textual_detail_get_context_hash
 swiftscan_swift_textual_detail_get_is_framework
 swiftscan_swift_textual_detail_get_swift_overlay_dependencies
+swiftscan_swift_textual_detail_get_cas_fs_root_id
+swiftscan_swift_textual_detail_get_module_cache_key
 swiftscan_swift_binary_detail_get_compiled_module_path
 swiftscan_swift_binary_detail_get_module_doc_path
 swiftscan_swift_binary_detail_get_module_source_info_path
 swiftscan_swift_binary_detail_get_is_framework
+swiftscan_swift_binary_detail_get_module_cache_key
 swiftscan_swift_placeholder_detail_get_compiled_module_path
 swiftscan_swift_placeholder_detail_get_module_doc_path
 swiftscan_swift_placeholder_detail_get_module_source_info_path
@@ -27,6 +31,8 @@ swiftscan_clang_detail_get_module_map_path
 swiftscan_clang_detail_get_context_hash
 swiftscan_clang_detail_get_command_line
 swiftscan_clang_detail_get_captured_pcm_args
+swiftscan_clang_detail_get_cas_fs_root_id
+swiftscan_clang_detail_get_module_cache_key
 swiftscan_batch_scan_input_set_modules
 swiftscan_batch_scan_entry_set_module_name
 swiftscan_batch_scan_entry_set_arguments
@@ -68,4 +74,8 @@ swiftscan_scanner_diagnostics_reset
 swiftscan_diagnostic_get_message
 swiftscan_diagnostic_get_severity
 swiftscan_diagnostics_set_dispose
+swiftscan_cas_create
+swiftscan_cas_dispose
+swiftscan_cas_store
+swiftscan_compute_cache_key
 invoke_swift_compiler