[Caching] Support CrossImport modules for caching build

Add support for cross import modules by ingesting swiftoverlay files for
the cross import into CAS file system.

The long-term better fix will be just passing the cross import
information from scanner to swift-frontend so frontend doesn't need to
read overlay files again to figure out the cross import module.

rdar://123839248

include/swift/AST/ModuleDependencies.h

@@ -172,6 +172,10 @@ public:
   /// The cache key for the produced module.
   std::string moduleCacheKey;
 
+  /// Auxiliary files that help to construct other dependencies (e.g.
+  /// command-line), no need to be saved to reconstruct from cache.
+  std::vector<std::string> auxiliaryFiles;
+
   /// The direct dependency of the module is resolved by scanner.
   bool resolved;
   /// ModuleDependencyInfo is finalized (with all transitive dependencies
@@ -661,6 +665,24 @@ public:
     llvm_unreachable("Unexpected type");
   }
 
+  void addAuxiliaryFile(const std::string &file) {
+    storage->auxiliaryFiles.emplace_back(file);
+  }
+
+  void updateCASFileSystemRootID(const std::string &rootID) {
+    if (isSwiftInterfaceModule())
+      cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())
+          ->textualModuleDetails.CASFileSystemRootID = rootID;
+    else if (isSwiftSourceModule())
+      cast<SwiftSourceModuleDependenciesStorage>(storage.get())
+          ->textualModuleDetails.CASFileSystemRootID = rootID;
+    else if (isClangModule())
+      cast<ClangModuleDependencyStorage>(storage.get())->CASFileSystemRootID =
+          rootID;
+    else
+      llvm_unreachable("Unexpected type");
+  }
+
   bool isResolved() const {
     return storage->resolved;
   }
@@ -790,7 +812,8 @@ public:
   /// Collect a map from a secondary module name to a list of cross-import
   /// overlays, when this current module serves as the primary module.
   llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
-  collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName) const;
+  collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName,
+                                 std::vector<std::string> &overlayFiles) const;
 };
 
 using ModuleDependencyVector = llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>;

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -72,9 +72,8 @@ public:
                           DiagnosticEngine &diags, bool ParallelScan);
 
   /// Identify the scanner invocation's main module's dependencies
-  llvm::ErrorOr<ModuleDependencyInfo> getMainModuleDependencyInfo(
-      ModuleDecl *mainModule,
-      std::optional<SwiftDependencyTracker> tracker = std::nullopt);
+  llvm::ErrorOr<ModuleDependencyInfo>
+  getMainModuleDependencyInfo(ModuleDecl *mainModule);
 
   /// Resolve module dependencies of the given module, computing a full
   /// transitive closure dependency graph.

include/swift/Serialization/ScanningLoaders.h

@@ -41,20 +41,17 @@ private:
   /// Location where pre-built moduels are to be built into.
   std::string moduleOutputPath;
 
-  std::optional<SwiftDependencyTracker> dependencyTracker;
-
 public:
   std::optional<ModuleDependencyInfo> dependencies;
 
-  SwiftModuleScanner(
-      ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName,
-      InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
-      ScannerKind kind = MDS_plain,
-      std::optional<SwiftDependencyTracker> tracker = std::nullopt)
+  SwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
+                     Identifier moduleName,
+                     InterfaceSubContextDelegate &astDelegate,
+                     StringRef moduleOutputPath, ScannerKind kind = MDS_plain)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
         kind(kind), moduleName(moduleName), astDelegate(astDelegate),
-        moduleOutputPath(moduleOutputPath), dependencyTracker(tracker) {}
+        moduleOutputPath(moduleOutputPath) {}
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -91,13 +88,13 @@ class PlaceholderSwiftModuleScanner : public SwiftModuleScanner {
   llvm::BumpPtrAllocator Allocator;
 
 public:
-  PlaceholderSwiftModuleScanner(
-      ASTContext &ctx, ModuleLoadingMode LoadMode, Identifier moduleName,
-      StringRef PlaceholderDependencyModuleMap,
-      InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
-      std::optional<SwiftDependencyTracker> tracker = std::nullopt)
+  PlaceholderSwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
+                                Identifier moduleName,
+                                StringRef PlaceholderDependencyModuleMap,
+                                InterfaceSubContextDelegate &astDelegate,
+                                StringRef moduleOutputPath)
       : SwiftModuleScanner(ctx, LoadMode, moduleName, astDelegate,
-                           moduleOutputPath, MDS_placeholder, tracker) {
+                           moduleOutputPath, MDS_placeholder) {
 
     // FIXME: Find a better place for this map to live, to avoid
     // doing the parsing on every module.

lib/AST/ModuleLoader.cpp

@@ -186,8 +186,9 @@ void ModuleLoader::findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module,
 }
 
 llvm::StringMap<llvm::SmallSetVector<Identifier, 4>>
-ModuleDependencyInfo::collectCrossImportOverlayNames(ASTContext &ctx,
-                                                     StringRef moduleName) const {
+ModuleDependencyInfo::collectCrossImportOverlayNames(
+    ASTContext &ctx, StringRef moduleName,
+    std::vector<std::string> &overlayFiles) const {
   using namespace llvm::sys;
   using namespace file_types;
   std::optional<std::string> modulePath;
@@ -239,6 +240,7 @@ ModuleDependencyInfo::collectCrossImportOverlayNames(ASTContext &ctx,
       ModuleDecl::collectCrossImportOverlay(ctx, file, moduleName,
                                             bystandingModule);
     result[bystandingModule] = std::move(overlayNames);
+    overlayFiles.push_back(file.str());
   });
   return result;
 }

lib/DependencyScan/ModuleDependencyScanner.cpp

@@ -326,8 +326,7 @@ static void findAllImportedClangModules(StringRef moduleName,
 }
 
 llvm::ErrorOr<ModuleDependencyInfo>
-ModuleDependencyScanner::getMainModuleDependencyInfo(
-    ModuleDecl *mainModule, std::optional<SwiftDependencyTracker> tracker) {
+ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
   // Main module file name.
   auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
   llvm::SmallString<32> mainModulePath = mainModule->getName().str();
@@ -345,35 +344,9 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
         ExtraPCMArgs.begin(),
         {"-Xcc", "-target", "-Xcc", ScanASTContext.LangOpts.Target.str()});
 
-  std::string rootID;
-  std::vector<std::string> buildArgs;
   auto clangImporter =
       static_cast<ClangImporter *>(ScanASTContext.getClangModuleLoader());
-  if (tracker) {
-    tracker->startTracking();
-    for (auto fileUnit : mainModule->getFiles()) {
-      auto sf = dyn_cast<SourceFile>(fileUnit);
-      if (!sf)
-        continue;
-      tracker->trackFile(sf->getFilename());
-    }
-    tracker->addCommonSearchPathDeps(
-        ScanCompilerInvocation.getSearchPathOptions());
-    // Fetch some dependency files from clang importer.
-    std::vector<std::string> clangDependencyFiles;
-    clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
-    llvm::for_each(clangDependencyFiles,
-                   [&](std::string &file) { tracker->trackFile(file); });
-
-    auto root = tracker->createTreeFromDependencies();
-    if (!root) {
-      Diagnostics.diagnose(SourceLoc(), diag::error_cas,
-                           toString(root.takeError()));
-      return std::make_error_code(std::errc::io_error);
-    }
-    rootID = root->getID().toString();
-  }
-
+  std::vector<std::string> buildArgs;
   if (ScanASTContext.ClangImporterOpts.ClangImporterDirectCC1Scan) {
     buildArgs.push_back("-direct-clang-cc1-module-build");
     for (auto &arg : clangImporter->getSwiftExplicitModuleDirectCC1Args()) {
@@ -389,7 +362,15 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(
   });
 
   auto mainDependencies = ModuleDependencyInfo::forSwiftSourceModule(
-      rootID, buildCommands, {}, ExtraPCMArgs);
+      {}, buildCommands, {}, ExtraPCMArgs);
+
+  if (ScanASTContext.CASOpts.EnableCaching) {
+    std::vector<std::string> clangDependencyFiles;
+    clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
+    llvm::for_each(clangDependencyFiles, [&](std::string &file) {
+      mainDependencies.addAuxiliaryFile(file);
+    });
+  }
 
   llvm::StringSet<> alreadyAddedModules;
   // Compute Implicit dependencies of the main module
@@ -808,6 +789,7 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
     llvm::function_ref<void(ModuleDependencyID)> action) {
   // Modules explicitly imported. Only these can be secondary module.
   llvm::SetVector<Identifier> newOverlays;
+  std::vector<std::string> overlayFiles;
   for (auto dep : allDependencies) {
     auto moduleName = dep.ModuleName;
     // Do not look for overlays of main module under scan
@@ -817,7 +799,7 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
     auto dependencies = cache.findDependency(moduleName, dep.Kind).value();
     // Collect a map from secondary module name to cross-import overlay names.
     auto overlayMap = dependencies->collectCrossImportOverlayNames(
-        ScanASTContext, moduleName);
+        ScanASTContext, moduleName, overlayFiles);
     if (overlayMap.empty())
       continue;
     for (const auto &dependencyId : allDependencies) {
@@ -879,11 +861,9 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
 
   // Update main module's dependencies to include these new overlays.
   auto resolvedDummyDep =
-      *(cache.findDependency(dummyMainName, ModuleDependencyKind::SwiftSource)
-            .value());
+      **cache.findDependency(dummyMainName, ModuleDependencyKind::SwiftSource);
   auto mainDep =
-      *(cache.findDependency(mainModuleName, ModuleDependencyKind::SwiftSource)
-            .value());
+      **cache.findDependency(mainModuleName, ModuleDependencyKind::SwiftSource);
   auto newOverlayDeps = resolvedDummyDep.getDirectModuleDependencies();
   auto existingMainDeps = mainDep.getDirectModuleDependencies();
   ModuleDependencyIDSet existingMainDepsSet(existingMainDeps.begin(),
@@ -895,6 +875,11 @@ void ModuleDependencyScanner::discoverCrossImportOverlayDependencies(
                   if (!existingMainDepsSet.count(crossImportOverlayModID))
                     mainDep.addModuleDependency(crossImportOverlayModID);
                 });
+
+  llvm::for_each(overlayFiles, [&mainDep](const std::string &file) {
+    mainDep.addAuxiliaryFile(file);
+  });
+
   cache.updateDependency(
       {mainModuleName.str(), ModuleDependencyKind::SwiftSource}, mainDep);
 

lib/DependencyScan/ScanDependencies.cpp

@@ -207,13 +207,11 @@ static llvm::Error resolveExplicitModuleInputs(
   auto &service = cache.getScanService();
   auto remapPath = [&](StringRef path) { return service.remapPath(path); };
   std::vector<std::string> rootIDs;
-  if (auto ID = resolvingDepInfo.getCASFSRootID())
-    rootIDs.push_back(*ID);
-
   std::vector<std::string> includeTrees;
   if (auto ID = resolvingDepInfo.getClangIncludeTree())
     includeTrees.push_back(*ID);
 
+  auto tracker = cache.getScanService().createSwiftDependencyTracker();
   auto addBridgingHeaderDeps =
       [&](const ModuleDependencyInfo &depInfo) -> llvm::Error {
     auto sourceDepDetails = depInfo.getAsSwiftSourceModule();
@@ -223,8 +221,7 @@ static llvm::Error resolveExplicitModuleInputs(
     if (sourceDepDetails->textualModuleDetails
             .CASBridgingHeaderIncludeTreeRootID.empty()) {
       if (!sourceDepDetails->textualModuleDetails.bridgingSourceFiles.empty()) {
-        if (auto tracker =
-                cache.getScanService().createSwiftDependencyTracker()) {
+        if (tracker) {
           tracker->startTracking();
           for (auto &file :
                sourceDepDetails->textualModuleDetails.bridgingSourceFiles)
@@ -329,6 +326,41 @@ static llvm::Error resolveExplicitModuleInputs(
     auto &CASFS = cache.getScanService().getSharedCachingFS();
     auto &CAS = CASFS.getCAS();
 
+    assert(tracker && "no caching tracker is available");
+    // Compute the CASFS root ID for the resolving dependency.
+    if (auto *sourceDep = resolvingDepInfo.getAsSwiftSourceModule()) {
+      tracker->startTracking();
+      tracker->addCommonSearchPathDeps(
+          instance.getInvocation().getSearchPathOptions());
+      llvm::for_each(
+          sourceDep->sourceFiles,
+          [&tracker](const std::string &file) { tracker->trackFile(file); });
+      llvm::for_each(
+          sourceDep->auxiliaryFiles,
+          [&tracker](const std::string &file) { tracker->trackFile(file); });
+      auto root = tracker->createTreeFromDependencies();
+      if (!root)
+        return root.takeError();
+      auto rootID = root->getID().toString();
+      dependencyInfoCopy.updateCASFileSystemRootID(rootID);
+      rootIDs.push_back(rootID);
+    } else if (auto *textualDep =
+                   resolvingDepInfo.getAsSwiftInterfaceModule()) {
+      tracker->startTracking();
+      tracker->addCommonSearchPathDeps(
+          instance.getInvocation().getSearchPathOptions());
+      tracker->trackFile(textualDep->swiftInterfaceFile);
+      llvm::for_each(
+          textualDep->auxiliaryFiles,
+          [&tracker](const std::string &file) { tracker->trackFile(file); });
+      auto root = tracker->createTreeFromDependencies();
+      if (!root)
+        return root.takeError();
+      auto rootID = root->getID().toString();
+      dependencyInfoCopy.updateCASFileSystemRootID(rootID);
+      rootIDs.push_back(rootID);
+    }
+
     // Update build command line.
     if (resolvingDepInfo.isSwiftInterfaceModule() ||
         resolvingDepInfo.isSwiftSourceModule()) {
@@ -1898,9 +1930,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
 
   // Identify imports of the main module and add an entry for it
   // to the dependency graph.
-  auto mainModuleDepInfo = scanner.getMainModuleDependencyInfo(
-      instance.getMainModule(),
-      cache.getScanService().createSwiftDependencyTracker());
+  auto mainModuleDepInfo =
+      scanner.getMainModuleDependencyInfo(instance.getMainModule());
   auto mainModuleName = instance.getMainModule()->getNameStr();
   auto mainModuleID = ModuleDependencyID{mainModuleName.str(),
                                          ModuleDependencyKind::SwiftSource};

lib/Serialization/ScanningLoaders.cpp

@@ -205,29 +205,21 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
             *moduleDecl, SourceFileKind::Interface, bufferID, parsingOpts);
         moduleDecl->addAuxiliaryFile(*sourceFile);
 
-        std::string RootID;
-        if (dependencyTracker) {
-          dependencyTracker->startTracking();
-          dependencyTracker->addCommonSearchPathDeps(Ctx.SearchPathOpts);
+        std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
+        Result = ModuleDependencyInfo::forSwiftInterfaceModule(
+            outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs,
+            PCMArgs, Hash, isFramework, {}, /*module-cache-key*/ "");
+
+        if (Ctx.CASOpts.EnableCaching) {
           std::vector<std::string> clangDependencyFiles;
           auto clangImporter =
               static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
           clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
           llvm::for_each(clangDependencyFiles, [&](std::string &file) {
-            dependencyTracker->trackFile(file);
+            Result->addAuxiliaryFile(file);
           });
-          dependencyTracker->trackFile(moduleInterfacePath);
-          auto RootOrError = dependencyTracker->createTreeFromDependencies();
-          if (!RootOrError)
-            return llvm::errorToErrorCode(RootOrError.takeError());
-          RootID = RootOrError->getID().toString();
         }
 
-        std::vector<StringRef> ArgsRefs(Args.begin(), Args.end());
-        Result = ModuleDependencyInfo::forSwiftInterfaceModule(
-            outputPathBase.str().str(), InPath, compiledCandidates, ArgsRefs,
-            PCMArgs, Hash, isFramework, RootID, /*module-cache-key*/ "");
-
         // Walk the source file to find the import declarations.
         llvm::StringSet<> alreadyAddedModules;
         Result->addModuleImport(*sourceFile, alreadyAddedModules);
@@ -260,9 +252,6 @@ ModuleDependencyVector SerializedModuleLoaderBase::getModuleDependencies(
   ImportPath::Module::Builder builder(moduleName);
   auto modulePath = builder.get();
   auto moduleId = modulePath.front().Item;
-  std::optional<SwiftDependencyTracker> tracker = std::nullopt;
-  if (CacheFS)
-    tracker = SwiftDependencyTracker(*CacheFS, mapper);
 
   // Do not load interface module if it is testable import.
   ModuleLoadingMode MLM =
@@ -277,10 +266,10 @@ ModuleDependencyVector SerializedModuleLoaderBase::getModuleDependencies(
   // FIXME: submodules?
   scanners.push_back(std::make_unique<PlaceholderSwiftModuleScanner>(
       Ctx, MLM, moduleId, Ctx.SearchPathOpts.PlaceholderDependencyModuleMap,
-      delegate, moduleOutputPath, tracker));
+      delegate, moduleOutputPath));
   scanners.push_back(std::make_unique<SwiftModuleScanner>(
       Ctx, MLM, moduleId, delegate, moduleOutputPath,
-      SwiftModuleScanner::MDS_plain, tracker));
+      SwiftModuleScanner::MDS_plain));
 
   // Check whether there is a module with this name that we can import.
   assert(isa<PlaceholderSwiftModuleScanner>(scanners[0].get()) &&

test/CAS/cross_import.swift

@@ -0,0 +1,67 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -emit-module -module-name A -o %t/A.swiftmodule -swift-version 5 \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -emit-module-interface-path %t/A.swiftinterface -enable-library-evolution -I %t %t/A.swift
+
+// RUN: %target-swift-frontend -emit-module -module-name B -o %t/A.swiftmodule -swift-version 5 \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -emit-module-interface-path %t/B.swiftinterface -enable-library-evolution -I %t %t/B.swift
+
+// RUN: %target-swift-frontend -emit-module -module-name _Cross -o %t/A.swiftmodule -swift-version 5 \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -emit-module-interface-path %t/_Cross.swiftinterface -enable-library-evolution -I %t %t/cross.swift
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache %t/main.swift \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -o %t/deps.json -I %t -cache-compile-job -cas-path %t/cas -swift-version 5 -enable-cross-import-overlays
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json A > %t/A.cmd
+// RUN: %swift_frontend_plain @%t/A.cmd
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json B > %t/B.cmd
+// RUN: %swift_frontend_plain @%t/B.cmd
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json _Cross > %t/Cross.cmd
+// RUN: %swift_frontend_plain @%t/Cross.cmd
+
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
+
+// RUN: %target-swift-frontend -emit-module -o %t/Test.swiftmodule  \
+// RUN:   -emit-module-interface-path %t/Test.swiftinterface \
+// RUN:   -cache-compile-job -cas-path %t/cas \
+// RUN:   -disable-implicit-swift-modules -swift-version 5 -enable-cross-import-overlays \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
+// RUN:   -module-name Test -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   %t/main.swift @%t/MyApp.cmd
+
+// RUN: %FileCheck %s --input-file=%t/Test.swiftinterface
+
+/// Check to make sure the implicit cross import turned into explicit import in the interface file.
+// CHECK: import _Cross
+
+//--- A.swift
+public func a() {}
+
+//--- B.swift
+public func b() {}
+
+//--- cross.swift
+public func cross() {}
+
+//--- main.swift
+import A
+import B
+
+func test () {
+  cross()
+}
+
+//--- B.swiftcrossimport/A.swiftoverlay
+%YAML 1.2
+---
+version: 1
+modules:
+  - name: _Cross