[CAS][DependencyScanning] Don't keep a shared state of common file deps

Unlike `swift-frontend -scan-dependencies` option, when dependency
scanner is used as a library by swift driver, the SwiftScanningService
is shared for multiple driver invocations. It can't keep states (like
common file dependencies) that can change from one invocation to
another.

Instead, the clang/SDK file dependencies are computed from each driver
invocations to avoid out-of-date information when scanning service is
reused.

The test case for a shared Service will be added to swift-driver repo
since there is no tool to test it within swift compiler.

include/swift/AST/ModuleDependencies.h

@@ -18,8 +18,9 @@
 #ifndef SWIFT_AST_MODULE_DEPENDENCIES_H
 #define SWIFT_AST_MODULE_DEPENDENCIES_H
 
-#include "swift/Basic/LLVM.h"
 #include "swift/AST/Import.h"
+#include "swift/AST/SearchPathOptions.h"
+#include "swift/Basic/LLVM.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
@@ -31,12 +32,12 @@
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
-#include "llvm/Support/Mutex.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/Mutex.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
-#include <vector>
 #include <unordered_map>
+#include <vector>
 
 namespace swift {
 
@@ -733,17 +734,16 @@ using ModuleDependenciesKindRefMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
-                         const std::vector<std::string> &CommonFiles)
-      : FS(FS.createProxyFS()), Files(CommonFiles) {}
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS)
+      : FS(FS.createProxyFS()) {}
 
   void startTracking();
+  void addCommonSearchPathDeps(const SearchPathOptions& Opts);
   void trackFile(const Twine &path) { (void)FS->status(path); }
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
   llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
-  const std::vector<std::string> &Files;
 };
 
 // MARK: SwiftDependencyScanningService
@@ -848,7 +848,7 @@ public:
     if (!CacheFS)
       return None;
 
-    return SwiftDependencyTracker(*CacheFS, CommonDependencyFiles);
+    return SwiftDependencyTracker(*CacheFS);
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {

lib/AST/ModuleDependencies.cpp

@@ -394,11 +394,31 @@ SwiftDependencyScanningService::SwiftDependencyScanningService() {
   SharedFilesystemCache.emplace();
 }
 
+void SwiftDependencyTracker::addCommonSearchPathDeps(
+    const SearchPathOptions &Opts) {
+  // Add SDKSetting file.
+  SmallString<256> SDKSettingPath;
+  llvm::sys::path::append(SDKSettingPath, Opts.getSDKPath(),
+                          "SDKSettings.json");
+  FS->status(SDKSettingPath);
+
+  // Add Legacy layout file.
+  const std::vector<std::string> AllSupportedArches = {
+      "arm64", "arm64e", "x86_64", "i386",
+      "armv7", "armv7s", "armv7k", "arm64_32"};
+
+  for (auto RuntimeLibPath : Opts.RuntimeLibraryPaths) {
+    std::error_code EC;
+    for (auto &Arch : AllSupportedArches) {
+      SmallString<256> LayoutFile(RuntimeLibPath);
+      llvm::sys::path::append(LayoutFile, "layout-" + Arch + ".yaml");
+      FS->status(LayoutFile);
+    }
+  }
+}
+
 void SwiftDependencyTracker::startTracking() {
   FS->trackNewAccesses();
-
-  for (auto &file : Files)
-    (void)FS->status(file);
 }
 
 llvm::Expected<llvm::cas::ObjectProxy>
@@ -459,11 +479,6 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
     }
   }
 
-  // Fetch some dependency files from clang importer.
-  auto clangImporter = static_cast<ClangImporter *>(
-      Instance.getASTContext().getClangModuleLoader());
-  clangImporter->addClangInvovcationDependencies(CommonDependencyFiles);
-
   auto CachingFS =
       llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
   if (!CachingFS) {

lib/DependencyScan/ScanDependencies.cpp

@@ -1626,7 +1626,7 @@ forEachBatchEntry(CompilerInstance &invocationInstance,
   return false;
 }
 
-static ModuleDependencyInfo identifyMainModuleDependencies(
+static llvm::ErrorOr<ModuleDependencyInfo> identifyMainModuleDependencies(
     CompilerInstance &instance,
     Optional<SwiftDependencyTracker> tracker = None) {
   ModuleDecl *mainModule = instance.getMainModule();
@@ -1659,8 +1659,23 @@ static ModuleDependencyInfo identifyMainModuleDependencies(
         continue;
       tracker->trackFile(sf->getFilename());
     }
-    auto root = cantFail(tracker->createTreeFromDependencies());
-    rootID = root.getID().toString();
+    tracker->addCommonSearchPathDeps(
+        instance.getInvocation().getSearchPathOptions());
+    // Fetch some dependency files from clang importer.
+    std::vector<std::string> clangDependencyFiles;
+    auto clangImporter = static_cast<ClangImporter *>(
+        instance.getASTContext().getClangModuleLoader());
+    clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
+    llvm::for_each(clangDependencyFiles,
+                   [&](std::string &file) { tracker->trackFile(file); });
+
+    auto root = tracker->createTreeFromDependencies();
+    if (!root) {
+      instance.getASTContext().Diags.diagnose(SourceLoc(), diag::error_cas,
+                                              toString(root.takeError()));
+      return std::make_error_code(std::errc::io_error);
+    }
+    rootID = root->getID().toString();
   }
 
   auto mainDependencies =
@@ -1927,11 +1942,14 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   // First, identify the dependencies of the main module
   auto mainDependencies = identifyMainModuleDependencies(
       instance, cache.getScanService().createSwiftDependencyTracker());
+  if (!mainDependencies)
+    return mainDependencies.getError();
   auto &ctx = instance.getASTContext();
 
   // Add the main module.
   StringRef mainModuleName = mainModule->getNameStr();
-  auto mainModuleID = ModuleDependencyID{mainModuleName.str(), mainDependencies.getKind()};
+  auto mainModuleID =
+      ModuleDependencyID{mainModuleName.str(), mainDependencies->getKind()};
 
   ModuleDependencyIDSetVector allModules;
   allModules.insert(mainModuleID);
@@ -1943,9 +1961,9 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
     cache.updateDependency(
         std::make_pair(mainModuleName.str(),
                        ModuleDependencyKind::SwiftSource),
-        std::move(mainDependencies));
+        std::move(*mainDependencies));
   } else {
-    cache.recordDependency(mainModuleName, std::move(mainDependencies));
+    cache.recordDependency(mainModuleName, std::move(*mainDependencies));
   }
 
   auto ModuleCachePath = getModuleCachePathFromClang(
@@ -2051,8 +2069,10 @@ llvm::ErrorOr<swiftscan_import_set_t>
 swift::dependencies::performModulePrescan(CompilerInstance &instance) {
   // Execute import prescan, and write JSON output to the output stream
   auto mainDependencies = identifyMainModuleDependencies(instance);
+  if (!mainDependencies)
+    return mainDependencies.getError();
   auto *importSet = new swiftscan_import_set_s;
-  importSet->imports = create_set(mainDependencies.getModuleImports());
+  importSet->imports = create_set(mainDependencies->getModuleImports());
   return importSet;
 }
 

lib/Serialization/ModuleDependencyScanner.cpp

@@ -174,6 +174,14 @@ ErrorOr<ModuleDependencyInfo> ModuleDependencyScanner::scanInterfaceFile(
     std::string RootID;
     if (dependencyTracker) {
       dependencyTracker->startTracking();
+      dependencyTracker->addCommonSearchPathDeps(Ctx.SearchPathOpts);
+      std::vector<std::string> clangDependencyFiles;
+      auto clangImporter =
+          static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
+      clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
+      llvm::for_each(clangDependencyFiles, [&](std::string &file) {
+        dependencyTracker->trackFile(file);
+      });
       dependencyTracker->trackFile(moduleInterfacePath);
       auto RootOrError = dependencyTracker->createTreeFromDependencies();
       if (!RootOrError)

test/CAS/module_deps_clang_extras.swift

@@ -0,0 +1,48 @@
+// REQUIRES: objc_interop
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/cas
+// RUN: split-file %s %t
+// RUN: %hmaptool write %t/hmap.json %t/empty.hmap
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache \
+// RUN:   %t/Test.swift -o %t/deps.json -cache-compile-job -cas-path %t/cas -clang-include-tree \
+// RUN:   -Xcc -fmodule-map-file=%t/module.modulemap -Xcc -ivfsoverlay -Xcc %t/empty.yaml \
+// RUN:   -Xcc -I%t/empty.hmap
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps casFSRootID > %t/fs.casid
+// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/fs.casid | %FileCheck %s -DDIR=%basename_t -check-prefix FS_ROOT
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:Dummy clangIncludeTree > %t/tree.casid
+// RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/tree.casid | %FileCheck %s -DDIR=%basename_t -check-prefix INCLUDE_TREE
+
+// FS_ROOT: [[DIR]].tmp/empty.hmap
+// FS_ROOT: [[DIR]].tmp/empty.yaml
+
+// INCLUDE_TREE: [[DIR]].tmp/Dummy.h
+
+//--- Test.swift
+import Dummy
+func test() {}
+
+//--- module.modulemap
+module Dummy {
+ umbrella header "Dummy.h"
+}
+
+//--- Dummy.h
+void dummy(void);
+
+//--- hmap.json
+{
+  "mappings": {} 
+}
+
+//--- empty.yaml
+{
+  "version": 0,
+  "case-sensitive": "false",
+  "use-external-names": true,
+  "roots": []
+}
+

test/lit.cfg

@@ -2640,6 +2640,7 @@ config.substitutions.append(('%target-resilience-test', config.target_resilience
 
 config.substitutions.append(('%llvm-profdata', config.llvm_profdata))
 config.substitutions.append(('%llvm-cov', config.llvm_cov))
+config.substitutions.append(('%hmaptool', os.path.join(config.llvm_src_root, '..', 'clang', 'utils', 'hmaptool', 'hmaptool')))
 
 # Set up the host library environment.
 if hasattr(config, 'target_library_path_var'):