[Dependency Scanning] Remove 'ClangImporter' instance from dependency scanning worker

Move relevant logic directly into the worker
2025-12-14 20:36:38 +01:00 · 2025-05-07 10:58:31 -07:00
parent d6ce27b0e6
commit fc23c6b460
11 changed files with 318 additions and 304 deletions
--- a/lib/DependencyScan/ModuleDependencyScanner.cpp
+++ b/lib/DependencyScan/ModuleDependencyScanner.cpp
@@ -179,13 +179,96 @@ static bool isSwiftDependencyKind(ModuleDependencyKind Kind) {
         Kind == ModuleDependencyKind::SwiftPlaceholder;
 }

+// The Swift compiler does not have a concept of a working directory.
+// It is instead handled by the Swift driver by resolving relative paths
+// according to the driver's notion of a working directory. On the other hand,
+// Clang does have a concept working directory which may be specified on a
+// Clang invocation with '-working-directory'. If so, it is crucial that we
+// use this directory as an argument to the Clang scanner invocation below.
+static std::string
+computeClangWorkingDirectory(const std::vector<std::string> &commandLineArgs,
+                             const ASTContext &ctx) {
+  std::string workingDir;
+  auto clangWorkingDirPos = std::find(
+      commandLineArgs.rbegin(), commandLineArgs.rend(), "-working-directory");
+  if (clangWorkingDirPos == commandLineArgs.rend())
+    workingDir =
+        ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get();
+  else {
+    if (clangWorkingDirPos - 1 == commandLineArgs.rend()) {
+      ctx.Diags.diagnose(SourceLoc(), diag::clang_dependency_scan_error,
+                         "Missing '-working-directory' argument");
+      workingDir =
+          ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get();
+    } else
+      workingDir = *(clangWorkingDirPos - 1);
+  }
+  return workingDir;
+}
+
+static std::string
+moduleCacheRelativeLookupModuleOutput(const clang::tooling::dependencies::ModuleDeps &MD,
+                                      clang::tooling::dependencies::ModuleOutputKind MOK,
+                                      const StringRef moduleCachePath,
+                                      const StringRef stableModuleCachePath,
+                                      const StringRef runtimeResourcePath) {
+  llvm::SmallString<128> outputPath(moduleCachePath);
+  if (MD.IsInStableDirectories)
+    outputPath = stableModuleCachePath;
+
+  // FIXME: This is a hack to treat Clang modules defined in the compiler's
+  // own resource directory as stable, when they are not reported as such
+  // by the Clang scanner.
+  if (!runtimeResourcePath.empty() &&
+      hasPrefix(llvm::sys::path::begin(MD.ClangModuleMapFile),
+                llvm::sys::path::end(MD.ClangModuleMapFile),
+                llvm::sys::path::begin(runtimeResourcePath),
+                llvm::sys::path::end(runtimeResourcePath)))
+    outputPath = stableModuleCachePath;
+
+  llvm::sys::path::append(outputPath, MD.ID.ModuleName + "-" + MD.ID.ContextHash);
+  switch (MOK) {
+  case clang::tooling::dependencies::ModuleOutputKind::ModuleFile:
+    llvm::sys::path::replace_extension(
+        outputPath, getExtension(swift::file_types::TY_ClangModuleFile));
+    break;
+  case clang::tooling::dependencies::ModuleOutputKind::DependencyFile:
+    llvm::sys::path::replace_extension(
+        outputPath, getExtension(swift::file_types::TY_Dependencies));
+    break;
+  case clang::tooling::dependencies::ModuleOutputKind::DependencyTargets:
+    return MD.ID.ModuleName + "-" + MD.ID.ContextHash;
+  case clang::tooling::dependencies::ModuleOutputKind::DiagnosticSerializationFile:
+    llvm::sys::path::replace_extension(
+        outputPath, getExtension(swift::file_types::TY_SerializedDiagnostics));
+    break;
+  }
+  return outputPath.str().str();
+}
+
+static std::vector<std::string> inputSpecificClangScannerCommand(
+    const std::vector<std::string> &baseCommandLineArgs,
+    std::optional<StringRef> sourceFileName) {
+  std::vector<std::string> result(baseCommandLineArgs.begin(),
+                                  baseCommandLineArgs.end());
+  auto sourceFilePos =
+      std::find(result.begin(), result.end(), "<swift-imported-modules>");
+  assert(sourceFilePos != result.end());
+  if (sourceFileName.has_value())
+    *sourceFilePos = sourceFileName->str();
+  else
+    result.erase(sourceFilePos);
+  return result;
+}
+
 ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
    SwiftDependencyScanningService &globalScanningService,
    const CompilerInvocation &ScanCompilerInvocation,
    const SILOptions &SILOptions, ASTContext &ScanASTContext,
    swift::DependencyTracker &DependencyTracker, DiagnosticEngine &Diagnostics)
    : clangScanningTool(*globalScanningService.ClangScanningService,
-                        globalScanningService.getClangScanningFS()) {
+                        globalScanningService.getClangScanningFS()),
+      swiftModuleClangCC1CommandLineArgs() {
  // Create a scanner-specific Invocation and ASTContext.
  workerCompilerInvocation =
      std::make_unique<CompilerInvocation>(ScanCompilerInvocation);
@@ -222,12 +305,33 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
  CAS = globalScanningService.CAS;
  ActionCache = globalScanningService.ActionCache;

-  // Set up the ClangImporter.
-  clangScannerModuleLoader = ClangImporter::create(
-      *workerASTContext, workerCompilerInvocation->getPCHHash(),
-      &DependencyTracker);
-  if (!clangScannerModuleLoader)
-    Diagnostics.diagnose(SourceLoc(), diag::error_clang_importer_create_fail);
+  // Set up the required command-line arguments and working directory
+  // configuration required for clang dependency scanner queries
+  auto scanClangImporter =
+      static_cast<ClangImporter *>(ScanASTContext.getClangModuleLoader());
+  clangScanningBaseCommandLineArgs =
+      scanClangImporter->getClangDepScanningInvocationArguments(ScanASTContext);
+  clangScanningModuleCommandLineArgs = inputSpecificClangScannerCommand(
+      clangScanningBaseCommandLineArgs, std::nullopt);
+  clangScanningWorkingDirectoryPath = computeClangWorkingDirectory(
+      clangScanningBaseCommandLineArgs, ScanASTContext);
+
+  // Handle clang arguments. For caching build, all arguments are passed
+  // with `-direct-clang-cc1-module-build`.
+  if (ScanASTContext.ClangImporterOpts.ClangImporterDirectCC1Scan) {
+    swiftModuleClangCC1CommandLineArgs.push_back("-direct-clang-cc1-module-build");
+    auto *importer =
+        static_cast<ClangImporter *>(ScanASTContext.getClangModuleLoader());
+    for (auto &Arg : importer->getSwiftExplicitModuleDirectCC1Args()) {
+      swiftModuleClangCC1CommandLineArgs.push_back("-Xcc");
+      swiftModuleClangCC1CommandLineArgs.push_back(Arg);
+    }
+  } else {
+    swiftModuleClangCC1CommandLineArgs.push_back("-Xcc");
+    swiftModuleClangCC1CommandLineArgs.push_back("-fno-implicit-modules");
+    swiftModuleClangCC1CommandLineArgs.push_back("-Xcc");
+    swiftModuleClangCC1CommandLineArgs.push_back("-fno-implicit-module-maps");
+  }

  // Set up the Swift interface loader for Swift scanning.
  swiftScannerModuleLoader = ModuleInterfaceLoader::create(
@@ -245,7 +349,7 @@ ModuleDependencyScanningWorker::scanFilesystemForSwiftModuleDependency(
    bool isTestableImport) {
  return swiftScannerModuleLoader->getModuleDependencies(
      moduleName, moduleOutputPath, sdkModuleOutputPath,
-      {}, clangScanningTool, *scanningASTDelegate,
+      {}, swiftModuleClangCC1CommandLineArgs, *scanningASTDelegate,
      prefixMapper, isTestableImport);
 }

@@ -253,12 +357,118 @@ ModuleDependencyVector
 ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency(
    Identifier moduleName, StringRef moduleOutputPath,
    StringRef sdkModuleOutputPath,
-    const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenModules,
+    const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
+        &alreadySeenModules,
    llvm::PrefixMapper *prefixMapper) {
-  return clangScannerModuleLoader->getModuleDependencies(
-      moduleName, moduleOutputPath, sdkModuleOutputPath,
-      alreadySeenModules, clangScanningTool,
-      *scanningASTDelegate, prefixMapper, false);
+  auto lookupModuleOutput =
+      [this, moduleOutputPath, sdkModuleOutputPath](
+          const clang::tooling::dependencies::ModuleDeps &MD,
+          const clang::tooling::dependencies::ModuleOutputKind MOK)
+      -> std::string {
+    return moduleCacheRelativeLookupModuleOutput(
+        MD, MOK, moduleOutputPath, sdkModuleOutputPath,
+        workerASTContext->SearchPathOpts.RuntimeResourcePath);
+  };
+
+  auto clangModuleDependencies = clangScanningTool.getModuleDependencies(
+      moduleName.str(), clangScanningModuleCommandLineArgs,
+      clangScanningWorkingDirectoryPath,
+      alreadySeenModules,
+      lookupModuleOutput);
+  if (!clangModuleDependencies) {
+    auto errorStr = toString(clangModuleDependencies.takeError());
+    // We ignore the "module 'foo' not found" error, the Swift dependency
+    // scanner will report such an error only if all of the module loaders
+    // fail as well.
+    if (errorStr.find("fatal error: module '" + moduleName.str().str() +
+                      "' not found") == std::string::npos)
+      workerASTContext->Diags.diagnose(
+          SourceLoc(), diag::clang_dependency_scan_error, errorStr);
+    return {};
+  }
+
+  return ClangImporter::bridgeClangModuleDependencies(
+      *workerASTContext, clangScanningTool, *clangModuleDependencies,
+      moduleOutputPath, sdkModuleOutputPath, lookupModuleOutput,
+      [&](StringRef path) {
+        if (prefixMapper)
+          return prefixMapper->mapToString(path);
+        return path.str();
+      });
+}
+
+bool ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule(
+    const ASTContext &ctx,
+    ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
+    std::optional<llvm::MemoryBufferRef> sourceBuffer,
+    ModuleDependenciesCache &cache,
+    ModuleDependencyIDSetVector &headerClangModuleDependencies,
+    std::vector<std::string> &headerFileInputs,
+    std::vector<std::string> &bridgingHeaderCommandLine,
+    std::optional<std::string> &includeTreeID) {
+  // Scan the specified textual header file and collect its dependencies
+  auto scanHeaderDependencies = [&]()
+      -> llvm::Expected<clang::tooling::dependencies::TranslationUnitDeps> {
+    auto moduleOutputPath = cache.getModuleOutputPath();
+    auto sdkModuleOutputPath = cache.getSDKModuleOutputPath();
+    auto lookupModuleOutput =
+        [moduleOutputPath, sdkModuleOutputPath,
+         &ctx](const clang::tooling::dependencies::ModuleDeps &MD,
+               const clang::tooling::dependencies::ModuleOutputKind MOK)
+        -> std::string {
+      return moduleCacheRelativeLookupModuleOutput(
+          MD, MOK, moduleOutputPath, sdkModuleOutputPath,
+          ctx.SearchPathOpts.RuntimeResourcePath);
+    };
+
+    auto dependencies = clangScanningTool.getTranslationUnitDependencies(
+        /*ACTODO:*/ inputSpecificClangScannerCommand(clangScanningBaseCommandLineArgs, headerPath),
+        clangScanningWorkingDirectoryPath,
+        cache.getAlreadySeenClangModules(), lookupModuleOutput, sourceBuffer);
+    if (!dependencies)
+      return dependencies.takeError();
+
+    // Record module dependencies for each new module we found.
+    auto bridgedDeps = ClangImporter::bridgeClangModuleDependencies(
+        ctx, clangScanningTool, dependencies->ModuleGraph, moduleOutputPath,
+        sdkModuleOutputPath, lookupModuleOutput, [&cache](StringRef path) {
+          return cache.getScanService().remapPath(path);
+        });
+    cache.recordDependencies(bridgedDeps, ctx.Diags);
+
+    llvm::copy(dependencies->FileDeps, std::back_inserter(headerFileInputs));
+    auto bridgedDependencyIDs =
+        llvm::map_range(dependencies->ClangModuleDeps, [](auto &input) {
+          return ModuleDependencyID{input.ModuleName,
+                                    ModuleDependencyKind::Clang};
+        });
+    headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(),
+                                         bridgedDependencyIDs.end());
+    return dependencies;
+  };
+
+  // - If a generated header is provided, scan the generated header.
+  // - Textual module dependencies require us to process their bridging header.
+  // - Binary module dependnecies may have arbitrary header inputs.
+  auto clangModuleDependencies = scanHeaderDependencies();
+  if (!clangModuleDependencies) {
+    // FIXME: Route this to a normal diagnostic.
+    llvm::logAllUnhandledErrors(clangModuleDependencies.takeError(),
+                                llvm::errs());
+    ctx.Diags.diagnose(SourceLoc(), diag::clang_dependency_scan_error,
+                       "failed to scan header dependencies");
+    return true;
+  }
+
+  auto targetModuleInfo = cache.findKnownDependency(moduleID);
+  if (!targetModuleInfo.isTextualSwiftModule())
+    return false;
+
+  if (auto TreeID = clangModuleDependencies->IncludeTreeID)
+    includeTreeID = TreeID;
+  ClangImporter::getBridgingHeaderOptions(ctx, *clangModuleDependencies,
+                                          bridgingHeaderCommandLine);
+  return false;
 }

 template <typename Function, typename... Args>
@@ -831,8 +1041,7 @@ ModuleDependencyScanner::resolveSwiftModuleDependencies(
  return;
 }

-void
-ModuleDependencyScanner::resolveAllClangModuleDependencies(
+void ModuleDependencyScanner::resolveAllClangModuleDependencies(
    ArrayRef<ModuleDependencyID> swiftModuleDependents,
    ModuleDependenciesCache &cache,
    ModuleDependencyIDSetVector &allDiscoveredClangModules) {
@@ -840,13 +1049,23 @@ ModuleDependencyScanner::resolveAllClangModuleDependencies(
  // Clang modules (since no Swift module for them was found).
  llvm::StringSet<> unresolvedImportIdentifiers;
  llvm::StringSet<> unresolvedOptionalImportIdentifiers;
-  std::unordered_map<ModuleDependencyID, std::vector<ScannerImportStatementInfo>> unresolvedImportsMap;
-  std::unordered_map<ModuleDependencyID, std::vector<ScannerImportStatementInfo>> unresolvedOptionalImportsMap;
+  std::unordered_map<ModuleDependencyID,
+                     std::vector<ScannerImportStatementInfo>>
+      unresolvedImportsMap;
+  std::unordered_map<ModuleDependencyID,
+                     std::vector<ScannerImportStatementInfo>>
+      unresolvedOptionalImportsMap;

  for (const auto &moduleID : swiftModuleDependents) {
    auto moduleDependencyInfo = cache.findKnownDependency(moduleID);
-    auto unresolvedImports = &unresolvedImportsMap.emplace(moduleID, std::vector<ScannerImportStatementInfo>()).first->second;
-    auto unresolvedOptionalImports = &unresolvedOptionalImportsMap.emplace(moduleID, std::vector<ScannerImportStatementInfo>()).first->second;
+    auto unresolvedImports =
+        &unresolvedImportsMap
+             .emplace(moduleID, std::vector<ScannerImportStatementInfo>())
+             .first->second;
+    auto unresolvedOptionalImports =
+        &unresolvedOptionalImportsMap
+             .emplace(moduleID, std::vector<ScannerImportStatementInfo>())
+             .first->second;

    // If we have already resolved Clang dependencies for this module,
    // then we have the entire dependency sub-graph already computed for
@@ -1226,15 +1445,13 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule(

  withDependencyScanningWorker(
      [&](ModuleDependencyScanningWorker *ScanningWorker) {
-        auto clangImporter = static_cast<ClangImporter *>(
-            ScanningWorker->clangScannerModuleLoader.get());
-
        std::vector<std::string> headerFileInputs;
        std::optional<std::string> includeTreeID;
        std::vector<std::string> bridgingHeaderCommandLine;
-        auto headerScan = clangImporter->getHeaderDependencies(
+        auto headerScan = ScanningWorker->scanHeaderDependenciesOfSwiftModule(
+            *ScanningWorker->workerASTContext,
            moduleID, headerPath, sourceBufferRef,
-            ScanningWorker->clangScanningTool, cache,
+            cache,
            headerClangModuleDependencies, headerFileInputs,
            bridgingHeaderCommandLine, includeTreeID);
        if (!headerScan) {
@@ -1502,14 +1719,12 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining(
  std::optional<std::string> includeTreeID;
  auto err = withDependencyScanningWorker(
      [&](ModuleDependencyScanningWorker *ScanningWorker) -> llvm::Error {
-        auto clangImporter = static_cast<ClangImporter *>(
-            ScanningWorker->clangScannerModuleLoader.get());
        std::vector<std::string> headerFileInputs;
        std::vector<std::string> bridgingHeaderCommandLine;
-        if (clangImporter->getHeaderDependencies(
+        if (ScanningWorker->scanHeaderDependenciesOfSwiftModule(
+                *ScanningWorker->workerASTContext,
                rootModuleID, /*headerPath=*/std::nullopt,
-                sourceBuffer->getMemBufferRef(),
-                ScanningWorker->clangScanningTool, cache,
+                sourceBuffer->getMemBufferRef(), cache,
                headerClangModuleDependencies, headerFileInputs,
                bridgingHeaderCommandLine, includeTreeID))
          return llvm::createStringError(