Merge pull request #83600 from artemcm/NoRedundantClangDepBridging

[Dependency Scanning] Bridge Clang dependency scanner results on-demand

include/swift/AST/ModuleDependencies.h

@@ -26,6 +26,7 @@
 #include "swift/Serialization/Validation.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
@@ -1028,6 +1029,8 @@ using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependencyKind, ModuleNameToDependencyMap,
                        ModuleDependencyKindHash>;
+using BridgeClangDependencyCallback = llvm::function_ref<ModuleDependencyInfo(
+    const clang::tooling::dependencies::ModuleDeps &clangModuleDep)>;
 
 // MARK: SwiftDependencyScanningService
 /// A carrier of state shared among possibly multiple invocations of the
@@ -1178,8 +1181,10 @@ public:
                         ModuleDependencyInfo dependencies);
 
   /// Record dependencies for the given collection of Clang modules.
-  void recordClangDependencies(ModuleDependencyVector moduleDependencies,
-                               DiagnosticEngine &diags);
+  void recordClangDependencies(
+      const clang::tooling::dependencies::ModuleDepsGraph &dependencies,
+      DiagnosticEngine &diags,
+      BridgeClangDependencyCallback bridgeClangModule);
 
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,

include/swift/ClangImporter/ClangImporter.h

@@ -492,19 +492,6 @@ public:
 
   void verifyAllModules() override;
 
-  using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  using LookupModuleOutputCallback =
-      llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
-                                     clang::tooling::dependencies::ModuleOutputKind)>;
-
-  static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  bridgeClangModuleDependencies(
-      const ASTContext &ctx,
-      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
-      clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      LookupModuleOutputCallback LookupModuleOutput,
-      RemapPathCallback remapPath = nullptr);
-
   static void getBridgingHeaderOptions(
       const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -25,6 +25,12 @@ class DependencyTracker;
 
 namespace swift {
 
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback = llvm::function_ref<std::string(
+    const clang::tooling::dependencies::ModuleDeps &,
+    clang::tooling::dependencies::ModuleOutputKind)>;
+using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
+
 /// A dependency scanning worker which performs filesystem lookup
 /// of a named module dependency.
 class ModuleDependencyScanningWorker {
@@ -39,16 +45,24 @@ public:
       llvm::PrefixMapper *mapper, DiagnosticEngine &diags);
 
 private:
-  /// Retrieve the module dependencies for the Clang module with the given name.
-  ClangModuleScannerQueryResult scanFilesystemForClangModuleDependency(
+  /// Query dependency information for a named Clang module
+  ///
+  /// \param moduleName moduel identifier for the query
+  ///
+  /// \param lookupModuleCallback a callback to compute a client-specific
+  /// module-cache-relative output path for discovered Clang module dependencies.
+  ///
+  /// \param alreadySeenModules a set of module dependencies previously seen
+  /// by the scanner, as to avoid processing them all over again
+  ///
+  /// \returns Clang dependency scanner's \c TranslationUnitDeps result
+  std::optional<clang::tooling::dependencies::TranslationUnitDeps>
+  scanFilesystemForClangModuleDependency(
       Identifier moduleName,
+      LookupModuleOutputCallback lookupModuleCallback,
       const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
           &alreadySeenModules);
 
-  /// Retrieve the module dependencies for the Swift module with the given name.
-  SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency(
-      Identifier moduleName, bool isTestableImport = false);
-
   /// Query dependency information for header dependencies
   /// of a binary Swift module.
   ///
@@ -56,25 +70,35 @@ private:
   /// information will be augmented with information about the given
   /// textual header inputs.
   ///
-  /// \param headerPath the path to the header to be scanned.
+  /// \param headerPath optional path to the header to be scanned.
   ///
-  /// \param clangScanningTool The clang dependency scanner.
+  /// \param sourceBuffer optional in-memory buffer of a header to be scanned.
   ///
-  /// \param cache The module dependencies cache to update, with information
-  /// about new Clang modules discovered along the way.
+  /// \param lookupModuleCallback a callback to compute a client-specific
+  /// module-cache-relative output path for discovered Clang module dependencies.
   ///
-  /// \returns \c true if an error occurred, \c false otherwise
-  bool scanHeaderDependenciesOfSwiftModule(
-      const ASTContext &ctx,
+  /// \param alreadySeenModules a set of module dependencies previously seen
+  /// by the scanner, as to avoid processing them all over again
+  ///
+  /// \returns Clang dependency scanner's \c TranslationUnitDeps result
+  std::optional<clang::tooling::dependencies::TranslationUnitDeps>
+  scanHeaderDependenciesOfSwiftModule(
       ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
       std::optional<llvm::MemoryBufferRef> sourceBuffer,
-      ModuleDependenciesCache &cache,
-      ModuleDependencyIDSetVector &headerClangModuleDependencies,
-      std::vector<std::string> &headerFileInputs,
-      std::vector<std::string> &bridgingHeaderCommandLine,
-      std::vector<std::string> &visibleClangModules,
-      std::optional<std::string> &includeTreeID);
+      LookupModuleOutputCallback lookupModuleCallback,
+      const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
+         &alreadySeenModules);
 
+  /// Query dependency information for a named Swift module
+  ///
+  /// \param moduleName moduel identifier for the query
+  ///
+  /// \param isTestableImport a boolean flag which indicates whether
+  /// this is an @testable dependency
+  ///
+  /// \returns a struct containing query results
+  SwiftModuleScannerQueryResult scanFilesystemForSwiftModuleDependency(
+      Identifier moduleName, bool isTestableImport = false);
 
   /// Store cache entry for include tree.
   llvm::Error
@@ -92,16 +116,9 @@ private:
   // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<SwiftModuleScanner> swiftModuleScannerLoader;
 
-  /// The location of where the explicitly-built modules will be output to
-  std::string moduleOutputPath;
-  /// The location of where the explicitly-built SDK modules will be output to
-  std::string sdkModuleOutputPath;
-
   // CAS instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;
-  /// File prefix mapper.
-  llvm::PrefixMapper *PrefixMapper;
 
   // Base command line invocation for clang scanner queries (both module and header)
   std::vector<std::string> clangScanningBaseCommandLineArgs;
@@ -222,11 +239,7 @@ public:
     return PrefixMapper && !PrefixMapper->getMappings().empty();
   }
   llvm::PrefixMapper *getPrefixMapper() const { return PrefixMapper.get(); }
-  std::string remapPath(StringRef Path) const {
-    if (!PrefixMapper)
-      return Path.str();
-    return PrefixMapper->mapToString(Path);
-  }
+  std::string remapPath(StringRef Path) const;
 
   /// CAS options.
   llvm::cas::ObjectStore &getCAS() const {
@@ -293,11 +306,22 @@ private:
                                 ModuleDependenciesCache &cache,
                                 ModuleDependencyIDSetVector &allModules);
 
+  /// Bridge Clang dependency scanner's dependency node
+  /// to the Swift scanner's `ModuleDependencyInfo`.
+  ModuleDependencyInfo
+  bridgeClangModuleDependency(
+      const clang::tooling::dependencies::ModuleDeps &clangDependency);
+
   /// Perform an operation utilizing one of the Scanning workers
   /// available to this scanner.
   template <typename Function, typename... Args>
   auto withDependencyScanningWorker(Function &&F, Args &&...ArgList);
 
+  /// Determine cache-relative output path for a given Clang module
+  std::string clangModuleOutputPathLookup(
+      const clang::tooling::dependencies::ModuleDeps &clangDep,
+      clang::tooling::dependencies::ModuleOutputKind moduleOutputKind) const;
+
   /// Use the scanner's ASTContext to construct an `Identifier`
   /// for a given module name.
   Identifier getModuleImportIdentifier(StringRef moduleName);
@@ -317,6 +341,11 @@ private:
   ASTContext &ScanASTContext;
   ModuleDependencyIssueReporter IssueReporter;
 
+  /// The location of where the explicitly-built modules will be output to
+  std::string ModuleOutputPath;
+  /// The location of where the explicitly-built SDK modules will be output to
+  std::string SDKModuleOutputPath;
+
   /// The available pool of workers for filesystem module search
   unsigned NumThreads;
   std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;

include/swift/Serialization/ScanningLoaders.h

@@ -40,18 +40,6 @@ struct SwiftModuleScannerQueryResult {
   std::vector<IncompatibleCandidate> incompatibleCandidates;
 };
 
-/// Result of looking up a Clang module on the current filesystem
-/// search paths.
-struct ClangModuleScannerQueryResult {
-  ClangModuleScannerQueryResult(const ModuleDependencyVector &dependencyModuleGraph,
-                                const std::vector<std::string> &visibleModuleIdentifiers)
-      : foundDependencyModuleGraph(dependencyModuleGraph),
-        visibleModuleIdentifiers(visibleModuleIdentifiers) {}
-
-  ModuleDependencyVector foundDependencyModuleGraph;
-  std::vector<std::string> visibleModuleIdentifiers;
-};
-
 /// A module "loader" that looks for .swiftinterface and .swiftmodule files
 /// for the purpose of determining dependencies, but does not attempt to
 /// load the module files.
@@ -89,10 +77,6 @@ private:
 
   /// AST delegate to be used for textual interface scanning
   InterfaceSubContextDelegate &astDelegate;
-  /// Location where pre-built modules are to be built into.
-  std::string moduleOutputPath;
-  /// Location where pre-built SDK modules are to be built into.
-  std::string sdkModuleOutputPath;
   /// Clang-specific (-Xcc) command-line flags to include on
   /// Swift module compilation commands
   std::vector<std::string> swiftModuleClangCC1CommandLineArgs;
@@ -108,14 +92,12 @@ private:
 public:
   SwiftModuleScanner(
       ASTContext &ctx, ModuleLoadingMode LoadMode,
-      InterfaceSubContextDelegate &astDelegate, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath,
+      InterfaceSubContextDelegate &astDelegate,
       std::vector<std::string> swiftModuleClangCC1CommandLineArgs,
       llvm::StringMap<std::string> &explicitSwiftModuleInputs)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
-        astDelegate(astDelegate), moduleOutputPath(moduleOutputPath),
-        sdkModuleOutputPath(sdkModuleOutputPath),
+        astDelegate(astDelegate),
         swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs),
         explicitSwiftModuleInputs(explicitSwiftModuleInputs) {
   }