[Dependency Scanning] Refactor Clang dependency bridging into a 'ModuleDependencyScanner' utility

This moves the functionality of 'bridgeClangModuleDependency' into a utility in the main scanner class because it relies on various objects whose lifetime is already tied to the scanner itself.
This commit is contained in:
Artem Chikin
2025-08-12 14:15:53 -07:00
parent 5015ba683a
commit 9f0083c7c0
5 changed files with 301 additions and 299 deletions

View File

@@ -137,28 +137,28 @@ computeClangWorkingDirectory(const std::vector<std::string> &commandLineArgs,
return workingDir;
}
static std::string moduleCacheRelativeLookupModuleOutput(
const clang::tooling::dependencies::ModuleDeps &MD,
clang::tooling::dependencies::ModuleOutputKind MOK,
const StringRef moduleCachePath, const StringRef stableModuleCachePath,
const StringRef runtimeResourcePath) {
llvm::SmallString<128> outputPath(moduleCachePath);
if (MD.IsInStableDirectories)
outputPath = stableModuleCachePath;
std::string ModuleDependencyScanner::clangModuleOutputPathLookup(
const clang::tooling::dependencies::ModuleDeps &clangDeps,
clang::tooling::dependencies::ModuleOutputKind moduleOutputKind) const {
llvm::SmallString<128> outputPath(ModuleOutputPath);
if (clangDeps.IsInStableDirectories)
outputPath = SDKModuleOutputPath;
auto runtimeResourcePath = ScanASTContext.SearchPathOpts.RuntimeResourcePath;
// FIXME: This is a hack to treat Clang modules defined in the compiler's
// own resource directory as stable, when they are not reported as such
// by the Clang scanner.
if (!runtimeResourcePath.empty() &&
hasPrefix(llvm::sys::path::begin(MD.ClangModuleMapFile),
llvm::sys::path::end(MD.ClangModuleMapFile),
hasPrefix(llvm::sys::path::begin(clangDeps.ClangModuleMapFile),
llvm::sys::path::end(clangDeps.ClangModuleMapFile),
llvm::sys::path::begin(runtimeResourcePath),
llvm::sys::path::end(runtimeResourcePath)))
outputPath = stableModuleCachePath;
outputPath = SDKModuleOutputPath;
llvm::sys::path::append(outputPath,
MD.ID.ModuleName + "-" + MD.ID.ContextHash);
switch (MOK) {
llvm::sys::path::append(outputPath, clangDeps.ID.ModuleName + "-" +
clangDeps.ID.ContextHash);
switch (moduleOutputKind) {
case clang::tooling::dependencies::ModuleOutputKind::ModuleFile:
llvm::sys::path::replace_extension(
outputPath, getExtension(swift::file_types::TY_ClangModuleFile));
@@ -168,7 +168,7 @@ static std::string moduleCacheRelativeLookupModuleOutput(
outputPath, getExtension(swift::file_types::TY_Dependencies));
break;
case clang::tooling::dependencies::ModuleOutputKind::DependencyTargets:
return MD.ID.ModuleName + "-" + MD.ID.ContextHash;
return clangDeps.ID.ModuleName + "-" + clangDeps.ID.ContextHash;
case clang::tooling::dependencies::ModuleOutputKind::
DiagnosticSerializationFile:
llvm::sys::path::replace_extension(
@@ -193,12 +193,6 @@ static std::vector<std::string> inputSpecificClangScannerCommand(
return result;
}
static std::string remapPath(llvm::PrefixMapper *PrefixMapper, StringRef Path) {
if (!PrefixMapper)
return Path.str();
return PrefixMapper->mapToString(Path);
}
static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
getClangScanningFS(std::shared_ptr<llvm::cas::ObjectStore> cas,
ASTContext &ctx) {
@@ -251,11 +245,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
RequireOSSAModules_t(SILOptions))),
clangScanningTool(*globalScanningService.ClangScanningService,
getClangScanningFS(CAS, ScanASTContext)),
moduleOutputPath(workerCompilerInvocation->getFrontendOptions()
.ExplicitModulesOutputPath),
sdkModuleOutputPath(workerCompilerInvocation->getFrontendOptions()
.ExplicitSDKModulesOutputPath),
CAS(CAS), ActionCache(ActionCache), PrefixMapper(Mapper) {
CAS(CAS), ActionCache(ActionCache) {
auto loader = std::make_unique<PluginLoader>(
*workerASTContext, /*DepTracker=*/nullptr,
workerCompilerInvocation->getFrontendOptions().CacheReplayPrefixMap,
@@ -293,8 +283,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker(
swiftModuleScannerLoader = std::make_unique<SwiftModuleScanner>(
*workerASTContext,
workerCompilerInvocation->getSearchPathOptions().ModuleLoadMode,
*scanningASTDelegate, moduleOutputPath, sdkModuleOutputPath,
swiftModuleClangCC1CommandLineArgs,
*scanningASTDelegate, swiftModuleClangCC1CommandLineArgs,
workerCompilerInvocation->getSearchPathOptions().ExplicitSwiftModuleInputs);
}
@@ -308,17 +297,9 @@ ModuleDependencyScanningWorker::scanFilesystemForSwiftModuleDependency(
std::optional<clang::tooling::dependencies::TranslationUnitDeps>
ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency(
Identifier moduleName,
LookupModuleOutputCallback lookupModuleOutput,
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
&alreadySeenModules) {
auto lookupModuleOutput =
[this](const clang::tooling::dependencies::ModuleDeps &MD,
const clang::tooling::dependencies::ModuleOutputKind MOK)
-> std::string {
return moduleCacheRelativeLookupModuleOutput(
MD, MOK, moduleOutputPath, sdkModuleOutputPath,
workerASTContext->SearchPathOpts.RuntimeResourcePath);
};
auto clangModuleDependencies = clangScanningTool.getModuleDependencies(
moduleName.str(), clangScanningModuleCommandLineArgs,
clangScanningWorkingDirectoryPath, alreadySeenModules,
@@ -338,50 +319,24 @@ ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency(
return clangModuleDependencies.get();
}
bool ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule(
const ASTContext &ctx, ModuleDependencyID moduleID,
std::optional<clang::tooling::dependencies::TranslationUnitDeps>
ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule(
ModuleDependencyID moduleID,
std::optional<StringRef> headerPath,
std::optional<llvm::MemoryBufferRef> sourceBuffer,
ModuleDependenciesCache &cache,
ModuleDependencyIDSetVector &headerClangModuleDependencies,
std::vector<std::string> &headerFileInputs,
std::vector<std::string> &bridgingHeaderCommandLine,
std::vector<std::string> &visibleClangModules,
std::optional<std::string> &includeTreeID) {
LookupModuleOutputCallback lookupModuleOutput,
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
&alreadySeenModules) {
// Scan the specified textual header file and collect its dependencies
auto scanHeaderDependencies = [&]()
-> llvm::Expected<clang::tooling::dependencies::TranslationUnitDeps> {
auto lookupModuleOutput =
[this, &ctx](const clang::tooling::dependencies::ModuleDeps &MD,
const clang::tooling::dependencies::ModuleOutputKind MOK)
-> std::string {
return moduleCacheRelativeLookupModuleOutput(
MD, MOK, moduleOutputPath, sdkModuleOutputPath,
ctx.SearchPathOpts.RuntimeResourcePath);
};
auto dependencies = clangScanningTool.getTranslationUnitDependencies(
inputSpecificClangScannerCommand(clangScanningBaseCommandLineArgs,
headerPath),
clangScanningWorkingDirectoryPath, cache.getAlreadySeenClangModules(),
clangScanningWorkingDirectoryPath, alreadySeenModules,
lookupModuleOutput, sourceBuffer);
if (!dependencies)
return dependencies.takeError();
// Record module dependencies for each new module we found.
cache.recordClangDependencies(
dependencies->ModuleGraph, ctx, lookupModuleOutput,
[this](StringRef path) { return remapPath(PrefixMapper, path); });
visibleClangModules = dependencies->VisibleModules;
llvm::copy(dependencies->FileDeps, std::back_inserter(headerFileInputs));
auto bridgedDependencyIDs =
llvm::map_range(dependencies->ClangModuleDeps, [](auto &input) {
return ModuleDependencyID{input.ModuleName,
ModuleDependencyKind::Clang};
});
headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(),
bridgedDependencyIDs.end());
return dependencies;
};
@@ -393,18 +348,10 @@ bool ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule(
auto errorStr = toString(clangModuleDependencies.takeError());
workerASTContext->Diags.diagnose(
SourceLoc(), diag::clang_header_dependency_scan_error, errorStr);
return true;
return std::nullopt;
}
auto targetModuleInfo = cache.findKnownDependency(moduleID);
if (!targetModuleInfo.isTextualSwiftModule())
return false;
if (auto TreeID = clangModuleDependencies->IncludeTreeID)
includeTreeID = TreeID;
ClangImporter::getBridgingHeaderOptions(ctx, *clangModuleDependencies,
bridgingHeaderCommandLine);
return false;
return *clangModuleDependencies;
}
template <typename Function, typename... Args>
@@ -576,6 +523,10 @@ ModuleDependencyScanner::ModuleDependencyScanner(
DiagnosticEngine &Diagnostics, bool ParallelScan)
: ScanCompilerInvocation(ScanCompilerInvocation),
ScanASTContext(ScanASTContext), IssueReporter(Diagnostics),
ModuleOutputPath(ScanCompilerInvocation.getFrontendOptions()
.ExplicitModulesOutputPath),
SDKModuleOutputPath(ScanCompilerInvocation.getFrontendOptions()
.ExplicitSDKModulesOutputPath),
NumThreads(ParallelScan
? llvm::hardware_concurrency().compute_thread_count()
: 1),
@@ -1112,17 +1063,18 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies(
llvm::StringMap<
std::optional<clang::tooling::dependencies::TranslationUnitDeps>>
moduleLookupResult;
const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
seenClangModules = cache.getAlreadySeenClangModules();
auto seenClangModules = cache.getAlreadySeenClangModules();
std::mutex resultAccessLock;
auto scanForClangModuleDependency = [this, &moduleLookupResult,
&resultAccessLock, &seenClangModules](
Identifier moduleIdentifier) {
auto scanResult = withDependencyScanningWorker(
[&seenClangModules,
moduleIdentifier](ModuleDependencyScanningWorker *ScanningWorker) {
[&](ModuleDependencyScanningWorker *ScanningWorker) {
auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto {
return clangModuleOutputPathLookup(cd, mok);
};
return ScanningWorker->scanFilesystemForClangModuleDependency(
moduleIdentifier, seenClangModules);
moduleIdentifier, lookupModuleOutput, seenClangModules);
});
{
std::lock_guard<std::mutex> guard(resultAccessLock);
@@ -1154,22 +1106,11 @@ void ModuleDependencyScanner::resolveAllClangModuleDependencies(
const auto &lookupResult =
moduleLookupResult.at(moduleImport.importIdentifier);
if (lookupResult.has_value()) {
auto lookupModuleOutput =
[this](const clang::tooling::dependencies::ModuleDeps &MD,
const clang::tooling::dependencies::ModuleOutputKind MOK)
-> std::string {
return moduleCacheRelativeLookupModuleOutput(
MD, MOK,
ScanCompilerInvocation.getFrontendOptions()
.ExplicitModulesOutputPath,
ScanCompilerInvocation.getFrontendOptions()
.ExplicitSDKModulesOutputPath,
ScanASTContext.SearchPathOpts.RuntimeResourcePath);
};
cache.recordClangDependencies(
lookupResult->ModuleGraph, ScanASTContext, lookupModuleOutput,
[this](StringRef path) { return remapPath(path); });
lookupResult->ModuleGraph, ScanASTContext.Diags,
[this](auto &clangDep) {
return bridgeClangModuleDependency(clangDep);
});
// Add the full transitive dependency set
for (const auto &dep : lookupResult->ModuleGraph)
@@ -1456,12 +1397,39 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule(
std::optional<std::string> includeTreeID;
std::vector<std::string> bridgingHeaderCommandLine;
std::vector<std::string> visibleClangModules;
auto headerScan = ScanningWorker->scanHeaderDependenciesOfSwiftModule(
*ScanningWorker->workerASTContext, moduleID, headerPath,
sourceBufferRef, cache, headerClangModuleDependencies,
headerFileInputs, bridgingHeaderCommandLine, visibleClangModules,
includeTreeID);
if (!headerScan) {
auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto {
return clangModuleOutputPathLookup(cd, mok);
};
auto headerScanResult =
ScanningWorker->scanHeaderDependenciesOfSwiftModule(
moduleID, headerPath, sourceBufferRef, lookupModuleOutput,
cache.getAlreadySeenClangModules());
if (headerScanResult) {
// Record module dependencies for each new module we found.
cache.recordClangDependencies(
headerScanResult->ModuleGraph, ScanASTContext.Diags,
[this](auto &clangDep) {
return bridgeClangModuleDependency(clangDep);
});
llvm::copy(headerScanResult->FileDeps,
std::back_inserter(headerFileInputs));
auto bridgedDependencyIDs = llvm::map_range(
headerScanResult->ClangModuleDeps, [](auto &input) {
return ModuleDependencyID{input.ModuleName,
ModuleDependencyKind::Clang};
});
headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(),
bridgedDependencyIDs.end());
auto targetModuleInfo = cache.findKnownDependency(moduleID);
if (targetModuleInfo.isTextualSwiftModule()) {
if (auto TreeID = headerScanResult->IncludeTreeID)
includeTreeID = TreeID;
ClangImporter::getBridgingHeaderOptions(
ScanASTContext, *headerScanResult, bridgingHeaderCommandLine);
}
// Record direct header Clang dependencies
moduleDependencyInfo.setHeaderClangDependencies(
headerClangModuleDependencies.getArrayRef());
@@ -1474,7 +1442,8 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule(
bridgingHeaderCommandLine);
moduleDependencyInfo.setHeaderSourceFiles(headerFileInputs);
// Update the set of visible Clang modules
moduleDependencyInfo.addVisibleClangModules(visibleClangModules);
moduleDependencyInfo.addVisibleClangModules(
headerScanResult->VisibleModules);
// Update the dependency in the cache
cache.updateDependency(moduleID, moduleDependencyInfo);
} else {
@@ -1766,14 +1735,41 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining(
std::vector<std::string> headerFileInputs;
std::vector<std::string> bridgingHeaderCommandLine;
std::vector<std::string> visibleClangModules;
if (ScanningWorker->scanHeaderDependenciesOfSwiftModule(
*ScanningWorker->workerASTContext, rootModuleID,
/*headerPath=*/std::nullopt, sourceBuffer->getMemBufferRef(),
cache, headerClangModuleDependencies, headerFileInputs,
bridgingHeaderCommandLine, visibleClangModules, includeTreeID))
auto lookupModuleOutput = [this](const auto &cd, auto mok) -> auto {
return clangModuleOutputPathLookup(cd, mok);
};
auto headerScanResult =
ScanningWorker->scanHeaderDependenciesOfSwiftModule(
rootModuleID, /*headerPath=*/std::nullopt,
sourceBuffer->getMemBufferRef(), lookupModuleOutput,
cache.getAlreadySeenClangModules());
if (!headerScanResult)
return llvm::createStringError(
"failed to scan generated bridging header " + outputPath);
// Record module dependencies for each new module we found.
cache.recordClangDependencies(
headerScanResult->ModuleGraph, ScanASTContext.Diags,
[this](auto &clangDep) {
return bridgeClangModuleDependency(clangDep);
});
llvm::copy(headerScanResult->FileDeps,
std::back_inserter(headerFileInputs));
auto bridgedDependencyIDs =
llvm::map_range(headerScanResult->ClangModuleDeps, [](auto &input) {
return ModuleDependencyID{input.ModuleName,
ModuleDependencyKind::Clang};
});
headerClangModuleDependencies.insert(bridgedDependencyIDs.begin(),
bridgedDependencyIDs.end());
// Update visible module set
if (auto TreeID = headerScanResult->IncludeTreeID)
includeTreeID = TreeID;
ClangImporter::getBridgingHeaderOptions(
ScanASTContext, *headerScanResult, bridgingHeaderCommandLine);
cache.setHeaderClangDependencies(
rootModuleID, headerClangModuleDependencies.getArrayRef());
// Record include Tree ID
@@ -1794,6 +1790,9 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining(
mainModuleDeps.setHeaderSourceFiles(headerFileInputs);
mainModuleDeps.setChainedBridgingHeaderBuffer(
outputPath, sourceBuffer->getBuffer());
// Update the set of visible Clang modules
mainModuleDeps.addVisibleClangModules(headerScanResult->VisibleModules);
// Update the dependency in the cache
cache.updateDependency(rootModuleID, mainModuleDeps);
return llvm::Error::success();
@@ -1812,6 +1811,118 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining(
return llvm::Error::success();
}
std::string ModuleDependencyScanner::remapPath(StringRef Path) const {
if (!PrefixMapper)
return Path.str();
return PrefixMapper->mapToString(Path);
}
ModuleDependencyInfo ModuleDependencyScanner::bridgeClangModuleDependency(
const clang::tooling::dependencies::ModuleDeps &clangModuleDep) {
// File dependencies for this module.
std::vector<std::string> fileDeps;
clangModuleDep.forEachFileDep(
[&fileDeps](StringRef fileDep) { fileDeps.emplace_back(fileDep); });
std::vector<std::string> swiftArgs;
auto addClangArg = [&](Twine arg) {
swiftArgs.push_back("-Xcc");
swiftArgs.push_back(arg.str());
};
// We are using Swift frontend mode.
swiftArgs.push_back("-frontend");
// Swift frontend action: -emit-pcm
swiftArgs.push_back("-emit-pcm");
swiftArgs.push_back("-module-name");
swiftArgs.push_back(clangModuleDep.ID.ModuleName);
auto pcmPath = clangModuleOutputPathLookup(
clangModuleDep,
clang::tooling::dependencies::ModuleOutputKind::ModuleFile);
swiftArgs.push_back("-o");
swiftArgs.push_back(pcmPath);
// Ensure that the resulting PCM build invocation uses Clang frontend
// directly
swiftArgs.push_back("-direct-clang-cc1-module-build");
// Swift frontend option for input file path (Foo.modulemap).
swiftArgs.push_back(remapPath(clangModuleDep.ClangModuleMapFile));
auto invocation = clangModuleDep.getUnderlyingCompilerInvocation();
// Clear some options from clang scanner.
invocation.getMutFrontendOpts().ModuleCacheKeys.clear();
invocation.getMutFrontendOpts().PathPrefixMappings.clear();
invocation.getMutFrontendOpts().OutputFile.clear();
// Reset CASOptions since that should be coming from swift.
invocation.getMutCASOpts() = clang::CASOptions();
invocation.getMutFrontendOpts().CASIncludeTreeID.clear();
// FIXME: workaround for rdar://105684525: find the -ivfsoverlay option
// from clang scanner and pass to swift.
if (!ScanASTContext.CASOpts.EnableCaching) {
auto &overlayFiles = invocation.getMutHeaderSearchOpts().VFSOverlayFiles;
for (auto overlay : overlayFiles) {
swiftArgs.push_back("-vfsoverlay");
swiftArgs.push_back(overlay);
}
}
// Add args reported by the scanner.
auto clangArgs = invocation.getCC1CommandLine();
llvm::for_each(clangArgs, addClangArg);
// CASFileSystemRootID.
std::string RootID = clangModuleDep.CASFileSystemRootID
? clangModuleDep.CASFileSystemRootID->toString()
: "";
std::string IncludeTree =
clangModuleDep.IncludeTreeID ? *clangModuleDep.IncludeTreeID : "";
ScanASTContext.CASOpts.enumerateCASConfigurationFlags(
[&](StringRef Arg) { swiftArgs.push_back(Arg.str()); });
if (!IncludeTree.empty()) {
swiftArgs.push_back("-clang-include-tree-root");
swiftArgs.push_back(IncludeTree);
}
std::string mappedPCMPath = remapPath(pcmPath);
std::vector<LinkLibrary> LinkLibraries;
for (const auto &ll : clangModuleDep.LinkLibraries)
LinkLibraries.emplace_back(ll.Library,
ll.IsFramework ? LibraryKind::Framework
: LibraryKind::Library,
/*static=*/false);
// Module-level dependencies.
llvm::StringSet<> alreadyAddedModules;
auto bridgedDependencyInfo = ModuleDependencyInfo::forClangModule(
pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, LinkLibraries, RootID,
IncludeTree, /*module-cache-key*/ "", clangModuleDep.IsSystem);
std::vector<ModuleDependencyID> directDependencyIDs;
for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
// FIXME: This assumes, conservatively, that all Clang module imports
// are exported. We need to fix this once the clang scanner gains the
// appropriate API to query this.
bridgedDependencyInfo.addModuleImport(
moduleName.ModuleName, /* isExported */ true, AccessLevel::Public,
&alreadyAddedModules);
// It is safe to assume that all dependencies of a Clang module are Clang
// modules.
directDependencyIDs.push_back(
{moduleName.ModuleName, ModuleDependencyKind::Clang});
}
bridgedDependencyInfo.setImportedClangDependencies(directDependencyIDs);
return bridgedDependencyInfo;
}
void ModuleDependencyIssueReporter::diagnoseModuleNotFoundFailure(
const ScannerImportStatementInfo &moduleImport,
const ModuleDependenciesCache &cache,
@@ -1970,8 +2081,7 @@ ModuleDependencyScanner::attemptToFindResolvingSerializedSearchPath(
// paths. That's fine because we are diagnosing a scan failure here, but
// worth being aware of.
result = withDependencyScanningWorker(
[this, &binaryModInfo, &moduleImport,
&binaryDepID](ModuleDependencyScanningWorker *ScanningWorker)
[&](ModuleDependencyScanningWorker *ScanningWorker)
-> std::optional<std::pair<ModuleDependencyID, std::string>> {
for (const auto &sp : binaryModInfo->serializedSearchPaths)
ScanningWorker->workerASTContext->addSearchPath(
@@ -1989,7 +2099,10 @@ ModuleDependencyScanner::attemptToFindResolvingSerializedSearchPath(
auto clangResult =
ScanningWorker->scanFilesystemForClangModuleDependency(
importIdentifier, {});
importIdentifier,
[this](const auto &cd, auto mok) -> std::string {
return clangModuleOutputPathLookup(cd, mok);
}, {});
if (clangResult)
return std::make_pair(
binaryDepID, clangResult->ModuleGraph[0].ClangModuleMapFile);