averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

[Dependency Scanning] Serialized Swift binary module serialized search paths

Browse Source
This commit is contained in:
Artem Chikin
2025-06-03 10:10:42 -07:00
parent 5dc42ffb0c
commit e67be5ad7a
5 changed files with 183 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/swift/AST/DiagnosticsSema.def
View File

@@ -2339,7 +2339,7 @@ NOTE(dependency_as_imported_by_main_module,none,
NOTE(dependency_as_imported_by, none,
"a dependency of %select{Swift|Clang}2 module '%0': '%1'", (StringRef, StringRef, bool))
NOTE(inherited_search_path_resolves_module,none,
"'%0' can be found on search path used to build module '%1': '%2'. "
"'%0' can be found on a search path used to build module '%1': '%2'. "
"These search paths are not inherited by the current compilation.", (StringRef, StringRef, StringRef))
ERROR(clang_dependency_scan_error, none, "Clang dependency scanner failure: %0", (StringRef))
ERROR(clang_header_dependency_scan_error, none, "Bridging header dependency scan failure: %0", (StringRef))

17
include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h
View File

@@ -41,7 +41,7 @@ using llvm::BCVBR;
const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 9;
/// Increment this on every change.
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 2;
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 3;
/// Various identifiers in this format will rely on having their strings mapped
/// using this ID.
@@ -78,6 +78,7 @@ using ModuleCacheKeyIDField = IdentifierIDField;
using ImportArrayIDField = IdentifierIDField;
using LinkLibrariesArrayIDField = IdentifierIDField;
using MacroDependenciesArrayIDField = IdentifierIDField;
using SearchPathArrayIDField = IdentifierIDField;
using FlagIDArrayIDField = IdentifierIDField;
using DependencyIDArrayIDField = IdentifierIDField;
using SourceLocationIDArrayIDField = IdentifierIDField;
@@ -101,6 +102,8 @@ enum {
LINK_LIBRARY_ARRAY_NODE,
MACRO_DEPENDENCY_NODE,
MACRO_DEPENDENCY_ARRAY_NODE,
SEARCH_PATH_NODE,
SEARCH_PATH_ARRAY_NODE,
IMPORT_STATEMENT_NODE,
IMPORT_STATEMENT_ARRAY_NODE,
OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE,
@@ -169,6 +172,17 @@ using MacroDependencyLayout =
using MacroDependencyArrayLayout =
BCRecordLayout<MACRO_DEPENDENCY_ARRAY_NODE, IdentifierIDArryField>;
// A record for a serialized search pathof a given dependency
// node (Swift binary module dependency only).
using SearchPathLayout =
BCRecordLayout<SEARCH_PATH_NODE, // ID
IdentifierIDField, // path
IsFrameworkField, // isFramework
IsSystemField // isSystem
>;
using SearchPathArrayLayout =
BCRecordLayout<SEARCH_PATH_ARRAY_NODE, IdentifierIDArryField>;
// A record capturing information about a given 'import' statement
// captured in a dependency node, including its source location.
using ImportStatementLayout =
@@ -248,6 +262,7 @@ using SwiftBinaryModuleDetailsLayout =
FileIDField, // definingInterfacePath
IdentifierIDField, // headerModuleDependencies
FileIDArrayIDField, // headerSourceFiles
SearchPathArrayIDField, // serializedSearchPaths
IsFrameworkField, // isFramework
IsStaticField, // isStatic
IdentifierIDField, // moduleCacheKey

127
lib/DependencyScan/ModuleDependencyCacheSerialization.cpp
View File

@@ -45,9 +45,11 @@ class ModuleDependenciesCacheDeserializer {
std::vector<LinkLibrary> LinkLibraries;
std::vector<std::vector<uint64_t>> ArraysOfLinkLibraryIDs;
std::vector<std::pair<std::string, MacroPluginDependency>> MacroDependencies;
std::vector<serialization::SearchPath> SearchPaths;
std::vector<std::vector<uint64_t>> ArraysOfMacroDependenciesIDs;
std::vector<ScannerImportStatementInfo> ImportStatements;
std::vector<std::vector<uint64_t>> ArraysOfImportStatementIDs;
std::vector<std::vector<uint64_t>> ArraysOfSearchPathIDs;
std::vector<std::vector<uint64_t>> ArraysOfOptionalImportStatementIDs;
llvm::BitstreamCursor Cursor;
@@ -66,6 +68,8 @@ class ModuleDependenciesCacheDeserializer {
std::optional<std::vector<LinkLibrary>> getLinkLibraryArray(unsigned n);
std::optional<std::vector<std::pair<std::string, MacroPluginDependency>>>
getMacroDependenciesArray(unsigned n);
std::optional<std::vector<serialization::SearchPath>>
getSearchPathArray(unsigned n);
std::optional<std::vector<ScannerImportStatementInfo>>
getImportStatementInfoArray(unsigned n);
std::optional<std::vector<ScannerImportStatementInfo>>
@@ -389,6 +393,24 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
break;
}
case SEARCH_PATH_NODE: {
unsigned pathStrID;
bool isFramework, isSystem;
SearchPathLayout::readRecord(Scratch, pathStrID, isFramework, isSystem);
auto pathStr = getIdentifier(pathStrID);
if (!pathStr)
llvm::report_fatal_error("Bad search path: no path string");
SearchPaths.push_back({*pathStr, isFramework, isSystem});
break;
}
case SEARCH_PATH_ARRAY_NODE: {
ArrayRef<uint64_t> identifierIDs;
SearchPathArrayLayout::readRecord(Scratch, identifierIDs);
ArraysOfSearchPathIDs.push_back(identifierIDs.vec());
break;
}
case IMPORT_STATEMENT_NODE: {
unsigned importIdentifierID, bufferIdentifierID;
unsigned lineNumber, columnNumber;
@@ -653,14 +675,15 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
llvm::report_fatal_error(
"Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
headerImportID, definingInterfacePathID,
headerImportID, definingInterfacePathID, searchPathArrayID,
headerModuleDependenciesArrayID, headerImportsSourceFilesArrayID,
isFramework, isStatic, moduleCacheKeyID, userModuleVersionID;
SwiftBinaryModuleDetailsLayout::readRecord(
Scratch, compiledModulePathID, moduleDocPathID,
moduleSourceInfoPathID, headerImportID, definingInterfacePathID,
headerModuleDependenciesArrayID, headerImportsSourceFilesArrayID,
isFramework, isStatic, moduleCacheKeyID, userModuleVersionID);
searchPathArrayID, isFramework, isStatic, moduleCacheKeyID,
userModuleVersionID);
auto compiledModulePath = getIdentifier(compiledModulePathID);
if (!compiledModulePath)
@@ -685,14 +708,17 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
if (!definingInterfacePath)
llvm::report_fatal_error(
"Bad binary direct dependencies: no defining interface path");
auto searchPaths = getSearchPathArray(searchPathArrayID);
if (!searchPaths)
llvm::report_fatal_error(
"Bad binary direct dependencies: no serialized search paths");
// Form the dependencies storage object
auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
*compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
importStatements, optionalImportStatements, linkLibraries,
{}, // TODO: serialized search path serialization
*headerImport, *definingInterfacePath, isFramework, isStatic,
*moduleCacheKey, *userModuleVersion);
*searchPaths, *headerImport, *definingInterfacePath, isFramework,
isStatic, *moduleCacheKey, *userModuleVersion);
addCommonDependencyInfo(moduleDep);
addSwiftCommonDependencyInfo(moduleDep);
@@ -909,6 +935,25 @@ ModuleDependenciesCacheDeserializer::getMacroDependenciesArray(unsigned n) {
return result;
}
std::optional<std::vector<serialization::SearchPath>>
ModuleDependenciesCacheDeserializer::getSearchPathArray(unsigned n) {
if (n == 0)
return std::vector<serialization::SearchPath>();
--n;
if (n >= ArraysOfSearchPathIDs.size())
return std::nullopt;
auto &llIDs = ArraysOfSearchPathIDs[n];
auto IDtoLLMap = [this](unsigned index) { return SearchPaths[index]; };
std::vector<serialization::SearchPath> result;
result.reserve(llIDs.size());
std::transform(llIDs.begin(), llIDs.end(), std::back_inserter(result),
IDtoLLMap);
return result;
}
std::optional<std::vector<ScannerImportStatementInfo>>
ModuleDependenciesCacheDeserializer::getImportStatementInfoArray(unsigned n) {
if (n == 0)
@@ -1084,6 +1129,7 @@ class ModuleDependenciesCacheSerializer {
std::unordered_map<ModuleDependencyID, unsigned> LinkLibraryArrayIDsMap;
std::unordered_map<ModuleDependencyID, unsigned> MacroDependenciesArrayIDsMap;
std::unordered_map<ModuleDependencyID, unsigned> SearchPathArrayIDsMap;
std::unordered_map<ModuleDependencyID, unsigned> ImportInfosArrayIDsMap;
std::unordered_map<ModuleDependencyID, unsigned>
OptionalImportInfosArrayIDsMap;
@@ -1110,6 +1156,7 @@ class ModuleDependenciesCacheSerializer {
ModuleIdentifierArrayKind arrayKind) const;
unsigned getLinkLibrariesArrayID(ModuleDependencyID moduleID) const;
unsigned getMacroDependenciesArrayID(ModuleDependencyID moduleID) const;
unsigned getSearchPathArrayID(ModuleDependencyID moduleID) const;
unsigned getImportStatementsArrayID(ModuleDependencyID moduleID) const;
unsigned
getOptionalImportStatementsArrayID(ModuleDependencyID moduleID) const;
@@ -1146,6 +1193,10 @@ class ModuleDependenciesCacheSerializer {
unsigned writeMacroDependencies(const ModuleDependencyInfo &dependencyInfo);
void writeMacroDependenciesArray(unsigned startIndex, unsigned count);
void writeSearchPaths(const ModuleDependenciesCache &cache);
unsigned writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo);
void writeSearchPathsArray(unsigned startIndex, unsigned count);
void writeImportStatementInfos(const ModuleDependenciesCache &cache);
unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo,
bool optional);
@@ -1206,6 +1257,8 @@ void ModuleDependenciesCacheSerializer::writeBlockInfoBlock() {
BLOCK_RECORD(graph_block, LINK_LIBRARY_ARRAY_NODE);
BLOCK_RECORD(graph_block, MACRO_DEPENDENCY_NODE);
BLOCK_RECORD(graph_block, MACRO_DEPENDENCY_ARRAY_NODE);
BLOCK_RECORD(graph_block, SEARCH_PATH_NODE);
BLOCK_RECORD(graph_block, SEARCH_PATH_ARRAY_NODE);
BLOCK_RECORD(graph_block, IMPORT_STATEMENT_NODE);
BLOCK_RECORD(graph_block, IMPORT_STATEMENT_ARRAY_NODE);
BLOCK_RECORD(graph_block, OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE);
@@ -1367,6 +1420,52 @@ void ModuleDependenciesCacheSerializer::writeMacroDependenciesArray(
Out, ScratchRecord, AbbrCodes[MacroDependencyArrayLayout::Code], vec);
}
void ModuleDependenciesCacheSerializer::writeSearchPaths(const ModuleDependenciesCache &cache) {
unsigned lastSPIndex = 0;
std::map<ModuleDependencyID, std::pair<unsigned, unsigned>>
moduleSearchPathArrayMap;
auto modMap = cache.getDependenciesMap(ModuleDependencyKind::SwiftBinary);
for (const auto &entry : modMap) {
ModuleDependencyID moduleID = {entry.getKey().str(), ModuleDependencyKind::SwiftBinary};
auto optionalDependencyInfo = cache.findDependency(moduleID);
assert(optionalDependencyInfo && "Expected dependency info.");
auto dependencyInfo = *optionalDependencyInfo;
unsigned numSPs = writeSearchPaths(*dependencyInfo->getAsSwiftBinaryModule());
moduleSearchPathArrayMap.insert({moduleID, std::make_pair(lastSPIndex, numSPs)});
lastSPIndex += numSPs;
}
unsigned lastSPArrayIndex = 1;
for (const auto &entry : modMap) {
ModuleDependencyID moduleID = {entry.getKey().str(), ModuleDependencyKind::SwiftBinary};
auto entries = moduleSearchPathArrayMap.at(moduleID);
if (entries.second == 0)
continue;
writeSearchPathsArray(entries.first, entries.second);
SearchPathArrayIDsMap.insert({moduleID, lastSPArrayIndex++});
}
}
unsigned ModuleDependenciesCacheSerializer::writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo) {
using namespace graph_block;
for (const auto &searchPath : dependencyInfo.serializedSearchPaths) {
SearchPathLayout::emitRecord(
Out, ScratchRecord, AbbrCodes[SearchPathLayout::Code],
getIdentifier(searchPath.Path),
searchPath.IsFramework,
searchPath.IsSystem);
}
return dependencyInfo.serializedSearchPaths.size();
}
void ModuleDependenciesCacheSerializer::writeSearchPathsArray(unsigned startIndex, unsigned count) {
using namespace graph_block;
std::vector<unsigned> vec(count);
std::iota(vec.begin(), vec.end(), startIndex);
SearchPathArrayLayout::emitRecord(
Out, ScratchRecord, AbbrCodes[SearchPathArrayLayout::Code], vec);
}
void ModuleDependenciesCacheSerializer::writeImportStatementInfos(
const ModuleDependenciesCache &cache) {
unsigned lastImportInfoIndex = 0;
@@ -1559,10 +1658,10 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
getIdentifierArrayID(
moduleID,
ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles),
getSearchPathArrayID(moduleID),
swiftBinDeps->isFramework, swiftBinDeps->isStatic,
getIdentifier(swiftBinDeps->moduleCacheKey),
getIdentifier(swiftBinDeps->userModuleVersion));
break;
}
case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1699,6 +1798,15 @@ unsigned ModuleDependenciesCacheSerializer::getMacroDependenciesArrayID(
return iter->second;
}
unsigned ModuleDependenciesCacheSerializer::getSearchPathArrayID(
ModuleDependencyID moduleID) const {
auto iter = SearchPathArrayIDsMap.find(moduleID);
if (iter == SearchPathArrayIDsMap.end())
return 0;
return iter->second;
}
unsigned ModuleDependenciesCacheSerializer::getImportStatementsArrayID(
ModuleDependencyID moduleID) const {
auto iter = ImportInfosArrayIDsMap.find(moduleID);
@@ -1820,6 +1928,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
moduleID,
ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles,
swiftBinDeps->headerSourceFiles);
llvm::for_each(swiftBinDeps->serializedSearchPaths,
[this](auto &sp) { addIdentifier(sp.Path); });
break;
}
case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1899,6 +2009,8 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
registerRecordAbbr<LinkLibraryArrayLayout>();
registerRecordAbbr<MacroDependencyLayout>();
registerRecordAbbr<MacroDependencyArrayLayout>();
registerRecordAbbr<SearchPathLayout>();
registerRecordAbbr<SearchPathArrayLayout>();
registerRecordAbbr<ImportStatementLayout>();
registerRecordAbbr<ImportStatementArrayLayout>();
registerRecordAbbr<ModuleInfoLayout>();
@@ -1933,6 +2045,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
// Write all the arrays of macro dependency infos for this graph
writeMacroDependencies(cache);
// Write all the arrays of binary-module-serialized search paths
writeSearchPaths(cache);
// Write the core graph
for (auto kind = ModuleDependencyKind::FirstKind;
kind != ModuleDependencyKind::LastKind; ++kind) {

44
test/ScanDependencies/cached-missing-module-found-in-serialized-paths.swift Normal file
View File

@@ -0,0 +1,44 @@
// RUN: %empty-directory(%t)
// RUN: %empty-directory(%t/module-cache)
// RUN: %empty-directory(%t/deps)
// RUN: %empty-directory(%t/moreDeps)
// RUN: split-file %s %t
// RUN: %target-swift-frontend -emit-module -emit-module-path %t/deps/B.swiftmodule -module-cache-path %t/module-cache %t/B.swift -module-name B -I %t/moreDeps
// Put the dependency module into a location discoverable by the first scan which will succeed and serialize scanner state
// RUN: cp %t/moreDeps/C.swiftinterface %t/deps/C.swiftinterface
// RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json %t/client.swift -I %t/deps -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache &> %t/initial_output.txt
// Remove the 'C' dependency module into a location not discoverable by the second scan in order to trigger a failure and use serialized scanner state
// to emit the diagnostic
// RUN: rm %t/deps/C.swiftinterface
// RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json %t/client.swift -I %t/deps -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -validate-prior-dependency-scan-cache &> %t/output.txt
// RUN: cat %t/output.txt | %FileCheck %s
// CHECK: remark: Incremental module scan: Re-using serialized module scanning dependency cache from: '{{.*}}cache.moddepcache'.
// CHECK: remark: Incremental module scan: Dependency info for module 'C' invalidated due to a modified input since last scan: '{{.*}}deps{{/|\\}}C.swiftinterface'.
// CHECK: remark: Incremental module scan: Dependency info for module 'deps' invalidated due to an out-of-date dependency.
// CHECK: error: Unable to find module dependency: 'C'
// CHECK: note: a dependency of main module 'deps'
// CHECK: note: 'C' can be found on a search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
//--- moreDeps/C.swiftinterface
// swift-interface-format-version: 1.0
// swift-module-flags: -module-name C -enable-library-evolution
public struct structC {}
//--- deps/A.swiftinterface
// swift-interface-format-version: 1.0
// swift-module-flags: -module-name A -enable-library-evolution
public func funcA() {}
//--- B.swift
public func funcB() {}
//--- client.swift
import A
import B
import C

2
test/ScanDependencies/diagnose-missing-module-found-in-serialized-paths.swift
View File

@@ -12,7 +12,7 @@
// CHECK: error: Unable to find module dependency: 'C'
// CHECK: note: a dependency of Swift module 'B': '{{.*}}B.swiftmodule'
// CHECK: note: a dependency of main module 'deps'
// CHECK: note: 'C' can be found on search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
// CHECK: note: 'C' can be found on a search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
//--- moreDeps/C.swiftinterface
// swift-interface-format-version: 1.0