averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Frontend: teach the compiler to use a backup directory to find .swiftinterface files to compile

Browse Source 
This mechanism allows the compiler to use a backup interface file to build into a binary module when
a corresponding interface file from the SDK is failing for whatever reasons. This mechansim should be entirely opaque
to end users except several diagnostic messages communicating backup interfaces are used.

Part of rdar://77676064
This commit is contained in:
Xi Ge
2021-05-10 16:31:58 -07:00
parent 7487c12d05
commit b6cd513534
20 changed files with 303 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
lib/Frontend/ModuleInterfaceLoader.cpp
View File

@@ -346,6 +346,7 @@ class ModuleInterfaceLoaderImpl {
const std::string interfacePath;
const StringRef moduleName;
const StringRef prebuiltCacheDir;
const StringRef backupInterfaceDir;
const StringRef cacheDir;
const SourceLoc diagnosticLoc;
DependencyTracker *const dependencyTracker;
@@ -356,13 +357,16 @@ class ModuleInterfaceLoaderImpl {
ModuleInterfaceLoaderImpl(
ASTContext &ctx, StringRef modulePath, StringRef interfacePath,
StringRef moduleName, StringRef cacheDir, StringRef prebuiltCacheDir,
StringRef backupInterfaceDir,
SourceLoc diagLoc, ModuleInterfaceLoaderOptions Opts,
RequireOSSAModules_t requiresOSSAModules,
DependencyTracker *dependencyTracker = nullptr,
ModuleLoadingMode loadMode = ModuleLoadingMode::PreferSerialized)
: ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), diags(ctx.Diags),
modulePath(modulePath), interfacePath(interfacePath),
moduleName(moduleName), prebuiltCacheDir(prebuiltCacheDir),
moduleName(moduleName),
prebuiltCacheDir(prebuiltCacheDir),
backupInterfaceDir(backupInterfaceDir),
cacheDir(cacheDir), diagnosticLoc(diagLoc),
dependencyTracker(dependencyTracker), loadMode(loadMode), Opts(Opts),
requiresOSSAModules(requiresOSSAModules) {}
@@ -420,6 +424,29 @@ class ModuleInterfaceLoaderImpl {
DependencyStatus::OutOfDate;
}
std::string getBackupPublicModuleInterfacePath() {
if (backupInterfaceDir.empty())
return std::string();
auto &fs = *ctx.SourceMgr.getFileSystem();
auto fileName = llvm::sys::path::filename(interfacePath);
{
llvm::SmallString<256> path(backupInterfaceDir);
llvm::sys::path::append(path, llvm::Twine(moduleName) + ".swiftmodule");
llvm::sys::path::append(path, fileName);
if (fs.exists(path.str())) {
return path.str().str();
}
}
{
llvm::SmallString<256> path(backupInterfaceDir);
llvm::sys::path::append(path, fileName);
if (fs.exists(path.str())) {
return path.str().str();
}
}
return std::string();
}
// Check if all the provided file dependencies are up-to-date compared to
// what's currently on disk.
bool dependenciesAreUpToDate(StringRef modulePath,
@@ -521,16 +548,22 @@ class ModuleInterfaceLoaderImpl {
return true;
}
bool canInterfaceHavePrebuiltModule() {
StringRef sdkPath = ctx.SearchPathOpts.SDKPath;
if (!sdkPath.empty() &&
hasPrefix(path::begin(interfacePath), path::end(interfacePath),
path::begin(sdkPath), path::end(sdkPath))) {
return !StringRef(interfacePath).endswith(".private.swiftinterface");
}
return false;
}
Optional<StringRef>
computePrebuiltModulePath(llvm::SmallString<256> &scratch) {
namespace path = llvm::sys::path;
StringRef sdkPath = ctx.SearchPathOpts.SDKPath;
// Check if this is a public interface file from the SDK.
if (sdkPath.empty() ||
!hasPrefix(path::begin(interfacePath), path::end(interfacePath),
path::begin(sdkPath), path::end(sdkPath)) ||
StringRef(interfacePath).endswith(".private.swiftinterface"))
if (!canInterfaceHavePrebuiltModule())
return None;
// Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or
@@ -862,9 +895,10 @@ class ModuleInterfaceLoaderImpl {
trackSystemDependencies = ClangDependencyTracker->needSystemDependencies();
}
InterfaceSubContextDelegateImpl astDelegate(
ctx.SourceMgr, ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
ctx.ClangImporterOpts, Opts,
/*buildModuleCacheDirIfAbsent*/ true, cacheDir, prebuiltCacheDir,
backupInterfaceDir,
/*serializeDependencyHashes*/ false, trackSystemDependencies,
requiresOSSAModules);
@@ -914,39 +948,91 @@ class ModuleInterfaceLoaderImpl {
return std::make_error_code(std::errc::not_supported);
}
// Set up a builder if we need to build the module. It'll also set up
// the genericSubInvocation we'll need to use to compute the cache paths.
ModuleInterfaceBuilder builder(
ctx.SourceMgr, ctx.Diags, astDelegate, interfacePath, moduleName, cacheDir,
prebuiltCacheDir,
Opts.disableInterfaceLock, diagnosticLoc,
dependencyTracker);
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
// We didn't discover a module corresponding to this interface.
// Diagnose that we didn't find a loadable module, if we were asked to.
auto remarkRebuild = [&]() {
rebuildInfo.diagnose(ctx, diagnosticLoc, moduleName,
interfacePath, prebuiltCacheDir);
};
// If we found an out-of-date .swiftmodule, we still want to add it as
// a dependency of the .swiftinterface. That way if it's updated, but
// the .swiftinterface remains the same, we invalidate the cache and
// check the new .swiftmodule, because it likely has more information
// about the state of the world.
if (rebuildInfo.sawOutOfDateModule(modulePath))
builder.addExtraDependency(modulePath);
if (builder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true,
&moduleBuffer,
Opts.remarkOnRebuildFromInterface ? remarkRebuild:
llvm::function_ref<void()>()))
bool failed = false;
std::string backupPath = getBackupPublicModuleInterfacePath();
{
DiagnosticEngine emptyDiags(ctx.SourceMgr);
std::unique_ptr<llvm::SaveAndRestore<DiagnosticEngine*>> saver;
DiagnosticEngine *diagsToUse = &ctx.Diags;
// Avoid emitting diagnostics if we have a backup interface to use.
// If we succeed in building this canonical interface, it's not interesting
// to see those diagnostics.
// If we failed in build, we will use the back up interface and it's interesting
// to see diagnostics there.
if (!backupPath.empty()) {
diagsToUse = &emptyDiags;
saver = std::make_unique<llvm::SaveAndRestore<DiagnosticEngine*>>(
astDelegate.Diags, diagsToUse);
}
// Set up a builder if we need to build the module. It'll also set up
// the genericSubInvocation we'll need to use to compute the cache paths.
ModuleInterfaceBuilder builder(
ctx.SourceMgr, diagsToUse,
astDelegate, interfacePath, moduleName, cacheDir,
prebuiltCacheDir, backupInterfaceDir,
Opts.disableInterfaceLock, diagnosticLoc,
dependencyTracker);
// If we found an out-of-date .swiftmodule, we still want to add it as
// a dependency of the .swiftinterface. That way if it's updated, but
// the .swiftinterface remains the same, we invalidate the cache and
// check the new .swiftmodule, because it likely has more information
// about the state of the world.
if (rebuildInfo.sawOutOfDateModule(modulePath))
builder.addExtraDependency(modulePath);
failed = builder.buildSwiftModule(cachedOutputPath,
/*shouldSerializeDeps*/true,
&moduleBuffer,
Opts.remarkOnRebuildFromInterface ?
remarkRebuild:
llvm::function_ref<void()>());
}
if (!failed) {
// If succeeded, we are done.
assert(moduleBuffer &&
"failed to write module buffer but returned success?");
return std::move(moduleBuffer);
} else if (backupPath.empty()) {
// If failed and we don't have a backup interface file, return error code.
return std::make_error_code(std::errc::invalid_argument);
assert(moduleBuffer &&
"failed to write module buffer but returned success?");
return std::move(moduleBuffer);
}
assert(failed);
assert(!backupPath.empty());
while (1) {
diags.diagnose(diagnosticLoc, diag::interface_file_backup_used,
interfacePath, backupPath);
// Set up a builder if we need to build the module. It'll also set up
// the genericSubInvocation we'll need to use to compute the cache paths.
ModuleInterfaceBuilder fallbackBuilder(
ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath, moduleName, cacheDir,
prebuiltCacheDir, backupInterfaceDir,
Opts.disableInterfaceLock, diagnosticLoc,
dependencyTracker);
if (rebuildInfo.sawOutOfDateModule(modulePath))
fallbackBuilder.addExtraDependency(modulePath);
// Add the canonical interface path as a dependency of this module.
// This ensures that after the user manually fixed the canonical interface
// file and removed the fallback interface file, we can rebuild the cache.
fallbackBuilder.addExtraDependency(interfacePath);
// Use cachedOutputPath as the output file path. This output path was
// calcualted using the canonical interface file path to make sure we
// can find it from the canonical interface file.
auto failedAgain = fallbackBuilder.buildSwiftModule(cachedOutputPath,
/*shouldSerializeDeps*/true, &moduleBuffer,
Opts.remarkOnRebuildFromInterface ? remarkRebuild:
llvm::function_ref<void()>());
if (failedAgain)
return std::make_error_code(std::errc::invalid_argument);
assert(moduleBuffer);
return std::move(moduleBuffer);
}
}
};
@@ -1004,7 +1090,8 @@ std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
auto ModuleName = ModuleID.Item.str();
ModuleInterfaceLoaderImpl Impl(
Ctx, ModPath, InPath, ModuleName, InterfaceChecker.CacheDir,
InterfaceChecker.PrebuiltCacheDir, ModuleID.Loc, InterfaceChecker.Opts,
InterfaceChecker.PrebuiltCacheDir, InterfaceChecker.BackupInterfaceDir,
ModuleID.Loc, InterfaceChecker.Opts,
InterfaceChecker.RequiresOSSAModules, dependencyTracker,
llvm::is_contained(PreferInterfaceForModules, ModuleName)
? ModuleLoadingMode::PreferInterface
@@ -1044,7 +1131,8 @@ ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(
llvm::SmallString<32> modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir, SourceLoc(), Opts,
CacheDir, PrebuiltCacheDir, BackupInterfaceDir,
SourceLoc(), Opts,
RequiresOSSAModules, nullptr,
ModuleLoadingMode::PreferSerialized);
std::vector<std::string> results;
@@ -1057,6 +1145,21 @@ ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(
return results;
}
std::string
ModuleInterfaceCheckerImpl::getBackupPublicModuleInterfacePath(StringRef moduleName,
StringRef interfacePath) {
// Derive .swiftmodule path from the .swiftinterface path.
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
llvm::SmallString<32> modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir, BackupInterfaceDir,
SourceLoc(), Opts,
RequiresOSSAModules, nullptr,
ModuleLoadingMode::PreferSerialized);
return Impl.getBackupPublicModuleInterfacePath();
}
bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule(
StringRef moduleName, StringRef interfacePath,
ArrayRef<std::string> candidates, StringRef outputPath) {
@@ -1065,7 +1168,8 @@ bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule(
llvm::SmallString<32> modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir, SourceLoc(), Opts,
CacheDir, PrebuiltCacheDir,
BackupInterfaceDir, SourceLoc(), Opts,
RequiresOSSAModules, nullptr,
ModuleLoadingMode::PreferSerialized);
SmallVector<FileDependency, 16> deps;
@@ -1092,16 +1196,19 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
SourceManager &SourceMgr, DiagnosticEngine &Diags,
const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts,
const ClangImporterOptions &ClangOpts, StringRef CacheDir,
StringRef PrebuiltCacheDir, StringRef ModuleName, StringRef InPath,
StringRef PrebuiltCacheDir, StringRef BackupInterfaceDir,
StringRef ModuleName, StringRef InPath,
StringRef OutPath, bool SerializeDependencyHashes,
bool TrackSystemDependencies, ModuleInterfaceLoaderOptions LoaderOpts,
RequireOSSAModules_t RequireOSSAModules) {
InterfaceSubContextDelegateImpl astDelegate(
SourceMgr, Diags, SearchPathOpts, LangOpts, ClangOpts, LoaderOpts,
SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, LoaderOpts,
/*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir,
BackupInterfaceDir,
SerializeDependencyHashes, TrackSystemDependencies, RequireOSSAModules);
ModuleInterfaceBuilder builder(SourceMgr, Diags, astDelegate, InPath,
ModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath,
ModuleName, CacheDir, PrebuiltCacheDir,
BackupInterfaceDir,
LoaderOpts.disableInterfaceLock);
// FIXME: We really only want to serialize 'important' dependencies here, if
// we want to ship the built swiftmodules to another machine.
@@ -1184,8 +1291,8 @@ bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
if (!FileOrError) {
// Don't use this->diagnose() because it'll just try to re-open
// interfacePath.
Diags.diagnose(diagnosticLoc, diag::error_open_input_file,
interfacePath, FileOrError.getError().message());
Diags->diagnose(diagnosticLoc, diag::error_open_input_file,
interfacePath, FileOrError.getError().message());
return true;
}
auto SB = FileOrError.get()->getBuffer();
@@ -1205,7 +1312,7 @@ bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
}
assert(VersMatches.size() == 2);
// FIXME We should diagnose this at a location that makes sense:
auto Vers = swift::version::Version(VersMatches[1], SourceLoc(), &Diags);
auto Vers = swift::version::Version(VersMatches[1], SourceLoc(), Diags);
if (CompRe.match(SB, &CompMatches)) {
assert(CompMatches.size() == 2);
@@ -1226,7 +1333,7 @@ bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
}
SmallString<32> ExpectedModuleName = subInvocation.getModuleName();
if (subInvocation.parseArgs(SubArgs, Diags)) {
if (subInvocation.parseArgs(SubArgs, *Diags)) {
return true;
}
@@ -1243,11 +1350,12 @@ bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
}
InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
SourceManager &SM, DiagnosticEngine &Diags,
SourceManager &SM, DiagnosticEngine *Diags,
const SearchPathOptions &searchPathOpts, const LangOptions &langOpts,
const ClangImporterOptions &clangImporterOpts,
ModuleInterfaceLoaderOptions LoaderOpts, bool buildModuleCacheDirIfAbsent,
StringRef moduleCachePath, StringRef prebuiltCachePath,
StringRef backupModuleInterfaceDir,
bool serializeDependencyHashes, bool trackSystemDependencies,
RequireOSSAModules_t requireOSSAModules)
: SM(SM), Diags(Diags), ArgSaver(Allocator) {
@@ -1264,6 +1372,10 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
genericSubInvocation.getFrontendOptions().PrebuiltModuleCachePath =
prebuiltCachePath.str();
}
if (!backupModuleInterfaceDir.empty()) {
genericSubInvocation.getFrontendOptions().BackupModuleInterfaceDir =
backupModuleInterfaceDir.str();
}
if (trackSystemDependencies) {
genericSubInvocation.getFrontendOptions().IntermoduleDependencyTracking =
IntermoduleDepTrackingMode::IncludeSystem;
@@ -1477,7 +1589,7 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName,
}
// Insert arguments collected from the interface file.
BuildArgs.insert(BuildArgs.end(), SubArgs.begin(), SubArgs.end());
if (subInvocation.parseArgs(SubArgs, Diags)) {
if (subInvocation.parseArgs(SubArgs, *Diags)) {
return std::make_error_code(std::errc::not_supported);
}
CompilerInstance subInstance;
@@ -1487,7 +1599,7 @@ InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName,
subInstance.getSourceMgr().setFileSystem(SM.getFileSystem());
ForwardingDiagnosticConsumer FDC(Diags);
ForwardingDiagnosticConsumer FDC(*Diags);
subInstance.addDiagnosticConsumer(&FDC);
if (subInstance.setup(subInvocation)) {
return std::make_error_code(std::errc::not_supported);