Files
swift-mirror/lib/Frontend/ModuleInterfaceLoader.cpp
Steven Wu 8e68fab034 Revert "[Caching][NFC] Using llvm::cas::CASConfiguration"
This reverts commit 4f059033bb. The change
is actually not NFC since previously, there is a cache in the
CompilerInvocation that prevents the same CAS from the same CASOptions
from being initialized multiple times, which was relied upon when
running inside sub invocation. When switching to a non-caching simple
CASOption types, it causes every single sub instance will create its own
CAS, and it can consume too many file descriptors and causing errors
during dependency scanning.

rdar://164903080
2025-11-17 12:21:53 -08:00

2952 lines
122 KiB
C++

//===------ ModuleInterfaceLoader.cpp - Loads .swiftinterface files -------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2019 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "textual-module-interface"
#include "swift/Frontend/ModuleInterfaceLoader.h"
#include "ModuleInterfaceBuilder.h"
#include "swift/AST/ASTContext.h"
#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/AST/DiagnosticsSema.h"
#include "swift/AST/FileSystem.h"
#include "swift/AST/Module.h"
#include "swift/AST/SearchPathOptions.h"
#include "swift/Basic/Assertions.h"
#include "swift/Basic/Platform.h"
#include "swift/Basic/StringExtras.h"
#include "swift/Frontend/CachingUtils.h"
#include "swift/Frontend/CompileJobCacheResult.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/ModuleInterfaceSupport.h"
#include "swift/Parse/ParseVersion.h"
#include "swift/Serialization/SerializationOptions.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "swift/Serialization/Validation.h"
#include "swift/Strings.h"
#include "clang/Basic/Module.h"
#include "clang/Frontend/CompileJobCacheResult.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/Twine.h"
#include "llvm/CAS/ActionCache.h"
#include "llvm/CAS/ObjectStore.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/VirtualOutputBackend.h"
#include "llvm/Support/YAMLParser.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/xxhash.h"
using namespace swift;
using FileDependency = SerializationOptions::FileDependency;
#pragma mark - Forwarding Modules
namespace {
/// Describes a "forwarding module", that is, a .swiftmodule that's actually
/// a YAML file inside, pointing to a the original .swiftmodule but describing
/// a different dependency resolution strategy.
struct ForwardingModule {
/// The path to the original .swiftmodule in the prebuilt cache.
std::string underlyingModulePath;
/// Describes a set of file-based dependencies with their size and
/// modification time stored. This is slightly different from
/// \c SerializationOptions::FileDependency, because this type needs to be
/// serializable to and from YAML.
struct Dependency {
std::string path;
uint64_t size;
uint64_t lastModificationTime;
bool isSDKRelative;
};
std::vector<Dependency> dependencies;
unsigned version = 1;
ForwardingModule() = default;
ForwardingModule(StringRef underlyingModulePath)
: underlyingModulePath(underlyingModulePath) {}
/// Loads the contents of the forwarding module whose contents lie in
/// the provided buffer, and returns a new \c ForwardingModule, or an error
/// if the YAML could not be parsed.
static llvm::ErrorOr<ForwardingModule> load(const llvm::MemoryBuffer &buf);
/// Adds a given dependency to the dependencies list.
void addDependency(StringRef path, bool isSDKRelative, uint64_t size,
uint64_t modTime) {
dependencies.push_back({path.str(), size, modTime, isSDKRelative});
}
};
} // end anonymous namespace
#pragma mark - YAML Serialization
namespace llvm {
namespace yaml {
template <>
struct MappingTraits<ForwardingModule::Dependency> {
static void mapping(IO &io, ForwardingModule::Dependency &dep) {
io.mapRequired("mtime", dep.lastModificationTime);
io.mapRequired("path", dep.path);
io.mapRequired("size", dep.size);
io.mapOptional("sdk_relative", dep.isSDKRelative, /*default*/false);
}
};
template <>
struct SequenceElementTraits<ForwardingModule::Dependency> {
static const bool flow = false;
};
template <>
struct MappingTraits<ForwardingModule> {
static void mapping(IO &io, ForwardingModule &module) {
io.mapRequired("path", module.underlyingModulePath);
io.mapRequired("dependencies", module.dependencies);
io.mapRequired("version", module.version);
}
};
}
} // end namespace llvm
llvm::ErrorOr<ForwardingModule>
ForwardingModule::load(const llvm::MemoryBuffer &buf) {
llvm::yaml::Input yamlIn(buf.getBuffer());
ForwardingModule fwd;
yamlIn >> fwd;
if (yamlIn.error())
return yamlIn.error();
// We only currently support Version 1 of the forwarding module format.
if (fwd.version != 1)
return std::make_error_code(std::errc::not_supported);
return std::move(fwd);
}
#pragma mark - Module Discovery
namespace {
/// The result of a search for a module either alongside an interface, in the
/// module cache, or in the prebuilt module cache.
class DiscoveredModule {
/// The kind of module we've found.
enum class Kind {
/// A module that's either alongside the swiftinterface or in the
/// module cache.
Normal,
/// A module that resides in the prebuilt cache, and has hash-based
/// dependencies.
Prebuilt,
/// A 'forwarded' module. This is a module in the prebuilt cache, but whose
/// dependencies live in a forwarding module.
/// \sa ForwardingModule.
Forwarded
};
/// The kind of module that's been discovered.
const Kind kind;
DiscoveredModule(StringRef path, Kind kind,
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer)
: kind(kind), moduleBuffer(std::move(moduleBuffer)), path(path) {}
public:
/// The contents of the .swiftmodule, if we've read it while validating
/// dependencies.
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
/// The path to the discovered serialized .swiftmodule on disk.
const std::string path;
/// Creates a \c Normal discovered module.
static DiscoveredModule normal(StringRef path,
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer) {
return { path, Kind::Normal, std::move(moduleBuffer) };
}
/// Creates a \c Prebuilt discovered module.
static DiscoveredModule prebuilt(
StringRef path, std::unique_ptr<llvm::MemoryBuffer> moduleBuffer) {
return { path, Kind::Prebuilt, std::move(moduleBuffer) };
}
/// Creates a \c Forwarded discovered module, whose dependencies have been
/// externally validated by a \c ForwardingModule.
static DiscoveredModule forwarded(
StringRef path, std::unique_ptr<llvm::MemoryBuffer> moduleBuffer) {
return { path, Kind::Forwarded, std::move(moduleBuffer) };
}
bool isNormal() const { return kind == Kind::Normal; }
bool isPrebuilt() const { return kind == Kind::Prebuilt; }
bool isForwarded() const { return kind == Kind::Forwarded; }
};
} // end anonymous namespace
#pragma mark - Common utilities
namespace path = llvm::sys::path;
static bool serializedASTLooksValid(const llvm::MemoryBuffer &buf,
StringRef requiredSDK) {
auto VI = serialization::validateSerializedAST(buf.getBuffer(),
requiredSDK);
return VI.status == serialization::Status::Valid;
}
#pragma mark - Module Loading
namespace {
/// Keeps track of the various reasons the module interface loader needed to
/// fall back and rebuild a module from its interface.
struct ModuleRebuildInfo {
enum class ModuleKind {
Normal,
Cached,
Forwarding,
Prebuilt
};
enum class ReasonIgnored {
NotIgnored,
PublicFramework,
PublicLibrary,
InterfacePreferred,
CompilerHostModule,
Blocklisted,
DistributedInterfaceByDefault,
};
// Keep aligned with diag::module_interface_ignored_reason.
enum class ReasonModuleInterfaceIgnored {
NotIgnored,
LocalModule,
Blocklisted,
Debugger,
};
struct CandidateModule {
std::string path;
std::optional<serialization::Status> serializationStatus;
ModuleKind kind;
ReasonIgnored reasonIgnored;
ReasonModuleInterfaceIgnored reasonModuleInterfaceIgnored;
SmallVector<std::string, 10> outOfDateDependencies;
SmallVector<std::string, 10> missingDependencies;
};
SmallVector<CandidateModule, 3> candidateModules;
CandidateModule &getOrInsertCandidateModule(StringRef path) {
for (auto &mod : candidateModules) {
if (mod.path == path) return mod;
}
candidateModules.push_back({path.str(),
std::nullopt,
ModuleKind::Normal,
ReasonIgnored::NotIgnored,
ReasonModuleInterfaceIgnored::NotIgnored,
{},
{}});
return candidateModules.back();
}
/// Sets the kind of a module that failed to load.
void setModuleKind(StringRef path, ModuleKind kind) {
getOrInsertCandidateModule(path).kind = kind;
}
/// Sets the serialization status of the module at \c path. If this is
/// anything other than \c Valid, a note will be added stating why the module
/// was invalid.
void setSerializationStatus(StringRef path, serialization::Status status) {
getOrInsertCandidateModule(path).serializationStatus = status;
}
/// Registers an out-of-date dependency at \c depPath for the module
/// at \c modulePath.
void addOutOfDateDependency(StringRef modulePath, StringRef depPath) {
getOrInsertCandidateModule(modulePath)
.outOfDateDependencies.push_back(depPath.str());
}
/// Registers a missing dependency at \c depPath for the module
/// at \c modulePath.
void addMissingDependency(StringRef modulePath, StringRef depPath) {
getOrInsertCandidateModule(modulePath)
.missingDependencies.push_back(depPath.str());
}
/// Sets the reason that the module at \c modulePath was ignored. If this is
/// anything besides \c NotIgnored a note will be added stating why the module
/// was ignored.
void addIgnoredModule(StringRef modulePath, ReasonIgnored reasonIgnored) {
getOrInsertCandidateModule(modulePath).reasonIgnored = reasonIgnored;
}
/// Record why no swiftinterfaces were preferred over the binary swiftmodule
/// at \c modulePath.
void addIgnoredModuleInterface(StringRef modulePath,
ReasonModuleInterfaceIgnored reasonIgnored) {
getOrInsertCandidateModule(modulePath).reasonModuleInterfaceIgnored =
reasonIgnored;
}
/// Determines if we saw the given module path and registered is as out of
/// date.
bool sawOutOfDateModule(StringRef modulePath) {
for (auto &mod : candidateModules)
if (mod.path == modulePath &&
mod.reasonIgnored == ReasonIgnored::NotIgnored)
return true;
return false;
}
/// Emits a diagnostic for all out-of-date compiled or forwarding modules
/// encountered while trying to load a module.
template<typename... DiagArgs>
void diagnose(ASTContext &ctx, DiagnosticEngine &diags,
StringRef prebuiltCacheDir, SourceLoc loc,
DiagArgs &&...diagArgs) {
diags.diagnose(loc, std::forward<DiagArgs>(diagArgs)...);
auto SDKVer = getSDKBuildVersion(ctx.SearchPathOpts.getSDKPath());
llvm::SmallString<64> buffer = prebuiltCacheDir;
llvm::sys::path::append(buffer, "SystemVersion.plist");
auto PBMVer = getSDKBuildVersionFromPlist(buffer.str());
if (!SDKVer.empty() && !PBMVer.empty()) {
// Remark the potential version difference.
diags.diagnose(loc, diag::sdk_version_pbm_version, SDKVer,
PBMVer);
}
// We may have found multiple failing modules, that failed for different
// reasons. Emit a note for each of them.
for (auto &mod : candidateModules) {
// If the compiled module was ignored, diagnose the reason.
if (mod.reasonIgnored != ReasonIgnored::NotIgnored) {
diags.diagnose(loc, diag::compiled_module_ignored_reason, mod.path,
(unsigned)mod.reasonIgnored);
} else {
diags.diagnose(loc, diag::out_of_date_module_here, (unsigned)mod.kind,
mod.path);
}
// Diagnose any out-of-date dependencies in this module.
for (auto &dep : mod.outOfDateDependencies) {
diags.diagnose(loc, diag::module_interface_dependency_out_of_date,
dep);
}
// Diagnose any missing dependencies in this module.
for (auto &dep : mod.missingDependencies) {
diags.diagnose(loc, diag::module_interface_dependency_missing, dep);
}
// If there was a compiled module that wasn't able to be read, diagnose
// the reason we couldn't read it.
if (auto status = mod.serializationStatus) {
if (auto reason = SerializedModuleLoaderBase::invalidModuleReason(*status)) {
diags.diagnose(loc, diag::compiled_module_invalid_reason,
mod.path, reason.value());
} else {
diags.diagnose(loc, diag::compiled_module_invalid, mod.path);
}
}
}
}
/// Emits a diagnostic for the reason why binary swiftmodules were preferred
/// over textual swiftinterfaces.
void diagnoseIgnoredModuleInterfaces(ASTContext &ctx, SourceLoc loc) {
for (auto &mod : candidateModules) {
auto interfaceIgnore = mod.reasonModuleInterfaceIgnored;
if (interfaceIgnore == ReasonModuleInterfaceIgnored::NotIgnored)
continue;
ctx.Diags.diagnose(loc, diag::module_interface_ignored_reason,
mod.path, (unsigned)interfaceIgnore);
}
}
};
/// Constructs the full path of the dependency \p dep by prepending the SDK
/// path if necessary.
StringRef getFullDependencyPath(const FileDependency &dep,
const ASTContext &ctx,
SmallVectorImpl<char> &scratch) {
if (!dep.isSDKRelative())
return dep.getPath();
path::native(ctx.SearchPathOpts.getSDKPath(), scratch);
llvm::sys::path::append(scratch, dep.getPath());
return StringRef(scratch.data(), scratch.size());
}
class UpToDateModuleCheker {
ASTContext &ctx;
llvm::vfs::FileSystem &fs;
public:
UpToDateModuleCheker(ASTContext &ctx)
: ctx(ctx),
fs(*ctx.SourceMgr.getFileSystem()) {}
// Check if all the provided file dependencies are up-to-date compared to
// what's currently on disk.
bool dependenciesAreUpToDate(StringRef modulePath,
ModuleRebuildInfo &rebuildInfo,
ArrayRef<FileDependency> deps,
bool skipSystemDependencies) {
SmallString<128> SDKRelativeBuffer;
for (auto &in : deps) {
if (skipSystemDependencies && in.isSDKRelative() &&
in.isModificationTimeBased()) {
continue;
}
StringRef fullPath = getFullDependencyPath(in, ctx, SDKRelativeBuffer);
switch (checkDependency(modulePath, in, fullPath)) {
case DependencyStatus::UpToDate:
LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is up to date\n");
break;
case DependencyStatus::OutOfDate:
LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is out of date\n");
rebuildInfo.addOutOfDateDependency(modulePath, fullPath);
return false;
case DependencyStatus::Missing:
LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is missing\n");
rebuildInfo.addMissingDependency(modulePath, fullPath);
return false;
}
}
return true;
}
// Check that the output .swiftmodule file is at least as new as all the
// dependencies it read when it was built last time.
bool serializedASTBufferIsUpToDate(
StringRef path, const llvm::MemoryBuffer &buf,
ModuleRebuildInfo &rebuildInfo,
SmallVectorImpl<FileDependency> &allDeps) {
// Clear the existing dependencies, because we're going to re-fill them
// in validateSerializedAST.
allDeps.clear();
LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << path << "\n");
auto validationInfo = serialization::validateSerializedAST(
buf.getBuffer(),
ctx.LangOpts.SDKName, /*ExtendedValidationInfo=*/nullptr, &allDeps);
if (validationInfo.status != serialization::Status::Valid) {
rebuildInfo.setSerializationStatus(path, validationInfo.status);
return false;
}
bool skipCheckingSystemDependencies =
ctx.SearchPathOpts.DisableModulesValidateSystemDependencies;
return dependenciesAreUpToDate(path, rebuildInfo, allDeps,
skipCheckingSystemDependencies);
}
// Check that the output .swiftmodule file is at least as new as all the
// dependencies it read when it was built last time.
bool swiftModuleIsUpToDate(
StringRef modulePath, ModuleRebuildInfo &rebuildInfo,
SmallVectorImpl<FileDependency> &AllDeps,
std::unique_ptr<llvm::MemoryBuffer> &moduleBuffer) {
auto OutBuf = fs.getBufferForFile(modulePath);
if (!OutBuf)
return false;
moduleBuffer = std::move(*OutBuf);
return serializedASTBufferIsUpToDate(modulePath, *moduleBuffer, rebuildInfo, AllDeps);
}
enum class DependencyStatus {
UpToDate,
OutOfDate,
Missing
};
// Checks that a dependency read from the cached module is up to date compared
// to the interface file it represents.
DependencyStatus checkDependency(StringRef modulePath,
const FileDependency &dep,
StringRef fullPath) {
auto status = fs.status(fullPath);
if (!status)
return DependencyStatus::Missing;
// If the sizes differ, then we know the file has changed.
if (status->getSize() != dep.getSize())
return DependencyStatus::OutOfDate;
// Otherwise, if this dependency is verified by modification time, check
// it vs. the modification time of the file.
if (dep.isModificationTimeBased()) {
uint64_t mtime =
status->getLastModificationTime().time_since_epoch().count();
return mtime == dep.getModificationTime() ?
DependencyStatus::UpToDate :
DependencyStatus::OutOfDate;
}
// Slow path: if the dependency is verified by content hash, check it vs.
// the hash of the file.
auto buf = fs.getBufferForFile(fullPath, /*FileSize=*/-1,
/*RequiresNullTerminator=*/false);
if (!buf)
return DependencyStatus::Missing;
return xxHash64(buf.get()->getBuffer()) == dep.getContentHash() ?
DependencyStatus::UpToDate :
DependencyStatus::OutOfDate;
}
};
/// Handles the details of loading module interfaces as modules, and will
/// do the necessary lookup to determine if we should be loading from the
/// normal cache, the prebuilt cache, a module adjacent to the interface, or
/// a module that we'll build from a module interface.
class ModuleInterfaceLoaderImpl {
friend class swift::ModuleInterfaceLoader;
friend class swift::ModuleInterfaceCheckerImpl;
ASTContext &ctx;
llvm::vfs::FileSystem &fs;
DiagnosticEngine &diags;
ModuleRebuildInfo rebuildInfo;
UpToDateModuleCheker upToDateChecker;
const StringRef modulePath;
const std::string interfacePath;
const StringRef moduleName;
const StringRef prebuiltCacheDir;
const StringRef backupInterfaceDir;
const StringRef cacheDir;
const SourceLoc diagnosticLoc;
DependencyTracker *const dependencyTracker;
const ModuleLoadingMode loadMode;
ModuleInterfaceLoaderOptions Opts;
ModuleInterfaceLoaderImpl(
ASTContext &ctx, StringRef modulePath, StringRef interfacePath,
StringRef moduleName, StringRef cacheDir, StringRef prebuiltCacheDir,
StringRef backupInterfaceDir,
SourceLoc diagLoc, ModuleInterfaceLoaderOptions Opts,
DependencyTracker *dependencyTracker = nullptr,
ModuleLoadingMode loadMode = ModuleLoadingMode::PreferSerialized)
: ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), diags(ctx.Diags),
upToDateChecker(ctx),
modulePath(modulePath), interfacePath(interfacePath),
moduleName(moduleName),
prebuiltCacheDir(prebuiltCacheDir),
backupInterfaceDir(backupInterfaceDir),
cacheDir(cacheDir), diagnosticLoc(diagLoc),
dependencyTracker(dependencyTracker), loadMode(loadMode), Opts(Opts) {}
std::string getBackupPublicModuleInterfacePath() {
return getBackupPublicModuleInterfacePath(ctx.SourceMgr, backupInterfaceDir,
moduleName, interfacePath);
}
static std::string getBackupPublicModuleInterfacePath(SourceManager &SM,
StringRef backupInterfaceDir,
StringRef moduleName,
StringRef interfacePath) {
if (backupInterfaceDir.empty())
return std::string();
auto &fs = *SM.getFileSystem();
auto fileName = llvm::sys::path::filename(interfacePath);
{
llvm::SmallString<256> path(backupInterfaceDir);
llvm::sys::path::append(path, llvm::Twine(moduleName) + ".swiftmodule");
llvm::sys::path::append(path, fileName);
if (fs.exists(path.str())) {
return path.str().str();
}
}
{
llvm::SmallString<256> path(backupInterfaceDir);
llvm::sys::path::append(path, fileName);
if (fs.exists(path.str())) {
return path.str().str();
}
}
return std::string();
}
// Check that a "forwarding" .swiftmodule file is at least as new as all the
// dependencies it read when it was built last time. Requires that the
// forwarding module has been loaded from disk.
bool forwardingModuleIsUpToDate(
StringRef path, const ForwardingModule &fwd,
SmallVectorImpl<FileDependency> &deps,
std::unique_ptr<llvm::MemoryBuffer> &moduleBuffer) {
// Clear the existing dependencies, because we're going to re-fill them
// from the forwarding module.
deps.clear();
LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << path << "\n");
// First, make sure the underlying module path exists and is valid.
auto modBuf = fs.getBufferForFile(fwd.underlyingModulePath);
if (!modBuf)
return false;
auto looksValid = serializedASTLooksValid(*modBuf.get(),
ctx.LangOpts.SDKName);
if (!looksValid)
return false;
// Next, check the dependencies in the forwarding file.
for (auto &dep : fwd.dependencies) {
deps.push_back(
FileDependency::modTimeBased(
dep.path, dep.isSDKRelative, dep.size, dep.lastModificationTime));
}
bool skipCheckingSystemDependencies =
ctx.SearchPathOpts.DisableModulesValidateSystemDependencies;
if (!upToDateChecker.dependenciesAreUpToDate(path, rebuildInfo, deps,
skipCheckingSystemDependencies))
return false;
moduleBuffer = std::move(*modBuf);
return true;
}
bool canInterfaceHavePrebuiltModule() {
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
if (!sdkPath.empty() &&
hasPrefix(path::begin(interfacePath), path::end(interfacePath),
path::begin(sdkPath), path::end(sdkPath))) {
return !(StringRef(interfacePath).ends_with(".private.swiftinterface") ||
StringRef(interfacePath).ends_with(".package.swiftinterface"));
}
return false;
}
std::optional<StringRef>
computePrebuiltModulePath(llvm::SmallString<256> &scratch) {
namespace path = llvm::sys::path;
// Check if this is a public interface file from the SDK.
if (!canInterfaceHavePrebuiltModule())
return std::nullopt;
// Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or
// $PREBUILT_CACHE/Foo.swiftmodule/arch.swiftmodule. Note that there's no
// cache key here.
scratch = prebuiltCacheDir;
// FIXME: Would it be possible to only have architecture-specific names
// here? Then we could skip this check.
StringRef inParentDirName =
path::filename(path::parent_path(interfacePath));
if (path::extension(inParentDirName) == ".swiftmodule") {
assert(path::stem(inParentDirName) ==
ctx.getRealModuleName(ctx.getIdentifier(moduleName)).str());
path::append(scratch, inParentDirName);
}
path::append(scratch, path::filename(modulePath));
// If there isn't a file at this location, skip returning a path.
if (!fs.exists(scratch))
return std::nullopt;
return scratch.str();
}
/// Hack to deal with build systems (including the Swift standard library, at
/// the time of this comment) that aren't yet using target-specific names for
/// multi-target swiftmodules, in case the prebuilt cache is.
std::optional<StringRef>
computeFallbackPrebuiltModulePath(llvm::SmallString<256> &scratch) {
namespace path = llvm::sys::path;
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
// Check if this is a public interface file from the SDK.
if (sdkPath.empty() ||
!hasPrefix(path::begin(interfacePath), path::end(interfacePath),
path::begin(sdkPath), path::end(sdkPath)) ||
StringRef(interfacePath).ends_with(".private.swiftinterface") ||
StringRef(interfacePath).ends_with(".package.swiftinterface"))
return std::nullopt;
// If the module isn't target-specific, there's no fallback path.
StringRef inParentDirName =
path::filename(path::parent_path(interfacePath));
if (path::extension(inParentDirName) != ".swiftmodule")
return std::nullopt;
// If the interface is already using the target-specific name, there's
// nothing else to try.
auto normalizedTarget = getTargetSpecificModuleTriple(ctx.LangOpts.Target);
if (path::stem(modulePath) == normalizedTarget.str())
return std::nullopt;
// Assemble the expected path:
// $PREBUILT_CACHE/Foo.swiftmodule/target.swiftmodule. Note that there's no
// cache key here.
scratch = prebuiltCacheDir;
path::append(scratch, inParentDirName);
path::append(scratch, normalizedTarget.str());
scratch += ".swiftmodule";
// If there isn't a file at this location, skip returning a path.
if (!fs.exists(scratch))
return std::nullopt;
return scratch.str();
}
bool isInResourceDir(StringRef path) {
StringRef resourceDir = ctx.SearchPathOpts.RuntimeResourcePath;
if (resourceDir.empty()) return false;
return pathStartsWith(resourceDir, path);
}
bool isInResourceHostDir(StringRef path) {
StringRef resourceDir = ctx.SearchPathOpts.RuntimeResourcePath;
if (resourceDir.empty()) return false;
SmallString<128> hostPath;
llvm::sys::path::append(hostPath,
resourceDir, "host");
return pathStartsWith(hostPath, path);
}
bool isInSDK(StringRef path) {
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
if (sdkPath.empty()) return false;
return pathStartsWith(sdkPath, path);
}
bool isInSystemFrameworks(StringRef path, bool publicFramework) {
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
if (sdkPath.empty()) return false;
SmallString<128> frameworksPath;
llvm::sys::path::append(frameworksPath,
sdkPath, "System", "Library",
publicFramework ? "Frameworks" : "PrivateFrameworks");
return pathStartsWith(frameworksPath, path);
}
bool isInSystemSubFrameworks(StringRef path) {
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
if (sdkPath.empty()) return false;
SmallString<128> frameworksPath;
llvm::sys::path::append(frameworksPath,
sdkPath, "System", "Library", "SubFrameworks");
return pathStartsWith(frameworksPath, path);
}
bool isInSystemLibraries(StringRef path) {
StringRef sdkPath = ctx.SearchPathOpts.getSDKPath();
if (sdkPath.empty()) return false;
SmallString<128> frameworksPath;
llvm::sys::path::append(frameworksPath,
sdkPath, "usr", "lib", "swift");
return pathStartsWith(frameworksPath, path);
}
std::pair<std::string, std::string> getCompiledModuleCandidates() {
using ReasonIgnored = ModuleRebuildInfo::ReasonIgnored;
using ReasonModuleInterfaceIgnored =
ModuleRebuildInfo::ReasonModuleInterfaceIgnored;
std::pair<std::string, std::string> result;
bool ignoreByDefault = ctx.blockListConfig.hasBlockListAction(
"Swift_UseSwiftinterfaceByDefault",
BlockListKeyKind::ModuleName,
BlockListAction::ShouldUseBinaryModule);
bool shouldLoadAdjacentModule;
if (ignoreByDefault) {
ReasonModuleInterfaceIgnored ignore =
ReasonModuleInterfaceIgnored::NotIgnored;
if (!isInSDK(modulePath) &&
!isInResourceHostDir(modulePath)) {
ignore = ReasonModuleInterfaceIgnored::LocalModule;
} else if (ctx.blockListConfig.hasBlockListAction(moduleName,
BlockListKeyKind::ModuleName,
BlockListAction::ShouldUseBinaryModule)) {
ignore = ReasonModuleInterfaceIgnored::Blocklisted;
} else if (ctx.LangOpts.DebuggerSupport) {
ignore = ReasonModuleInterfaceIgnored::Debugger;
}
shouldLoadAdjacentModule =
ignore != ReasonModuleInterfaceIgnored::NotIgnored;
if (shouldLoadAdjacentModule) {
// Prefer the swiftmodule.
rebuildInfo.addIgnoredModuleInterface(modulePath, ignore);
} else {
// Prefer the swiftinterface.
rebuildInfo.addIgnoredModule(modulePath,
ReasonIgnored::DistributedInterfaceByDefault);
}
} else {
// Should we attempt to load a swiftmodule adjacent to the swiftinterface?
shouldLoadAdjacentModule = !ctx.IgnoreAdjacentModules;
if (modulePath.contains(".sdk")) {
if (ctx.blockListConfig.hasBlockListAction(moduleName,
BlockListKeyKind::ModuleName,
BlockListAction::ShouldUseTextualModule)) {
shouldLoadAdjacentModule = false;
rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::Blocklisted);
}
}
// Don't use the adjacent swiftmodule for frameworks from the public
// Frameworks folder of the SDK.
bool blocklistSwiftmodule =
ctx.blockListConfig.hasBlockListAction(moduleName,
BlockListKeyKind::ModuleName,
BlockListAction::ShouldUseBinaryModule);
if ((isInSystemFrameworks(modulePath, /*publicFramework*/true) ||
isInSystemSubFrameworks(modulePath)) &&
!blocklistSwiftmodule) {
shouldLoadAdjacentModule = false;
rebuildInfo.addIgnoredModule(modulePath,
ReasonIgnored::PublicFramework);
} else if (isInSystemLibraries(modulePath) &&
moduleName != STDLIB_NAME &&
!blocklistSwiftmodule) {
shouldLoadAdjacentModule = false;
rebuildInfo.addIgnoredModule(modulePath,
ReasonIgnored::PublicLibrary);
} else if (isInResourceHostDir(modulePath)) {
shouldLoadAdjacentModule = false;
rebuildInfo.addIgnoredModule(modulePath,
ReasonIgnored::CompilerHostModule);
}
}
switch (loadMode) {
case ModuleLoadingMode::OnlyInterface:
// Always skip both the caches and adjacent modules, and always build the
// module interface.
return {};
case ModuleLoadingMode::PreferInterface:
// If we're in the load mode that prefers .swiftinterfaces, specifically
// skip the module adjacent to the interface, but use the caches if
// they're present.
shouldLoadAdjacentModule = false;
rebuildInfo.addIgnoredModule(modulePath,
ReasonIgnored::InterfacePreferred);
break;
case ModuleLoadingMode::PreferSerialized:
// The rest of the function should be covered by this.
break;
case ModuleLoadingMode::OnlySerialized:
llvm_unreachable("module interface loader should not have been created");
}
// [NOTE: ModuleInterfaceLoader-defer-to-ImplicitSerializedModuleLoader]
// If there's a module adjacent to the .swiftinterface that we can
// _likely_ load (it validates OK and is up to date), bail early with
// errc::not_supported, so the next (serialized) loader in the chain will
// load it.
// Alternately, if there's a .swiftmodule present but we can't even
// read it (for whatever reason), we should let the other module loader
// diagnose it.
if (shouldLoadAdjacentModule) {
if (fs.exists(modulePath)) {
result.first = modulePath.str();
}
}
// If we have a prebuilt cache path, check that too if the interface comes
// from the SDK.
if (!prebuiltCacheDir.empty()) {
llvm::SmallString<256> scratch;
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
std::optional<StringRef> path = computePrebuiltModulePath(scratch);
if (!path) {
// Hack: deal with prebuilds of modules that still use the target-based
// names.
path = computeFallbackPrebuiltModulePath(scratch);
}
if (path) {
if (fs.exists(*path)) {
result.second = path->str();
}
}
}
return result;
}
llvm::ErrorOr<DiscoveredModule>
discoverUpToDateCompiledModuleForInterface(SmallVectorImpl<FileDependency> &deps,
std::string &UsableModulePath) {
std::string adjacentMod, prebuiltMod;
std::tie(adjacentMod, prebuiltMod) = getCompiledModuleCandidates();
if (!adjacentMod.empty()) {
auto adjacentModuleBuffer = fs.getBufferForFile(adjacentMod);
if (adjacentModuleBuffer) {
if (upToDateChecker.serializedASTBufferIsUpToDate(adjacentMod, *adjacentModuleBuffer.get(),
rebuildInfo, deps)) {
LLVM_DEBUG(llvm::dbgs() << "Found up-to-date module at "
<< adjacentMod
<< "; deferring to serialized module loader\n");
UsableModulePath = adjacentMod;
return std::make_error_code(std::errc::not_supported);
} else if (isInResourceDir(adjacentMod) &&
loadMode == ModuleLoadingMode::PreferSerialized &&
version::getCurrentCompilerSerializationTag().empty() &&
rebuildInfo.getOrInsertCandidateModule(adjacentMod)
.serializationStatus !=
serialization::Status::SDKMismatch) {
// Special-case here: If we're loading a .swiftmodule from the resource
// dir adjacent to the compiler, defer to the serialized loader instead
// of falling back. This is to support local development of Swift,
// where one might change the module format version but forget to
// recompile the standard library. If that happens, don't fall back
// and silently recompile the standard library, raise an error
// instead.
//
// This logic is disabled for tagged compilers, so distributed
// compilers should ignore this restriction and rebuild all modules
// from a swiftinterface when required.
//
// Still accept modules built with a different SDK, allowing the use
// of one toolchain against a different SDK.
LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module in the "
"resource-dir at "
<< adjacentMod
<< "; deferring to serialized module loader "
"to diagnose\n");
return std::make_error_code(std::errc::not_supported);
} else {
LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module at "
<< adjacentMod << "\n");
rebuildInfo.setModuleKind(adjacentMod,
ModuleRebuildInfo::ModuleKind::Normal);
}
} else {
LLVM_DEBUG(llvm::dbgs() << "Found unreadable module at "
<< adjacentMod
<< "; deferring to serialized module loader\n");
return std::make_error_code(std::errc::not_supported);
}
}
if(!prebuiltMod.empty()) {
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
if (upToDateChecker.swiftModuleIsUpToDate(prebuiltMod, rebuildInfo,
deps, moduleBuffer)) {
LLVM_DEBUG(llvm::dbgs() << "Found up-to-date prebuilt module at "
<< prebuiltMod << "\n");
UsableModulePath = prebuiltMod;
return DiscoveredModule::prebuilt(prebuiltMod, std::move(moduleBuffer));
} else {
LLVM_DEBUG(llvm::dbgs() << "Found out-of-date prebuilt module at "
<< prebuiltMod << "\n");
rebuildInfo.setModuleKind(prebuiltMod,
ModuleRebuildInfo::ModuleKind::Prebuilt);
}
}
// We cannot find any proper compiled module to use.
return std::make_error_code(std::errc::no_such_file_or_directory);
}
/// Finds the most appropriate .swiftmodule, whose dependencies are up to
/// date, that we can load for the provided .swiftinterface file.
llvm::ErrorOr<DiscoveredModule> discoverUpToDateModuleForInterface(
StringRef cachedOutputPath,
SmallVectorImpl<FileDependency> &deps) {
// First, check the cached module path. Whatever's in this cache represents
// the most up-to-date knowledge we have about the module.
if (auto cachedBufOrError = fs.getBufferForFile(cachedOutputPath)) {
auto buf = std::move(*cachedBufOrError);
// Check to see if the module is a serialized AST. If it's not, then we're
// probably dealing with a Forwarding Module, which is a YAML file.
bool isForwardingModule =
!serialization::isSerializedAST(buf->getBuffer());
// If it's a forwarding module, load the YAML file from disk and check
// if it's up-to-date.
if (isForwardingModule) {
if (auto forwardingModule = ForwardingModule::load(*buf)) {
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
if (forwardingModuleIsUpToDate(cachedOutputPath,
*forwardingModule, deps,
moduleBuffer)) {
LLVM_DEBUG(llvm::dbgs() << "Found up-to-date forwarding module at "
<< cachedOutputPath << "\n");
return DiscoveredModule::forwarded(
forwardingModule->underlyingModulePath, std::move(moduleBuffer));
}
LLVM_DEBUG(llvm::dbgs() << "Found out-of-date forwarding module at "
<< cachedOutputPath << "\n");
rebuildInfo.setModuleKind(cachedOutputPath,
ModuleRebuildInfo::ModuleKind::Forwarding);
}
// Otherwise, check if the AST buffer itself is up to date.
} else if (upToDateChecker.serializedASTBufferIsUpToDate(cachedOutputPath, *buf,
rebuildInfo, deps)) {
LLVM_DEBUG(llvm::dbgs() << "Found up-to-date cached module at "
<< cachedOutputPath << "\n");
return DiscoveredModule::normal(cachedOutputPath, std::move(buf));
} else {
LLVM_DEBUG(llvm::dbgs() << "Found out-of-date cached module at "
<< cachedOutputPath << "\n");
rebuildInfo.setModuleKind(cachedOutputPath,
ModuleRebuildInfo::ModuleKind::Cached);
}
}
std::string usableModulePath;
return discoverUpToDateCompiledModuleForInterface(deps, usableModulePath);
}
/// Writes the "forwarding module" that will forward to a module in the
/// prebuilt cache.
///
/// Since forwarding modules track dependencies separately from the module
/// they point to, we'll need to grab the up-to-date file status while doing
/// this. If the write was successful, it also updates the
/// list of dependencies to match what was written to the forwarding file.
bool writeForwardingModuleAndUpdateDeps(
const DiscoveredModule &mod, llvm::vfs::OutputBackend &backend,
StringRef outputPath, SmallVectorImpl<FileDependency> &deps) {
assert(mod.isPrebuilt() &&
"cannot write forwarding file for non-prebuilt module");
ForwardingModule fwd(mod.path);
SmallVector<FileDependency, 16> depsAdjustedToMTime;
// FIXME: We need to avoid re-statting all these dependencies, otherwise
// we may record out-of-date information.
SmallString<128> SDKRelativeBuffer;
auto addDependency = [&](FileDependency dep) -> FileDependency {
auto status = fs.status(getFullDependencyPath(dep, ctx, SDKRelativeBuffer));
uint64_t mtime =
status->getLastModificationTime().time_since_epoch().count();
fwd.addDependency(dep.getPath(), dep.isSDKRelative(), status->getSize(),
mtime);
// Construct new FileDependency matching what we've added to the
// forwarding module.
return FileDependency::modTimeBased(dep.getPath(), dep.isSDKRelative(),
status->getSize(), mtime);
};
// Add the prebuilt module as a dependency of the forwarding module, but
// don't add it to the outer dependency list.
(void)addDependency(FileDependency::hashBased(fwd.underlyingModulePath,
/*SDKRelative*/false,
/*size*/0, /*hash*/0));
// Add all the dependencies from the prebuilt module, and update our list
// of dependencies to reflect what's recorded in the forwarding module.
for (auto dep : deps) {
auto adjustedDep = addDependency(dep);
depsAdjustedToMTime.push_back(adjustedDep);
}
auto hadError = withOutputPath(diags, backend, outputPath,
[&](llvm::raw_pwrite_stream &out) {
llvm::yaml::Output yamlWriter(out);
yamlWriter << fwd;
return false;
});
if (hadError)
return true;
// If and only if we succeeded writing the forwarding file, update the
// provided list of dependencies.
deps = depsAdjustedToMTime;
return false;
}
/// Looks up the best module to load for a given interface, and returns a
/// buffer of the module's contents. Also reports the module's dependencies
/// to the parent \c dependencyTracker if it came from the cache, or was built
/// from the given interface. See the main comment in
/// \c ModuleInterfaceLoader.h for an explanation of the module
/// loading strategy.
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
findOrBuildLoadableModule() {
// Track system dependencies if the parent tracker is set to do so.
bool trackSystemDependencies = false;
if (dependencyTracker) {
auto ClangDependencyTracker = dependencyTracker->getClangCollector();
trackSystemDependencies = ClangDependencyTracker->needSystemDependencies();
}
InterfaceSubContextDelegateImpl astDelegate(
ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts,
ctx.ClangImporterOpts, ctx.CASOpts, Opts,
/*buildModuleCacheDirIfAbsent*/ true, cacheDir, prebuiltCacheDir,
backupInterfaceDir, /*replayPrefixMap=*/{},
/*serializeDependencyHashes*/ false, trackSystemDependencies);
// Compute the output path if we're loading or emitting a cached module.
SwiftInterfaceModuleOutputPathResolution::ResultTy resolvedOutputPath;
astDelegate.getCachedOutputPath(resolvedOutputPath, moduleName,
interfacePath,
ctx.SearchPathOpts.getSDKPath());
auto &cachedOutputPath = resolvedOutputPath.outputPath;
// Try to find the right module for this interface, either alongside it,
// in the cache, or in the prebuilt cache.
SmallVector<FileDependency, 16> allDeps;
auto moduleOrErr =
discoverUpToDateModuleForInterface(cachedOutputPath, allDeps);
// If we errored with anything other than 'no such file or directory',
// fail this load and let the other module loader diagnose it.
if (!moduleOrErr &&
moduleOrErr.getError() != std::errc::no_such_file_or_directory) {
rebuildInfo.diagnoseIgnoredModuleInterfaces(ctx, diagnosticLoc);
return moduleOrErr.getError();
}
// We discovered a module! Return that module's buffer so we can load it.
if (moduleOrErr) {
auto module = std::move(moduleOrErr.get());
// If it's prebuilt, use this time to generate a forwarding module and
// update the dependencies to use modification times.
if (module.isPrebuilt())
if (writeForwardingModuleAndUpdateDeps(module, ctx.getOutputBackend(),
cachedOutputPath, allDeps))
return std::make_error_code(std::errc::not_supported);
// Report the module's dependencies to the dependencyTracker
if (dependencyTracker) {
SmallString<128> SDKRelativeBuffer;
for (auto &dep: allDeps) {
StringRef fullPath = getFullDependencyPath(dep, ctx, SDKRelativeBuffer);
dependencyTracker->addDependency(fullPath,
/*IsSystem=*/dep.isSDKRelative());
}
}
return std::move(module.moduleBuffer);
}
// If building from interface is disabled, return error.
if (Opts.disableBuildingInterface) {
return std::make_error_code(std::errc::not_supported);
}
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
// We didn't discover a module corresponding to this interface.
// Diagnose that we didn't find a loadable module, if we were asked to.
//
// Note that we use `diags` so that we emit this remark even when we're
// emitting other messages to `emptyDiags` (see below); these act as status
// messages to explain what's taking so long.
auto remarkRebuildAll = [&]() {
rebuildInfo.diagnose(ctx, diags, prebuiltCacheDir, diagnosticLoc,
diag::rebuilding_module_from_interface, moduleName,
interfacePath);
};
auto remarkRebuild = Opts.remarkOnRebuildFromInterface
? llvm::function_ref<void()>(remarkRebuildAll)
: nullptr;
bool failed = false;
std::string backupPath = getBackupPublicModuleInterfacePath();
{
DiagnosticEngine emptyDiags(ctx.SourceMgr);
std::unique_ptr<llvm::SaveAndRestore<DiagnosticEngine*>> saver;
DiagnosticEngine *diagsToUse = &ctx.Diags;
// Avoid emitting diagnostics if we have a backup interface to use.
// If we succeed in building this canonical interface, it's not interesting
// to see those diagnostics.
// If we failed in build, we will use the back up interface and it's interesting
// to see diagnostics there.
if (!backupPath.empty()) {
diagsToUse = &emptyDiags;
saver = std::make_unique<llvm::SaveAndRestore<DiagnosticEngine*>>(
astDelegate.Diags, diagsToUse);
}
// Set up a builder if we need to build the module. It'll also set up
// the genericSubInvocation we'll need to use to compute the cache paths.
Identifier realName = ctx.getRealModuleName(ctx.getIdentifier(moduleName));
ImplicitModuleInterfaceBuilder builder(
ctx.SourceMgr, diagsToUse,
astDelegate, interfacePath,
ctx.SearchPathOpts.getSDKPath(),
ctx.SearchPathOpts.getSysRoot(),
realName.str(), cacheDir,
prebuiltCacheDir, backupInterfaceDir, StringRef(),
Opts.disableInterfaceLock,
ctx.IgnoreAdjacentModules, diagnosticLoc,
dependencyTracker);
// If we found an out-of-date .swiftmodule, we still want to add it as
// a dependency of the .swiftinterface. That way if it's updated, but
// the .swiftinterface remains the same, we invalidate the cache and
// check the new .swiftmodule, because it likely has more information
// about the state of the world.
if (rebuildInfo.sawOutOfDateModule(modulePath))
builder.addExtraDependency(modulePath);
failed = builder.buildSwiftModule(cachedOutputPath,
/*shouldSerializeDeps*/true,
&moduleBuffer, remarkRebuild);
}
if (!failed) {
// If succeeded, we are done.
assert(moduleBuffer &&
"failed to write module buffer but returned success?");
return std::move(moduleBuffer);
} else if (backupPath.empty()) {
// If failed and we don't have a backup interface file, return error code.
return std::make_error_code(std::errc::invalid_argument);
}
assert(failed);
assert(!backupPath.empty());
while (1) {
diags.diagnose(diagnosticLoc, diag::interface_file_backup_used,
interfacePath, backupPath);
// Set up a builder if we need to build the module. It'll also set up
// the genericSubInvocation we'll need to use to compute the cache paths.
ImplicitModuleInterfaceBuilder fallbackBuilder(
ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath,
ctx.SearchPathOpts.getSDKPath(),
ctx.SearchPathOpts.getSysRoot(),
moduleName, cacheDir,
prebuiltCacheDir, backupInterfaceDir, StringRef(),
Opts.disableInterfaceLock,
ctx.IgnoreAdjacentModules, diagnosticLoc,
dependencyTracker);
if (rebuildInfo.sawOutOfDateModule(modulePath))
fallbackBuilder.addExtraDependency(modulePath);
// Add the canonical interface path as a dependency of this module.
// This ensures that after the user manually fixed the canonical interface
// file and removed the fallback interface file, we can rebuild the cache.
fallbackBuilder.addExtraDependency(interfacePath);
// Use cachedOutputPath as the output file path. This output path was
// calculated using the canonical interface file path to make sure we
// can find it from the canonical interface file.
auto failedAgain = fallbackBuilder.buildSwiftModule(cachedOutputPath,
/*shouldSerializeDeps*/true,
&moduleBuffer,
remarkRebuild);
if (failedAgain)
return std::make_error_code(std::errc::invalid_argument);
assert(moduleBuffer);
return std::move(moduleBuffer);
}
}
};
} // end anonymous namespace
bool ModuleInterfaceCheckerImpl::isCached(StringRef DepPath) {
if (!CacheDir.empty() && DepPath.starts_with(CacheDir))
return true;
return !PrebuiltCacheDir.empty() && DepPath.starts_with(PrebuiltCacheDir);
}
static FrontendOptions::ActionType
reqestedModuleLoaderActionForFrontendOpts(const FrontendOptions &Opts) {
switch (Opts.RequestedAction) {
case FrontendOptions::ActionType::TypecheckModuleFromInterface:
return FrontendOptions::ActionType::Typecheck;
case FrontendOptions::ActionType::ScanDependencies:
return Opts.RequestedAction;
default:
return FrontendOptions::ActionType::EmitModuleOnly;
}
}
ModuleInterfaceLoaderOptions::ModuleInterfaceLoaderOptions(
const FrontendOptions &Opts, bool inheritDebuggingOpts)
: requestedAction(reqestedModuleLoaderActionForFrontendOpts(Opts)),
remarkOnRebuildFromInterface(Opts.RemarkOnRebuildFromModuleInterface),
disableInterfaceLock(Opts.DisableInterfaceFileLock),
disableImplicitSwiftModule(Opts.DisableImplicitModules),
disableBuildingInterface(Opts.DisableBuildingInterface),
downgradeInterfaceVerificationError(
Opts.DowngradeInterfaceVerificationError),
strictImplicitModuleContext(Opts.StrictImplicitModuleContext),
mainExecutablePath(Opts.MainExecutablePath) {
if (inheritDebuggingOpts) {
compilerDebuggingOptions = Opts.CompilerDebuggingOpts;
}
}
bool ModuleInterfaceLoader::isCached(StringRef DepPath) {
return InterfaceChecker.isCached(DepPath);
}
/// Load a .swiftmodule associated with a .swiftinterface either from a
/// cache or by converting it in a subordinate \c CompilerInstance, caching
/// the results.
std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
bool IsCanImportLookup, bool IsFramework,
bool IsTestableImport) {
// If running in OnlySerialized mode, ModuleInterfaceLoader
// should not have been constructed at all.
assert(LoadMode != ModuleLoadingMode::OnlySerialized);
std::string ModPath{BaseName.getName(file_types::TY_SwiftModuleFile)};
// First check to see if the .swiftinterface exists at all. Bail if not.
auto &fs = *Ctx.SourceMgr.getFileSystem();
auto InPath = BaseName.findInterfacePath(fs, Ctx);
if (!InPath) {
if (fs.exists(ModPath)) {
LLVM_DEBUG(llvm::dbgs()
<< "No .swiftinterface file found adjacent to module file "
<< ModPath << "\n");
return std::make_error_code(std::errc::not_supported);
}
return std::make_error_code(std::errc::no_such_file_or_directory);
}
if (ModuleInterfaceSourcePath)
ModuleInterfaceSourcePath->assign(InPath->begin(), InPath->end());
// If we are currently answering a `canImport` query, it is enough to have
// found the interface.
if (IsCanImportLookup) {
if (ModuleInterfacePath)
ModuleInterfacePath->assign(InPath->begin(), InPath->end());
return std::error_code();
}
// Create an instance of the Impl to do the heavy lifting.
auto ModuleName = ModuleID.Item.str();
ModuleInterfaceLoaderImpl Impl(
Ctx, ModPath, *InPath, ModuleName, InterfaceChecker.CacheDir,
InterfaceChecker.PrebuiltCacheDir, InterfaceChecker.BackupInterfaceDir,
ModuleID.Loc, InterfaceChecker.Opts,
dependencyTracker,
llvm::is_contained(PreferInterfaceForModules, ModuleName)
? ModuleLoadingMode::PreferInterface
: LoadMode);
// Ask the impl to find us a module that we can load or give us an error
// telling us that we couldn't load it.
auto ModuleBufferOrErr = Impl.findOrBuildLoadableModule();
if (!ModuleBufferOrErr)
return ModuleBufferOrErr.getError();
if (ModuleBuffer) {
*ModuleBuffer = std::move(*ModuleBufferOrErr);
if (ModuleInterfacePath)
ModuleInterfacePath->assign(InPath->begin(), InPath->end());
}
// Open .swiftsourceinfo file if it's present.
if (auto SourceInfoError = openModuleSourceInfoFileIfPresent(ModuleID,
BaseName,
ModuleSourceInfoBuffer))
return SourceInfoError;
// Delegate back to the serialized module loader to load the module doc.
if (auto DocLoadErr = openModuleDocFileIfPresent(ModuleID, BaseName,
ModuleDocBuffer))
return DocLoadErr;
return std::error_code();
}
std::vector<std::string>
ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(
StringRef moduleName, StringRef interfacePath) {
// Derive .swiftmodule path from the .swiftinterface path.
auto interfaceExt = file_types::getExtension(file_types::TY_SwiftModuleInterfaceFile);
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
llvm::SmallString<32> modulePath;
// When looking up the module for a private or package interface, strip
// the '.private.' or '.package.'section of the base name
if (interfacePath.ends_with(".private." + interfaceExt.str()) ||
interfacePath.ends_with(".package." + interfaceExt.str())) {
auto newBaseName = llvm::sys::path::stem(llvm::sys::path::stem(interfacePath));
modulePath = llvm::sys::path::parent_path(interfacePath);
llvm::sys::path::append(modulePath, newBaseName + "." + newExt.str());
} else {
modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
}
ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir, BackupInterfaceDir,
SourceLoc(), Opts,
nullptr, Ctx.SearchPathOpts.ModuleLoadMode);
std::vector<std::string> results;
std::string adjacentMod, prebuiltMod;
std::tie(adjacentMod, prebuiltMod) = Impl.getCompiledModuleCandidates();
auto validateModule = [&](StringRef modulePath) {
// Legacy behavior do not validate module.
if (!Ctx.SearchPathOpts.ScannerModuleValidation)
return true;
// If we picked the other module already, no need to validate this one since
// it should not be used anyway.
if (!results.empty())
return false;
SmallVector<FileDependency, 16> deps;
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
return Impl.upToDateChecker.swiftModuleIsUpToDate(
modulePath, Impl.rebuildInfo, deps, moduleBuffer);
};
// Add compiled module candidates only when they are non-empty and up-to-date.
if (!adjacentMod.empty() && validateModule(adjacentMod))
results.push_back(adjacentMod);
if (!prebuiltMod.empty() && validateModule(prebuiltMod))
results.push_back(prebuiltMod);
return results;
}
bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule(
StringRef moduleName, StringRef interfacePath,
ArrayRef<std::string> candidates, llvm::vfs::OutputBackend &backend,
StringRef outputPath) {
// Derive .swiftmodule path from the .swiftinterface path.
auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
llvm::SmallString<32> modulePath = interfacePath;
llvm::sys::path::replace_extension(modulePath, newExt);
ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName,
CacheDir, PrebuiltCacheDir,
BackupInterfaceDir, SourceLoc(), Opts,
nullptr,
ModuleLoadingMode::PreferSerialized);
SmallVector<FileDependency, 16> deps;
std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
for (auto mod: candidates) {
// Check if the candidate compiled module is still up-to-date.
if (Impl.upToDateChecker.swiftModuleIsUpToDate(mod, Impl.rebuildInfo,
deps, moduleBuffer)) {
// If so, emit a forwarding module to the candidate.
ForwardingModule FM(mod);
auto hadError = withOutputPath(Ctx.Diags, backend, outputPath,
[&](llvm::raw_pwrite_stream &out) {
llvm::yaml::Output yamlWriter(out);
yamlWriter << FM;
return false;
});
if (!hadError)
return true;
}
}
return false;
}
bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
SourceManager &SourceMgr, DiagnosticEngine &Diags,
const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts,
const ClangImporterOptions &ClangOpts, const CASOptions &CASOpts,
StringRef CacheDir, StringRef PrebuiltCacheDir,
StringRef BackupInterfaceDir, StringRef ModuleName, StringRef InPath,
StringRef OutPath, StringRef ABIOutputPath,
ArrayRef<std::pair<std::string, std::string>> replayPrefixMap,
bool SerializeDependencyHashes, bool TrackSystemDependencies,
ModuleInterfaceLoaderOptions LoaderOpts,
bool silenceInterfaceDiagnostics) {
InterfaceSubContextDelegateImpl astDelegate(
SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, CASOpts,
LoaderOpts,
/*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir,
BackupInterfaceDir, replayPrefixMap, SerializeDependencyHashes,
TrackSystemDependencies);
ImplicitModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath,
SearchPathOpts.getSDKPath(),
SearchPathOpts.getSysRoot(),
ModuleName, CacheDir, PrebuiltCacheDir,
BackupInterfaceDir, ABIOutputPath,
LoaderOpts.disableInterfaceLock,
silenceInterfaceDiagnostics);
// FIXME: We really only want to serialize 'important' dependencies here, if
// we want to ship the built swiftmodules to another machine.
auto failed = builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
/*ModuleBuffer*/nullptr, nullptr,
SearchPathOpts.CandidateCompiledModules);
if (!failed)
return false;
auto backInPath =
ModuleInterfaceLoaderImpl::getBackupPublicModuleInterfacePath(SourceMgr,
BackupInterfaceDir, ModuleName, InPath);
if (backInPath.empty())
return true;
assert(failed);
assert(!backInPath.empty());
ImplicitModuleInterfaceBuilder backupBuilder(SourceMgr, &Diags, astDelegate, backInPath,
SearchPathOpts.getSDKPath(),
SearchPathOpts.getSysRoot(),
ModuleName, CacheDir, PrebuiltCacheDir,
BackupInterfaceDir, ABIOutputPath,
LoaderOpts.disableInterfaceLock,
silenceInterfaceDiagnostics);
// Ensure we can rebuild module after user changed the original interface file.
backupBuilder.addExtraDependency(InPath);
// FIXME: We really only want to serialize 'important' dependencies here, if
// we want to ship the built swiftmodules to another machine.
return backupBuilder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
/*ModuleBuffer*/nullptr, nullptr,
SearchPathOpts.CandidateCompiledModules);
}
static bool readSwiftInterfaceVersionAndArgs(
SourceManager &SM, DiagnosticEngine &Diags, llvm::StringSaver &ArgSaver,
SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath,
SourceLoc diagnosticLoc, llvm::Triple preferredTarget) {
llvm::vfs::FileSystem &fs = *SM.getFileSystem();
auto FileOrError = swift::vfs::getFileOrSTDIN(fs, interfacePath);
if (!FileOrError) {
// Don't use this->diagnose() because it'll just try to re-open
// interfacePath.
Diags.diagnose(diagnosticLoc, diag::error_open_input_file, interfacePath,
FileOrError.getError().message());
return true;
}
auto SB = FileOrError.get()->getBuffer();
auto VersRe = getSwiftInterfaceFormatVersionRegex();
auto CompRe = getSwiftInterfaceCompilerVersionRegex();
SmallVector<StringRef, 2> VersMatches, CompMatches;
if (!VersRe.match(SB, &VersMatches)) {
InterfaceSubContextDelegateImpl::diagnose(
interfacePath, diagnosticLoc, SM, &Diags,
diag::error_extracting_version_from_module_interface);
return true;
}
if (extractCompilerFlagsFromInterface(interfacePath, SB, ArgSaver,
interfaceInfo.Arguments,
preferredTarget, &Diags)) {
InterfaceSubContextDelegateImpl::diagnose(
interfacePath, diagnosticLoc, SM, &Diags,
diag::error_extracting_version_from_module_interface);
return true;
}
assert(VersMatches.size() == 2);
// FIXME We should diagnose this at a location that makes sense:
auto Vers =
VersionParser::parseVersionString(VersMatches[1], SourceLoc(), &Diags);
if (!Vers.has_value()) {
InterfaceSubContextDelegateImpl::diagnose(
interfacePath, diagnosticLoc, SM, &Diags,
diag::error_extracting_version_from_module_interface);
return true;
}
if (CompRe.match(SB, &CompMatches)) {
assert(CompMatches.size() == 2);
interfaceInfo.CompilerVersion = ArgSaver.save(CompMatches[1]).str();
// For now, successfully parsing the tools version out of the interface is
// optional.
auto ToolsVersRe = getSwiftInterfaceCompilerToolsVersionRegex();
SmallVector<StringRef, 2> VendorToolsVersMatches;
if (ToolsVersRe.match(interfaceInfo.CompilerVersion,
&VendorToolsVersMatches)) {
interfaceInfo.CompilerToolsVersion = VersionParser::parseVersionString(
VendorToolsVersMatches[1], SourceLoc(), nullptr);
}
} else {
// Don't diagnose; handwritten module interfaces don't include this field.
interfaceInfo.CompilerVersion = "(unspecified, file possibly handwritten)";
}
// For now: we support anything with the same "major version" and assume
// minor versions might be interesting for debugging, or special-casing a
// compatible field variant.
if (Vers->asMajorVersion() != InterfaceFormatVersion.asMajorVersion()) {
InterfaceSubContextDelegateImpl::diagnose(
interfacePath, diagnosticLoc, SM, &Diags,
diag::unsupported_version_of_module_interface, interfacePath, *Vers);
return true;
}
return false;
}
bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface(
CompilerInstance &Instance, const StringRef moduleCachePath,
const StringRef backupInterfaceDir, const StringRef prebuiltCachePath,
const StringRef ABIDescriptorPath, StringRef interfacePath,
StringRef outputPath, bool ShouldSerializeDeps,
ArrayRef<std::string> CompiledCandidates,
DependencyTracker *tracker) {
// Read out the compiler version.
llvm::BumpPtrAllocator alloc;
llvm::StringSaver ArgSaver(alloc);
SwiftInterfaceInfo InterfaceInfo;
readSwiftInterfaceVersionAndArgs(
Instance.getSourceMgr(), Instance.getDiags(), ArgSaver, InterfaceInfo,
interfacePath, SourceLoc(),
Instance.getInvocation().getLangOptions().Target);
auto Builder = ExplicitModuleInterfaceBuilder(
Instance, &Instance.getDiags(), Instance.getSourceMgr(),
moduleCachePath, backupInterfaceDir, prebuiltCachePath,
ABIDescriptorPath, {});
auto error = Builder.buildSwiftModuleFromInterface(
interfacePath, outputPath, ShouldSerializeDeps, /*ModuleBuffer*/nullptr,
CompiledCandidates, InterfaceInfo.CompilerVersion);
if (!error)
return false;
else
return true;
}
void ModuleInterfaceLoader::collectVisibleTopLevelModuleNames(
SmallVectorImpl<Identifier> &names) const {
collectVisibleTopLevelModuleNamesImpl(
names,
file_types::getExtension(file_types::TY_SwiftModuleInterfaceFile));
}
void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface(
FrontendOptions::ActionType requestedAction,
const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts,
const ClangImporterOptions &clangImporterOpts, const CASOptions &casOpts,
bool suppressNotes, bool suppressRemarks,
PrintDiagnosticNamesMode printDiagnosticNames) {
GenericArgs.push_back("-frontend");
// Start with a genericSubInvocation that copies various state from our
// invoking ASTContext.
GenericArgs.push_back("-compile-module-from-interface");
genericSubInvocation.setTargetTriple(LangOpts.Target);
auto triple = ArgSaver.save(genericSubInvocation.getTargetTriple());
if (!triple.empty()) {
GenericArgs.push_back("-target");
GenericArgs.push_back(triple);
}
if (LangOpts.ClangTarget.has_value()) {
genericSubInvocation.getLangOptions().ClangTarget = LangOpts.ClangTarget;
auto triple = ArgSaver.save(genericSubInvocation.getLangOptions()
.ClangTarget->getTriple());
assert(!triple.empty());
// In explicit module build, all PCMs will be built using the given clang target.
// So the Swift interface should know that as well to load these PCMs properly.
GenericArgs.push_back("-clang-target");
GenericArgs.push_back(triple);
}
if (LangOpts.ClangTargetVariant.has_value()) {
genericSubInvocation.getLangOptions().ClangTargetVariant = LangOpts.ClangTargetVariant;
auto variantTriple = ArgSaver.save(genericSubInvocation.getLangOptions()
.ClangTargetVariant->getTriple());
assert(!variantTriple.empty());
GenericArgs.push_back("-clang-target-variant");
GenericArgs.push_back(variantTriple);
}
// Inherit the target SDK name and version
if (!LangOpts.SDKName.empty()) {
genericSubInvocation.getLangOptions().SDKName = LangOpts.SDKName;
GenericArgs.push_back("-target-sdk-name");
GenericArgs.push_back(ArgSaver.save(LangOpts.SDKName));
}
if (LangOpts.SDKVersion.has_value()) {
genericSubInvocation.getLangOptions().SDKVersion = LangOpts.SDKVersion;
GenericArgs.push_back("-target-sdk-version");
GenericArgs.push_back(ArgSaver.save(LangOpts.SDKVersion.value()
.getAsString()));
}
// Inherit the Swift language version
genericSubInvocation.getLangOptions().EffectiveLanguageVersion =
LangOpts.EffectiveLanguageVersion;
GenericArgs.push_back("-swift-version");
GenericArgs.push_back(ArgSaver.save(genericSubInvocation.getLangOptions()
.EffectiveLanguageVersion.asAPINotesVersionString()));
genericSubInvocation.setImportSearchPaths(
SearchPathOpts.getImportSearchPaths());
genericSubInvocation.setFrameworkSearchPaths(
SearchPathOpts.getFrameworkSearchPaths());
if (!SearchPathOpts.getSDKPath().empty()) {
// Add -sdk arguments to the module building commands.
// Module building commands need this because dependencies sometimes use
// sdk-relative paths (prebuilt modules for example). Without -sdk, the command
// will not be able to local these dependencies, leading to unnecessary
// building from textual interfaces.
GenericArgs.push_back("-sdk");
GenericArgs.push_back(ArgSaver.save(SearchPathOpts.getSDKPath()));
genericSubInvocation.setSDKPath(SearchPathOpts.getSDKPath().str());
}
if (SearchPathOpts.PlatformAvailabilityInheritanceMapPath) {
GenericArgs.push_back("-platform-availability-inheritance-map-path");
GenericArgs.push_back(ArgSaver.save(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath));
genericSubInvocation.setPlatformAvailabilityInheritanceMapPath(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath);
}
// Inherit the plugin search opts but do not inherit the arguments.
genericSubInvocation.getSearchPathOptions().PluginSearchOpts =
SearchPathOpts.PluginSearchOpts;
genericSubInvocation.getFrontendOptions().InputMode
= FrontendOptions::ParseInputMode::SwiftModuleInterface;
if (!SearchPathOpts.RuntimeResourcePath.empty()) {
genericSubInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath);
}
// Inhibit warnings from the genericSubInvocation since we are assuming the
// user is not in a position to address them.
genericSubInvocation.getDiagnosticOptions().SuppressWarnings = true;
GenericArgs.push_back("-suppress-warnings");
// Inherit the parent invocation's setting on whether to suppress remarks
if (suppressNotes) {
genericSubInvocation.getDiagnosticOptions().SuppressNotes = true;
GenericArgs.push_back("-suppress-notes");
}
// Inherit the parent invocation's setting on whether to suppress remarks
if (suppressRemarks) {
genericSubInvocation.getDiagnosticOptions().SuppressRemarks = true;
GenericArgs.push_back("-suppress-remarks");
}
// Inherit the parent invocation's setting for printing diagnostic IDs.
genericSubInvocation.getDiagnosticOptions().PrintDiagnosticNames =
printDiagnosticNames;
switch (printDiagnosticNames) {
case PrintDiagnosticNamesMode::None:
break;
case PrintDiagnosticNamesMode::Identifier:
GenericArgs.push_back("-debug-diagnostic-names");
break;
case PrintDiagnosticNamesMode::Group:
// FIXME: Currently no flag for Group mode
break;
}
// Inherit this setting down so that it can affect error diagnostics (mostly
// by making them non-fatal).
genericSubInvocation.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport;
if (LangOpts.DebuggerSupport) {
GenericArgs.push_back("-debugger-support");
}
// Disable this; deinitializers always get printed with `@objc` even in
// modules that don't import Foundation.
genericSubInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false;
GenericArgs.push_back("-disable-objc-attr-requires-foundation-module");
if (LangOpts.DisableAvailabilityChecking) {
genericSubInvocation.getLangOptions().DisableAvailabilityChecking = true;
GenericArgs.push_back("-disable-availability-checking");
}
// Pass-down the obfuscators so we can get the serialized search paths properly.
genericSubInvocation.setSerializedPathObfuscator(
SearchPathOpts.DeserializedPathRecoverer);
SearchPathOpts.DeserializedPathRecoverer
.forEachPair([&](StringRef lhs, StringRef rhs) {
GenericArgs.push_back("-serialized-path-obfuscate");
std::string pair = (llvm::Twine(lhs) + "=" + rhs).str();
GenericArgs.push_back(ArgSaver.save(pair));
});
genericSubInvocation.getClangImporterOptions().DirectClangCC1ModuleBuild =
clangImporterOpts.DirectClangCC1ModuleBuild;
genericSubInvocation.getClangImporterOptions().ClangImporterDirectCC1Scan =
clangImporterOpts.ClangImporterDirectCC1Scan;
genericSubInvocation.getSearchPathOptions().ScannerPrefixMapper =
SearchPathOpts.ScannerPrefixMapper;
// Validate Clang modules once per-build session flags must be consistent
// across all module sub-invocations
if (clangImporterOpts.ValidateModulesOnce) {
genericSubInvocation.getClangImporterOptions().ValidateModulesOnce = true;
genericSubInvocation.getClangImporterOptions().BuildSessionFilePath = clangImporterOpts.BuildSessionFilePath;
GenericArgs.push_back("-validate-clang-modules-once");
GenericArgs.push_back("-clang-build-session-file");
GenericArgs.push_back(clangImporterOpts.BuildSessionFilePath);
}
if (casOpts.EnableCaching) {
genericSubInvocation.getCASOptions().EnableCaching = casOpts.EnableCaching;
genericSubInvocation.getCASOptions().CASOpts = casOpts.CASOpts;
genericSubInvocation.getCASOptions().HasImmutableFileSystem =
casOpts.HasImmutableFileSystem;
casOpts.enumerateCASConfigurationFlags(
[&](StringRef Arg) { GenericArgs.push_back(ArgSaver.save(Arg)); });
}
}
bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs(
CompilerInvocation &subInvocation, DiagnosticEngine &subInstanceDiags,
SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath,
SourceLoc diagnosticLoc) {
if (readSwiftInterfaceVersionAndArgs(SM, *Diags, ArgSaver, interfaceInfo,
interfacePath, diagnosticLoc,
subInvocation.getLangOptions().Target))
return true;
// Prior to Swift 5.9, swiftinterfaces were always built (accidentally) with
// `-target-min-inlining-version target` prepended to the argument list. To
// preserve compatibility we must continue to prepend those flags to the
// invocation when the interface was generated by an older compiler.
if (auto toolsVersion = interfaceInfo.CompilerToolsVersion) {
if (toolsVersion < version::Version{5, 9}) {
interfaceInfo.Arguments.push_back("-target-min-inlining-version");
interfaceInfo.Arguments.push_back("target");
}
}
SmallString<32> ExpectedModuleName = subInvocation.getModuleName();
if (subInvocation.parseArgs(interfaceInfo.Arguments, subInstanceDiags)) {
return true;
}
if (subInvocation.getModuleName() != ExpectedModuleName) {
auto DiagKind = diag::serialization_name_mismatch;
if (subInvocation.getLangOptions().DebuggerSupport)
DiagKind = diag::serialization_name_mismatch_repl;
diagnose(interfacePath, diagnosticLoc,
DiagKind, subInvocation.getModuleName(), ExpectedModuleName);
return true;
}
return false;
}
InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
SourceManager &SM, DiagnosticEngine *Diags,
const SearchPathOptions &searchPathOpts, const LangOptions &langOpts,
const ClangImporterOptions &clangImporterOpts, const CASOptions &casOpts,
ModuleInterfaceLoaderOptions LoaderOpts, bool buildModuleCacheDirIfAbsent,
StringRef moduleCachePath, StringRef prebuiltCachePath,
StringRef backupModuleInterfaceDir,
ArrayRef<std::pair<std::string, std::string>> replayPrefixMap,
bool serializeDependencyHashes, bool trackSystemDependencies)
: SM(SM), Diags(Diags), ArgSaver(Allocator) {
genericSubInvocation.setMainExecutablePath(LoaderOpts.mainExecutablePath);
inheritOptionsForBuildingInterface(LoaderOpts.requestedAction, searchPathOpts,
langOpts, clangImporterOpts, casOpts,
Diags->getSuppressNotes(),
Diags->getSuppressRemarks(),
Diags->getPrintDiagnosticNamesMode());
// Configure front-end input.
auto &SubFEOpts = genericSubInvocation.getFrontendOptions();
SubFEOpts.RequestedAction = LoaderOpts.requestedAction;
SubFEOpts.StrictImplicitModuleContext =
LoaderOpts.strictImplicitModuleContext;
SubFEOpts.CompilerDebuggingOpts = LoaderOpts.compilerDebuggingOptions;
if (!moduleCachePath.empty()) {
genericSubInvocation.setClangModuleCachePath(moduleCachePath);
}
if (!prebuiltCachePath.empty()) {
genericSubInvocation.getFrontendOptions().PrebuiltModuleCachePath =
prebuiltCachePath.str();
}
if (!backupModuleInterfaceDir.empty()) {
genericSubInvocation.getFrontendOptions().BackupModuleInterfaceDir =
backupModuleInterfaceDir.str();
}
if (trackSystemDependencies) {
genericSubInvocation.getFrontendOptions().IntermoduleDependencyTracking =
IntermoduleDepTrackingMode::IncludeSystem;
GenericArgs.push_back("-track-system-dependencies");
} else {
// Always track at least the non-system dependencies for interface building.
genericSubInvocation.getFrontendOptions().IntermoduleDependencyTracking =
IntermoduleDepTrackingMode::ExcludeSystem;
}
if (LoaderOpts.disableImplicitSwiftModule) {
genericSubInvocation.getFrontendOptions().DisableImplicitModules = true;
GenericArgs.push_back("-disable-implicit-swift-modules");
GenericArgs.push_back("-Xcc");
GenericArgs.push_back("-fno-implicit-modules");
GenericArgs.push_back("-Xcc");
GenericArgs.push_back("-fno-implicit-module-maps");
}
// If building an application extension, make sure API use
// is restricted accordingly in downstream dependnecies.
if (langOpts.EnableAppExtensionRestrictions) {
genericSubInvocation.getLangOptions().EnableAppExtensionRestrictions = true;
GenericArgs.push_back("-application-extension");
}
SubFEOpts.CacheReplayPrefixMap = replayPrefixMap;
// Pass down -explicit-swift-module-map-file
StringRef explicitSwiftModuleMap = searchPathOpts.ExplicitSwiftModuleMapPath;
genericSubInvocation.getSearchPathOptions().ExplicitSwiftModuleMapPath =
explicitSwiftModuleMap.str();
// Pass down VFSOverlay flags (do not need to inherit the options because
// FileSystem is shared).
for (auto &Overlay : searchPathOpts.VFSOverlayFiles) {
GenericArgs.push_back("-vfsoverlay");
GenericArgs.push_back(Overlay);
}
// Load plugin libraries for macro expression as default arguments
genericSubInvocation.getSearchPathOptions().PluginSearchOpts =
searchPathOpts.PluginSearchOpts;
genericSubInvocation.getSearchPathOptions().ResolvedPluginVerification =
searchPathOpts.ResolvedPluginVerification;
// Get module loading behavior options.
genericSubInvocation.getSearchPathOptions().ScannerModuleValidation = searchPathOpts.ScannerModuleValidation;
genericSubInvocation.getSearchPathOptions().ModuleLoadMode =
searchPathOpts.ModuleLoadMode;
auto &subClangImporterOpts = genericSubInvocation.getClangImporterOptions();
// Respect the detailed-record preprocessor setting of the parent context.
// This, and the "raw" clang module format it implicitly enables, are
// required by sourcekitd.
subClangImporterOpts.DetailedPreprocessingRecord =
clangImporterOpts.DetailedPreprocessingRecord;
std::vector<std::string> inheritedParentContextClangArgs;
if (LoaderOpts.requestedAction ==
FrontendOptions::ActionType::ScanDependencies) {
// For a dependency scanning action, interface build command generation must
// inherit `-Xcc` flags used for configuration of the building instance's
// `ClangImporter`. However, we can ignore Clang search path flags because
// explicit Swift module build tasks will not rely on them and they may be
// source-target-context-specific and hinder module sharing across
// compilation source targets.
// Clang module dependecies of this Swift dependency will be distinguished
// by their context hash for different variants, so would still cause a
// difference in the Swift compile commands, when different.
inheritedParentContextClangArgs =
clangImporterOpts.getReducedExtraArgsForSwiftModuleDependency();
genericSubInvocation.getFrontendOptions()
.DependencyScanningSubInvocation = true;
} else if (LoaderOpts.strictImplicitModuleContext ||
// Explicit module Interface verification jobs still spawn a
// sub-instance and we must ensure this sub-instance gets all of
// the Xcc flags.
LoaderOpts.disableImplicitSwiftModule ||
// If using direct cc1 argument mode for lldb or typecheck
// interface, inherit all clang arguments.
clangImporterOpts.DirectClangCC1ModuleBuild) {
// If the compiler has been asked to be strict with ensuring downstream
// dependencies get the parent invocation's context, inherit the extra Clang
// arguments also. Inherit any clang-specific state of the compilation
// (macros, clang flags, etc.)
inheritedParentContextClangArgs = clangImporterOpts.ExtraArgs;
}
subClangImporterOpts.ExtraArgs = inheritedParentContextClangArgs;
// If using DirectCC1Scan, the command-line reduction is handled inside
// `getSwiftExplicitModuleDirectCC1Args()`, there is no need to inherit
// anything here as the ExtraArgs from the invocation are clang driver
// options, not cc1 options.
if (!clangImporterOpts.ClangImporterDirectCC1Scan) {
for (auto arg : subClangImporterOpts.ExtraArgs) {
GenericArgs.push_back("-Xcc");
GenericArgs.push_back(ArgSaver.save(arg));
}
}
subClangImporterOpts.EnableClangSPI = clangImporterOpts.EnableClangSPI;
if (!subClangImporterOpts.EnableClangSPI) {
GenericArgs.push_back("-disable-clang-spi");
}
// Tell the genericSubInvocation to serialize dependency hashes if asked to do
// so.
auto &frontendOpts = genericSubInvocation.getFrontendOptions();
frontendOpts.SerializeModuleInterfaceDependencyHashes =
serializeDependencyHashes;
if (serializeDependencyHashes) {
GenericArgs.push_back("-serialize-module-interface-dependency-hashes");
}
// Tell the genericSubInvocation to remark on rebuilds from an interface if asked
// to do so.
frontendOpts.RemarkOnRebuildFromModuleInterface =
LoaderOpts.remarkOnRebuildFromInterface;
if (LoaderOpts.remarkOnRebuildFromInterface) {
GenericArgs.push_back("-Rmodule-interface-rebuild");
}
// This flag only matters when we are verifying an textual interface.
frontendOpts.DowngradeInterfaceVerificationError =
LoaderOpts.downgradeInterfaceVerificationError;
// Note that we don't assume cachePath is the same as the Clang
// module cache path at this point.
if (buildModuleCacheDirIfAbsent && !moduleCachePath.empty())
(void)llvm::sys::fs::create_directories(moduleCachePath);
// Inherit all block list configuration files
frontendOpts.BlocklistConfigFilePaths = langOpts.BlocklistConfigFilePaths;
for (auto &blocklist: langOpts.BlocklistConfigFilePaths) {
GenericArgs.push_back("-blocklist-file");
GenericArgs.push_back(blocklist);
}
// Inherit APINotes processing method
if (clangImporterOpts.LoadVersionIndependentAPINotes) {
GenericArgs.push_back("-version-independent-apinotes");
genericSubInvocation.getClangImporterOptions().LoadVersionIndependentAPINotes = true;
}
// Inherit the C++ interoperability mode.
if (langOpts.EnableCXXInterop) {
// Modelled after a reverse of validateCxxInteropCompatibilityMode
genericSubInvocation.getLangOptions().EnableCXXInterop = true;
genericSubInvocation.getLangOptions().cxxInteropCompatVersion =
langOpts.cxxInteropCompatVersion;
std::string compatVersion;
if (langOpts.cxxInteropCompatVersion.empty())
compatVersion = "default";
else if (langOpts.cxxInteropCompatVersion[0] == 5)
compatVersion = "swift-5.9";
else if (langOpts.cxxInteropCompatVersion[0] == 6)
compatVersion = "swift-6";
else if (langOpts.cxxInteropCompatVersion[0] ==
version::getUpcomingCxxInteropCompatVersion())
compatVersion = "upcoming-swift";
else // TODO: This may need to be updated once more versions are added
compatVersion = "default";
GenericArgs.push_back(
ArgSaver.save("-cxx-interoperability-mode=" + compatVersion));
if (!langOpts.isUsingPlatformDefaultCXXStdlib() &&
langOpts.CXXStdlib == CXXStdlibKind::Libcxx) {
genericSubInvocation.getLangOptions().CXXStdlib = CXXStdlibKind::Libcxx;
genericSubInvocation.getClangImporterOptions().ExtraArgs.push_back(
"-stdlib=libc++");
GenericArgs.push_back("-Xcc");
GenericArgs.push_back("-stdlib=libc++");
}
}
// Inherit Embedded Swift
if (langOpts.hasFeature(Feature::Embedded)) {
genericSubInvocation.getLangOptions().enableFeature(Feature::Embedded);
GenericArgs.push_back("-enable-experimental-feature");
GenericArgs.push_back("Embedded");
}
if (langOpts.DebuggerSupport) {
if (clangImporterOpts.DirectClangCC1ModuleBuild) {
subClangImporterOpts.ExtraArgs.push_back("-dwarf-ext-refs");
subClangImporterOpts.ExtraArgs.push_back("-fmodule-format=obj");
subClangImporterOpts.ExtraArgs.push_back("-debug-info-kind=standalone");
} else {
subClangImporterOpts.ExtraArgs.push_back("-gmodules");
subClangImporterOpts.ExtraArgs.push_back("-g");
}
}
}
/// Calculate an output filename in \p genericSubInvocation's cache path that
/// includes a hash of relevant key data.
void InterfaceSubContextDelegateImpl::getCachedOutputPath(
SwiftInterfaceModuleOutputPathResolution::ResultTy &resolvedOutputPath,
StringRef moduleName, StringRef interfacePath, StringRef sdkPath) {
SwiftInterfaceModuleOutputPathResolution::setOutputPath(
resolvedOutputPath, moduleName, interfacePath, sdkPath,
genericSubInvocation,
genericSubInvocation.getClangImporterOptions()
.getReducedExtraArgsForSwiftModuleDependency());
}
std::error_code
InterfaceSubContextDelegateImpl::runInSubContext(StringRef moduleName,
StringRef interfacePath,
StringRef sdkPath,
std::optional<StringRef> sysroot,
StringRef outputPath,
SourceLoc diagLoc,
llvm::function_ref<std::error_code(ASTContext&, ModuleDecl*, ArrayRef<StringRef>,
StringRef, StringRef)> action) {
return runInSubCompilerInstance(moduleName, interfacePath, sdkPath, sysroot,
outputPath, diagLoc, /*silenceErrors=*/false,
[&](SubCompilerInstanceInfo &info){
std::string UserModuleVer = info.Instance->getInvocation().getFrontendOptions()
.UserModuleVersion.getAsString();
return action(info.Instance->getASTContext(),
info.Instance->getMainModule(),
info.BuildArguments,
info.Hash,
UserModuleVer);
});
}
std::error_code
InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName,
StringRef interfacePath,
StringRef sdkPath,
std::optional<StringRef> sysroot,
StringRef outputPath,
SourceLoc diagLoc,
bool silenceErrors,
llvm::function_ref<std::error_code(SubCompilerInstanceInfo&)> action) {
// We are about to mess up the compiler invocation by using the compiler
// arguments in the textual interface file. So copy to use a new compiler
// invocation.
CompilerInvocation subInvocation = genericSubInvocation;
// save `StrictImplicitModuleContext`
bool StrictImplicitModuleContext =
subInvocation.getFrontendOptions().StrictImplicitModuleContext;
// It isn't appropriate to restrict use of experimental features in another
// module since it may have been built with a different compiler that allowed
// the use of the feature.
subInvocation.getLangOptions().RestrictNonProductionExperimentalFeatures =
false;
// Save the target triple from the original context.
llvm::Triple originalTargetTriple(subInvocation.getLangOptions().Target);
std::vector<StringRef> BuildArgs(GenericArgs.begin(), GenericArgs.end());
assert(BuildArgs.size() == GenericArgs.size());
// Configure inputs
subInvocation.getFrontendOptions().InputsAndOutputs
.addInputFile(interfacePath);
BuildArgs.push_back(interfacePath);
subInvocation.setModuleName(moduleName);
BuildArgs.push_back("-module-name");
BuildArgs.push_back(moduleName);
if (sysroot) {
subInvocation.setSysRoot(sysroot.value());
}
// Calculate output path of the module.
SwiftInterfaceModuleOutputPathResolution::ResultTy resolvedOutputPath;
getCachedOutputPath(resolvedOutputPath, moduleName, interfacePath, sdkPath);
// If no specific output path is given, use the hashed output path.
if (outputPath.empty()) {
outputPath = resolvedOutputPath.outputPath;
}
// Configure the outputs in front-end options. There must be an equal number of
// inputs and outputs.
std::vector<std::string> outputFiles{"/<unused>"};
std::vector<SupplementaryOutputPaths> ModuleOutputPaths;
ModuleOutputPaths.emplace_back();
if (subInvocation.getFrontendOptions().RequestedAction ==
FrontendOptions::ActionType::EmitModuleOnly) {
ModuleOutputPaths.back().ModuleOutputPath = outputPath.str();
}
assert(subInvocation.getFrontendOptions().InputsAndOutputs.isWholeModule());
subInvocation.getFrontendOptions().InputsAndOutputs
.setMainAndSupplementaryOutputs(outputFiles, ModuleOutputPaths);
CompilerInstance subInstance;
ForwardingDiagnosticConsumer FDC(*Diags);
NullDiagnosticConsumer noopConsumer;
if (!silenceErrors) {
subInstance.addDiagnosticConsumer(&FDC);
} else {
subInstance.addDiagnosticConsumer(&noopConsumer);
}
SwiftInterfaceInfo interfaceInfo;
// Extract compiler arguments from the interface file and use them to configure
// the compiler invocation.
if (extractSwiftInterfaceVersionAndArgs(subInvocation, subInstance.getDiags(),
interfaceInfo, interfacePath,
diagLoc)) {
return std::make_error_code(std::errc::not_supported);
}
// Insert arguments collected from the interface file.
BuildArgs.insert(BuildArgs.end(), interfaceInfo.Arguments.begin(),
interfaceInfo.Arguments.end());
// restore `StrictImplicitModuleContext`
subInvocation.getFrontendOptions().StrictImplicitModuleContext =
StrictImplicitModuleContext;
SubCompilerInstanceInfo info;
info.Instance = &subInstance;
info.CompilerVersion = interfaceInfo.CompilerVersion;
subInstance.getSourceMgr().setFileSystem(SM.getFileSystem());
std::string InstanceSetupError;
if (subInstance.setup(subInvocation, InstanceSetupError)) {
return std::make_error_code(std::errc::not_supported);
}
info.BuildArguments = BuildArgs;
info.Hash = resolvedOutputPath.hash;
// Run the action under the sub compiler instance.
return action(info);
}
static void addModuleAliasesFromExplicitSwiftModuleMap(
ASTContext &Ctx, llvm::StringMap<std::string> ModuleAliases) {
for (auto &entry : ModuleAliases) {
Ctx.addModuleAlias(/*moduleAlias=*/entry.getKey(),
/*realModule=*/entry.getValue());
}
}
struct ExplicitSwiftModuleLoader::Implementation {
ASTContext &Ctx;
llvm::BumpPtrAllocator Allocator;
llvm::StringMap<ExplicitSwiftModuleInputInfo> ExplicitModuleMap;
Implementation(ASTContext &Ctx) : Ctx(Ctx) {}
void parseSwiftExplicitModuleMap(StringRef fileName) {
ExplicitModuleMapParser parser(Allocator);
llvm::StringMap<ExplicitClangModuleInputInfo> ExplicitClangModuleMap;
llvm::StringMap<std::string> ModuleAliases;
// Load the input file.
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
llvm::MemoryBuffer::getFile(fileName);
if (!fileBufOrErr) {
Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing,
fileName);
return;
}
auto error = parser.parseSwiftExplicitModuleMap(
(*fileBufOrErr)->getMemBufferRef(), ExplicitModuleMap,
ExplicitClangModuleMap, ModuleAliases);
llvm::handleAllErrors(std::move(error), [this, &fileName](
const llvm::StringError &E) {
Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted,
fileName, E.getMessage());
});
// A single module map can define multiple modules; keep track of the ones
// we've seen so that we don't generate duplicate flags.
std::set<std::string> moduleMapsSeen;
std::vector<std::string> &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs;
for (auto &entry : ExplicitClangModuleMap) {
const auto &moduleMapPath = entry.getValue().moduleMapPath;
if (!moduleMapPath.empty() &&
entry.getValue().isBridgingHeaderDependency &&
moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) {
moduleMapsSeen.insert(moduleMapPath);
extraClangArgs.push_back(
(Twine("-fmodule-map-file=") + moduleMapPath).str());
}
const auto &modulePath = entry.getValue().modulePath;
if (!modulePath.empty()) {
extraClangArgs.push_back(
(Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath)
.str());
}
}
addModuleAliasesFromExplicitSwiftModuleMap(Ctx, ModuleAliases);
}
void addCommandLineExplicitInputs(
const llvm::StringMap<std::string> &commandLineExplicitInputs) {
for (const auto &moduleInput : commandLineExplicitInputs) {
ExplicitSwiftModuleInputInfo entry(moduleInput.getValue(), {}, {}, {});
ExplicitModuleMap.try_emplace(moduleInput.first(), std::move(entry));
}
}
};
ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
ASTContext &ctx,
DependencyTracker *tracker,
ModuleLoadingMode loadMode,
bool IgnoreSwiftSourceInfoFile):
SerializedModuleLoaderBase(ctx, tracker, loadMode,
IgnoreSwiftSourceInfoFile),
Impl(*new Implementation(ctx)) {}
ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; }
bool ExplicitSwiftModuleLoader::findModule(
ImportPath::Element ModuleID, SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
std::string *cacheKey, bool IsCanImportLookup,
bool isTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) {
// Find a module with an actual, physical name on disk, in case
// -module-alias is used (otherwise same).
//
// For example, if '-module-alias Foo=Bar' is passed in to the frontend, and an
// input file has 'import Foo', a module called Bar (real name) should be searched.
StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str();
auto it = Impl.ExplicitModuleMap.find(moduleName);
// If no explicit module path is given matches the name, return with an
// error code.
if (it == Impl.ExplicitModuleMap.end()) {
return false;
}
auto &moduleInfo = it->getValue();
// Set IsFramework bit according to the moduleInfo
IsFramework = moduleInfo.isFramework;
IsSystemModule = moduleInfo.isSystem;
auto &fs = *Ctx.SourceMgr.getFileSystem();
// Open .swiftmodule file
auto moduleBuf = fs.getBufferForFile(moduleInfo.modulePath);
if (!moduleBuf) {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
assert(moduleBuf);
const bool isForwardingModule = !serialization::isSerializedAST(moduleBuf
.get()->getBuffer());
// If the module is a forwarding module, read the actual content from the path
// encoded in the forwarding module as the actual module content.
if (isForwardingModule) {
auto forwardingModule = ForwardingModule::load(*moduleBuf.get());
if (forwardingModule) {
moduleBuf = fs.getBufferForFile(forwardingModule->underlyingModulePath);
if (!moduleBuf) {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
} else {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
}
assert(moduleBuf);
// Move the opened module buffer to the caller.
*ModuleBuffer = std::move(moduleBuf.get());
// Open .swiftdoc file
if (moduleInfo.moduleDocPath.has_value()) {
auto moduleDocBuf = fs.getBufferForFile(moduleInfo.moduleDocPath.value());
if (moduleBuf)
*ModuleDocBuffer = std::move(moduleDocBuf.get());
}
// Open .swiftsourceinfo file
if (moduleInfo.moduleSourceInfoPath.has_value()) {
auto moduleSourceInfoBuf = fs.getBufferForFile(moduleInfo.moduleSourceInfoPath.value());
if (moduleSourceInfoBuf)
*ModuleSourceInfoBuffer = std::move(moduleSourceInfoBuf.get());
}
return true;
}
std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
bool IsCanImportLookup, bool IsFramework,
bool IsTestableDependencyLookup) {
llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
return std::make_error_code(std::errc::not_supported);
}
bool ExplicitSwiftModuleLoader::canImportModule(
ImportPath::Module path, SourceLoc loc, ModuleVersionInfo *versionInfo,
bool isTestableDependencyLookup) {
// FIXME: Swift submodules?
if (path.hasSubmodule())
return false;
ImportPath::Element mID = path.front();
// Look up the module with the real name (physical name on disk);
// in case `-module-alias` is used, the name appearing in source files
// and the real module name are different. For example, '-module-alias Foo=Bar'
// maps Foo appearing in source files, e.g. 'import Foo', to the real module
// name Bar (on-disk name), which should be searched for loading.
StringRef moduleName = Ctx.getRealModuleName(mID.Item).str();
auto it = Impl.ExplicitModuleMap.find(moduleName);
// If no provided explicit module matches the name, then it cannot be imported.
if (it == Impl.ExplicitModuleMap.end()) {
return false;
}
// If the caller doesn't want version info we're done.
if (!versionInfo)
return true;
// Open .swiftmodule file and read out the version
auto &fs = *Ctx.SourceMgr.getFileSystem();
auto moduleBuf = fs.getBufferForFile(it->second.modulePath);
if (!moduleBuf) {
Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file,
it->second.modulePath);
return false;
}
// If it's a forwarding module, load the YAML file from disk and get the path
// to the actual module for the version check.
if (!serialization::isSerializedAST((*moduleBuf)->getBuffer())) {
if (auto forwardingModule = ForwardingModule::load(**moduleBuf)) {
moduleBuf = fs.getBufferForFile(forwardingModule->underlyingModulePath);
if (!moduleBuf) {
Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file,
forwardingModule->underlyingModulePath);
return false;
}
}
}
auto metaData = serialization::validateSerializedAST(
(*moduleBuf)->getBuffer(),
Ctx.LangOpts.SDKName);
versionInfo->setVersion(metaData.userModuleVersion,
ModuleVersionSourceKind::SwiftBinaryModule);
return true;
}
void ExplicitSwiftModuleLoader::collectVisibleTopLevelModuleNames(
SmallVectorImpl<Identifier> &names) const {
for (auto &entry: Impl.ExplicitModuleMap) {
names.push_back(Ctx.getIdentifier(entry.getKey()));
}
}
std::unique_ptr<ExplicitSwiftModuleLoader>
ExplicitSwiftModuleLoader::create(ASTContext &ctx,
DependencyTracker *tracker, ModuleLoadingMode loadMode,
StringRef ExplicitSwiftModuleMap,
const llvm::StringMap<std::string> &ExplicitSwiftModuleInputs,
bool IgnoreSwiftSourceInfoFile) {
auto result = std::unique_ptr<ExplicitSwiftModuleLoader>(
new ExplicitSwiftModuleLoader(ctx, tracker, loadMode,
IgnoreSwiftSourceInfoFile));
auto &Impl = result->Impl;
// If the explicit module map is given, try parse it.
if (!ExplicitSwiftModuleMap.empty()) {
// Parse a JSON file to collect explicitly built modules.
Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap);
}
// If some modules are provided with explicit
// '-swift-module-file' options, add those as well.
if (!ExplicitSwiftModuleInputs.empty()) {
Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs);
}
return result;
}
void ExplicitSwiftModuleLoader::addExplicitModulePath(StringRef name,
std::string path) {
ExplicitSwiftModuleInputInfo entry(path, {}, {}, {});
Impl.ExplicitModuleMap.try_emplace(name, std::move(entry));
}
struct ExplicitCASModuleLoader::Implementation {
ASTContext &Ctx;
llvm::BumpPtrAllocator Allocator;
llvm::cas::ObjectStore &CAS;
llvm::cas::ActionCache &Cache;
llvm::StringMap<ExplicitSwiftModuleInputInfo> ExplicitModuleMap;
Implementation(ASTContext &Ctx, llvm::cas::ObjectStore &CAS,
llvm::cas::ActionCache &Cache)
: Ctx(Ctx), CAS(CAS), Cache(Cache) {}
std::unique_ptr<llvm::MemoryBuffer> loadBuffer(StringRef ID) {
auto key = CAS.parseID(ID);
if (!key) {
Ctx.Diags.diagnose(SourceLoc(), diag::error_invalid_cas_id, ID,
toString(key.takeError()));
return nullptr;
}
auto ref = CAS.getReference(*key);
if (!ref)
return nullptr;
auto loaded = CAS.getProxy(*ref);
if (!loaded) {
Ctx.Diags.diagnose(SourceLoc(), diag::error_cas,
"loading module map from CAS",
toString(loaded.takeError()));
return nullptr;
}
return loaded->getMemoryBuffer();
}
// Same as the regular explicit module map but must be loaded from
// CAS, instead of a file that is not tracked by the dependency.
void parseSwiftExplicitModuleMap(StringRef ID) {
ExplicitModuleMapParser parser(Allocator);
llvm::StringMap<ExplicitClangModuleInputInfo> ExplicitClangModuleMap;
llvm::StringMap<std::string> ModuleAliases;
auto buf = loadBuffer(ID);
if (!buf) {
Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing,
ID);
return;
}
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
llvm::MemoryBuffer::getFile(ID);
auto error = parser.parseSwiftExplicitModuleMap(
buf->getMemBufferRef(), ExplicitModuleMap, ExplicitClangModuleMap,
ModuleAliases);
llvm::handleAllErrors(std::move(error), [this,
&ID](const llvm::StringError &E) {
Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted,
ID, E.getMessage());
});
std::set<std::string> moduleMapsSeen;
std::vector<std::string> &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs;
// Append -Xclang if we are not in direct cc1 mode.
auto appendXclang = [&]() {
if (!Ctx.ClangImporterOpts.DirectClangCC1ModuleBuild)
extraClangArgs.push_back("-Xclang");
};
for (auto &entry : ExplicitClangModuleMap) {
const auto &moduleMapPath = entry.getValue().moduleMapPath;
if (!moduleMapPath.empty() && !Ctx.CASOpts.EnableCaching &&
moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) {
moduleMapsSeen.insert(moduleMapPath);
extraClangArgs.push_back(
(Twine("-fmodule-map-file=") + moduleMapPath).str());
}
const auto &modulePath = entry.getValue().modulePath;
if (!modulePath.empty()) {
extraClangArgs.push_back(
(Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath)
.str());
}
auto cachePath = entry.getValue().moduleCacheKey;
if (cachePath) {
appendXclang();
extraClangArgs.push_back("-fmodule-file-cache-key");
appendXclang();
extraClangArgs.push_back(modulePath);
appendXclang();
extraClangArgs.push_back(*cachePath);
}
}
addModuleAliasesFromExplicitSwiftModuleMap(Ctx, ModuleAliases);
}
void addCommandLineExplicitInputs(
const llvm::StringMap<std::string> &commandLineExplicitInputs) {
for (const auto &moduleInput : commandLineExplicitInputs) {
ExplicitSwiftModuleInputInfo entry(moduleInput.getValue(), {}, {}, {});
ExplicitModuleMap.try_emplace(moduleInput.getKey(), std::move(entry));
}
}
std::unique_ptr<llvm::MemoryBuffer>
loadFileBuffer(SourceLoc Loc, StringRef ID, StringRef Name) {
auto outputMissing = [&]() {
Ctx.Diags.diagnose(Loc, diag::error_opening_explicit_module_file, Name);
return nullptr;
};
auto casError = [&](StringRef Stage, llvm::Error Err) {
Ctx.Diags.diagnose(Loc, diag::error_cas, Stage, toString(std::move(Err)));
return nullptr;
};
auto key = CAS.parseID(ID);
if (!key) {
Ctx.Diags.diagnose(Loc, diag::error_invalid_cas_id, ID,
toString(key.takeError()));
return nullptr;
}
auto moduleLookup = Cache.get(*key);
if (!moduleLookup)
return casError("looking up module output cache",
moduleLookup.takeError());
if (!*moduleLookup)
return outputMissing();
auto moduleRef = CAS.getReference(**moduleLookup);
if (!moduleRef)
return outputMissing();
auto proxy = CAS.getProxy(*moduleRef);
if (!proxy)
return casError("loading module build outputs", proxy.takeError());
swift::cas::CompileJobResultSchema schema(CAS);
if (!schema.isRootNode(*proxy))
return outputMissing();
auto result = schema.load(*moduleRef);
if (!result)
return casError("loading module schema", result.takeError());
auto output = result->getOutput(file_types::ID::TY_SwiftModuleFile);
if (!output)
return outputMissing();
auto buf = CAS.getProxy(output->Object);
if (!buf)
return casError("loading dependency module", result.takeError());
return buf->getMemoryBuffer(Name);
}
llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
loadModuleFromPath(StringRef Path, DiagnosticEngine &Diags) {
for (auto &Deps : ExplicitModuleMap) {
if (Deps.second.modulePath == Path) {
if (!Deps.second.moduleCacheKey)
return nullptr;
return loadCachedCompileResultFromCacheKey(
CAS, Cache, Diags, *Deps.second.moduleCacheKey,
file_types::ID::TY_SwiftModuleFile, Path);
}
}
return nullptr;
}
};
ExplicitCASModuleLoader::ExplicitCASModuleLoader(ASTContext &ctx,
llvm::cas::ObjectStore &CAS,
llvm::cas::ActionCache &cache,
DependencyTracker *tracker,
ModuleLoadingMode loadMode,
bool IgnoreSwiftSourceInfoFile)
: SerializedModuleLoaderBase(ctx, tracker, loadMode,
IgnoreSwiftSourceInfoFile),
Impl(*new Implementation(ctx, CAS, cache)) {}
ExplicitCASModuleLoader::~ExplicitCASModuleLoader() { delete &Impl; }
bool ExplicitCASModuleLoader::findModule(
ImportPath::Element ModuleID, SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
std::string *CacheKey, bool IsCanImportLookup,
bool IsTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) {
// Find a module with an actual, physical name on disk, in case
// -module-alias is used (otherwise same).
//
// For example, if '-module-alias Foo=Bar' is passed in to the frontend, and
// an input file has 'import Foo', a module called Bar (real name) should be
// searched.
StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str();
auto it = Impl.ExplicitModuleMap.find(moduleName);
// If no explicit module path is given matches the name, return with an
// error code.
if (it == Impl.ExplicitModuleMap.end()) {
return false;
}
auto &moduleInfo = it->getValue();
// Set IsFramework bit according to the moduleInfo
IsFramework = moduleInfo.isFramework;
IsSystemModule = moduleInfo.isSystem;
if (CacheKey && moduleInfo.moduleCacheKey)
*CacheKey = *moduleInfo.moduleCacheKey;
// Fallback check for module cache key passed on command-line as module path.
std::string moduleCASID = moduleInfo.moduleCacheKey
? *moduleInfo.moduleCacheKey
: moduleInfo.modulePath;
// FIXME: the loaded module buffer doesn't set an identifier so it
// is not tracked in dependency tracker, which doesn't handle modules
// that are not located on disk.
auto moduleBuf = loadCachedCompileResultFromCacheKey(
Impl.CAS, Impl.Cache, Ctx.Diags, moduleCASID,
file_types::ID::TY_SwiftModuleFile, moduleInfo.modulePath);
if (!moduleBuf) {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
const bool isForwardingModule =
!serialization::isSerializedAST(moduleBuf->getBuffer());
// If the module is a forwarding module, read the actual content from the path
// encoded in the forwarding module as the actual module content.
if (isForwardingModule) {
auto forwardingModule = ForwardingModule::load(*moduleBuf.get());
if (forwardingModule) {
// Look through ExplicitModuleMap for paths.
// TODO: need to have dependency scanner reports forwarded module as
// dependency for this compilation and ingested into CAS.
auto moduleOrErr = Impl.loadModuleFromPath(
forwardingModule->underlyingModulePath, Ctx.Diags);
if (!moduleOrErr) {
llvm::consumeError(moduleOrErr.takeError());
Ctx.Diags.diagnose(SourceLoc(),
diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
moduleBuf = std::move(*moduleOrErr);
if (!moduleBuf) {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(),
diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
} else {
// We cannot read the module content, diagnose.
Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file,
moduleInfo.modulePath);
return false;
}
}
assert(moduleBuf);
// Move the opened module buffer to the caller.
*ModuleBuffer = std::move(moduleBuf);
// TODO: support .swiftdoc file and .swiftsourceinfo file
return true;
}
std::error_code ExplicitCASModuleLoader::findModuleFilesInDirectory(
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
bool IsCanImportLookup, bool IsFramework,
bool IsTestableDependencyLookup) {
llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
return std::make_error_code(std::errc::not_supported);
}
bool ExplicitCASModuleLoader::canImportModule(
ImportPath::Module path, SourceLoc loc, ModuleVersionInfo *versionInfo,
bool isTestableDependencyLookup) {
// FIXME: Swift submodules?
if (path.hasSubmodule())
return false;
ImportPath::Element mID = path.front();
// Look up the module with the real name (physical name on disk);
// in case `-module-alias` is used, the name appearing in source files
// and the real module name are different. For example, '-module-alias
// Foo=Bar' maps Foo appearing in source files, e.g. 'import Foo', to the real
// module name Bar (on-disk name), which should be searched for loading.
StringRef moduleName = Ctx.getRealModuleName(mID.Item).str();
auto it = Impl.ExplicitModuleMap.find(moduleName);
// If no provided explicit module matches the name, then it cannot be
// imported.
if (it == Impl.ExplicitModuleMap.end()) {
return false;
}
// If the caller doesn't want version info we're done.
if (!versionInfo)
return true;
// Open .swiftmodule file and read out the version
std::string moduleCASID = it->second.moduleCacheKey
? *it->second.moduleCacheKey
: it->second.modulePath;
auto moduleBuf = Impl.loadFileBuffer(loc, moduleCASID, it->second.modulePath);
if (!moduleBuf) {
Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file,
it->second.modulePath);
return false;
}
auto metaData = serialization::validateSerializedAST(
moduleBuf->getBuffer(),
Ctx.LangOpts.SDKName);
versionInfo->setVersion(metaData.userModuleVersion,
ModuleVersionSourceKind::SwiftBinaryModule);
return true;
}
void ExplicitCASModuleLoader::collectVisibleTopLevelModuleNames(
SmallVectorImpl<Identifier> &names) const {
for (auto &entry : Impl.ExplicitModuleMap) {
names.push_back(Ctx.getIdentifier(entry.getKey()));
}
}
std::unique_ptr<ExplicitCASModuleLoader> ExplicitCASModuleLoader::create(
ASTContext &ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &cache,
DependencyTracker *tracker, ModuleLoadingMode loadMode,
StringRef ExplicitSwiftModuleMap,
const llvm::StringMap<std::string> &ExplicitSwiftModuleInputs,
bool IgnoreSwiftSourceInfoFile) {
auto result =
std::unique_ptr<ExplicitCASModuleLoader>(new ExplicitCASModuleLoader(
ctx, CAS, cache, tracker, loadMode, IgnoreSwiftSourceInfoFile));
auto &Impl = result->Impl;
// If the explicit module map is given, try parse it.
if (!ExplicitSwiftModuleMap.empty()) {
// Parse a JSON file to collect explicitly built modules.
Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap);
}
// If some modules are provided with explicit
// '-swift-module-file' options, add those as well.
if (!ExplicitSwiftModuleInputs.empty()) {
Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs);
}
return result;
}
namespace swift::SwiftInterfaceModuleOutputPathResolution {
/// Construct a key for the .swiftmodule being generated. There is a
/// balance to be struck here between things that go in the cache key and
/// things that go in the "up to date" check of the cache entry. We want to
/// avoid fighting over a single cache entry too much when (say) running
/// different compiler versions on the same machine or different inputs
/// that happen to have the same short module name, so we will disambiguate
/// those in the key. But we want to invalidate and rebuild a cache entry
/// -- rather than making a new one and potentially filling up the cache
/// with dead entries -- when other factors change, such as the contents of
/// the .swiftinterface input or its dependencies.
static std::string getContextHash(const CompilerInvocation &CI,
const StringRef &interfacePath,
const StringRef &sdkPath,
const ArgListTy &extraArgs) {
// When doing dependency scanning for explicit module, use strict context hash
// to ensure sound module hash.
bool useStrictCacheHash = CI.getFrontendOptions().RequestedAction ==
FrontendOptions::ActionType::ScanDependencies;
// Include the normalized target triple when not using strict hash.
// Otherwise, use the full target to ensure soundness of the hash. In
// practice, .swiftinterface files will be in target-specific subdirectories
// and would have target-specific pieces #if'd out. However, it doesn't hurt
// to include it, and it guards against mistakenly reusing cached modules
// across targets. Note that this normalization explicitly doesn't include the
// minimum deployment target (e.g. the '12.0' in 'ios12.0').
auto targetToHash =
useStrictCacheHash
? CI.getLangOptions().Target
: getTargetSpecificModuleTriple(CI.getLangOptions().Target);
std::string sdkBuildVersion = getSDKBuildVersion(sdkPath);
llvm::hash_code H = llvm::hash_combine(
// Start with the compiler version (which will be either tag names or
// revs). Explicitly don't pass in the "effective" language version --
// this would mean modules built in different -swift-version modes would
// rebuild their dependencies.
swift::version::getSwiftFullVersion(),
// Simplest representation of input "identity" (not content) is just a
// pathname, and probably all we can get from the VFS in this regard
// anyways.
interfacePath,
// The target triple to hash.
targetToHash.str(),
// The SDK path is going to affect how this module is imported, so
// include it.
CI.getSDKPath(),
// The SDK build version may identify differences in headers
// that affects references serialized in the cached file.
sdkBuildVersion,
// Applying the distribution channel of the current compiler enables
// different compilers to share a module cache location.
version::getCurrentCompilerChannel(),
// Whether or not we're tracking system dependencies affects the
// invalidation behavior of this cache item.
CI.getFrontendOptions().shouldTrackSystemDependencies(),
// Whether or not caching is enabled affects if the instance is able to
// correctly load the dependencies.
CI.getCASOptions().getModuleScanningHashComponents(),
// Take care of any extra arguments that should affect the hash.
llvm::hash_combine_range(extraArgs.begin(), extraArgs.end()),
// Application extension.
unsigned(CI.getLangOptions().EnableAppExtensionRestrictions),
// Is the C++ interop enabled?
unsigned(CI.getLangOptions().EnableCXXInterop),
// Is Embedded Swift enabled?
unsigned(CI.getLangOptions().hasFeature(Feature::Embedded))
);
return llvm::toString(llvm::APInt(64, H), 36, /*Signed=*/false);
}
void setOutputPath(ResultTy &resolvedOutputPath, const StringRef &moduleName,
const StringRef &interfacePath, const StringRef &sdkPath,
const CompilerInvocation &CI, const ArgListTy &extraArgs) {
auto &outputPath = resolvedOutputPath.outputPath;
outputPath = CI.getClangModuleCachePath();
auto isPrefixedWith = [interfacePath](StringRef path) {
return !path.empty() &&
hasPrefix(llvm::sys::path::begin(interfacePath.str()),
llvm::sys::path::end(interfacePath.str()),
llvm::sys::path::begin(path), llvm::sys::path::end(path));
};
// Dependency-scanner-specific module output path handling
if ((CI.getFrontendOptions().RequestedAction ==
FrontendOptions::ActionType::ScanDependencies)) {
auto runtimeResourcePath = CI.getSearchPathOptions().RuntimeResourcePath;
if (isPrefixedWith(sdkPath) || isPrefixedWith(runtimeResourcePath))
outputPath = CI.getFrontendOptions().ExplicitSDKModulesOutputPath;
else
outputPath = CI.getFrontendOptions().ExplicitModulesOutputPath;
}
llvm::sys::path::append(outputPath, moduleName);
outputPath.append("-");
auto hashStart = outputPath.size();
outputPath.append(getContextHash(CI, interfacePath, sdkPath, extraArgs));
resolvedOutputPath.hash = outputPath.str().substr(hashStart);
outputPath.append(".");
auto outExt = file_types::getExtension(file_types::TY_SwiftModuleFile);
outputPath.append(outExt);
return;
}
} // namespace swift::SwiftInterfaceModuleOutputPathResolution