//===------ ModuleInterfaceLoader.cpp - Loads .swiftinterface files -------===// // // This source file is part of the Swift.org open source project // // Copyright (c) 2019 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "textual-module-interface" #include "swift/Frontend/ModuleInterfaceLoader.h" #include "ModuleInterfaceBuilder.h" #include "swift/AST/ASTContext.h" #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" #include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/AST/SearchPathOptions.h" #include "swift/Basic/Assertions.h" #include "swift/Basic/Platform.h" #include "swift/Basic/StringExtras.h" #include "swift/Frontend/CachingUtils.h" #include "swift/Frontend/CompileJobCacheResult.h" #include "swift/Frontend/Frontend.h" #include "swift/Frontend/ModuleInterfaceSupport.h" #include "swift/Parse/ParseVersion.h" #include "swift/Serialization/SerializationOptions.h" #include "swift/Serialization/SerializedModuleLoader.h" #include "swift/Serialization/Validation.h" #include "swift/Strings.h" #include "clang/Basic/Module.h" #include "clang/Frontend/CompileJobCacheResult.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/xxhash.h" using namespace swift; using FileDependency = SerializationOptions::FileDependency; #pragma mark - Forwarding Modules namespace { /// Describes a "forwarding module", that is, a .swiftmodule that's actually /// a YAML file inside, pointing to a the original .swiftmodule but describing /// a different dependency resolution strategy. struct ForwardingModule { /// The path to the original .swiftmodule in the prebuilt cache. std::string underlyingModulePath; /// Describes a set of file-based dependencies with their size and /// modification time stored. This is slightly different from /// \c SerializationOptions::FileDependency, because this type needs to be /// serializable to and from YAML. struct Dependency { std::string path; uint64_t size; uint64_t lastModificationTime; bool isSDKRelative; }; std::vector dependencies; unsigned version = 1; ForwardingModule() = default; ForwardingModule(StringRef underlyingModulePath) : underlyingModulePath(underlyingModulePath) {} /// Loads the contents of the forwarding module whose contents lie in /// the provided buffer, and returns a new \c ForwardingModule, or an error /// if the YAML could not be parsed. static llvm::ErrorOr load(const llvm::MemoryBuffer &buf); /// Adds a given dependency to the dependencies list. void addDependency(StringRef path, bool isSDKRelative, uint64_t size, uint64_t modTime) { dependencies.push_back({path.str(), size, modTime, isSDKRelative}); } }; } // end anonymous namespace #pragma mark - YAML Serialization namespace llvm { namespace yaml { template <> struct MappingTraits { static void mapping(IO &io, ForwardingModule::Dependency &dep) { io.mapRequired("mtime", dep.lastModificationTime); io.mapRequired("path", dep.path); io.mapRequired("size", dep.size); io.mapOptional("sdk_relative", dep.isSDKRelative, /*default*/false); } }; template <> struct SequenceElementTraits { static const bool flow = false; }; template <> struct MappingTraits { static void mapping(IO &io, ForwardingModule &module) { io.mapRequired("path", module.underlyingModulePath); io.mapRequired("dependencies", module.dependencies); io.mapRequired("version", module.version); } }; } } // end namespace llvm llvm::ErrorOr ForwardingModule::load(const llvm::MemoryBuffer &buf) { llvm::yaml::Input yamlIn(buf.getBuffer()); ForwardingModule fwd; yamlIn >> fwd; if (yamlIn.error()) return yamlIn.error(); // We only currently support Version 1 of the forwarding module format. if (fwd.version != 1) return std::make_error_code(std::errc::not_supported); return std::move(fwd); } #pragma mark - Module Discovery namespace { /// The result of a search for a module either alongside an interface, in the /// module cache, or in the prebuilt module cache. class DiscoveredModule { /// The kind of module we've found. enum class Kind { /// A module that's either alongside the swiftinterface or in the /// module cache. Normal, /// A module that resides in the prebuilt cache, and has hash-based /// dependencies. Prebuilt, /// A 'forwarded' module. This is a module in the prebuilt cache, but whose /// dependencies live in a forwarding module. /// \sa ForwardingModule. Forwarded }; /// The kind of module that's been discovered. const Kind kind; DiscoveredModule(StringRef path, Kind kind, std::unique_ptr moduleBuffer) : kind(kind), moduleBuffer(std::move(moduleBuffer)), path(path) {} public: /// The contents of the .swiftmodule, if we've read it while validating /// dependencies. std::unique_ptr moduleBuffer; /// The path to the discovered serialized .swiftmodule on disk. const std::string path; /// Creates a \c Normal discovered module. static DiscoveredModule normal(StringRef path, std::unique_ptr moduleBuffer) { return { path, Kind::Normal, std::move(moduleBuffer) }; } /// Creates a \c Prebuilt discovered module. static DiscoveredModule prebuilt( StringRef path, std::unique_ptr moduleBuffer) { return { path, Kind::Prebuilt, std::move(moduleBuffer) }; } /// Creates a \c Forwarded discovered module, whose dependencies have been /// externally validated by a \c ForwardingModule. static DiscoveredModule forwarded( StringRef path, std::unique_ptr moduleBuffer) { return { path, Kind::Forwarded, std::move(moduleBuffer) }; } bool isNormal() const { return kind == Kind::Normal; } bool isPrebuilt() const { return kind == Kind::Prebuilt; } bool isForwarded() const { return kind == Kind::Forwarded; } }; } // end anonymous namespace #pragma mark - Common utilities namespace path = llvm::sys::path; static bool serializedASTLooksValid(const llvm::MemoryBuffer &buf, bool requiresOSSAModules, StringRef requiredSDK) { auto VI = serialization::validateSerializedAST(buf.getBuffer(), requiresOSSAModules, requiredSDK); return VI.status == serialization::Status::Valid; } #pragma mark - Module Loading namespace { /// Keeps track of the various reasons the module interface loader needed to /// fall back and rebuild a module from its interface. struct ModuleRebuildInfo { enum class ModuleKind { Normal, Cached, Forwarding, Prebuilt }; enum class ReasonIgnored { NotIgnored, PublicFramework, PublicLibrary, InterfacePreferred, CompilerHostModule, Blocklisted, DistributedInterfaceByDefault, }; // Keep aligned with diag::module_interface_ignored_reason. enum class ReasonModuleInterfaceIgnored { NotIgnored, LocalModule, Blocklisted, Debugger, }; struct CandidateModule { std::string path; std::optional serializationStatus; ModuleKind kind; ReasonIgnored reasonIgnored; ReasonModuleInterfaceIgnored reasonModuleInterfaceIgnored; SmallVector outOfDateDependencies; SmallVector missingDependencies; }; SmallVector candidateModules; CandidateModule &getOrInsertCandidateModule(StringRef path) { for (auto &mod : candidateModules) { if (mod.path == path) return mod; } candidateModules.push_back({path.str(), std::nullopt, ModuleKind::Normal, ReasonIgnored::NotIgnored, ReasonModuleInterfaceIgnored::NotIgnored, {}, {}}); return candidateModules.back(); } /// Sets the kind of a module that failed to load. void setModuleKind(StringRef path, ModuleKind kind) { getOrInsertCandidateModule(path).kind = kind; } /// Sets the serialization status of the module at \c path. If this is /// anything other than \c Valid, a note will be added stating why the module /// was invalid. void setSerializationStatus(StringRef path, serialization::Status status) { getOrInsertCandidateModule(path).serializationStatus = status; } /// Registers an out-of-date dependency at \c depPath for the module /// at \c modulePath. void addOutOfDateDependency(StringRef modulePath, StringRef depPath) { getOrInsertCandidateModule(modulePath) .outOfDateDependencies.push_back(depPath.str()); } /// Registers a missing dependency at \c depPath for the module /// at \c modulePath. void addMissingDependency(StringRef modulePath, StringRef depPath) { getOrInsertCandidateModule(modulePath) .missingDependencies.push_back(depPath.str()); } /// Sets the reason that the module at \c modulePath was ignored. If this is /// anything besides \c NotIgnored a note will be added stating why the module /// was ignored. void addIgnoredModule(StringRef modulePath, ReasonIgnored reasonIgnored) { getOrInsertCandidateModule(modulePath).reasonIgnored = reasonIgnored; } /// Record why no swiftinterfaces were preferred over the binary swiftmodule /// at \c modulePath. void addIgnoredModuleInterface(StringRef modulePath, ReasonModuleInterfaceIgnored reasonIgnored) { getOrInsertCandidateModule(modulePath).reasonModuleInterfaceIgnored = reasonIgnored; } /// Determines if we saw the given module path and registered is as out of /// date. bool sawOutOfDateModule(StringRef modulePath) { for (auto &mod : candidateModules) if (mod.path == modulePath && mod.reasonIgnored == ReasonIgnored::NotIgnored) return true; return false; } /// Emits a diagnostic for all out-of-date compiled or forwarding modules /// encountered while trying to load a module. template void diagnose(ASTContext &ctx, DiagnosticEngine &diags, StringRef prebuiltCacheDir, SourceLoc loc, DiagArgs &&...diagArgs) { diags.diagnose(loc, std::forward(diagArgs)...); auto SDKVer = getSDKBuildVersion(ctx.SearchPathOpts.getSDKPath()); llvm::SmallString<64> buffer = prebuiltCacheDir; llvm::sys::path::append(buffer, "SystemVersion.plist"); auto PBMVer = getSDKBuildVersionFromPlist(buffer.str()); if (!SDKVer.empty() && !PBMVer.empty()) { // Remark the potential version difference. diags.diagnose(loc, diag::sdk_version_pbm_version, SDKVer, PBMVer); } // We may have found multiple failing modules, that failed for different // reasons. Emit a note for each of them. for (auto &mod : candidateModules) { // If the compiled module was ignored, diagnose the reason. if (mod.reasonIgnored != ReasonIgnored::NotIgnored) { diags.diagnose(loc, diag::compiled_module_ignored_reason, mod.path, (unsigned)mod.reasonIgnored); } else { diags.diagnose(loc, diag::out_of_date_module_here, (unsigned)mod.kind, mod.path); } // Diagnose any out-of-date dependencies in this module. for (auto &dep : mod.outOfDateDependencies) { diags.diagnose(loc, diag::module_interface_dependency_out_of_date, dep); } // Diagnose any missing dependencies in this module. for (auto &dep : mod.missingDependencies) { diags.diagnose(loc, diag::module_interface_dependency_missing, dep); } // If there was a compiled module that wasn't able to be read, diagnose // the reason we couldn't read it. if (auto status = mod.serializationStatus) { if (auto reason = SerializedModuleLoaderBase::invalidModuleReason(*status)) { diags.diagnose(loc, diag::compiled_module_invalid_reason, mod.path, reason.value()); } else { diags.diagnose(loc, diag::compiled_module_invalid, mod.path); } } } } /// Emits a diagnostic for the reason why binary swiftmodules were preferred /// over textual swiftinterfaces. void diagnoseIgnoredModuleInterfaces(ASTContext &ctx, SourceLoc loc) { for (auto &mod : candidateModules) { auto interfaceIgnore = mod.reasonModuleInterfaceIgnored; if (interfaceIgnore == ReasonModuleInterfaceIgnored::NotIgnored) continue; ctx.Diags.diagnose(loc, diag::module_interface_ignored_reason, mod.path, (unsigned)interfaceIgnore); } } }; /// Constructs the full path of the dependency \p dep by prepending the SDK /// path if necessary. StringRef getFullDependencyPath(const FileDependency &dep, const ASTContext &ctx, SmallVectorImpl &scratch) { if (!dep.isSDKRelative()) return dep.getPath(); path::native(ctx.SearchPathOpts.getSDKPath(), scratch); llvm::sys::path::append(scratch, dep.getPath()); return StringRef(scratch.data(), scratch.size()); } class UpToDateModuleCheker { ASTContext &ctx; llvm::vfs::FileSystem &fs; RequireOSSAModules_t requiresOSSAModules; public: UpToDateModuleCheker(ASTContext &ctx, RequireOSSAModules_t requiresOSSAModules) : ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), requiresOSSAModules(requiresOSSAModules) {} // Check if all the provided file dependencies are up-to-date compared to // what's currently on disk. bool dependenciesAreUpToDate(StringRef modulePath, ModuleRebuildInfo &rebuildInfo, ArrayRef deps, bool skipSystemDependencies) { SmallString<128> SDKRelativeBuffer; for (auto &in : deps) { if (skipSystemDependencies && in.isSDKRelative() && in.isModificationTimeBased()) { continue; } StringRef fullPath = getFullDependencyPath(in, ctx, SDKRelativeBuffer); switch (checkDependency(modulePath, in, fullPath)) { case DependencyStatus::UpToDate: LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is up to date\n"); break; case DependencyStatus::OutOfDate: LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is out of date\n"); rebuildInfo.addOutOfDateDependency(modulePath, fullPath); return false; case DependencyStatus::Missing: LLVM_DEBUG(llvm::dbgs() << "Dep " << fullPath << " is missing\n"); rebuildInfo.addMissingDependency(modulePath, fullPath); return false; } } return true; } // Check that the output .swiftmodule file is at least as new as all the // dependencies it read when it was built last time. bool serializedASTBufferIsUpToDate( StringRef path, const llvm::MemoryBuffer &buf, ModuleRebuildInfo &rebuildInfo, SmallVectorImpl &allDeps) { // Clear the existing dependencies, because we're going to re-fill them // in validateSerializedAST. allDeps.clear(); LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << path << "\n"); auto validationInfo = serialization::validateSerializedAST( buf.getBuffer(), requiresOSSAModules, ctx.LangOpts.SDKName, /*ExtendedValidationInfo=*/nullptr, &allDeps); if (validationInfo.status != serialization::Status::Valid) { rebuildInfo.setSerializationStatus(path, validationInfo.status); return false; } bool skipCheckingSystemDependencies = ctx.SearchPathOpts.DisableModulesValidateSystemDependencies; return dependenciesAreUpToDate(path, rebuildInfo, allDeps, skipCheckingSystemDependencies); } // Check that the output .swiftmodule file is at least as new as all the // dependencies it read when it was built last time. bool swiftModuleIsUpToDate( StringRef modulePath, ModuleRebuildInfo &rebuildInfo, SmallVectorImpl &AllDeps, std::unique_ptr &moduleBuffer) { auto OutBuf = fs.getBufferForFile(modulePath); if (!OutBuf) return false; moduleBuffer = std::move(*OutBuf); return serializedASTBufferIsUpToDate(modulePath, *moduleBuffer, rebuildInfo, AllDeps); } enum class DependencyStatus { UpToDate, OutOfDate, Missing }; // Checks that a dependency read from the cached module is up to date compared // to the interface file it represents. DependencyStatus checkDependency(StringRef modulePath, const FileDependency &dep, StringRef fullPath) { auto status = fs.status(fullPath); if (!status) return DependencyStatus::Missing; // If the sizes differ, then we know the file has changed. if (status->getSize() != dep.getSize()) return DependencyStatus::OutOfDate; // Otherwise, if this dependency is verified by modification time, check // it vs. the modification time of the file. if (dep.isModificationTimeBased()) { uint64_t mtime = status->getLastModificationTime().time_since_epoch().count(); return mtime == dep.getModificationTime() ? DependencyStatus::UpToDate : DependencyStatus::OutOfDate; } // Slow path: if the dependency is verified by content hash, check it vs. // the hash of the file. auto buf = fs.getBufferForFile(fullPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false); if (!buf) return DependencyStatus::Missing; return xxHash64(buf.get()->getBuffer()) == dep.getContentHash() ? DependencyStatus::UpToDate : DependencyStatus::OutOfDate; } }; /// Handles the details of loading module interfaces as modules, and will /// do the necessary lookup to determine if we should be loading from the /// normal cache, the prebuilt cache, a module adjacent to the interface, or /// a module that we'll build from a module interface. class ModuleInterfaceLoaderImpl { friend class swift::ModuleInterfaceLoader; friend class swift::ModuleInterfaceCheckerImpl; ASTContext &ctx; llvm::vfs::FileSystem &fs; DiagnosticEngine &diags; ModuleRebuildInfo rebuildInfo; UpToDateModuleCheker upToDateChecker; const StringRef modulePath; const std::string interfacePath; const StringRef moduleName; const StringRef prebuiltCacheDir; const StringRef backupInterfaceDir; const StringRef cacheDir; const SourceLoc diagnosticLoc; DependencyTracker *const dependencyTracker; const ModuleLoadingMode loadMode; ModuleInterfaceLoaderOptions Opts; RequireOSSAModules_t requiresOSSAModules; ModuleInterfaceLoaderImpl( ASTContext &ctx, StringRef modulePath, StringRef interfacePath, StringRef moduleName, StringRef cacheDir, StringRef prebuiltCacheDir, StringRef backupInterfaceDir, SourceLoc diagLoc, ModuleInterfaceLoaderOptions Opts, RequireOSSAModules_t requiresOSSAModules, DependencyTracker *dependencyTracker = nullptr, ModuleLoadingMode loadMode = ModuleLoadingMode::PreferSerialized) : ctx(ctx), fs(*ctx.SourceMgr.getFileSystem()), diags(ctx.Diags), upToDateChecker(ctx, requiresOSSAModules), modulePath(modulePath), interfacePath(interfacePath), moduleName(moduleName), prebuiltCacheDir(prebuiltCacheDir), backupInterfaceDir(backupInterfaceDir), cacheDir(cacheDir), diagnosticLoc(diagLoc), dependencyTracker(dependencyTracker), loadMode(loadMode), Opts(Opts), requiresOSSAModules(requiresOSSAModules) {} std::string getBackupPublicModuleInterfacePath() { return getBackupPublicModuleInterfacePath(ctx.SourceMgr, backupInterfaceDir, moduleName, interfacePath); } static std::string getBackupPublicModuleInterfacePath(SourceManager &SM, StringRef backupInterfaceDir, StringRef moduleName, StringRef interfacePath) { if (backupInterfaceDir.empty()) return std::string(); auto &fs = *SM.getFileSystem(); auto fileName = llvm::sys::path::filename(interfacePath); { llvm::SmallString<256> path(backupInterfaceDir); llvm::sys::path::append(path, llvm::Twine(moduleName) + ".swiftmodule"); llvm::sys::path::append(path, fileName); if (fs.exists(path.str())) { return path.str().str(); } } { llvm::SmallString<256> path(backupInterfaceDir); llvm::sys::path::append(path, fileName); if (fs.exists(path.str())) { return path.str().str(); } } return std::string(); } // Check that a "forwarding" .swiftmodule file is at least as new as all the // dependencies it read when it was built last time. Requires that the // forwarding module has been loaded from disk. bool forwardingModuleIsUpToDate( StringRef path, const ForwardingModule &fwd, SmallVectorImpl &deps, std::unique_ptr &moduleBuffer) { // Clear the existing dependencies, because we're going to re-fill them // from the forwarding module. deps.clear(); LLVM_DEBUG(llvm::dbgs() << "Validating deps of " << path << "\n"); // First, make sure the underlying module path exists and is valid. auto modBuf = fs.getBufferForFile(fwd.underlyingModulePath); if (!modBuf) return false; auto looksValid = serializedASTLooksValid(*modBuf.get(), requiresOSSAModules, ctx.LangOpts.SDKName); if (!looksValid) return false; // Next, check the dependencies in the forwarding file. for (auto &dep : fwd.dependencies) { deps.push_back( FileDependency::modTimeBased( dep.path, dep.isSDKRelative, dep.size, dep.lastModificationTime)); } bool skipCheckingSystemDependencies = ctx.SearchPathOpts.DisableModulesValidateSystemDependencies; if (!upToDateChecker.dependenciesAreUpToDate(path, rebuildInfo, deps, skipCheckingSystemDependencies)) return false; moduleBuffer = std::move(*modBuf); return true; } bool canInterfaceHavePrebuiltModule() { StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); if (!sdkPath.empty() && hasPrefix(path::begin(interfacePath), path::end(interfacePath), path::begin(sdkPath), path::end(sdkPath))) { return !(StringRef(interfacePath).ends_with(".private.swiftinterface") || StringRef(interfacePath).ends_with(".package.swiftinterface")); } return false; } std::optional computePrebuiltModulePath(llvm::SmallString<256> &scratch) { namespace path = llvm::sys::path; // Check if this is a public interface file from the SDK. if (!canInterfaceHavePrebuiltModule()) return std::nullopt; // Assemble the expected path: $PREBUILT_CACHE/Foo.swiftmodule or // $PREBUILT_CACHE/Foo.swiftmodule/arch.swiftmodule. Note that there's no // cache key here. scratch = prebuiltCacheDir; // FIXME: Would it be possible to only have architecture-specific names // here? Then we could skip this check. StringRef inParentDirName = path::filename(path::parent_path(interfacePath)); if (path::extension(inParentDirName) == ".swiftmodule") { assert(path::stem(inParentDirName) == ctx.getRealModuleName(ctx.getIdentifier(moduleName)).str()); path::append(scratch, inParentDirName); } path::append(scratch, path::filename(modulePath)); // If there isn't a file at this location, skip returning a path. if (!fs.exists(scratch)) return std::nullopt; return scratch.str(); } /// Hack to deal with build systems (including the Swift standard library, at /// the time of this comment) that aren't yet using target-specific names for /// multi-target swiftmodules, in case the prebuilt cache is. std::optional computeFallbackPrebuiltModulePath(llvm::SmallString<256> &scratch) { namespace path = llvm::sys::path; StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); // Check if this is a public interface file from the SDK. if (sdkPath.empty() || !hasPrefix(path::begin(interfacePath), path::end(interfacePath), path::begin(sdkPath), path::end(sdkPath)) || StringRef(interfacePath).ends_with(".private.swiftinterface") || StringRef(interfacePath).ends_with(".package.swiftinterface")) return std::nullopt; // If the module isn't target-specific, there's no fallback path. StringRef inParentDirName = path::filename(path::parent_path(interfacePath)); if (path::extension(inParentDirName) != ".swiftmodule") return std::nullopt; // If the interface is already using the target-specific name, there's // nothing else to try. auto normalizedTarget = getTargetSpecificModuleTriple(ctx.LangOpts.Target); if (path::stem(modulePath) == normalizedTarget.str()) return std::nullopt; // Assemble the expected path: // $PREBUILT_CACHE/Foo.swiftmodule/target.swiftmodule. Note that there's no // cache key here. scratch = prebuiltCacheDir; path::append(scratch, inParentDirName); path::append(scratch, normalizedTarget.str()); scratch += ".swiftmodule"; // If there isn't a file at this location, skip returning a path. if (!fs.exists(scratch)) return std::nullopt; return scratch.str(); } bool isInResourceDir(StringRef path) { StringRef resourceDir = ctx.SearchPathOpts.RuntimeResourcePath; if (resourceDir.empty()) return false; return pathStartsWith(resourceDir, path); } bool isInResourceHostDir(StringRef path) { StringRef resourceDir = ctx.SearchPathOpts.RuntimeResourcePath; if (resourceDir.empty()) return false; SmallString<128> hostPath; llvm::sys::path::append(hostPath, resourceDir, "host"); return pathStartsWith(hostPath, path); } bool isInSDK(StringRef path) { StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); if (sdkPath.empty()) return false; return pathStartsWith(sdkPath, path); } bool isInSystemFrameworks(StringRef path, bool publicFramework) { StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); if (sdkPath.empty()) return false; SmallString<128> frameworksPath; llvm::sys::path::append(frameworksPath, sdkPath, "System", "Library", publicFramework ? "Frameworks" : "PrivateFrameworks"); return pathStartsWith(frameworksPath, path); } bool isInSystemSubFrameworks(StringRef path) { StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); if (sdkPath.empty()) return false; SmallString<128> frameworksPath; llvm::sys::path::append(frameworksPath, sdkPath, "System", "Library", "SubFrameworks"); return pathStartsWith(frameworksPath, path); } bool isInSystemLibraries(StringRef path) { StringRef sdkPath = ctx.SearchPathOpts.getSDKPath(); if (sdkPath.empty()) return false; SmallString<128> frameworksPath; llvm::sys::path::append(frameworksPath, sdkPath, "usr", "lib", "swift"); return pathStartsWith(frameworksPath, path); } std::pair getCompiledModuleCandidates() { using ReasonIgnored = ModuleRebuildInfo::ReasonIgnored; using ReasonModuleInterfaceIgnored = ModuleRebuildInfo::ReasonModuleInterfaceIgnored; std::pair result; bool ignoreByDefault = ctx.blockListConfig.hasBlockListAction( "Swift_UseSwiftinterfaceByDefault", BlockListKeyKind::ModuleName, BlockListAction::ShouldUseBinaryModule); bool shouldLoadAdjacentModule; if (ignoreByDefault) { ReasonModuleInterfaceIgnored ignore = ReasonModuleInterfaceIgnored::NotIgnored; if (!isInSDK(modulePath) && !isInResourceHostDir(modulePath)) { ignore = ReasonModuleInterfaceIgnored::LocalModule; } else if (ctx.blockListConfig.hasBlockListAction(moduleName, BlockListKeyKind::ModuleName, BlockListAction::ShouldUseBinaryModule)) { ignore = ReasonModuleInterfaceIgnored::Blocklisted; } else if (ctx.LangOpts.DebuggerSupport) { ignore = ReasonModuleInterfaceIgnored::Debugger; } shouldLoadAdjacentModule = ignore != ReasonModuleInterfaceIgnored::NotIgnored; if (shouldLoadAdjacentModule) { // Prefer the swiftmodule. rebuildInfo.addIgnoredModuleInterface(modulePath, ignore); } else { // Prefer the swiftinterface. rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::DistributedInterfaceByDefault); } } else { // Should we attempt to load a swiftmodule adjacent to the swiftinterface? shouldLoadAdjacentModule = !ctx.IgnoreAdjacentModules; if (modulePath.contains(".sdk")) { if (ctx.blockListConfig.hasBlockListAction(moduleName, BlockListKeyKind::ModuleName, BlockListAction::ShouldUseTextualModule)) { shouldLoadAdjacentModule = false; rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::Blocklisted); } } // Don't use the adjacent swiftmodule for frameworks from the public // Frameworks folder of the SDK. bool blocklistSwiftmodule = ctx.blockListConfig.hasBlockListAction(moduleName, BlockListKeyKind::ModuleName, BlockListAction::ShouldUseBinaryModule); if ((isInSystemFrameworks(modulePath, /*publicFramework*/true) || isInSystemSubFrameworks(modulePath)) && !blocklistSwiftmodule) { shouldLoadAdjacentModule = false; rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::PublicFramework); } else if (isInSystemLibraries(modulePath) && moduleName != STDLIB_NAME && !blocklistSwiftmodule) { shouldLoadAdjacentModule = false; rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::PublicLibrary); } else if (isInResourceHostDir(modulePath)) { shouldLoadAdjacentModule = false; rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::CompilerHostModule); } } switch (loadMode) { case ModuleLoadingMode::OnlyInterface: // Always skip both the caches and adjacent modules, and always build the // module interface. return {}; case ModuleLoadingMode::PreferInterface: // If we're in the load mode that prefers .swiftinterfaces, specifically // skip the module adjacent to the interface, but use the caches if // they're present. shouldLoadAdjacentModule = false; rebuildInfo.addIgnoredModule(modulePath, ReasonIgnored::InterfacePreferred); break; case ModuleLoadingMode::PreferSerialized: // The rest of the function should be covered by this. break; case ModuleLoadingMode::OnlySerialized: llvm_unreachable("module interface loader should not have been created"); } // [NOTE: ModuleInterfaceLoader-defer-to-ImplicitSerializedModuleLoader] // If there's a module adjacent to the .swiftinterface that we can // _likely_ load (it validates OK and is up to date), bail early with // errc::not_supported, so the next (serialized) loader in the chain will // load it. // Alternately, if there's a .swiftmodule present but we can't even // read it (for whatever reason), we should let the other module loader // diagnose it. if (shouldLoadAdjacentModule) { if (fs.exists(modulePath)) { result.first = modulePath.str(); } } // If we have a prebuilt cache path, check that too if the interface comes // from the SDK. if (!prebuiltCacheDir.empty()) { llvm::SmallString<256> scratch; std::unique_ptr moduleBuffer; std::optional path = computePrebuiltModulePath(scratch); if (!path) { // Hack: deal with prebuilds of modules that still use the target-based // names. path = computeFallbackPrebuiltModulePath(scratch); } if (path) { if (fs.exists(*path)) { result.second = path->str(); } } } return result; } llvm::ErrorOr discoverUpToDateCompiledModuleForInterface(SmallVectorImpl &deps, std::string &UsableModulePath) { std::string adjacentMod, prebuiltMod; std::tie(adjacentMod, prebuiltMod) = getCompiledModuleCandidates(); if (!adjacentMod.empty()) { auto adjacentModuleBuffer = fs.getBufferForFile(adjacentMod); if (adjacentModuleBuffer) { if (upToDateChecker.serializedASTBufferIsUpToDate(adjacentMod, *adjacentModuleBuffer.get(), rebuildInfo, deps)) { LLVM_DEBUG(llvm::dbgs() << "Found up-to-date module at " << adjacentMod << "; deferring to serialized module loader\n"); UsableModulePath = adjacentMod; return std::make_error_code(std::errc::not_supported); } else if (isInResourceDir(adjacentMod) && loadMode == ModuleLoadingMode::PreferSerialized && version::getCurrentCompilerSerializationTag().empty() && rebuildInfo.getOrInsertCandidateModule(adjacentMod) .serializationStatus != serialization::Status::SDKMismatch && // FIXME (meg-gupta): We need to support recompilation of // modules in the resource directory if the mismatch is due // to importing a non-ossa module to an ossa module. This is // needed during ossa bringup, once -enable-ossa-modules is // on by default, this can be deleted. rebuildInfo.getOrInsertCandidateModule(adjacentMod) .serializationStatus != serialization::Status::NotInOSSA) { // Special-case here: If we're loading a .swiftmodule from the resource // dir adjacent to the compiler, defer to the serialized loader instead // of falling back. This is to support local development of Swift, // where one might change the module format version but forget to // recompile the standard library. If that happens, don't fall back // and silently recompile the standard library, raise an error // instead. // // This logic is disabled for tagged compilers, so distributed // compilers should ignore this restriction and rebuild all modules // from a swiftinterface when required. // // Still accept modules built with a different SDK, allowing the use // of one toolchain against a different SDK. LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module in the " "resource-dir at " << adjacentMod << "; deferring to serialized module loader " "to diagnose\n"); return std::make_error_code(std::errc::not_supported); } else { LLVM_DEBUG(llvm::dbgs() << "Found out-of-date module at " << adjacentMod << "\n"); rebuildInfo.setModuleKind(adjacentMod, ModuleRebuildInfo::ModuleKind::Normal); } } else { LLVM_DEBUG(llvm::dbgs() << "Found unreadable module at " << adjacentMod << "; deferring to serialized module loader\n"); return std::make_error_code(std::errc::not_supported); } } if(!prebuiltMod.empty()) { std::unique_ptr moduleBuffer; if (upToDateChecker.swiftModuleIsUpToDate(prebuiltMod, rebuildInfo, deps, moduleBuffer)) { LLVM_DEBUG(llvm::dbgs() << "Found up-to-date prebuilt module at " << prebuiltMod << "\n"); UsableModulePath = prebuiltMod; return DiscoveredModule::prebuilt(prebuiltMod, std::move(moduleBuffer)); } else { LLVM_DEBUG(llvm::dbgs() << "Found out-of-date prebuilt module at " << prebuiltMod << "\n"); rebuildInfo.setModuleKind(prebuiltMod, ModuleRebuildInfo::ModuleKind::Prebuilt); } } // We cannot find any proper compiled module to use. return std::make_error_code(std::errc::no_such_file_or_directory); } /// Finds the most appropriate .swiftmodule, whose dependencies are up to /// date, that we can load for the provided .swiftinterface file. llvm::ErrorOr discoverUpToDateModuleForInterface( StringRef cachedOutputPath, SmallVectorImpl &deps) { // First, check the cached module path. Whatever's in this cache represents // the most up-to-date knowledge we have about the module. if (auto cachedBufOrError = fs.getBufferForFile(cachedOutputPath)) { auto buf = std::move(*cachedBufOrError); // Check to see if the module is a serialized AST. If it's not, then we're // probably dealing with a Forwarding Module, which is a YAML file. bool isForwardingModule = !serialization::isSerializedAST(buf->getBuffer()); // If it's a forwarding module, load the YAML file from disk and check // if it's up-to-date. if (isForwardingModule) { if (auto forwardingModule = ForwardingModule::load(*buf)) { std::unique_ptr moduleBuffer; if (forwardingModuleIsUpToDate(cachedOutputPath, *forwardingModule, deps, moduleBuffer)) { LLVM_DEBUG(llvm::dbgs() << "Found up-to-date forwarding module at " << cachedOutputPath << "\n"); return DiscoveredModule::forwarded( forwardingModule->underlyingModulePath, std::move(moduleBuffer)); } LLVM_DEBUG(llvm::dbgs() << "Found out-of-date forwarding module at " << cachedOutputPath << "\n"); rebuildInfo.setModuleKind(cachedOutputPath, ModuleRebuildInfo::ModuleKind::Forwarding); } // Otherwise, check if the AST buffer itself is up to date. } else if (upToDateChecker.serializedASTBufferIsUpToDate(cachedOutputPath, *buf, rebuildInfo, deps)) { LLVM_DEBUG(llvm::dbgs() << "Found up-to-date cached module at " << cachedOutputPath << "\n"); return DiscoveredModule::normal(cachedOutputPath, std::move(buf)); } else { LLVM_DEBUG(llvm::dbgs() << "Found out-of-date cached module at " << cachedOutputPath << "\n"); rebuildInfo.setModuleKind(cachedOutputPath, ModuleRebuildInfo::ModuleKind::Cached); } } std::string usableModulePath; return discoverUpToDateCompiledModuleForInterface(deps, usableModulePath); } /// Writes the "forwarding module" that will forward to a module in the /// prebuilt cache. /// /// Since forwarding modules track dependencies separately from the module /// they point to, we'll need to grab the up-to-date file status while doing /// this. If the write was successful, it also updates the /// list of dependencies to match what was written to the forwarding file. bool writeForwardingModuleAndUpdateDeps( const DiscoveredModule &mod, llvm::vfs::OutputBackend &backend, StringRef outputPath, SmallVectorImpl &deps) { assert(mod.isPrebuilt() && "cannot write forwarding file for non-prebuilt module"); ForwardingModule fwd(mod.path); SmallVector depsAdjustedToMTime; // FIXME: We need to avoid re-statting all these dependencies, otherwise // we may record out-of-date information. SmallString<128> SDKRelativeBuffer; auto addDependency = [&](FileDependency dep) -> FileDependency { auto status = fs.status(getFullDependencyPath(dep, ctx, SDKRelativeBuffer)); uint64_t mtime = status->getLastModificationTime().time_since_epoch().count(); fwd.addDependency(dep.getPath(), dep.isSDKRelative(), status->getSize(), mtime); // Construct new FileDependency matching what we've added to the // forwarding module. return FileDependency::modTimeBased(dep.getPath(), dep.isSDKRelative(), status->getSize(), mtime); }; // Add the prebuilt module as a dependency of the forwarding module, but // don't add it to the outer dependency list. (void)addDependency(FileDependency::hashBased(fwd.underlyingModulePath, /*SDKRelative*/false, /*size*/0, /*hash*/0)); // Add all the dependencies from the prebuilt module, and update our list // of dependencies to reflect what's recorded in the forwarding module. for (auto dep : deps) { auto adjustedDep = addDependency(dep); depsAdjustedToMTime.push_back(adjustedDep); } auto hadError = withOutputPath(diags, backend, outputPath, [&](llvm::raw_pwrite_stream &out) { llvm::yaml::Output yamlWriter(out); yamlWriter << fwd; return false; }); if (hadError) return true; // If and only if we succeeded writing the forwarding file, update the // provided list of dependencies. deps = depsAdjustedToMTime; return false; } /// Looks up the best module to load for a given interface, and returns a /// buffer of the module's contents. Also reports the module's dependencies /// to the parent \c dependencyTracker if it came from the cache, or was built /// from the given interface. See the main comment in /// \c ModuleInterfaceLoader.h for an explanation of the module /// loading strategy. llvm::ErrorOr> findOrBuildLoadableModule() { // Track system dependencies if the parent tracker is set to do so. bool trackSystemDependencies = false; if (dependencyTracker) { auto ClangDependencyTracker = dependencyTracker->getClangCollector(); trackSystemDependencies = ClangDependencyTracker->needSystemDependencies(); } InterfaceSubContextDelegateImpl astDelegate( ctx.SourceMgr, &ctx.Diags, ctx.SearchPathOpts, ctx.LangOpts, ctx.ClangImporterOpts, ctx.CASOpts, Opts, /*buildModuleCacheDirIfAbsent*/ true, cacheDir, prebuiltCacheDir, backupInterfaceDir, /*serializeDependencyHashes*/ false, trackSystemDependencies, requiresOSSAModules); // Compute the output path if we're loading or emitting a cached module. SwiftInterfaceModuleOutputPathResolution::ResultTy resolvedOutputPath; astDelegate.getCachedOutputPath(resolvedOutputPath, moduleName, interfacePath, ctx.SearchPathOpts.getSDKPath()); auto &cachedOutputPath = resolvedOutputPath.outputPath; // Try to find the right module for this interface, either alongside it, // in the cache, or in the prebuilt cache. SmallVector allDeps; auto moduleOrErr = discoverUpToDateModuleForInterface(cachedOutputPath, allDeps); // If we errored with anything other than 'no such file or directory', // fail this load and let the other module loader diagnose it. if (!moduleOrErr && moduleOrErr.getError() != std::errc::no_such_file_or_directory) { rebuildInfo.diagnoseIgnoredModuleInterfaces(ctx, diagnosticLoc); return moduleOrErr.getError(); } // We discovered a module! Return that module's buffer so we can load it. if (moduleOrErr) { auto module = std::move(moduleOrErr.get()); // If it's prebuilt, use this time to generate a forwarding module and // update the dependencies to use modification times. if (module.isPrebuilt()) if (writeForwardingModuleAndUpdateDeps(module, ctx.getOutputBackend(), cachedOutputPath, allDeps)) return std::make_error_code(std::errc::not_supported); // Report the module's dependencies to the dependencyTracker if (dependencyTracker) { SmallString<128> SDKRelativeBuffer; for (auto &dep: allDeps) { StringRef fullPath = getFullDependencyPath(dep, ctx, SDKRelativeBuffer); dependencyTracker->addDependency(fullPath, /*IsSystem=*/dep.isSDKRelative()); } } return std::move(module.moduleBuffer); } // If building from interface is disabled, return error. if (Opts.disableBuildingInterface) { return std::make_error_code(std::errc::not_supported); } std::unique_ptr moduleBuffer; // We didn't discover a module corresponding to this interface. // Diagnose that we didn't find a loadable module, if we were asked to. // // Note that we use `diags` so that we emit this remark even when we're // emitting other messages to `emptyDiags` (see below); these act as status // messages to explain what's taking so long. auto remarkRebuildAll = [&]() { rebuildInfo.diagnose(ctx, diags, prebuiltCacheDir, diagnosticLoc, diag::rebuilding_module_from_interface, moduleName, interfacePath); }; auto remarkRebuild = Opts.remarkOnRebuildFromInterface ? llvm::function_ref(remarkRebuildAll) : nullptr; bool failed = false; std::string backupPath = getBackupPublicModuleInterfacePath(); { DiagnosticEngine emptyDiags(ctx.SourceMgr); std::unique_ptr> saver; DiagnosticEngine *diagsToUse = &ctx.Diags; // Avoid emitting diagnostics if we have a backup interface to use. // If we succeed in building this canonical interface, it's not interesting // to see those diagnostics. // If we failed in build, we will use the back up interface and it's interesting // to see diagnostics there. if (!backupPath.empty()) { diagsToUse = &emptyDiags; saver = std::make_unique>( astDelegate.Diags, diagsToUse); } // Set up a builder if we need to build the module. It'll also set up // the genericSubInvocation we'll need to use to compute the cache paths. Identifier realName = ctx.getRealModuleName(ctx.getIdentifier(moduleName)); ImplicitModuleInterfaceBuilder builder( ctx.SourceMgr, diagsToUse, astDelegate, interfacePath, ctx.SearchPathOpts.getSDKPath(), ctx.SearchPathOpts.getSysRoot(), realName.str(), cacheDir, prebuiltCacheDir, backupInterfaceDir, StringRef(), Opts.disableInterfaceLock, ctx.IgnoreAdjacentModules, diagnosticLoc, dependencyTracker); // If we found an out-of-date .swiftmodule, we still want to add it as // a dependency of the .swiftinterface. That way if it's updated, but // the .swiftinterface remains the same, we invalidate the cache and // check the new .swiftmodule, because it likely has more information // about the state of the world. if (rebuildInfo.sawOutOfDateModule(modulePath)) builder.addExtraDependency(modulePath); failed = builder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true, &moduleBuffer, remarkRebuild); } if (!failed) { // If succeeded, we are done. assert(moduleBuffer && "failed to write module buffer but returned success?"); return std::move(moduleBuffer); } else if (backupPath.empty()) { // If failed and we don't have a backup interface file, return error code. return std::make_error_code(std::errc::invalid_argument); } assert(failed); assert(!backupPath.empty()); while (1) { diags.diagnose(diagnosticLoc, diag::interface_file_backup_used, interfacePath, backupPath); // Set up a builder if we need to build the module. It'll also set up // the genericSubInvocation we'll need to use to compute the cache paths. ImplicitModuleInterfaceBuilder fallbackBuilder( ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath, ctx.SearchPathOpts.getSDKPath(), ctx.SearchPathOpts.getSysRoot(), moduleName, cacheDir, prebuiltCacheDir, backupInterfaceDir, StringRef(), Opts.disableInterfaceLock, ctx.IgnoreAdjacentModules, diagnosticLoc, dependencyTracker); if (rebuildInfo.sawOutOfDateModule(modulePath)) fallbackBuilder.addExtraDependency(modulePath); // Add the canonical interface path as a dependency of this module. // This ensures that after the user manually fixed the canonical interface // file and removed the fallback interface file, we can rebuild the cache. fallbackBuilder.addExtraDependency(interfacePath); // Use cachedOutputPath as the output file path. This output path was // calculated using the canonical interface file path to make sure we // can find it from the canonical interface file. auto failedAgain = fallbackBuilder.buildSwiftModule(cachedOutputPath, /*shouldSerializeDeps*/true, &moduleBuffer, remarkRebuild); if (failedAgain) return std::make_error_code(std::errc::invalid_argument); assert(moduleBuffer); return std::move(moduleBuffer); } } }; } // end anonymous namespace bool ModuleInterfaceCheckerImpl::isCached(StringRef DepPath) { if (!CacheDir.empty() && DepPath.starts_with(CacheDir)) return true; return !PrebuiltCacheDir.empty() && DepPath.starts_with(PrebuiltCacheDir); } static FrontendOptions::ActionType reqestedModuleLoaderActionForFrontendOpts(const FrontendOptions &Opts) { switch (Opts.RequestedAction) { case FrontendOptions::ActionType::TypecheckModuleFromInterface: return FrontendOptions::ActionType::Typecheck; case FrontendOptions::ActionType::ScanDependencies: return Opts.RequestedAction; default: return FrontendOptions::ActionType::EmitModuleOnly; } } ModuleInterfaceLoaderOptions::ModuleInterfaceLoaderOptions( const FrontendOptions &Opts, bool inheritDebuggingOpts) : requestedAction(reqestedModuleLoaderActionForFrontendOpts(Opts)), remarkOnRebuildFromInterface(Opts.RemarkOnRebuildFromModuleInterface), disableInterfaceLock(Opts.DisableInterfaceFileLock), disableImplicitSwiftModule(Opts.DisableImplicitModules), disableBuildingInterface(Opts.DisableBuildingInterface), downgradeInterfaceVerificationError( Opts.DowngradeInterfaceVerificationError), strictImplicitModuleContext(Opts.StrictImplicitModuleContext), mainExecutablePath(Opts.MainExecutablePath) { if (inheritDebuggingOpts) { compilerDebuggingOptions = Opts.CompilerDebuggingOpts; } } bool ModuleInterfaceLoader::isCached(StringRef DepPath) { return InterfaceChecker.isCached(DepPath); } /// Load a .swiftmodule associated with a .swiftinterface either from a /// cache or by converting it in a subordinate \c CompilerInstance, caching /// the results. std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory( ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName, SmallVectorImpl *ModuleInterfacePath, SmallVectorImpl *ModuleInterfaceSourcePath, std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer, std::unique_ptr *ModuleSourceInfoBuffer, bool skipBuildingInterface, bool IsFramework, bool isTestableImport) { // If running in OnlySerialized mode, ModuleInterfaceLoader // should not have been constructed at all. assert(LoadMode != ModuleLoadingMode::OnlySerialized); std::string ModPath{BaseName.getName(file_types::TY_SwiftModuleFile)}; // First check to see if the .swiftinterface exists at all. Bail if not. auto &fs = *Ctx.SourceMgr.getFileSystem(); auto InPath = BaseName.findInterfacePath(fs, Ctx); if (!InPath) { if (fs.exists(ModPath)) { LLVM_DEBUG(llvm::dbgs() << "No .swiftinterface file found adjacent to module file " << ModPath << "\n"); return std::make_error_code(std::errc::not_supported); } return std::make_error_code(std::errc::no_such_file_or_directory); } if (ModuleInterfaceSourcePath) ModuleInterfaceSourcePath->assign(InPath->begin(), InPath->end()); // If we've been told to skip building interfaces, we are done here and do // not need to have the module actually built. For example, if we are // currently answering a `canImport` query, it is enough to have found // the interface. if (skipBuildingInterface) { if (ModuleInterfacePath) ModuleInterfacePath->assign(InPath->begin(), InPath->end()); return std::error_code(); } // Create an instance of the Impl to do the heavy lifting. auto ModuleName = ModuleID.Item.str(); ModuleInterfaceLoaderImpl Impl( Ctx, ModPath, *InPath, ModuleName, InterfaceChecker.CacheDir, InterfaceChecker.PrebuiltCacheDir, InterfaceChecker.BackupInterfaceDir, ModuleID.Loc, InterfaceChecker.Opts, InterfaceChecker.RequiresOSSAModules, dependencyTracker, llvm::is_contained(PreferInterfaceForModules, ModuleName) ? ModuleLoadingMode::PreferInterface : LoadMode); // Ask the impl to find us a module that we can load or give us an error // telling us that we couldn't load it. auto ModuleBufferOrErr = Impl.findOrBuildLoadableModule(); if (!ModuleBufferOrErr) return ModuleBufferOrErr.getError(); if (ModuleBuffer) { *ModuleBuffer = std::move(*ModuleBufferOrErr); if (ModuleInterfacePath) ModuleInterfacePath->assign(InPath->begin(), InPath->end()); } // Open .swiftsourceinfo file if it's present. if (auto SourceInfoError = openModuleSourceInfoFileIfPresent(ModuleID, BaseName, ModuleSourceInfoBuffer)) return SourceInfoError; // Delegate back to the serialized module loader to load the module doc. if (auto DocLoadErr = openModuleDocFileIfPresent(ModuleID, BaseName, ModuleDocBuffer)) return DocLoadErr; return std::error_code(); } std::vector ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface( StringRef moduleName, StringRef interfacePath) { // Derive .swiftmodule path from the .swiftinterface path. auto interfaceExt = file_types::getExtension(file_types::TY_SwiftModuleInterfaceFile); auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); llvm::SmallString<32> modulePath; // When looking up the module for a private or package interface, strip // the '.private.' or '.package.'section of the base name if (interfacePath.ends_with(".private." + interfaceExt.str()) || interfacePath.ends_with(".package." + interfaceExt.str())) { auto newBaseName = llvm::sys::path::stem(llvm::sys::path::stem(interfacePath)); modulePath = llvm::sys::path::parent_path(interfacePath); llvm::sys::path::append(modulePath, newBaseName + "." + newExt.str()); } else { modulePath = interfacePath; llvm::sys::path::replace_extension(modulePath, newExt); } ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName, CacheDir, PrebuiltCacheDir, BackupInterfaceDir, SourceLoc(), Opts, RequiresOSSAModules, nullptr, Ctx.SearchPathOpts.ModuleLoadMode); std::vector results; std::string adjacentMod, prebuiltMod; std::tie(adjacentMod, prebuiltMod) = Impl.getCompiledModuleCandidates(); auto validateModule = [&](StringRef modulePath) { // Legacy behavior do not validate module. if (!Ctx.SearchPathOpts.ScannerModuleValidation) return true; // If we picked the other module already, no need to validate this one since // it should not be used anyway. if (!results.empty()) return false; SmallVector deps; std::unique_ptr moduleBuffer; return Impl.upToDateChecker.swiftModuleIsUpToDate( modulePath, Impl.rebuildInfo, deps, moduleBuffer); }; // Add compiled module candidates only when they are non-empty and up-to-date. if (!adjacentMod.empty() && validateModule(adjacentMod)) results.push_back(adjacentMod); if (!prebuiltMod.empty() && validateModule(prebuiltMod)) results.push_back(prebuiltMod); return results; } bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( StringRef moduleName, StringRef interfacePath, ArrayRef candidates, llvm::vfs::OutputBackend &backend, StringRef outputPath) { // Derive .swiftmodule path from the .swiftinterface path. auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); llvm::SmallString<32> modulePath = interfacePath; llvm::sys::path::replace_extension(modulePath, newExt); ModuleInterfaceLoaderImpl Impl(Ctx, modulePath, interfacePath, moduleName, CacheDir, PrebuiltCacheDir, BackupInterfaceDir, SourceLoc(), Opts, RequiresOSSAModules, nullptr, ModuleLoadingMode::PreferSerialized); SmallVector deps; std::unique_ptr moduleBuffer; for (auto mod: candidates) { // Check if the candidate compiled module is still up-to-date. if (Impl.upToDateChecker.swiftModuleIsUpToDate(mod, Impl.rebuildInfo, deps, moduleBuffer)) { // If so, emit a forwarding module to the candidate. ForwardingModule FM(mod); auto hadError = withOutputPath(Ctx.Diags, backend, outputPath, [&](llvm::raw_pwrite_stream &out) { llvm::yaml::Output yamlWriter(out); yamlWriter << FM; return false; }); if (!hadError) return true; } } return false; } bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface( SourceManager &SourceMgr, DiagnosticEngine &Diags, const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts, const ClangImporterOptions &ClangOpts, const CASOptions &CASOpts, StringRef CacheDir, StringRef PrebuiltCacheDir, StringRef BackupInterfaceDir, StringRef ModuleName, StringRef InPath, StringRef OutPath, StringRef ABIOutputPath, bool SerializeDependencyHashes, bool TrackSystemDependencies, ModuleInterfaceLoaderOptions LoaderOpts, RequireOSSAModules_t RequireOSSAModules, bool silenceInterfaceDiagnostics) { InterfaceSubContextDelegateImpl astDelegate( SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, CASOpts, LoaderOpts, /*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir, BackupInterfaceDir, SerializeDependencyHashes, TrackSystemDependencies, RequireOSSAModules); ImplicitModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath, SearchPathOpts.getSDKPath(), SearchPathOpts.getSysRoot(), ModuleName, CacheDir, PrebuiltCacheDir, BackupInterfaceDir, ABIOutputPath, LoaderOpts.disableInterfaceLock, silenceInterfaceDiagnostics); // FIXME: We really only want to serialize 'important' dependencies here, if // we want to ship the built swiftmodules to another machine. auto failed = builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true, /*ModuleBuffer*/nullptr, nullptr, SearchPathOpts.CandidateCompiledModules); if (!failed) return false; auto backInPath = ModuleInterfaceLoaderImpl::getBackupPublicModuleInterfacePath(SourceMgr, BackupInterfaceDir, ModuleName, InPath); if (backInPath.empty()) return true; assert(failed); assert(!backInPath.empty()); ImplicitModuleInterfaceBuilder backupBuilder(SourceMgr, &Diags, astDelegate, backInPath, SearchPathOpts.getSDKPath(), SearchPathOpts.getSysRoot(), ModuleName, CacheDir, PrebuiltCacheDir, BackupInterfaceDir, ABIOutputPath, LoaderOpts.disableInterfaceLock, silenceInterfaceDiagnostics); // Ensure we can rebuild module after user changed the original interface file. backupBuilder.addExtraDependency(InPath); // FIXME: We really only want to serialize 'important' dependencies here, if // we want to ship the built swiftmodules to another machine. return backupBuilder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true, /*ModuleBuffer*/nullptr, nullptr, SearchPathOpts.CandidateCompiledModules); } static bool readSwiftInterfaceVersionAndArgs( SourceManager &SM, DiagnosticEngine &Diags, llvm::StringSaver &ArgSaver, SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath, SourceLoc diagnosticLoc, llvm::Triple preferredTarget) { llvm::vfs::FileSystem &fs = *SM.getFileSystem(); auto FileOrError = swift::vfs::getFileOrSTDIN(fs, interfacePath); if (!FileOrError) { // Don't use this->diagnose() because it'll just try to re-open // interfacePath. Diags.diagnose(diagnosticLoc, diag::error_open_input_file, interfacePath, FileOrError.getError().message()); return true; } auto SB = FileOrError.get()->getBuffer(); auto VersRe = getSwiftInterfaceFormatVersionRegex(); auto CompRe = getSwiftInterfaceCompilerVersionRegex(); SmallVector VersMatches, CompMatches; if (!VersRe.match(SB, &VersMatches)) { InterfaceSubContextDelegateImpl::diagnose( interfacePath, diagnosticLoc, SM, &Diags, diag::error_extracting_version_from_module_interface); return true; } if (extractCompilerFlagsFromInterface(interfacePath, SB, ArgSaver, interfaceInfo.Arguments, preferredTarget, &Diags)) { InterfaceSubContextDelegateImpl::diagnose( interfacePath, diagnosticLoc, SM, &Diags, diag::error_extracting_version_from_module_interface); return true; } assert(VersMatches.size() == 2); // FIXME We should diagnose this at a location that makes sense: auto Vers = VersionParser::parseVersionString(VersMatches[1], SourceLoc(), &Diags); if (!Vers.has_value()) { InterfaceSubContextDelegateImpl::diagnose( interfacePath, diagnosticLoc, SM, &Diags, diag::error_extracting_version_from_module_interface); return true; } if (CompRe.match(SB, &CompMatches)) { assert(CompMatches.size() == 2); interfaceInfo.CompilerVersion = ArgSaver.save(CompMatches[1]).str(); // For now, successfully parsing the tools version out of the interface is // optional. auto ToolsVersRe = getSwiftInterfaceCompilerToolsVersionRegex(); SmallVector VendorToolsVersMatches; if (ToolsVersRe.match(interfaceInfo.CompilerVersion, &VendorToolsVersMatches)) { interfaceInfo.CompilerToolsVersion = VersionParser::parseVersionString( VendorToolsVersMatches[1], SourceLoc(), nullptr); } } else { // Don't diagnose; handwritten module interfaces don't include this field. interfaceInfo.CompilerVersion = "(unspecified, file possibly handwritten)"; } // For now: we support anything with the same "major version" and assume // minor versions might be interesting for debugging, or special-casing a // compatible field variant. if (Vers->asMajorVersion() != InterfaceFormatVersion.asMajorVersion()) { InterfaceSubContextDelegateImpl::diagnose( interfacePath, diagnosticLoc, SM, &Diags, diag::unsupported_version_of_module_interface, interfacePath, *Vers); return true; } return false; } bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface( CompilerInstance &Instance, const StringRef moduleCachePath, const StringRef backupInterfaceDir, const StringRef prebuiltCachePath, const StringRef ABIDescriptorPath, StringRef interfacePath, StringRef outputPath, bool ShouldSerializeDeps, ArrayRef CompiledCandidates, DependencyTracker *tracker) { if (!Instance.getInvocation().getIRGenOptions().AlwaysCompile) { // First, check if the expected output already exists and possibly // up-to-date w.r.t. all of the dependencies it was built with. If so, early // exit. UpToDateModuleCheker checker( Instance.getASTContext(), RequireOSSAModules_t(Instance.getSILOptions())); ModuleRebuildInfo rebuildInfo; SmallVector allDeps; std::unique_ptr moduleBuffer; if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, moduleBuffer)) { if (Instance.getASTContext() .LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { Instance.getDiags().diagnose( SourceLoc(), diag::explicit_interface_build_skipped, outputPath); } return false; } } // Read out the compiler version. llvm::BumpPtrAllocator alloc; llvm::StringSaver ArgSaver(alloc); SwiftInterfaceInfo InterfaceInfo; readSwiftInterfaceVersionAndArgs( Instance.getSourceMgr(), Instance.getDiags(), ArgSaver, InterfaceInfo, interfacePath, SourceLoc(), Instance.getInvocation().getLangOptions().Target); auto Builder = ExplicitModuleInterfaceBuilder( Instance, &Instance.getDiags(), Instance.getSourceMgr(), moduleCachePath, backupInterfaceDir, prebuiltCachePath, ABIDescriptorPath, {}); auto error = Builder.buildSwiftModuleFromInterface( interfacePath, outputPath, ShouldSerializeDeps, /*ModuleBuffer*/nullptr, CompiledCandidates, InterfaceInfo.CompilerVersion); if (!error) return false; else return true; } void ModuleInterfaceLoader::collectVisibleTopLevelModuleNames( SmallVectorImpl &names) const { collectVisibleTopLevelModuleNamesImpl( names, file_types::getExtension(file_types::TY_SwiftModuleInterfaceFile)); } void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( FrontendOptions::ActionType requestedAction, const SearchPathOptions &SearchPathOpts, const LangOptions &LangOpts, const ClangImporterOptions &clangImporterOpts, const CASOptions &casOpts, bool suppressRemarks, RequireOSSAModules_t RequireOSSAModules) { GenericArgs.push_back("-frontend"); // Start with a genericSubInvocation that copies various state from our // invoking ASTContext. GenericArgs.push_back("-compile-module-from-interface"); genericSubInvocation.setTargetTriple(LangOpts.Target); auto triple = ArgSaver.save(genericSubInvocation.getTargetTriple()); if (!triple.empty()) { GenericArgs.push_back("-target"); GenericArgs.push_back(triple); } if (LangOpts.ClangTarget.has_value()) { genericSubInvocation.getLangOptions().ClangTarget = LangOpts.ClangTarget; auto triple = ArgSaver.save(genericSubInvocation.getLangOptions() .ClangTarget->getTriple()); assert(!triple.empty()); // In explicit module build, all PCMs will be built using the given clang target. // So the Swift interface should know that as well to load these PCMs properly. GenericArgs.push_back("-clang-target"); GenericArgs.push_back(triple); } if (LangOpts.ClangTargetVariant.has_value()) { genericSubInvocation.getLangOptions().ClangTargetVariant = LangOpts.ClangTargetVariant; auto variantTriple = ArgSaver.save(genericSubInvocation.getLangOptions() .ClangTargetVariant->getTriple()); assert(!variantTriple.empty()); GenericArgs.push_back("-clang-target-variant"); GenericArgs.push_back(variantTriple); } // Inherit the target SDK name and version if (!LangOpts.SDKName.empty()) { genericSubInvocation.getLangOptions().SDKName = LangOpts.SDKName; GenericArgs.push_back("-target-sdk-name"); GenericArgs.push_back(ArgSaver.save(LangOpts.SDKName)); } if (LangOpts.SDKVersion.has_value()) { genericSubInvocation.getLangOptions().SDKVersion = LangOpts.SDKVersion; GenericArgs.push_back("-target-sdk-version"); GenericArgs.push_back(ArgSaver.save(LangOpts.SDKVersion.value() .getAsString())); } // Inherit the Swift language version genericSubInvocation.getLangOptions().EffectiveLanguageVersion = LangOpts.EffectiveLanguageVersion; GenericArgs.push_back("-swift-version"); GenericArgs.push_back(ArgSaver.save(genericSubInvocation.getLangOptions() .EffectiveLanguageVersion.asAPINotesVersionString())); genericSubInvocation.setImportSearchPaths( SearchPathOpts.getImportSearchPaths()); genericSubInvocation.setFrameworkSearchPaths( SearchPathOpts.getFrameworkSearchPaths()); if (!SearchPathOpts.getSDKPath().empty()) { // Add -sdk arguments to the module building commands. // Module building commands need this because dependencies sometimes use // sdk-relative paths (prebuilt modules for example). Without -sdk, the command // will not be able to local these dependencies, leading to unnecessary // building from textual interfaces. GenericArgs.push_back("-sdk"); GenericArgs.push_back(ArgSaver.save(SearchPathOpts.getSDKPath())); genericSubInvocation.setSDKPath(SearchPathOpts.getSDKPath().str()); } if (SearchPathOpts.PlatformAvailabilityInheritanceMapPath) { GenericArgs.push_back("-platform-availability-inheritance-map-path"); GenericArgs.push_back(ArgSaver.save(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath)); genericSubInvocation.setPlatformAvailabilityInheritanceMapPath(*SearchPathOpts.PlatformAvailabilityInheritanceMapPath); } // Inherit the plugin search opts but do not inherit the arguments. genericSubInvocation.getSearchPathOptions().PluginSearchOpts = SearchPathOpts.PluginSearchOpts; genericSubInvocation.getFrontendOptions().InputMode = FrontendOptions::ParseInputMode::SwiftModuleInterface; if (!SearchPathOpts.RuntimeResourcePath.empty()) { genericSubInvocation.setRuntimeResourcePath(SearchPathOpts.RuntimeResourcePath); } // Inhibit warnings from the genericSubInvocation since we are assuming the // user is not in a position to address them. genericSubInvocation.getDiagnosticOptions().SuppressWarnings = true; GenericArgs.push_back("-suppress-warnings"); // Inherit the parent invocation's setting on whether to suppress remarks if (suppressRemarks) { genericSubInvocation.getDiagnosticOptions().SuppressRemarks = true; GenericArgs.push_back("-suppress-remarks"); } // Inherit this setting down so that it can affect error diagnostics (mostly // by making them non-fatal). genericSubInvocation.getLangOptions().DebuggerSupport = LangOpts.DebuggerSupport; if (LangOpts.DebuggerSupport) { GenericArgs.push_back("-debugger-support"); } // Disable this; deinitializers always get printed with `@objc` even in // modules that don't import Foundation. genericSubInvocation.getLangOptions().EnableObjCAttrRequiresFoundation = false; GenericArgs.push_back("-disable-objc-attr-requires-foundation-module"); // If we are supposed to use RequireOSSAModules, do so. if (RequireOSSAModules) { genericSubInvocation.getSILOptions().EnableOSSAModules = true; GenericArgs.push_back("-enable-ossa-modules"); } if (LangOpts.DisableAvailabilityChecking) { genericSubInvocation.getLangOptions().DisableAvailabilityChecking = true; GenericArgs.push_back("-disable-availability-checking"); } // Pass-down the obfuscators so we can get the serialized search paths properly. genericSubInvocation.setSerializedPathObfuscator( SearchPathOpts.DeserializedPathRecoverer); SearchPathOpts.DeserializedPathRecoverer .forEachPair([&](StringRef lhs, StringRef rhs) { GenericArgs.push_back("-serialized-path-obfuscate"); std::string pair = (llvm::Twine(lhs) + "=" + rhs).str(); GenericArgs.push_back(ArgSaver.save(pair)); }); genericSubInvocation.getClangImporterOptions().DirectClangCC1ModuleBuild = clangImporterOpts.DirectClangCC1ModuleBuild; genericSubInvocation.getClangImporterOptions().ClangImporterDirectCC1Scan = clangImporterOpts.ClangImporterDirectCC1Scan; genericSubInvocation.getSearchPathOptions().ScannerPrefixMapper = SearchPathOpts.ScannerPrefixMapper; // Validate Clang modules once per-build session flags must be consistent // across all module sub-invocations if (clangImporterOpts.ValidateModulesOnce) { genericSubInvocation.getClangImporterOptions().ValidateModulesOnce = true; genericSubInvocation.getClangImporterOptions().BuildSessionFilePath = clangImporterOpts.BuildSessionFilePath; GenericArgs.push_back("-validate-clang-modules-once"); GenericArgs.push_back("-clang-build-session-file"); GenericArgs.push_back(clangImporterOpts.BuildSessionFilePath); } if (casOpts.EnableCaching) { genericSubInvocation.getCASOptions().EnableCaching = casOpts.EnableCaching; genericSubInvocation.getCASOptions().CASOpts = casOpts.CASOpts; genericSubInvocation.getCASOptions().HasImmutableFileSystem = casOpts.HasImmutableFileSystem; casOpts.enumerateCASConfigurationFlags( [&](StringRef Arg) { GenericArgs.push_back(ArgSaver.save(Arg)); }); } } bool InterfaceSubContextDelegateImpl::extractSwiftInterfaceVersionAndArgs( CompilerInvocation &subInvocation, DiagnosticEngine &subInstanceDiags, SwiftInterfaceInfo &interfaceInfo, StringRef interfacePath, SourceLoc diagnosticLoc) { if (readSwiftInterfaceVersionAndArgs(SM, *Diags, ArgSaver, interfaceInfo, interfacePath, diagnosticLoc, subInvocation.getLangOptions().Target)) return true; // Prior to Swift 5.9, swiftinterfaces were always built (accidentally) with // `-target-min-inlining-version target` prepended to the argument list. To // preserve compatibility we must continue to prepend those flags to the // invocation when the interface was generated by an older compiler. if (auto toolsVersion = interfaceInfo.CompilerToolsVersion) { if (toolsVersion < version::Version{5, 9}) { interfaceInfo.Arguments.push_back("-target-min-inlining-version"); interfaceInfo.Arguments.push_back("target"); } } SmallString<32> ExpectedModuleName = subInvocation.getModuleName(); if (subInvocation.parseArgs(interfaceInfo.Arguments, subInstanceDiags)) { return true; } if (subInvocation.getModuleName() != ExpectedModuleName) { auto DiagKind = diag::serialization_name_mismatch; if (subInvocation.getLangOptions().DebuggerSupport) DiagKind = diag::serialization_name_mismatch_repl; diagnose(interfacePath, diagnosticLoc, DiagKind, subInvocation.getModuleName(), ExpectedModuleName); return true; } return false; } InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl( SourceManager &SM, DiagnosticEngine *Diags, const SearchPathOptions &searchPathOpts, const LangOptions &langOpts, const ClangImporterOptions &clangImporterOpts, const CASOptions &casOpts, ModuleInterfaceLoaderOptions LoaderOpts, bool buildModuleCacheDirIfAbsent, StringRef moduleCachePath, StringRef prebuiltCachePath, StringRef backupModuleInterfaceDir, bool serializeDependencyHashes, bool trackSystemDependencies, RequireOSSAModules_t requireOSSAModules) : SM(SM), Diags(Diags), ArgSaver(Allocator) { genericSubInvocation.setMainExecutablePath(LoaderOpts.mainExecutablePath); inheritOptionsForBuildingInterface(LoaderOpts.requestedAction, searchPathOpts, langOpts, clangImporterOpts, casOpts, Diags->getSuppressRemarks(), requireOSSAModules); // Configure front-end input. auto &SubFEOpts = genericSubInvocation.getFrontendOptions(); SubFEOpts.RequestedAction = LoaderOpts.requestedAction; SubFEOpts.StrictImplicitModuleContext = LoaderOpts.strictImplicitModuleContext; SubFEOpts.CompilerDebuggingOpts = LoaderOpts.compilerDebuggingOptions; if (!moduleCachePath.empty()) { genericSubInvocation.setClangModuleCachePath(moduleCachePath); } if (!prebuiltCachePath.empty()) { genericSubInvocation.getFrontendOptions().PrebuiltModuleCachePath = prebuiltCachePath.str(); } if (!backupModuleInterfaceDir.empty()) { genericSubInvocation.getFrontendOptions().BackupModuleInterfaceDir = backupModuleInterfaceDir.str(); } if (trackSystemDependencies) { genericSubInvocation.getFrontendOptions().IntermoduleDependencyTracking = IntermoduleDepTrackingMode::IncludeSystem; GenericArgs.push_back("-track-system-dependencies"); } else { // Always track at least the non-system dependencies for interface building. genericSubInvocation.getFrontendOptions().IntermoduleDependencyTracking = IntermoduleDepTrackingMode::ExcludeSystem; } if (LoaderOpts.disableImplicitSwiftModule) { genericSubInvocation.getFrontendOptions().DisableImplicitModules = true; GenericArgs.push_back("-disable-implicit-swift-modules"); GenericArgs.push_back("-Xcc"); GenericArgs.push_back("-fno-implicit-modules"); GenericArgs.push_back("-Xcc"); GenericArgs.push_back("-fno-implicit-module-maps"); } // If building an application extension, make sure API use // is restricted accordingly in downstream dependnecies. if (langOpts.EnableAppExtensionRestrictions) { genericSubInvocation.getLangOptions().EnableAppExtensionRestrictions = true; GenericArgs.push_back("-application-extension"); } // Pass down -explicit-swift-module-map-file StringRef explicitSwiftModuleMap = searchPathOpts.ExplicitSwiftModuleMapPath; genericSubInvocation.getSearchPathOptions().ExplicitSwiftModuleMapPath = explicitSwiftModuleMap.str(); // Pass down VFSOverlay flags (do not need to inherit the options because // FileSystem is shared). for (auto &Overlay : searchPathOpts.VFSOverlayFiles) { GenericArgs.push_back("-vfsoverlay"); GenericArgs.push_back(Overlay); } // Load plugin libraries for macro expression as default arguments genericSubInvocation.getSearchPathOptions().PluginSearchOpts = searchPathOpts.PluginSearchOpts; genericSubInvocation.getSearchPathOptions().ResolvedPluginVerification = searchPathOpts.ResolvedPluginVerification; // Get module loading behavior options. genericSubInvocation.getSearchPathOptions().ScannerModuleValidation = searchPathOpts.ScannerModuleValidation; genericSubInvocation.getSearchPathOptions().ModuleLoadMode = searchPathOpts.ModuleLoadMode; auto &subClangImporterOpts = genericSubInvocation.getClangImporterOptions(); // Respect the detailed-record preprocessor setting of the parent context. // This, and the "raw" clang module format it implicitly enables, are // required by sourcekitd. subClangImporterOpts.DetailedPreprocessingRecord = clangImporterOpts.DetailedPreprocessingRecord; std::vector inheritedParentContextClangArgs; if (LoaderOpts.requestedAction == FrontendOptions::ActionType::ScanDependencies) { // For a dependency scanning action, interface build command generation must // inherit `-Xcc` flags used for configuration of the building instance's // `ClangImporter`. However, we can ignore Clang search path flags because // explicit Swift module build tasks will not rely on them and they may be // source-target-context-specific and hinder module sharing across // compilation source targets. // Clang module dependecies of this Swift dependency will be distinguished // by their context hash for different variants, so would still cause a // difference in the Swift compile commands, when different. inheritedParentContextClangArgs = clangImporterOpts.getReducedExtraArgsForSwiftModuleDependency(); genericSubInvocation.getFrontendOptions() .DependencyScanningSubInvocation = true; } else if (LoaderOpts.strictImplicitModuleContext || // Explicit module Interface verification jobs still spawn a // sub-instance and we must ensure this sub-instance gets all of // the Xcc flags. LoaderOpts.disableImplicitSwiftModule || // If using direct cc1 argument mode for lldb or typecheck // interface, inherit all clang arguments. clangImporterOpts.DirectClangCC1ModuleBuild) { // If the compiler has been asked to be strict with ensuring downstream // dependencies get the parent invocation's context, inherit the extra Clang // arguments also. Inherit any clang-specific state of the compilation // (macros, clang flags, etc.) inheritedParentContextClangArgs = clangImporterOpts.ExtraArgs; } subClangImporterOpts.ExtraArgs = inheritedParentContextClangArgs; // If using DirectCC1Scan, the command-line reduction is handled inside // `getSwiftExplicitModuleDirectCC1Args()`, there is no need to inherit // anything here as the ExtraArgs from the invocation are clang driver // options, not cc1 options. if (!clangImporterOpts.ClangImporterDirectCC1Scan) { for (auto arg : subClangImporterOpts.ExtraArgs) { GenericArgs.push_back("-Xcc"); GenericArgs.push_back(ArgSaver.save(arg)); } } subClangImporterOpts.EnableClangSPI = clangImporterOpts.EnableClangSPI; if (!subClangImporterOpts.EnableClangSPI) { GenericArgs.push_back("-disable-clang-spi"); } // Tell the genericSubInvocation to serialize dependency hashes if asked to do // so. auto &frontendOpts = genericSubInvocation.getFrontendOptions(); frontendOpts.SerializeModuleInterfaceDependencyHashes = serializeDependencyHashes; if (serializeDependencyHashes) { GenericArgs.push_back("-serialize-module-interface-dependency-hashes"); } // Tell the genericSubInvocation to remark on rebuilds from an interface if asked // to do so. frontendOpts.RemarkOnRebuildFromModuleInterface = LoaderOpts.remarkOnRebuildFromInterface; if (LoaderOpts.remarkOnRebuildFromInterface) { GenericArgs.push_back("-Rmodule-interface-rebuild"); } // This flag only matters when we are verifying an textual interface. frontendOpts.DowngradeInterfaceVerificationError = LoaderOpts.downgradeInterfaceVerificationError; // Note that we don't assume cachePath is the same as the Clang // module cache path at this point. if (buildModuleCacheDirIfAbsent && !moduleCachePath.empty()) (void)llvm::sys::fs::create_directories(moduleCachePath); // Inherit all block list configuration files frontendOpts.BlocklistConfigFilePaths = langOpts.BlocklistConfigFilePaths; for (auto &blocklist: langOpts.BlocklistConfigFilePaths) { GenericArgs.push_back("-blocklist-file"); GenericArgs.push_back(blocklist); } // Inherit APINotes processing method if (clangImporterOpts.LoadVersionIndependentAPINotes) { GenericArgs.push_back("-version-independent-apinotes"); genericSubInvocation.getClangImporterOptions().LoadVersionIndependentAPINotes = true; } // Inherit the C++ interoperability mode. if (langOpts.EnableCXXInterop) { // Modelled after a reverse of validateCxxInteropCompatibilityMode genericSubInvocation.getLangOptions().EnableCXXInterop = true; genericSubInvocation.getLangOptions().cxxInteropCompatVersion = langOpts.cxxInteropCompatVersion; std::string compatVersion; if (langOpts.cxxInteropCompatVersion.empty()) compatVersion = "default"; else if (langOpts.cxxInteropCompatVersion[0] == 5) compatVersion = "swift-5.9"; else if (langOpts.cxxInteropCompatVersion[0] == 6) compatVersion = "swift-6"; else if (langOpts.cxxInteropCompatVersion[0] == version::getUpcomingCxxInteropCompatVersion()) compatVersion = "upcoming-swift"; else // TODO: This may need to be updated once more versions are added compatVersion = "default"; GenericArgs.push_back( ArgSaver.save("-cxx-interoperability-mode=" + compatVersion)); if (!langOpts.isUsingPlatformDefaultCXXStdlib() && langOpts.CXXStdlib == CXXStdlibKind::Libcxx) { genericSubInvocation.getLangOptions().CXXStdlib = CXXStdlibKind::Libcxx; genericSubInvocation.getClangImporterOptions().ExtraArgs.push_back( "-stdlib=libc++"); GenericArgs.push_back("-Xcc"); GenericArgs.push_back("-stdlib=libc++"); } } } /// Calculate an output filename in \p genericSubInvocation's cache path that /// includes a hash of relevant key data. void InterfaceSubContextDelegateImpl::getCachedOutputPath( SwiftInterfaceModuleOutputPathResolution::ResultTy &resolvedOutputPath, StringRef moduleName, StringRef interfacePath, StringRef sdkPath) { SwiftInterfaceModuleOutputPathResolution::setOutputPath( resolvedOutputPath, moduleName, interfacePath, sdkPath, genericSubInvocation, genericSubInvocation.getClangImporterOptions() .getReducedExtraArgsForSwiftModuleDependency()); } std::error_code InterfaceSubContextDelegateImpl::runInSubContext(StringRef moduleName, StringRef interfacePath, StringRef sdkPath, std::optional sysroot, StringRef outputPath, SourceLoc diagLoc, llvm::function_ref, StringRef, StringRef)> action) { return runInSubCompilerInstance(moduleName, interfacePath, sdkPath, sysroot, outputPath, diagLoc, /*silenceErrors=*/false, [&](SubCompilerInstanceInfo &info){ std::string UserModuleVer = info.Instance->getInvocation().getFrontendOptions() .UserModuleVersion.getAsString(); return action(info.Instance->getASTContext(), info.Instance->getMainModule(), info.BuildArguments, info.Hash, UserModuleVer); }); } std::error_code InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleName, StringRef interfacePath, StringRef sdkPath, std::optional sysroot, StringRef outputPath, SourceLoc diagLoc, bool silenceErrors, llvm::function_ref action) { // We are about to mess up the compiler invocation by using the compiler // arguments in the textual interface file. So copy to use a new compiler // invocation. CompilerInvocation subInvocation = genericSubInvocation; // save `StrictImplicitModuleContext` bool StrictImplicitModuleContext = subInvocation.getFrontendOptions().StrictImplicitModuleContext; // It isn't appropriate to restrict use of experimental features in another // module since it may have been built with a different compiler that allowed // the use of the feature. subInvocation.getLangOptions().RestrictNonProductionExperimentalFeatures = false; // Save the target triple from the original context. llvm::Triple originalTargetTriple(subInvocation.getLangOptions().Target); std::vector BuildArgs(GenericArgs.begin(), GenericArgs.end()); assert(BuildArgs.size() == GenericArgs.size()); // Configure inputs subInvocation.getFrontendOptions().InputsAndOutputs .addInputFile(interfacePath); BuildArgs.push_back(interfacePath); subInvocation.setModuleName(moduleName); BuildArgs.push_back("-module-name"); BuildArgs.push_back(moduleName); if (sysroot) { subInvocation.setSysRoot(sysroot.value()); } // Calculate output path of the module. SwiftInterfaceModuleOutputPathResolution::ResultTy resolvedOutputPath; getCachedOutputPath(resolvedOutputPath, moduleName, interfacePath, sdkPath); // If no specific output path is given, use the hashed output path. if (outputPath.empty()) { outputPath = resolvedOutputPath.outputPath; } // Configure the outputs in front-end options. There must be an equal number of // inputs and outputs. std::vector outputFiles{"/"}; std::vector ModuleOutputPaths; ModuleOutputPaths.emplace_back(); if (subInvocation.getFrontendOptions().RequestedAction == FrontendOptions::ActionType::EmitModuleOnly) { ModuleOutputPaths.back().ModuleOutputPath = outputPath.str(); } assert(subInvocation.getFrontendOptions().InputsAndOutputs.isWholeModule()); subInvocation.getFrontendOptions().InputsAndOutputs .setMainAndSupplementaryOutputs(outputFiles, ModuleOutputPaths); CompilerInstance subInstance; ForwardingDiagnosticConsumer FDC(*Diags); NullDiagnosticConsumer noopConsumer; if (!silenceErrors) { subInstance.addDiagnosticConsumer(&FDC); } else { subInstance.addDiagnosticConsumer(&noopConsumer); } SwiftInterfaceInfo interfaceInfo; // Extract compiler arguments from the interface file and use them to configure // the compiler invocation. if (extractSwiftInterfaceVersionAndArgs(subInvocation, subInstance.getDiags(), interfaceInfo, interfacePath, diagLoc)) { return std::make_error_code(std::errc::not_supported); } // Insert arguments collected from the interface file. BuildArgs.insert(BuildArgs.end(), interfaceInfo.Arguments.begin(), interfaceInfo.Arguments.end()); // restore `StrictImplicitModuleContext` subInvocation.getFrontendOptions().StrictImplicitModuleContext = StrictImplicitModuleContext; SubCompilerInstanceInfo info; info.Instance = &subInstance; info.CompilerVersion = interfaceInfo.CompilerVersion; subInstance.getSourceMgr().setFileSystem(SM.getFileSystem()); std::string InstanceSetupError; if (subInstance.setup(subInvocation, InstanceSetupError)) { return std::make_error_code(std::errc::not_supported); } info.BuildArguments = BuildArgs; info.Hash = resolvedOutputPath.hash; // Run the action under the sub compiler instance. return action(info); } static void addModuleAliasesFromExplicitSwiftModuleMap( ASTContext &Ctx, llvm::StringMap ModuleAliases) { for (auto &entry : ModuleAliases) { Ctx.addModuleAlias(/*moduleAlias=*/entry.getKey(), /*realModule=*/entry.getValue()); } } struct ExplicitSwiftModuleLoader::Implementation { ASTContext &Ctx; llvm::BumpPtrAllocator Allocator; llvm::StringMap ExplicitModuleMap; Implementation(ASTContext &Ctx) : Ctx(Ctx) {} void parseSwiftExplicitModuleMap(StringRef fileName) { ExplicitModuleMapParser parser(Allocator); llvm::StringMap ExplicitClangModuleMap; llvm::StringMap ModuleAliases; // Load the input file. llvm::ErrorOr> fileBufOrErr = llvm::MemoryBuffer::getFile(fileName); if (!fileBufOrErr) { Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing, fileName); return; } auto hasError = parser.parseSwiftExplicitModuleMap( (*fileBufOrErr)->getMemBufferRef(), ExplicitModuleMap, ExplicitClangModuleMap, ModuleAliases); if (hasError) Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted, fileName); // A single module map can define multiple modules; keep track of the ones // we've seen so that we don't generate duplicate flags. std::set moduleMapsSeen; std::vector &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs; for (auto &entry : ExplicitClangModuleMap) { const auto &moduleMapPath = entry.getValue().moduleMapPath; if (!moduleMapPath.empty() && entry.getValue().isBridgingHeaderDependency && moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) { moduleMapsSeen.insert(moduleMapPath); extraClangArgs.push_back( (Twine("-fmodule-map-file=") + moduleMapPath).str()); } const auto &modulePath = entry.getValue().modulePath; if (!modulePath.empty()) { extraClangArgs.push_back( (Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath) .str()); } } addModuleAliasesFromExplicitSwiftModuleMap(Ctx, ModuleAliases); } void addCommandLineExplicitInputs( const llvm::StringMap &commandLineExplicitInputs) { for (const auto &moduleInput : commandLineExplicitInputs) { ExplicitSwiftModuleInputInfo entry(moduleInput.getValue(), {}, {}, {}); ExplicitModuleMap.try_emplace(moduleInput.first(), std::move(entry)); } } }; ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader( ASTContext &ctx, DependencyTracker *tracker, ModuleLoadingMode loadMode, bool IgnoreSwiftSourceInfoFile): SerializedModuleLoaderBase(ctx, tracker, loadMode, IgnoreSwiftSourceInfoFile), Impl(*new Implementation(ctx)) {} ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; } bool ExplicitSwiftModuleLoader::findModule( ImportPath::Element ModuleID, SmallVectorImpl *ModuleInterfacePath, SmallVectorImpl *ModuleInterfaceSourcePath, std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer, std::unique_ptr *ModuleSourceInfoBuffer, bool skipBuildingInterface, bool isTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) { // Find a module with an actual, physical name on disk, in case // -module-alias is used (otherwise same). // // For example, if '-module-alias Foo=Bar' is passed in to the frontend, and an // input file has 'import Foo', a module called Bar (real name) should be searched. StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str(); auto it = Impl.ExplicitModuleMap.find(moduleName); // If no explicit module path is given matches the name, return with an // error code. if (it == Impl.ExplicitModuleMap.end()) { return false; } auto &moduleInfo = it->getValue(); // Set IsFramework bit according to the moduleInfo IsFramework = moduleInfo.isFramework; IsSystemModule = moduleInfo.isSystem; auto &fs = *Ctx.SourceMgr.getFileSystem(); // Open .swiftmodule file auto moduleBuf = fs.getBufferForFile(moduleInfo.modulePath); if (!moduleBuf) { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } assert(moduleBuf); const bool isForwardingModule = !serialization::isSerializedAST(moduleBuf .get()->getBuffer()); // If the module is a forwarding module, read the actual content from the path // encoded in the forwarding module as the actual module content. if (isForwardingModule) { auto forwardingModule = ForwardingModule::load(*moduleBuf.get()); if (forwardingModule) { moduleBuf = fs.getBufferForFile(forwardingModule->underlyingModulePath); if (!moduleBuf) { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } } else { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } } assert(moduleBuf); // Move the opened module buffer to the caller. *ModuleBuffer = std::move(moduleBuf.get()); // Open .swiftdoc file if (moduleInfo.moduleDocPath.has_value()) { auto moduleDocBuf = fs.getBufferForFile(moduleInfo.moduleDocPath.value()); if (moduleBuf) *ModuleDocBuffer = std::move(moduleDocBuf.get()); } // Open .swiftsourceinfo file if (moduleInfo.moduleSourceInfoPath.has_value()) { auto moduleSourceInfoBuf = fs.getBufferForFile(moduleInfo.moduleSourceInfoPath.value()); if (moduleSourceInfoBuf) *ModuleSourceInfoBuffer = std::move(moduleSourceInfoBuf.get()); } return true; } std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory( ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName, SmallVectorImpl *ModuleInterfacePath, SmallVectorImpl *ModuleInterfaceSourcePath, std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer, std::unique_ptr *ModuleSourceInfoBuffer, bool skipBuildingInterface, bool IsFramework, bool IsTestableDependencyLookup) { llvm_unreachable("Not supported in the Explicit Swift Module Loader."); return std::make_error_code(std::errc::not_supported); } bool ExplicitSwiftModuleLoader::canImportModule( ImportPath::Module path, SourceLoc loc, ModuleVersionInfo *versionInfo, bool isTestableDependencyLookup) { // FIXME: Swift submodules? if (path.hasSubmodule()) return false; ImportPath::Element mID = path.front(); // Look up the module with the real name (physical name on disk); // in case `-module-alias` is used, the name appearing in source files // and the real module name are different. For example, '-module-alias Foo=Bar' // maps Foo appearing in source files, e.g. 'import Foo', to the real module // name Bar (on-disk name), which should be searched for loading. StringRef moduleName = Ctx.getRealModuleName(mID.Item).str(); auto it = Impl.ExplicitModuleMap.find(moduleName); // If no provided explicit module matches the name, then it cannot be imported. if (it == Impl.ExplicitModuleMap.end()) { return false; } // If the caller doesn't want version info we're done. if (!versionInfo) return true; // Open .swiftmodule file and read out the version auto &fs = *Ctx.SourceMgr.getFileSystem(); auto moduleBuf = fs.getBufferForFile(it->second.modulePath); if (!moduleBuf) { Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file, it->second.modulePath); return false; } // If it's a forwarding module, load the YAML file from disk and get the path // to the actual module for the version check. if (!serialization::isSerializedAST((*moduleBuf)->getBuffer())) { if (auto forwardingModule = ForwardingModule::load(**moduleBuf)) { moduleBuf = fs.getBufferForFile(forwardingModule->underlyingModulePath); if (!moduleBuf) { Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file, forwardingModule->underlyingModulePath); return false; } } } auto metaData = serialization::validateSerializedAST( (*moduleBuf)->getBuffer(), Ctx.SILOpts.EnableOSSAModules, Ctx.LangOpts.SDKName); versionInfo->setVersion(metaData.userModuleVersion, ModuleVersionSourceKind::SwiftBinaryModule); return true; } void ExplicitSwiftModuleLoader::collectVisibleTopLevelModuleNames( SmallVectorImpl &names) const { for (auto &entry: Impl.ExplicitModuleMap) { names.push_back(Ctx.getIdentifier(entry.getKey())); } } std::unique_ptr ExplicitSwiftModuleLoader::create(ASTContext &ctx, DependencyTracker *tracker, ModuleLoadingMode loadMode, StringRef ExplicitSwiftModuleMap, const llvm::StringMap &ExplicitSwiftModuleInputs, bool IgnoreSwiftSourceInfoFile) { auto result = std::unique_ptr( new ExplicitSwiftModuleLoader(ctx, tracker, loadMode, IgnoreSwiftSourceInfoFile)); auto &Impl = result->Impl; // If the explicit module map is given, try parse it. if (!ExplicitSwiftModuleMap.empty()) { // Parse a JSON file to collect explicitly built modules. Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap); } // If some modules are provided with explicit // '-swift-module-file' options, add those as well. if (!ExplicitSwiftModuleInputs.empty()) { Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs); } return result; } struct ExplicitCASModuleLoader::Implementation { ASTContext &Ctx; llvm::BumpPtrAllocator Allocator; llvm::cas::ObjectStore &CAS; llvm::cas::ActionCache &Cache; llvm::StringMap ExplicitModuleMap; Implementation(ASTContext &Ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache) : Ctx(Ctx), CAS(CAS), Cache(Cache) {} std::unique_ptr loadBuffer(StringRef ID) { auto key = CAS.parseID(ID); if (!key) { Ctx.Diags.diagnose(SourceLoc(), diag::error_invalid_cas_id, ID, toString(key.takeError())); return nullptr; } auto ref = CAS.getReference(*key); if (!ref) return nullptr; auto loaded = CAS.getProxy(*ref); if (!loaded) { Ctx.Diags.diagnose(SourceLoc(), diag::error_cas, "loading module map from CAS", toString(loaded.takeError())); return nullptr; } return loaded->getMemoryBuffer(); } // Same as the regular explicit module map but must be loaded from // CAS, instead of a file that is not tracked by the dependency. void parseSwiftExplicitModuleMap(StringRef ID) { ExplicitModuleMapParser parser(Allocator); llvm::StringMap ExplicitClangModuleMap; llvm::StringMap ModuleAliases; auto buf = loadBuffer(ID); if (!buf) { Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_missing, ID); return; } llvm::ErrorOr> fileBufOrErr = llvm::MemoryBuffer::getFile(ID); auto hasError = parser.parseSwiftExplicitModuleMap( buf->getMemBufferRef(), ExplicitModuleMap, ExplicitClangModuleMap, ModuleAliases); if (hasError) Ctx.Diags.diagnose(SourceLoc(), diag::explicit_swift_module_map_corrupted, ID); std::set moduleMapsSeen; std::vector &extraClangArgs = Ctx.ClangImporterOpts.ExtraArgs; // Append -Xclang if we are not in direct cc1 mode. auto appendXclang = [&]() { if (!Ctx.ClangImporterOpts.DirectClangCC1ModuleBuild) extraClangArgs.push_back("-Xclang"); }; for (auto &entry : ExplicitClangModuleMap) { const auto &moduleMapPath = entry.getValue().moduleMapPath; if (!moduleMapPath.empty() && !Ctx.CASOpts.EnableCaching && moduleMapsSeen.find(moduleMapPath) == moduleMapsSeen.end()) { moduleMapsSeen.insert(moduleMapPath); extraClangArgs.push_back( (Twine("-fmodule-map-file=") + moduleMapPath).str()); } const auto &modulePath = entry.getValue().modulePath; if (!modulePath.empty()) { extraClangArgs.push_back( (Twine("-fmodule-file=") + entry.getKey() + "=" + modulePath) .str()); } auto cachePath = entry.getValue().moduleCacheKey; if (cachePath) { appendXclang(); extraClangArgs.push_back("-fmodule-file-cache-key"); appendXclang(); extraClangArgs.push_back(modulePath); appendXclang(); extraClangArgs.push_back(*cachePath); } } addModuleAliasesFromExplicitSwiftModuleMap(Ctx, ModuleAliases); } void addCommandLineExplicitInputs( const llvm::StringMap &commandLineExplicitInputs) { for (const auto &moduleInput : commandLineExplicitInputs) { ExplicitSwiftModuleInputInfo entry(moduleInput.getValue(), {}, {}, {}); ExplicitModuleMap.try_emplace(moduleInput.getKey(), std::move(entry)); } } std::unique_ptr loadFileBuffer(SourceLoc Loc, StringRef ID, StringRef Name) { auto outputMissing = [&]() { Ctx.Diags.diagnose(Loc, diag::error_opening_explicit_module_file, Name); return nullptr; }; auto casError = [&](StringRef Stage, llvm::Error Err) { Ctx.Diags.diagnose(Loc, diag::error_cas, Stage, toString(std::move(Err))); return nullptr; }; auto key = CAS.parseID(ID); if (!key) { Ctx.Diags.diagnose(Loc, diag::error_invalid_cas_id, ID, toString(key.takeError())); return nullptr; } auto moduleLookup = Cache.get(*key); if (!moduleLookup) return casError("looking up module output cache", moduleLookup.takeError()); if (!*moduleLookup) return outputMissing(); auto moduleRef = CAS.getReference(**moduleLookup); if (!moduleRef) return outputMissing(); auto proxy = CAS.getProxy(*moduleRef); if (!proxy) return casError("loading module build outputs", proxy.takeError()); swift::cas::CompileJobResultSchema schema(CAS); if (!schema.isRootNode(*proxy)) return outputMissing(); auto result = schema.load(*moduleRef); if (!result) return casError("loading module schema", result.takeError()); auto output = result->getOutput(file_types::ID::TY_SwiftModuleFile); if (!output) return outputMissing(); auto buf = CAS.getProxy(output->Object); if (!buf) return casError("loading dependency module", result.takeError()); return buf->getMemoryBuffer(Name); } llvm::Expected> loadModuleFromPath(StringRef Path, DiagnosticEngine &Diags) { for (auto &Deps : ExplicitModuleMap) { if (Deps.second.modulePath == Path) { if (!Deps.second.moduleCacheKey) return nullptr; return loadCachedCompileResultFromCacheKey( CAS, Cache, Diags, *Deps.second.moduleCacheKey, file_types::ID::TY_SwiftModuleFile, Path); } } return nullptr; } }; ExplicitCASModuleLoader::ExplicitCASModuleLoader(ASTContext &ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &cache, DependencyTracker *tracker, ModuleLoadingMode loadMode, bool IgnoreSwiftSourceInfoFile) : SerializedModuleLoaderBase(ctx, tracker, loadMode, IgnoreSwiftSourceInfoFile), Impl(*new Implementation(ctx, CAS, cache)) {} ExplicitCASModuleLoader::~ExplicitCASModuleLoader() { delete &Impl; } bool ExplicitCASModuleLoader::findModule( ImportPath::Element ModuleID, SmallVectorImpl *ModuleInterfacePath, SmallVectorImpl *ModuleInterfaceSourcePath, std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer, std::unique_ptr *ModuleSourceInfoBuffer, bool skipBuildingInterface, bool isTestableDependencyLookup, bool &IsFramework, bool &IsSystemModule) { // Find a module with an actual, physical name on disk, in case // -module-alias is used (otherwise same). // // For example, if '-module-alias Foo=Bar' is passed in to the frontend, and // an input file has 'import Foo', a module called Bar (real name) should be // searched. StringRef moduleName = Ctx.getRealModuleName(ModuleID.Item).str(); auto it = Impl.ExplicitModuleMap.find(moduleName); // If no explicit module path is given matches the name, return with an // error code. if (it == Impl.ExplicitModuleMap.end()) { return false; } auto &moduleInfo = it->getValue(); // Set IsFramework bit according to the moduleInfo IsFramework = moduleInfo.isFramework; IsSystemModule = moduleInfo.isSystem; // Fallback check for module cache key passed on command-line as module path. std::string moduleCASID = moduleInfo.moduleCacheKey ? *moduleInfo.moduleCacheKey : moduleInfo.modulePath; // FIXME: the loaded module buffer doesn't set an identifier so it // is not tracked in dependency tracker, which doesn't handle modules // that are not located on disk. auto moduleBuf = loadCachedCompileResultFromCacheKey( Impl.CAS, Impl.Cache, Ctx.Diags, moduleCASID, file_types::ID::TY_SwiftModuleFile, moduleInfo.modulePath); if (!moduleBuf) { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } const bool isForwardingModule = !serialization::isSerializedAST(moduleBuf->getBuffer()); // If the module is a forwarding module, read the actual content from the path // encoded in the forwarding module as the actual module content. if (isForwardingModule) { auto forwardingModule = ForwardingModule::load(*moduleBuf.get()); if (forwardingModule) { // Look through ExplicitModuleMap for paths. // TODO: need to have dependency scanner reports forwarded module as // dependency for this compilation and ingested into CAS. auto moduleOrErr = Impl.loadModuleFromPath( forwardingModule->underlyingModulePath, Ctx.Diags); if (!moduleOrErr) { llvm::consumeError(moduleOrErr.takeError()); Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } moduleBuf = std::move(*moduleOrErr); if (!moduleBuf) { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } } else { // We cannot read the module content, diagnose. Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_explicit_module_file, moduleInfo.modulePath); return false; } } assert(moduleBuf); // Move the opened module buffer to the caller. *ModuleBuffer = std::move(moduleBuf); // TODO: support .swiftdoc file and .swiftsourceinfo file return true; } std::error_code ExplicitCASModuleLoader::findModuleFilesInDirectory( ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName, SmallVectorImpl *ModuleInterfacePath, SmallVectorImpl *ModuleInterfaceSourcePath, std::unique_ptr *ModuleBuffer, std::unique_ptr *ModuleDocBuffer, std::unique_ptr *ModuleSourceInfoBuffer, bool skipBuildingInterface, bool IsFramework, bool IsTestableDependencyLookup) { llvm_unreachable("Not supported in the Explicit Swift Module Loader."); return std::make_error_code(std::errc::not_supported); } bool ExplicitCASModuleLoader::canImportModule( ImportPath::Module path, SourceLoc loc, ModuleVersionInfo *versionInfo, bool isTestableDependencyLookup) { // FIXME: Swift submodules? if (path.hasSubmodule()) return false; ImportPath::Element mID = path.front(); // Look up the module with the real name (physical name on disk); // in case `-module-alias` is used, the name appearing in source files // and the real module name are different. For example, '-module-alias // Foo=Bar' maps Foo appearing in source files, e.g. 'import Foo', to the real // module name Bar (on-disk name), which should be searched for loading. StringRef moduleName = Ctx.getRealModuleName(mID.Item).str(); auto it = Impl.ExplicitModuleMap.find(moduleName); // If no provided explicit module matches the name, then it cannot be // imported. if (it == Impl.ExplicitModuleMap.end()) { return false; } // If the caller doesn't want version info we're done. if (!versionInfo) return true; // Open .swiftmodule file and read out the version std::string moduleCASID = it->second.moduleCacheKey ? *it->second.moduleCacheKey : it->second.modulePath; auto moduleBuf = Impl.loadFileBuffer(loc, moduleCASID, it->second.modulePath); if (!moduleBuf) { Ctx.Diags.diagnose(loc, diag::error_opening_explicit_module_file, it->second.modulePath); return false; } auto metaData = serialization::validateSerializedAST( moduleBuf->getBuffer(), Ctx.SILOpts.EnableOSSAModules, Ctx.LangOpts.SDKName); versionInfo->setVersion(metaData.userModuleVersion, ModuleVersionSourceKind::SwiftBinaryModule); return true; } void ExplicitCASModuleLoader::collectVisibleTopLevelModuleNames( SmallVectorImpl &names) const { for (auto &entry : Impl.ExplicitModuleMap) { names.push_back(Ctx.getIdentifier(entry.getKey())); } } std::unique_ptr ExplicitCASModuleLoader::create( ASTContext &ctx, llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &cache, DependencyTracker *tracker, ModuleLoadingMode loadMode, StringRef ExplicitSwiftModuleMap, const llvm::StringMap &ExplicitSwiftModuleInputs, bool IgnoreSwiftSourceInfoFile) { auto result = std::unique_ptr(new ExplicitCASModuleLoader( ctx, CAS, cache, tracker, loadMode, IgnoreSwiftSourceInfoFile)); auto &Impl = result->Impl; // If the explicit module map is given, try parse it. if (!ExplicitSwiftModuleMap.empty()) { // Parse a JSON file to collect explicitly built modules. Impl.parseSwiftExplicitModuleMap(ExplicitSwiftModuleMap); } // If some modules are provided with explicit // '-swift-module-file' options, add those as well. if (!ExplicitSwiftModuleInputs.empty()) { Impl.addCommandLineExplicitInputs(ExplicitSwiftModuleInputs); } return result; } namespace swift::SwiftInterfaceModuleOutputPathResolution { /// Construct a key for the .swiftmodule being generated. There is a /// balance to be struck here between things that go in the cache key and /// things that go in the "up to date" check of the cache entry. We want to /// avoid fighting over a single cache entry too much when (say) running /// different compiler versions on the same machine or different inputs /// that happen to have the same short module name, so we will disambiguate /// those in the key. But we want to invalidate and rebuild a cache entry /// -- rather than making a new one and potentially filling up the cache /// with dead entries -- when other factors change, such as the contents of /// the .swiftinterface input or its dependencies. static std::string getContextHash(const CompilerInvocation &CI, const StringRef &interfacePath, const StringRef &sdkPath, const ArgListTy &extraArgs) { // When doing dependency scanning for explicit module, use strict context hash // to ensure sound module hash. bool useStrictCacheHash = CI.getFrontendOptions().RequestedAction == FrontendOptions::ActionType::ScanDependencies; // Include the normalized target triple when not using strict hash. // Otherwise, use the full target to ensure soundness of the hash. In // practice, .swiftinterface files will be in target-specific subdirectories // and would have target-specific pieces #if'd out. However, it doesn't hurt // to include it, and it guards against mistakenly reusing cached modules // across targets. Note that this normalization explicitly doesn't include the // minimum deployment target (e.g. the '12.0' in 'ios12.0'). auto targetToHash = useStrictCacheHash ? CI.getLangOptions().Target : getTargetSpecificModuleTriple(CI.getLangOptions().Target); std::string sdkBuildVersion = getSDKBuildVersion(sdkPath); llvm::hash_code H = llvm::hash_combine( // Start with the compiler version (which will be either tag names or // revs). Explicitly don't pass in the "effective" language version -- // this would mean modules built in different -swift-version modes would // rebuild their dependencies. swift::version::getSwiftFullVersion(), // Simplest representation of input "identity" (not content) is just a // pathname, and probably all we can get from the VFS in this regard // anyways. interfacePath, // The target triple to hash. targetToHash.str(), // The SDK path is going to affect how this module is imported, so // include it. CI.getSDKPath(), // The SDK build version may identify differences in headers // that affects references serialized in the cached file. sdkBuildVersion, // Applying the distribution channel of the current compiler enables // different compilers to share a module cache location. version::getCurrentCompilerChannel(), // Whether or not we're tracking system dependencies affects the // invalidation behavior of this cache item. CI.getFrontendOptions().shouldTrackSystemDependencies(), // Whether or not caching is enabled affects if the instance is able to // correctly load the dependencies. CI.getCASOptions().getModuleScanningHashComponents(), // Take care of any extra arguments that should affect the hash. llvm::hash_combine_range(extraArgs.begin(), extraArgs.end()), // Application extension. unsigned(CI.getLangOptions().EnableAppExtensionRestrictions), // Whether or not OSSA modules are enabled. // // If OSSA modules are enabled, we use a separate namespace of modules to // ensure that we compile all swift interface files with the option set. unsigned(CI.getSILOptions().EnableOSSAModules), // Is the C++ interop enabled? unsigned(CI.getLangOptions().EnableCXXInterop) ); return llvm::toString(llvm::APInt(64, H), 36, /*Signed=*/false); } void setOutputPath(ResultTy &resolvedOutputPath, const StringRef &moduleName, const StringRef &interfacePath, const StringRef &sdkPath, const CompilerInvocation &CI, const ArgListTy &extraArgs) { auto &outputPath = resolvedOutputPath.outputPath; outputPath = CI.getClangModuleCachePath(); auto isPrefixedWith = [interfacePath](StringRef path) { return !path.empty() && hasPrefix(llvm::sys::path::begin(interfacePath.str()), llvm::sys::path::end(interfacePath.str()), llvm::sys::path::begin(path), llvm::sys::path::end(path)); }; // Dependency-scanner-specific module output path handling if ((CI.getFrontendOptions().RequestedAction == FrontendOptions::ActionType::ScanDependencies)) { auto runtimeResourcePath = CI.getSearchPathOptions().RuntimeResourcePath; if (isPrefixedWith(sdkPath) || isPrefixedWith(runtimeResourcePath)) outputPath = CI.getFrontendOptions().ExplicitSDKModulesOutputPath; else outputPath = CI.getFrontendOptions().ExplicitModulesOutputPath; } llvm::sys::path::append(outputPath, moduleName); outputPath.append("-"); auto hashStart = outputPath.size(); outputPath.append(getContextHash(CI, interfacePath, sdkPath, extraArgs)); resolvedOutputPath.hash = outputPath.str().substr(hashStart); outputPath.append("."); auto outExt = file_types::getExtension(file_types::TY_SwiftModuleFile); outputPath.append(outExt); return; } } // namespace swift::SwiftInterfaceModuleOutputPathResolution