averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

[CAS] swift dependency scanning using CAS for compiler caching (#66366)

Browse Source 
Teach swift dependency scanner to use CAS to capture the full dependencies for a build and construct build commands with immutable inputs from CAS.

This allows swift compilation caching using CAS.
This commit is contained in:
Steven Wu
2023-06-12 10:55:53 -07:00
committed by GitHub
parent 2db4a038c3
commit b1f99b8e93
34 changed files with 2671 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
include/swift/Frontend/ModuleInterfaceLoader.h
View File

@@ -111,9 +111,17 @@
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/ModuleInterfaceSupport.h"
#include "swift/Serialization/SerializedModuleLoader.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/YAMLTraits.h"
namespace llvm {
namespace cas {
class ObjectStore;
class ActionCache;
} // namespace cas
} // namespace llvm
namespace clang {
class CompilerInstance;
}
@@ -178,6 +186,55 @@ public:
~ExplicitSwiftModuleLoader();
};
class ExplicitCASModuleLoader : public SerializedModuleLoaderBase {
explicit ExplicitCASModuleLoader(ASTContext &ctx, llvm::cas::ObjectStore &CAS,
llvm::cas::ActionCache &cache,
DependencyTracker *tracker,
ModuleLoadingMode loadMode,
bool IgnoreSwiftSourceInfoFile);
bool findModule(ImportPath::Element moduleID,
SmallVectorImpl<char> *moduleInterfacePath,
SmallVectorImpl<char> *moduleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
bool skipBuildingInterface, bool isTestableDependencyLookup,
bool &isFramework, bool &isSystemModule) override;
std::error_code findModuleFilesInDirectory(
ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
SmallVectorImpl<char> *ModuleInterfacePath,
SmallVectorImpl<char> *ModuleInterfaceSourcePath,
std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
bool SkipBuildingInterface, bool IsFramework,
bool IsTestableDependencyLookup = false) override;
bool canImportModule(ImportPath::Module named,
ModuleVersionInfo *versionInfo,
bool isTestableDependencyLookup = false) override;
struct Implementation;
Implementation &Impl;
public:
static std::unique_ptr<ExplicitCASModuleLoader>
create(ASTContext &ctx, llvm::cas::ObjectStore &CAS,
llvm::cas::ActionCache &cache, DependencyTracker *tracker,
ModuleLoadingMode loadMode, StringRef ExplicitSwiftModuleMap,
const std::vector<std::pair<std::string, std::string>>
&ExplicitSwiftModuleInputs,
bool IgnoreSwiftSourceInfoFile);
/// Append visible module names to \p names. Note that names are possibly
/// duplicated, and not guaranteed to be ordered in any way.
void collectVisibleTopLevelModuleNames(
SmallVectorImpl<Identifier> &names) const override;
~ExplicitCASModuleLoader();
};
// Explicitly-specified Swift module inputs
struct ExplicitSwiftModuleInputInfo {
@@ -185,12 +242,14 @@ struct ExplicitSwiftModuleInputInfo {
llvm::Optional<std::string> moduleDocPath,
llvm::Optional<std::string> moduleSourceInfoPath,
bool isFramework = false,
bool isSystem = false)
bool isSystem = false,
llvm::Optional<std::string> moduleCacheKey = None)
: modulePath(modulePath),
moduleDocPath(moduleDocPath),
moduleSourceInfoPath(moduleSourceInfoPath),
isFramework(isFramework),
isSystem(isSystem) {}
isSystem(isSystem),
moduleCacheKey(moduleCacheKey) {}
// Path of the .swiftmodule file.
std::string modulePath;
// Path of the .swiftmoduledoc file.
@@ -201,6 +260,8 @@ struct ExplicitSwiftModuleInputInfo {
bool isFramework = false;
// A flag that indicates whether this module is a system module
bool isSystem = false;
// The cache key for clang module.
llvm::Optional<std::string> moduleCacheKey;
};
// Explicitly-specified Clang module inputs
@@ -208,11 +269,13 @@ struct ExplicitClangModuleInputInfo {
ExplicitClangModuleInputInfo(std::string moduleMapPath,
std::string modulePath,
bool isFramework = false,
bool isSystem = false)
bool isSystem = false,
llvm::Optional<std::string> moduleCacheKey = None)
: moduleMapPath(moduleMapPath),
modulePath(modulePath),
isFramework(isFramework),
isSystem(isSystem) {}
isSystem(isSystem),
moduleCacheKey(moduleCacheKey) {}
// Path of the Clang module map file.
std::string moduleMapPath;
// Path of a compiled Clang explicit module file (pcm).
@@ -221,6 +284,8 @@ struct ExplicitClangModuleInputInfo {
bool isFramework = false;
// A flag that indicates whether this module is a system module
bool isSystem = false;
// The cache key for clang module.
llvm::Optional<std::string> moduleCacheKey;
};
/// Parser of explicit module maps passed into the compiler.
@@ -233,6 +298,8 @@ struct ExplicitClangModuleInputInfo {
// "isFramework": false,
// "clangModuleMapPath": "A/module.modulemap",
// "clangModulePath": "A.pcm",
// "moduleCacheKey": "llvmcas://<hash>",
// "clangModuleCacheKey": "llvmcas://<hash>",
// },
// {
// "moduleName": "B",
@@ -242,6 +309,8 @@ struct ExplicitClangModuleInputInfo {
// "isFramework": false,
// "clangModuleMapPath": "B/module.modulemap",
// "clangModulePath": "B.pcm",
// "moduleCacheKey": "llvmcas://<hash>",
// "clangModuleCacheKey": "llvmcas://<hash>",
// }
// ]
class ExplicitModuleMapParser {
@@ -249,21 +318,14 @@ public:
ExplicitModuleMapParser(llvm::BumpPtrAllocator &Allocator) : Saver(Allocator) {}
std::error_code
parseSwiftExplicitModuleMap(llvm::StringRef fileName,
parseSwiftExplicitModuleMap(llvm::MemoryBufferRef BufferRef,
llvm::StringMap<ExplicitSwiftModuleInputInfo> &swiftModuleMap,
llvm::StringMap<ExplicitClangModuleInputInfo> &clangModuleMap) {
using namespace llvm::yaml;
// Load the input file.
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
llvm::MemoryBuffer::getFile(fileName);
if (!fileBufOrErr) {
return std::make_error_code(std::errc::no_such_file_or_directory);
}
StringRef Buffer = fileBufOrErr->get()->getBuffer();
// Use a new source manager instead of the one from ASTContext because we
// don't want the JSON file to be persistent.
llvm::SourceMgr SM;
Stream Stream(llvm::MemoryBufferRef(Buffer, fileName), SM);
Stream Stream(BufferRef, SM);
for (auto DI = Stream.begin(); DI != Stream.end(); ++DI) {
assert(DI != Stream.end() && "Failed to read a document");
if (auto *MN = dyn_cast_or_null<SequenceNode>(DI->getRoot())) {
@@ -305,7 +367,8 @@ private:
return true;
StringRef moduleName;
llvm::Optional<std::string> swiftModulePath, swiftModuleDocPath,
swiftModuleSourceInfoPath;
swiftModuleSourceInfoPath, swiftModuleCacheKey,
clangModuleCacheKey;
std::string clangModuleMapPath = "", clangModulePath = "";
bool isFramework = false, isSystem = false;
for (auto &entry : *mapNode) {
@@ -327,6 +390,10 @@ private:
clangModuleMapPath = val.str();
} else if (key == "clangModulePath") {
clangModulePath = val.str();
} else if (key == "moduleCacheKey") {
swiftModuleCacheKey = val.str();
} else if (key == "clangModuleCacheKey") {
clangModuleCacheKey = val.str();
} else {
// Being forgiving for future fields.
continue;
@@ -343,7 +410,8 @@ private:
swiftModuleDocPath,
swiftModuleSourceInfoPath,
isFramework,
isSystem);
isSystem,
swiftModuleCacheKey);
swiftModuleMap.try_emplace(moduleName, std::move(entry));
} else {
assert((!clangModuleMapPath.empty() ||
@@ -352,7 +420,8 @@ private:
ExplicitClangModuleInputInfo entry(clangModuleMapPath,
clangModulePath,
isFramework,
isSystem);
isSystem,
clangModuleCacheKey);
clangModuleMap.try_emplace(moduleName, std::move(entry));
}