averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
76bde39ee72d4a8440d44f665fdd7ee6f60a74a9
swift-mirror/tools/libSwiftScan/SwiftCaching.cpp
Steven Wu 76bde39ee7 [Caching] Encoding cache key for input file with index instead of path 
Avoid path encoding difference (for example, real_path vs. path from
symlink) by eliminating the path from cache key. Cache key is now
encoded with the index of the input file from all the input files from
the command-line, reguardless if those inputs will produce output or
not. This is to ensure stable ordering even the batching is different.

Add a new cache computation API that is preferred for using input index
directly. Old API for cache key is deprecated but still updated to
fallback to real_path comparsion if needed.

As a result of swift scan API change, rename the feature in JSON file to
avoid version confusion between swift-driver and libSwiftScan.

rdar://119387650
2023-12-18 14:06:00 -08:00

1010 lines
34 KiB
C++
Raw Blame History

//===------------ SwiftCaching.cpp - Swift Compiler -----------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// Implementation of the caching APIs in libSwiftScan
//
//===----------------------------------------------------------------------===//
#include "swift/AST/DiagnosticEngine.h"
#include "swift/AST/DiagnosticsFrontend.h"
#include "swift/Basic/Defer.h"
#include "swift/Basic/FileTypes.h"
#include "swift/Basic/SourceManager.h"
#include "swift/DependencyScan/DependencyScanImpl.h"
#include "swift/DependencyScan/StringUtils.h"
#include "swift/Frontend/CachedDiagnostics.h"
#include "swift/Frontend/CachingUtils.h"
#include "swift/Frontend/CompileJobCacheKey.h"
#include "swift/Frontend/CompileJobCacheResult.h"
#include "swift/Frontend/Frontend.h"
#include "swift/Frontend/PrintingDiagnosticConsumer.h"
#include "swift/Option/Options.h"
#include "clang/CAS/CASOptions.h"
#include "clang/Frontend/CompileJobCacheResult.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CAS/ActionCache.h"
#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
#include "llvm/CAS/CASReference.h"
#include "llvm/CAS/ObjectStore.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/PrefixMapper.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/VirtualOutputBackend.h"
#include "llvm/Support/VirtualOutputBackends.h"
#include "llvm/Support/raw_ostream.h"
#include <variant>
namespace {
/// Helper class to manage CAS/Caching from libSwiftScan C APIs.
class SwiftScanCAS {
public:
llvm::cas::ObjectStore &getCAS() const { return *CAS; }
llvm::cas::ActionCache &getCache() const { return *Cache; }
// Construct SwiftScanCAS.
static llvm::Expected<SwiftScanCAS *>
createSwiftScanCAS(llvm::StringRef Path);
static llvm::Expected<SwiftScanCAS *>
createSwiftScanCAS(clang::CASOptions &CASOpts);
private:
SwiftScanCAS(std::shared_ptr<llvm::cas::ObjectStore> CAS,
std::shared_ptr<llvm::cas::ActionCache> Cache)
: CAS(CAS), Cache(Cache) {}
std::shared_ptr<llvm::cas::ObjectStore> CAS;
std::shared_ptr<llvm::cas::ActionCache> Cache;
};
struct SwiftCachedCompilationHandle {
SwiftCachedCompilationHandle(llvm::cas::ObjectRef Key,
llvm::cas::ObjectRef Output,
clang::cas::CompileJobCacheResult &&Result,
unsigned InputIndex, SwiftScanCAS &CAS)
: Key(Key), Output(Output), InputIndex(InputIndex), Result(Result),
DB(CAS) {}
SwiftCachedCompilationHandle(llvm::cas::ObjectRef Key,
llvm::cas::ObjectRef Output,
swift::cas::CompileJobCacheResult &&Result,
unsigned InputIndex, SwiftScanCAS &CAS)
: Key(Key), Output(Output), InputIndex(InputIndex), Result(Result),
DB(CAS) {}
llvm::cas::ObjectRef Key;
llvm::cas::ObjectRef Output;
unsigned InputIndex;
std::variant<swift::cas::CompileJobCacheResult,
clang::cas::CompileJobCacheResult>
Result;
SwiftScanCAS &DB;
};
struct SwiftCachedOutputHandle {
llvm::cas::ObjectRef Ref;
swift::file_types::ID Kind;
llvm::cas::ObjectStore &CAS;
std::optional<llvm::cas::ObjectProxy> LoadedProxy;
SwiftCachedOutputHandle(llvm::cas::ObjectRef Ref, swift::file_types::ID Kind,
llvm::cas::ObjectStore &CAS)
: Ref(Ref), Kind(Kind), CAS(CAS) {}
};
struct SwiftScanReplayInstance {
swift::CompilerInvocation Invocation;
llvm::BumpPtrAllocator StringAlloc;
std::vector<const char *> Args;
};
struct SwiftCachedReplayResult {
llvm::SmallVector<char> outMsg;
llvm::SmallVector<char> errMsg;
llvm::raw_svector_ostream outOS;
llvm::raw_svector_ostream errOS;
SwiftCachedReplayResult() : outOS(outMsg), errOS(errMsg) {}
};
} // namespace
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(clang::CASOptions, swiftscan_cas_options_t)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftScanCAS, swiftscan_cas_t)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftCachedCompilationHandle,
swiftscan_cached_compilation_t)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftCachedOutputHandle,
swiftscan_cached_output_t)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftScanReplayInstance,
swiftscan_cache_replay_instance_t)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftCachedReplayResult,
swiftscan_cache_replay_result_t)
//=== CAS Functions ----------------------------------------------------------//
swiftscan_cas_options_t swiftscan_cas_options_create() {
clang::CASOptions *CASOpts = new clang::CASOptions();
return wrap(CASOpts);
}
void swiftscan_cas_options_dispose(swiftscan_cas_options_t options) {
delete unwrap(options);
}
void swiftscan_cas_options_set_ondisk_path(swiftscan_cas_options_t options,
const char *path) {
unwrap(options)->CASPath = path;
}
void swiftscan_cas_options_set_plugin_path(swiftscan_cas_options_t options,
const char *path) {
unwrap(options)->PluginPath = path;
}
bool swiftscan_cas_options_set_plugin_option(swiftscan_cas_options_t options,
const char *name,
const char *value,
swiftscan_string_ref_t *error) {
unwrap(options)->PluginOptions.emplace_back(name, value);
return false;
}
swiftscan_cas_t
swiftscan_cas_create_from_options(swiftscan_cas_options_t options,
swiftscan_string_ref_t *error) {
clang::CASOptions *opts = unwrap(options);
auto cas = SwiftScanCAS::createSwiftScanCAS(*opts);
if (!cas) {
*error =
swift::c_string_utils::create_clone(toString(cas.takeError()).c_str());
return nullptr;
}
return wrap(*cas);
}
void swiftscan_cas_dispose(swiftscan_cas_t cas) { delete unwrap(cas); }
swiftscan_string_ref_t swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data,
unsigned size,
swiftscan_string_ref_t *error) {
auto failure = [&](llvm::Error E) {
*error =
swift::c_string_utils::create_clone(toString(std::move(E)).c_str());
return swift::c_string_utils::create_null();
};
auto &CAS = unwrap(cas)->getCAS();
llvm::StringRef StrContent((char *)data, size);
auto Result = CAS.storeFromString({}, StrContent);
if (!Result)
return failure(Result.takeError());
*error = swift::c_string_utils::create_null();
return swift::c_string_utils::create_clone(
CAS.getID(*Result).toString().c_str());
}
static llvm::Expected<std::string>
computeCacheKey(llvm::cas::ObjectStore &CAS, llvm::ArrayRef<const char *> Args,
llvm::StringRef InputPath) {
auto BaseKey = swift::createCompileJobBaseCacheKey(CAS, Args);
if (!BaseKey)
return BaseKey.takeError();
// Parse the arguments to figure out the index for the input.
swift::CompilerInvocation Invocation;
swift::SourceManager SourceMgr;
swift::DiagnosticEngine Diags(SourceMgr);
llvm::SmallString<128> workingDirectory;
llvm::sys::fs::current_path(workingDirectory);
llvm::SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 4>
configurationFileBuffers;
std::string MainExecutablePath = llvm::sys::fs::getMainExecutable(
"swift-frontend", (void *)swiftscan_cache_replay_compilation);
// Drop the `-frontend` option if it is passed.
if (llvm::StringRef(Args.front()) == "-frontend")
Args = Args.drop_front();
if (Invocation.parseArgs(Args, Diags, &configurationFileBuffers,
workingDirectory, MainExecutablePath))
return llvm::createStringError(llvm::inconvertibleErrorCode(),
"Argument parsing failed");
auto computeKey = [&](unsigned Index) -> llvm::Expected<std::string> {
auto Key = swift::createCompileJobCacheKeyForOutput(CAS, *BaseKey, Index);
if (!Key)
return Key.takeError();
return CAS.getID(*Key).toString();
};
auto AllInputs =
Invocation.getFrontendOptions().InputsAndOutputs.getAllInputs();
// First pass, check for path equal.
for (unsigned Idx = 0; Idx < AllInputs.size(); ++Idx) {
if (AllInputs[Idx].getFileName() == InputPath)
return computeKey(Idx);
}
// If not found, slow second iteration with real_path.
llvm::SmallString<256> InputRealPath;
llvm::sys::fs::real_path(InputPath, InputRealPath, true);
for (unsigned Idx = 0; Idx < AllInputs.size(); ++Idx) {
llvm::SmallString<256> TestRealPath;
llvm::sys::fs::real_path(AllInputs[Idx].getFileName(), TestRealPath, true);
if (InputRealPath == TestRealPath)
return computeKey(Idx);
}
return llvm::createStringError(llvm::inconvertibleErrorCode(),
"requested input not found from invocation");
}
static llvm::Expected<std::string>
computeCacheKeyFromIndex(llvm::cas::ObjectStore &CAS,
llvm::ArrayRef<const char *> Args,
unsigned InputIndex) {
auto BaseKey = swift::createCompileJobBaseCacheKey(CAS, Args);
if (!BaseKey)
return BaseKey.takeError();
auto Key =
swift::createCompileJobCacheKeyForOutput(CAS, *BaseKey, InputIndex);
if (!Key)
return Key.takeError();
return CAS.getID(*Key).toString();
}
swiftscan_string_ref_t
swiftscan_cache_compute_key(swiftscan_cas_t cas, int argc, const char **argv,
const char *input, swiftscan_string_ref_t *error) {
std::vector<const char *> Compilation;
for (int i = 0; i < argc; ++i)
Compilation.push_back(argv[i]);
auto ID = computeCacheKey(unwrap(cas)->getCAS(), Compilation, input);
if (!ID) {
*error =
swift::c_string_utils::create_clone(toString(ID.takeError()).c_str());
return swift::c_string_utils::create_null();
}
*error = swift::c_string_utils::create_null();
return swift::c_string_utils::create_clone(ID->c_str());
}
swiftscan_string_ref_t
swiftscan_cache_compute_key_from_input_index(swiftscan_cas_t cas, int argc,
const char **argv,
unsigned input_index,
swiftscan_string_ref_t *error) {
std::vector<const char *> Compilation;
for (int i = 0; i < argc; ++i)
Compilation.push_back(argv[i]);
auto ID =
computeCacheKeyFromIndex(unwrap(cas)->getCAS(), Compilation, input_index);
if (!ID) {
*error =
swift::c_string_utils::create_clone(toString(ID.takeError()).c_str());
return swift::c_string_utils::create_null();
}
*error = swift::c_string_utils::create_null();
return swift::c_string_utils::create_clone(ID->c_str());
}
// Create a non-owning string ref that is used in call backs.
static swiftscan_string_ref_t createNonOwningString(llvm::StringRef Str) {
if (Str.empty())
return swift::c_string_utils::create_null();
swiftscan_string_ref_t Result;
Result.data = Str.data();
Result.length = Str.size();
return Result;
}
static llvm::Error createUnsupportedSchemaError() {
return llvm::createStringError(llvm::inconvertibleErrorCode(),
"unsupported compile result schema found");
}
static llvm::Expected<SwiftCachedCompilationHandle *>
createCachedCompilation(SwiftScanCAS &CAS, const llvm::cas::CASID &ID,
const llvm::cas::CASID &Key) {
auto Ref = CAS.getCAS().getReference(ID);
if (!Ref)
return nullptr;
auto Proxy = CAS.getCAS().getProxy(*Ref);
if (!Proxy)
return Proxy.takeError();
// Load input file name from the key CAS object. Input file name is the data
// blob in the root node.
auto KeyProxy = CAS.getCAS().getProxy(Key);
if (!KeyProxy)
return KeyProxy.takeError();
auto Input = KeyProxy->getData();
unsigned Index =
llvm::support::endian::read<uint32_t, llvm::support::little,
llvm::support::unaligned>(Input.data());
{
swift::cas::CompileJobResultSchema Schema(CAS.getCAS());
if (Schema.isRootNode(*Proxy)) {
auto Result = Schema.load(Proxy->getRef());
if (!Result)
return Result.takeError();
return new SwiftCachedCompilationHandle(KeyProxy->getRef(), *Ref,
std::move(*Result), Index, CAS);
}
}
{
clang::cas::CompileJobResultSchema Schema(CAS.getCAS());
if (Schema.isRootNode(*Proxy)) {
auto Result = Schema.load(Proxy->getRef());
if (!Result)
return Result.takeError();
return new SwiftCachedCompilationHandle(KeyProxy->getRef(), *Ref,
std::move(*Result), Index, CAS);
}
}
return createUnsupportedSchemaError();
}
swiftscan_cached_compilation_t
swiftscan_cache_query(swiftscan_cas_t cas, const char *key, bool globally,
swiftscan_string_ref_t *error) {
auto failure = [&](llvm::Error E) {
*error =
swift::c_string_utils::create_clone(toString(std::move(E)).c_str());
return nullptr;
};
auto notfound = [&]() {
*error = swift::c_string_utils::create_null();
return nullptr;
};
auto &CAS = unwrap(cas)->getCAS();
auto &Cache = unwrap(cas)->getCache();
auto ID = CAS.parseID(key);
if (!ID)
return failure(ID.takeError());
auto Result = Cache.get(*ID, globally);
if (!Result)
return failure(Result.takeError());
if (!*Result)
return notfound();
auto Cached = createCachedCompilation(*unwrap(cas), **Result, *ID);
if (!Cached)
return failure(Cached.takeError());
if (!*Cached)
return notfound();
*error = swift::c_string_utils::create_null();
return wrap(*Cached);
}
void swiftscan_cache_query_async(
swiftscan_cas_t cas, const char *key, bool globally, void *ctx,
void (*callback)(void *ctx, swiftscan_cached_compilation_t,
swiftscan_string_ref_t error),
swiftscan_cache_cancellation_token_t *token) {
if (token)
*token = nullptr;
auto &CAS = unwrap(cas)->getCAS();
auto &Cache = unwrap(cas)->getCache();
auto failure = [ctx, callback](llvm::Error E) {
std::string ErrMsg = toString(std::move(E));
return callback(ctx, nullptr, createNonOwningString(ErrMsg));
};
auto notfound = [ctx, callback]() {
return callback(ctx, nullptr, swift::c_string_utils::create_null());
};
auto success = [ctx, callback](swiftscan_cached_compilation_t comp) {
return callback(ctx, comp, swift::c_string_utils::create_null());
};
auto ID = CAS.parseID(key);
if (!ID)
return failure(ID.takeError());
auto KeyID = *ID;
Cache.getAsync(*ID, globally,
[failure, notfound, success, cas, KeyID](
llvm::Expected<std::optional<llvm::cas::CASID>> Result) {
if (!Result)
return failure(Result.takeError());
if (!*Result)
return notfound();
auto Cached =
createCachedCompilation(*unwrap(cas), **Result, KeyID);
if (!Cached)
return failure(Cached.takeError());
if (!*Cached)
return notfound();
return success(wrap(*Cached));
});
}
unsigned swiftscan_cached_compilation_get_num_outputs(
swiftscan_cached_compilation_t id) {
auto *Comp = unwrap(id);
if (auto *Result =
std::get_if<swift::cas::CompileJobCacheResult>(&Comp->Result))
return Result->getNumOutputs();
auto *Result = std::get_if<clang::cas::CompileJobCacheResult>(&Comp->Result);
assert(Result && "unexpected variant");
return Result->getNumOutputs();
}
swiftscan_cached_output_t
swiftscan_cached_compilation_get_output(swiftscan_cached_compilation_t id,
unsigned idx) {
auto *Comp = unwrap(id);
if (auto *Result =
std::get_if<swift::cas::CompileJobCacheResult>(&Comp->Result))
return wrap(new SwiftCachedOutputHandle(Result->getOutput(idx).Object,
Result->getOutput(idx).Kind,
Comp->DB.getCAS()));
auto *Result = std::get_if<clang::cas::CompileJobCacheResult>(&Comp->Result);
assert(Result && "unexpected variant");
swift::file_types::ID Kind = swift::file_types::TY_INVALID;
switch (Result->getOutput(idx).Kind) {
case clang::cas::CompileJobCacheResult::OutputKind::MainOutput:
Kind = swift::file_types::TY_ClangModuleFile;
break;
case clang::cas::CompileJobCacheResult::OutputKind::Dependencies:
Kind = swift::file_types::TY_Dependencies;
break;
case clang::cas::CompileJobCacheResult::OutputKind::SerializedDiagnostics:
Kind = swift::file_types::TY_CachedDiagnostics;
break;
}
assert(Kind != swift::file_types::TY_INVALID && "Invalid kind");
return wrap(new SwiftCachedOutputHandle(Result->getOutput(idx).Object, Kind,
Comp->DB.getCAS()));
}
bool swiftscan_cached_compilation_is_uncacheable(
swiftscan_cached_compilation_t id) {
// Currently, all compilations are cacheable.
return false;
}
void swiftscan_cached_compilation_dispose(swiftscan_cached_compilation_t id) {
delete unwrap(id);
}
bool swiftscan_cached_output_load(swiftscan_cached_output_t output,
swiftscan_string_ref_t *error) {
auto *Out = unwrap(output);
auto failure = [&](llvm::Error E) {
*error =
swift::c_string_utils::create_clone(toString(std::move(E)).c_str());
return false;
};
auto notfound = [&]() {
*error = swift::c_string_utils::create_null();
return false;
};
auto success = [&]() {
*error = swift::c_string_utils::create_null();
return true;
};
// If proxy exists, there is nothing to be done.
if (Out->LoadedProxy)
return success();
if (auto Err = Out->CAS.getProxyIfExists(Out->Ref).moveInto(Out->LoadedProxy))
return failure(std::move(Err));
if (!Out->LoadedProxy)
return notfound();
llvm::DenseSet<llvm::cas::ObjectRef> Visited;
llvm::SmallVector<llvm::cas::ObjectRef> WorkList;
auto addToWorkList = [&](llvm::cas::ObjectRef Item) {
if (Visited.insert(Item).second)
WorkList.push_back(Item);
};
addToWorkList(Out->LoadedProxy->getRef());
while (!WorkList.empty()) {
auto Current = WorkList.pop_back_val();
auto Proxy = Out->CAS.getProxyIfExists(Current);
if (!Proxy)
return failure(Proxy.takeError());
if (!*Proxy)
return notfound();
if (auto Err = (*Proxy)->forEachReference(
[&](llvm::cas::ObjectRef Ref) -> llvm::Error {
addToWorkList(Ref);
return llvm::Error::success();
}))
return failure(std::move(Err));
}
return success();
}
namespace {
/// Asynchronously visits the graph of the object node to ensure it's fully
/// materialized.
class AsyncObjectLoader
: public std::enable_shared_from_this<AsyncObjectLoader> {
void *Ctx;
void (*Callback)(void *Ctx, bool, swiftscan_string_ref_t);
llvm::cas::ObjectStore &CAS;
// The output handle to update after load if set.
SwiftCachedOutputHandle *OutputHandle;
llvm::SmallDenseSet<llvm::cas::ObjectRef> ObjectsSeen;
unsigned NumPending = 0;
std::optional<llvm::cas::ObjectProxy> RootObj;
std::atomic<bool> MissingNode{false};
/// The first error that occurred.
std::optional<llvm::Error> ErrOccurred;
std::mutex Mutex;
public:
AsyncObjectLoader(void *Ctx,
void (*Callback)(void *Ctx, bool, swiftscan_string_ref_t),
llvm::cas::ObjectStore &CAS,
SwiftCachedOutputHandle *Handle = nullptr)
: Ctx(Ctx), Callback(Callback), CAS(CAS), OutputHandle(Handle) {}
void visit(llvm::cas::ObjectRef Ref, bool IsRootNode) {
{
std::lock_guard<std::mutex> Guard(Mutex);
if (!ObjectsSeen.insert(Ref).second)
return;
++NumPending;
}
auto This = shared_from_this();
CAS.getProxyAsync(
Ref, [This, IsRootNode](
llvm::Expected<std::optional<llvm::cas::ObjectProxy>> Obj) {
SWIFT_DEFER { This->finishedNode(); };
if (!Obj) {
This->encounteredError(Obj.takeError());
return;
}
if (!*Obj) {
This->MissingNode = true;
return;
}
if (IsRootNode)
This->RootObj = *Obj;
if (auto Err = (*Obj)->forEachReference(
[&This](llvm::cas::ObjectRef Sub) -> llvm::Error {
This->visit(Sub, /*IsRootNode*/ false);
return llvm::Error::success();
})) {
This->encounteredError(std::move(Err));
return;
}
});
}
void finishedNode() {
{
std::lock_guard<std::mutex> Guard(Mutex);
assert(NumPending);
--NumPending;
if (NumPending != 0)
return;
}
if (ErrOccurred) {
std::string ErrMsg = toString(std::move(*ErrOccurred));
return Callback(Ctx, false, createNonOwningString(ErrMsg));
}
if (MissingNode)
return Callback(Ctx, false, swift::c_string_utils::create_null());
if (OutputHandle)
OutputHandle->LoadedProxy = RootObj;
return Callback(Ctx, true, swift::c_string_utils::create_null());
}
/// Only keeps the first error that occurred.
void encounteredError(llvm::Error &&E) {
std::lock_guard<std::mutex> Guard(Mutex);
if (ErrOccurred) {
llvm::consumeError(std::move(E));
return;
}
ErrOccurred = std::move(E);
}
};
} // namespace
void swiftscan_cached_output_load_async(
swiftscan_cached_output_t output, void *ctx,
void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error),
swiftscan_cache_cancellation_token_t *token) {
if (token)
*token = nullptr;
auto *Out = unwrap(output);
if (Out->LoadedProxy)
return callback(ctx, true, swift::c_string_utils::create_null());
auto Loader =
std::make_shared<AsyncObjectLoader>(ctx, callback, Out->CAS, Out);
Loader->visit(Out->Ref, /*IsRootNode*/ true);
}
void swiftscan_cache_download_cas_object_async(
swiftscan_cas_t cas, const char *id, void *ctx,
void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error),
swiftscan_cache_cancellation_token_t *token) {
if (token)
*token = nullptr;
auto failure = [ctx, callback](llvm::Error E) {
std::string ErrMsg = toString(std::move(E));
return callback(ctx, false, createNonOwningString(ErrMsg));
};
auto &CAS = unwrap(cas)->getCAS();
auto ID = CAS.parseID(id);
if (!ID)
return failure(ID.takeError());
auto Ref = CAS.getReference(*ID);
if (!Ref)
return callback(ctx, false, swift::c_string_utils::create_null());
auto Loader =
std::make_shared<AsyncObjectLoader>(ctx, callback, CAS);
Loader->visit(*Ref, /*IsRootNode*/ true);
}
bool swiftscan_cached_output_is_materialized(swiftscan_cached_output_t output) {
auto *Out = unwrap(output);
// Already loaded.
if (Out->LoadedProxy)
return true;
auto Loaded = Out->CAS.isMaterialized(Out->Ref);
if (!Loaded) {
// There is an error loading, output is not materialized. Discard error and
// return false.
consumeError(Loaded.takeError());
return false;
}
return *Loaded;
}
swiftscan_string_ref_t
swiftscan_cached_output_get_casid(swiftscan_cached_output_t output) {
auto *Out = unwrap(output);
auto ID = Out->CAS.getID(Out->Ref);
return swift::c_string_utils::create_clone(ID.toString().c_str());
}
void swiftscan_cached_compilation_make_global_async(
swiftscan_cached_compilation_t comp, void *ctx,
void (*callback)(void *ctx, swiftscan_string_ref_t error),
swiftscan_cache_cancellation_token_t *token) {
if (token)
*token = nullptr;
auto failure = [ctx, callback](llvm::Error E) {
std::string ErrMsg = toString(std::move(E));
return callback(ctx, createNonOwningString(ErrMsg));
};
auto success = [ctx, callback]() {
return callback(ctx, swift::c_string_utils::create_null());
};
auto *Compilation = unwrap(comp);
auto &CAS = Compilation->DB.getCAS();
auto &Cache = Compilation->DB.getCache();
auto ID = CAS.getID(Compilation->Key);
auto Result = CAS.getID(Compilation->Output);
Cache.putAsync(ID, Result, /*globally=*/true,
[failure, success](llvm::Error E) {
if (E)
return failure(std::move(E));
return success();
});
}
swiftscan_string_ref_t
swiftscan_cached_output_get_name(swiftscan_cached_output_t output) {
auto *Out = unwrap(output);
return swift::c_string_utils::create_clone(
swift::file_types::getTypeName(Out->Kind).str().c_str());
}
swiftscan_cache_replay_instance_t
swiftscan_cache_replay_instance_create(int argc, const char **argv,
swiftscan_string_ref_t *error) {
auto *Instance = new SwiftScanReplayInstance();
llvm::StringSaver Saver(Instance->StringAlloc);
for (int i = 0; i < argc; ++i) {
auto Str = Saver.save(argv[i]);
Instance->Args.push_back(Str.data());
}
// Capture the diagnostics when creating invocation.
std::string err_msg;
llvm::raw_string_ostream OS(err_msg);
swift::PrintingDiagnosticConsumer Diags(OS);
swift::SourceManager SrcMgr;
swift::DiagnosticEngine DE(SrcMgr);
DE.addConsumer(Diags);
std::string MainExecutablePath = llvm::sys::fs::getMainExecutable(
"swift-frontend", (void *)swiftscan_cache_replay_compilation);
if (Instance->Invocation.parseArgs(Instance->Args, DE, nullptr, {},
MainExecutablePath)) {
delete Instance;
*error = swift::c_string_utils::create_clone(err_msg.c_str());
return nullptr;
}
if (!Instance->Invocation.getFrontendOptions().EnableCaching) {
delete Instance;
*error = swift::c_string_utils::create_clone(
"caching is not enabled from command-line");
return nullptr;
}
return wrap(Instance);
}
void swiftscan_cache_replay_instance_dispose(
swiftscan_cache_replay_instance_t instance) {
delete unwrap(instance);
}
namespace {
class StreamOutputFileImpl final
: public llvm::RTTIExtends<StreamOutputFileImpl,
llvm::vfs::OutputFileImpl> {
public:
StreamOutputFileImpl(llvm::raw_pwrite_stream &OS) : OS(OS) {}
llvm::Error keep() final { return llvm::Error::success(); }
llvm::Error discard() final { return llvm::Error::success(); }
llvm::raw_pwrite_stream &getOS() final { return OS; }
private:
llvm::raw_pwrite_stream &OS;
};
// The replay output backend. Currently, it redirects "-" to the
// stdout provided.
struct ReplayOutputBackend : public llvm::vfs::ProxyOutputBackend {
llvm::Expected<std::unique_ptr<llvm::vfs::OutputFileImpl>>
createFileImpl(llvm::StringRef Path,
std::optional<llvm::vfs::OutputConfig> Config) override {
if (Path == "-")
return std::make_unique<StreamOutputFileImpl>(StdOut);
return llvm::vfs::ProxyOutputBackend::createFileImpl(Path, Config);
}
llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend>
cloneImpl() const override {
return llvm::makeIntrusiveRefCnt<ReplayOutputBackend>(
getUnderlyingBackend().clone(), StdOut);
}
ReplayOutputBackend(
llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> UnderlyingBackend,
llvm::raw_pwrite_stream &StdOut)
: ProxyOutputBackend(UnderlyingBackend), StdOut(StdOut) {}
llvm::raw_pwrite_stream &StdOut;
};
} // namespace
static llvm::Error replayCompilation(SwiftScanReplayInstance &Instance,
llvm::cas::ObjectStore &CAS,
SwiftCachedCompilationHandle &Comp,
llvm::raw_pwrite_stream &Out,
llvm::raw_pwrite_stream &Err) {
using namespace swift;
using namespace llvm;
CompilerInstance Inst;
CompilerInvocation &Invocation = Instance.Invocation;
// Find the input file from the invocation.
auto &InputsAndOutputs =
Instance.Invocation.getFrontendOptions().InputsAndOutputs;
auto AllInputs = InputsAndOutputs.getAllInputs();
if (Comp.InputIndex >= AllInputs.size())
return createStringError(inconvertibleErrorCode(),
"InputFile index too large for compilation");
const auto &Input = AllInputs[Comp.InputIndex];
// Setup DiagnosticsConsumers.
// FIXME: Reduce code duplication against `performFrontend()` and add support
// for JSONFIXIT, SerializedDiagnostics, etc.
PrintingDiagnosticConsumer PDC(Err);
Inst.addDiagnosticConsumer(&PDC);
if (Invocation.getDiagnosticOptions().UseColor)
PDC.forceColors();
PDC.setPrintEducationalNotes(
Invocation.getDiagnosticOptions().PrintEducationalNotes);
PDC.setFormattingStyle(
Invocation.getDiagnosticOptions().PrintedFormattingStyle);
PDC.setEmitMacroExpansionFiles(
Invocation.getDiagnosticOptions().EmitMacroExpansionFiles);
std::string InstanceSetupError;
if (Inst.setup(Instance.Invocation, InstanceSetupError, Instance.Args))
return createStringError(inconvertibleErrorCode(), InstanceSetupError);
auto *CDP = Inst.getCachingDiagnosticsProcessor();
assert(CDP && "CachingDiagnosticsProcessor needs to be setup for replay");
// No diags are captured in replay instance.
CDP->endDiagnosticCapture();
// Collect the file that needs to write.
DenseMap<file_types::ID, std::string> Outputs;
if (!Input.outputFilename().empty())
Outputs.try_emplace(InputsAndOutputs.getPrincipalOutputType(),
Input.outputFilename());
Input.getPrimarySpecificPaths().SupplementaryOutputs.forEachSetOutputAndType(
[&](const std::string &File, file_types::ID ID) {
if (file_types::isProducedFromDiagnostics(ID))
return;
Outputs.try_emplace(ID, File);
});
Outputs.try_emplace(file_types::TY_CachedDiagnostics, "<cached-diagnostics>");
// Load all the output buffer.
bool Remarks = Instance.Invocation.getFrontendOptions().EnableCachingRemarks;
struct OutputEntry {
std::string Path;
llvm::cas::ObjectProxy Proxy;
};
SmallVector<OutputEntry> OutputProxies;
std::optional<llvm::cas::ObjectProxy> DiagnosticsOutput;
swift::cas::CachedResultLoader Loader(CAS, Comp.Output);
if (auto Err = Loader.replay(
[&](file_types::ID Kind, llvm::cas::ObjectRef Ref) -> Error {
auto OutputPath = Outputs.find(Kind);
if (OutputPath == Outputs.end())
return createStringError(
inconvertibleErrorCode(),
"unexpected output kind in the cached output");
auto Proxy = CAS.getProxy(Ref);
if (!Proxy)
return Proxy.takeError();
if (Kind == file_types::ID::TY_CachedDiagnostics) {
assert(!DiagnosticsOutput && "more than 1 diagnotics found");
DiagnosticsOutput = std::move(*Proxy);
} else
OutputProxies.emplace_back(
OutputEntry{OutputPath->second, std::move(*Proxy)});
return Error::success();
}))
return Err;
// Replay diagnostics first.
// FIXME: Currently, the diagnostics is replay from the first file.
if (DiagnosticsOutput) {
if (auto E = CDP->replayCachedDiagnostics(DiagnosticsOutput->getData()))
return E;
if (Remarks)
Inst.getDiags().diagnose(SourceLoc(), diag::replay_output,
"<cached-diagnostics>",
CAS.getID(Comp.Key).toString());
}
// OutputBackend for replay.
ReplayOutputBackend Backend(
makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>(), Out);
for (auto &Output : OutputProxies) {
auto File = Backend.createFile(Output.Path);
if (!File)
return File.takeError();
*File << Output.Proxy.getData();
if (auto E = File->keep())
return E;
if (Remarks)
Inst.getDiags().diagnose(SourceLoc(), diag::replay_output, Output.Path,
CAS.getID(Comp.Key).toString());
}
return llvm::Error::success();
}
swiftscan_cache_replay_result_t
swiftscan_cache_replay_compilation(swiftscan_cache_replay_instance_t instance,
swiftscan_cached_compilation_t comp,
swiftscan_string_ref_t *error) {
SwiftScanReplayInstance &Instance = *unwrap(instance);
SwiftCachedCompilationHandle &Comp = *unwrap(comp);
SwiftCachedReplayResult *Result = new SwiftCachedReplayResult();
if (auto err = replayCompilation(Instance, Comp.DB.getCAS(), Comp,
Result->outOS, Result->errOS)) {
*error =
swift::c_string_utils::create_clone(toString(std::move(err)).c_str());
delete Result;
return nullptr;
}
*error = swift::c_string_utils::create_null();
return wrap(Result);
}
swiftscan_string_ref_t swiftscan_cache_replay_result_get_stdout(
swiftscan_cache_replay_result_t result) {
return createNonOwningString(unwrap(result)->outOS.str());
}
swiftscan_string_ref_t swiftscan_cache_replay_result_get_stderr(
swiftscan_cache_replay_result_t result) {
return createNonOwningString(unwrap(result)->errOS.str());
}
void swiftscan_cached_output_dispose(swiftscan_cached_output_t out) {
delete unwrap(out);
}
void swiftscan_cache_replay_result_dispose(
swiftscan_cache_replay_result_t result) {
delete unwrap(result);
}
// FIXME: implement cancellation.
void swiftscan_cache_action_cancel(swiftscan_cache_cancellation_token_t) {}
void swiftscan_cache_cancellation_token_dispose(
swiftscan_cache_cancellation_token_t) {}
llvm::Expected<SwiftScanCAS *>
SwiftScanCAS::createSwiftScanCAS(llvm::StringRef Path) {
clang::CASOptions Opts;
Opts.CASPath = Path;
return createSwiftScanCAS(Opts);
}
llvm::Expected<SwiftScanCAS *>
SwiftScanCAS::createSwiftScanCAS(clang::CASOptions &CASOpts) {
auto DB = CASOpts.getOrCreateDatabases();
if (!DB)
return DB.takeError();
return new SwiftScanCAS(std::move(DB->first), std::move(DB->second));
}
Reference in New Issue View Git Blame Copy Permalink