mirror of
https://github.com/apple/swift.git
synced 2025-12-14 20:36:38 +01:00
f0aa023489
Temporarily disable replaying optimization remarks when cache hit. Optimization remarks file are not setup to be cached because: * the write of such files are not going through output backend * when optimization remarks files are requested, it produces files that are not intended to be produced in SupplementoryOutputs, which confuses compiler when storing and loading compilation results, and causes cache misses all the time when optimization remark file is reuqested. Disable optimization remarks for cache replay so it is currently only produced when compiler actually did the compile locally.
378 lines
15 KiB
C++
378 lines
15 KiB
C++
//===--- CASOutputBackends.cpp ----------------------------------*- C++ -*-===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "swift/Frontend/CASOutputBackends.h"
|
|
|
|
#include "swift/Basic/Assertions.h"
|
|
#include "swift/Basic/FileTypes.h"
|
|
#include "swift/Frontend/CachingUtils.h"
|
|
#include "swift/Frontend/CompileJobCacheKey.h"
|
|
#include "swift/Frontend/CompileJobCacheResult.h"
|
|
#include "clang/Frontend/CompileJobCacheResult.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/CAS/HierarchicalTreeBuilder.h"
|
|
#include "llvm/CAS/ObjectStore.h"
|
|
#include "llvm/Support/Debug.h"
|
|
#include "llvm/Support/Mutex.h"
|
|
#include <optional>
|
|
|
|
#define DEBUG_TYPE "swift-cas-backend"
|
|
|
|
using namespace swift;
|
|
using namespace llvm;
|
|
using namespace llvm::cas;
|
|
using namespace llvm::vfs;
|
|
|
|
namespace {
|
|
class SwiftCASOutputFile final : public OutputFileImpl {
|
|
public:
|
|
Error keep() override { return OnKeep(Path, Bytes); }
|
|
Error discard() override { return Error::success(); }
|
|
raw_pwrite_stream &getOS() override { return OS; }
|
|
|
|
using OnKeepType = llvm::unique_function<Error(StringRef, StringRef)>;
|
|
SwiftCASOutputFile(StringRef Path, OnKeepType OnKeep)
|
|
: Path(Path.str()), OS(Bytes), OnKeep(std::move(OnKeep)) {}
|
|
|
|
private:
|
|
std::string Path;
|
|
SmallString<16> Bytes;
|
|
raw_svector_ostream OS;
|
|
OnKeepType OnKeep;
|
|
};
|
|
} // namespace
|
|
|
|
namespace swift::cas {
|
|
void SwiftCASOutputBackend::anchor() {}
|
|
|
|
class SwiftCASOutputBackend::Implementation {
|
|
public:
|
|
Implementation(ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
|
|
const FrontendInputsAndOutputs &InputsAndOutputs,
|
|
const FrontendOptions &Opts,
|
|
FrontendOptions::ActionType Action)
|
|
: CAS(CAS), Cache(Cache), BaseKey(BaseKey),
|
|
InputsAndOutputs(InputsAndOutputs), Opts(Opts), Action(Action) {
|
|
initBackend(InputsAndOutputs);
|
|
}
|
|
|
|
llvm::Expected<std::unique_ptr<llvm::vfs::OutputFileImpl>>
|
|
createFileImpl(llvm::StringRef ResolvedPath,
|
|
std::optional<llvm::vfs::OutputConfig> Config) {
|
|
auto ProducingInput = OutputToInputMap.find(ResolvedPath);
|
|
if (ProducingInput == OutputToInputMap.end() &&
|
|
ResolvedPath.starts_with(Opts.SymbolGraphOutputDir)) {
|
|
return std::make_unique<SwiftCASOutputFile>(
|
|
ResolvedPath, [this](StringRef Path, StringRef Bytes) -> Error {
|
|
bool Shortened = Path.consume_front(Opts.SymbolGraphOutputDir);
|
|
assert(Shortened && "symbol graph path outside output dir");
|
|
(void)Shortened;
|
|
std::optional<ObjectRef> PathRef;
|
|
if (Error E =
|
|
CAS.storeFromString(std::nullopt, Path).moveInto(PathRef))
|
|
return E;
|
|
std::optional<ObjectRef> BytesRef;
|
|
if (Error E =
|
|
CAS.storeFromString(std::nullopt, Bytes).moveInto(BytesRef))
|
|
return E;
|
|
auto Ref = CAS.store({*PathRef, *BytesRef}, {});
|
|
if (!Ref)
|
|
return Ref.takeError();
|
|
SymbolGraphOutputRefs.push_back(*Ref);
|
|
return Error::success();
|
|
});
|
|
}
|
|
assert(ProducingInput != OutputToInputMap.end() && "Unknown output file");
|
|
|
|
unsigned InputIndex = ProducingInput->second.first;
|
|
auto OutputType = ProducingInput->second.second;
|
|
|
|
// Uncached output kind.
|
|
if (!isStoredDirectly(OutputType))
|
|
return std::make_unique<llvm::vfs::NullOutputFileImpl>();
|
|
|
|
return std::make_unique<SwiftCASOutputFile>(
|
|
ResolvedPath,
|
|
[this, InputIndex, OutputType](StringRef Path,
|
|
StringRef Bytes) -> Error {
|
|
return storeImpl(Path, Bytes, InputIndex, OutputType);
|
|
});
|
|
}
|
|
|
|
void initBackend(const FrontendInputsAndOutputs &InputsAndOutputs);
|
|
|
|
Error storeImpl(StringRef Path, StringRef Bytes, unsigned InputIndex,
|
|
file_types::ID OutputKind);
|
|
|
|
// Return true if all the outputs are produced for the given index.
|
|
bool addProducedOutput(unsigned InputIndex, file_types::ID OutputKind,
|
|
ObjectRef BytesRef);
|
|
Error finalizeCacheKeysFor(unsigned InputIndex);
|
|
|
|
private:
|
|
friend class SwiftCASOutputBackend;
|
|
ObjectStore &CAS;
|
|
ActionCache &Cache;
|
|
ObjectRef BaseKey;
|
|
const FrontendInputsAndOutputs &InputsAndOutputs;
|
|
const FrontendOptions &Opts;
|
|
FrontendOptions::ActionType Action;
|
|
|
|
// Lock for updating output file status.
|
|
llvm::sys::SmartMutex<true> OutputLock;
|
|
|
|
// Map from output path to the input index and output kind.
|
|
StringMap<std::pair<unsigned, file_types::ID>> OutputToInputMap;
|
|
|
|
// A vector of output refs where the index is the input index.
|
|
SmallVector<DenseMap<file_types::ID, ObjectRef>> OutputRefs;
|
|
|
|
SmallVector<ObjectRef> SymbolGraphOutputRefs;
|
|
};
|
|
|
|
SwiftCASOutputBackend::SwiftCASOutputBackend(
|
|
ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
|
|
const FrontendInputsAndOutputs &InputsAndOutputs,
|
|
const FrontendOptions &Opts, FrontendOptions::ActionType Action)
|
|
: Impl(*new SwiftCASOutputBackend::Implementation(
|
|
CAS, Cache, BaseKey, InputsAndOutputs, Opts, Action)) {}
|
|
|
|
SwiftCASOutputBackend::~SwiftCASOutputBackend() { delete &Impl; }
|
|
|
|
bool SwiftCASOutputBackend::isStoredDirectly(file_types::ID Kind) {
|
|
return !file_types::isProducedFromDiagnostics(Kind) &&
|
|
Kind != file_types::TY_Dependencies;
|
|
}
|
|
|
|
IntrusiveRefCntPtr<OutputBackend> SwiftCASOutputBackend::cloneImpl() const {
|
|
return makeIntrusiveRefCnt<SwiftCASOutputBackend>(
|
|
Impl.CAS, Impl.Cache, Impl.BaseKey, Impl.InputsAndOutputs, Impl.Opts,
|
|
Impl.Action);
|
|
}
|
|
|
|
Expected<std::unique_ptr<OutputFileImpl>>
|
|
SwiftCASOutputBackend::createFileImpl(StringRef ResolvedPath,
|
|
std::optional<OutputConfig> Config) {
|
|
return Impl.createFileImpl(ResolvedPath, Config);
|
|
}
|
|
|
|
file_types::ID SwiftCASOutputBackend::getOutputFileType(StringRef Path) const {
|
|
return file_types::lookupTypeForExtension(llvm::sys::path::extension(Path));
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::storeImpl(StringRef Path, StringRef Bytes,
|
|
unsigned InputIndex,
|
|
file_types::ID OutputKind) {
|
|
return Impl.storeImpl(Path, Bytes, InputIndex, OutputKind);
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::storeCachedDiagnostics(unsigned InputIndex,
|
|
StringRef Bytes) {
|
|
return storeImpl("<cached-diagnostics>", Bytes, InputIndex,
|
|
file_types::ID::TY_CachedDiagnostics);
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::storeMakeDependenciesFile(StringRef OutputFilename,
|
|
llvm::StringRef Bytes) {
|
|
auto Input = Impl.OutputToInputMap.find(OutputFilename);
|
|
if (Input == Impl.OutputToInputMap.end())
|
|
return llvm::createStringError("InputIndex for output file not found!");
|
|
auto InputIndex = Input->second.first;
|
|
assert(Input->second.second == file_types::TY_Dependencies &&
|
|
"wrong output type");
|
|
return storeImpl(OutputFilename, Bytes, InputIndex,
|
|
file_types::TY_Dependencies);
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::storeMCCASObjectID(StringRef OutputFilename,
|
|
llvm::cas::CASID ID) {
|
|
auto Input = Impl.OutputToInputMap.find(OutputFilename);
|
|
if (Input == Impl.OutputToInputMap.end())
|
|
return llvm::createStringError("InputIndex for output file not found!");
|
|
auto InputIndex = Input->second.first;
|
|
auto MCRef = Impl.CAS.getReference(ID);
|
|
if (!MCRef)
|
|
return createStringError("Invalid CASID: " + ID.toString() +
|
|
". No associated ObjectRef found!");
|
|
|
|
if (Impl.addProducedOutput(InputIndex, file_types::TY_Object, *MCRef))
|
|
return Impl.finalizeCacheKeysFor(InputIndex);
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
void SwiftCASOutputBackend::Implementation::initBackend(
|
|
const FrontendInputsAndOutputs &InputsAndOutputs) {
|
|
// FIXME: The output to input map might not be enough for example all the
|
|
// outputs can be written to `-`, but the backend cannot distinguish which
|
|
// input it actually comes from. Maybe the solution is just not to cache
|
|
// any commands write output to `-`.
|
|
file_types::ID mainOutputType = InputsAndOutputs.getPrincipalOutputType();
|
|
auto addInput = [&](const InputFile &Input, unsigned Index) {
|
|
// Ignore the outputFilename for typecheck action since it is not producing
|
|
// an output file for that.
|
|
if (!Input.outputFilename().empty() &&
|
|
Action != FrontendOptions::ActionType::Typecheck)
|
|
OutputToInputMap.insert(
|
|
{Input.outputFilename(), {Index, mainOutputType}});
|
|
Input.getPrimarySpecificPaths()
|
|
.SupplementaryOutputs.forEachSetOutputAndType(
|
|
[&](const std::string &Out, file_types::ID ID) {
|
|
// FIXME: Opt Remarks are not setup correctly to be cached and
|
|
// didn't go through the output backend. Do not cache them.
|
|
if (ID == file_types::ID::TY_YAMLOptRecord ||
|
|
ID == file_types::ID::TY_BitstreamOptRecord)
|
|
return;
|
|
|
|
if (!file_types::isProducedFromDiagnostics(ID))
|
|
OutputToInputMap.insert({Out, {Index, ID}});
|
|
});
|
|
};
|
|
|
|
for (unsigned idx = 0; idx < InputsAndOutputs.getAllInputs().size(); ++idx)
|
|
addInput(InputsAndOutputs.getAllInputs()[idx], idx);
|
|
|
|
// FIXME: Cached diagnostics is associated with the first output producing
|
|
// input file.
|
|
OutputToInputMap.insert(
|
|
{"<cached-diagnostics>",
|
|
{InputsAndOutputs.getIndexOfFirstOutputProducingInput(),
|
|
file_types::TY_CachedDiagnostics}});
|
|
|
|
// Resize the output refs to hold all inputs.
|
|
OutputRefs.resize(InputsAndOutputs.getAllInputs().size());
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::Implementation::storeImpl(
|
|
StringRef Path, StringRef Bytes, unsigned InputIndex,
|
|
file_types::ID OutputKind) {
|
|
std::optional<ObjectRef> BytesRef;
|
|
if (Error E = CAS.storeFromString(std::nullopt, Bytes).moveInto(BytesRef))
|
|
return E;
|
|
|
|
LLVM_DEBUG(llvm::dbgs() << "DEBUG: producing CAS output of type \'"
|
|
<< file_types::getTypeName(OutputKind)
|
|
<< "\' for input \'" << InputIndex << "\': \'"
|
|
<< CAS.getID(*BytesRef).toString() << "\'\n";);
|
|
|
|
if (addProducedOutput(InputIndex, OutputKind, *BytesRef))
|
|
return finalizeCacheKeysFor(InputIndex);
|
|
return Error::success();
|
|
}
|
|
|
|
bool SwiftCASOutputBackend::Implementation::addProducedOutput(
|
|
unsigned InputIndex, file_types::ID OutputKind,
|
|
ObjectRef BytesRef) {
|
|
llvm::sys::SmartScopedLock<true> LockOutput(OutputLock);
|
|
auto &ProducedOutputs = OutputRefs[InputIndex];
|
|
ProducedOutputs.insert({OutputKind, BytesRef});
|
|
|
|
return llvm::all_of(OutputToInputMap, [&](auto &E) {
|
|
return (E.second.first != InputIndex ||
|
|
ProducedOutputs.count(E.second.second));
|
|
});
|
|
}
|
|
|
|
Error SwiftCASOutputBackend::Implementation::finalizeCacheKeysFor(
|
|
unsigned InputIndex) {
|
|
const auto &ProducedOutputs = OutputRefs[InputIndex];
|
|
assert(!ProducedOutputs.empty() && "Expect outputs for this input");
|
|
|
|
std::vector<std::pair<file_types::ID, ObjectRef>> OutputsForInput;
|
|
llvm::for_each(ProducedOutputs, [&OutputsForInput](auto E) {
|
|
OutputsForInput.emplace_back(E.first, E.second);
|
|
});
|
|
if (InputIndex == InputsAndOutputs.getIndexOfFirstOutputProducingInput() &&
|
|
!SymbolGraphOutputRefs.empty()) {
|
|
auto SGRef = CAS.store(SymbolGraphOutputRefs, {});
|
|
if (!SGRef)
|
|
return SGRef.takeError();
|
|
OutputsForInput.emplace_back(file_types::ID::TY_SymbolGraphFile, *SGRef);
|
|
}
|
|
// Sort to a stable ordering for deterministic output cache object.
|
|
llvm::sort(OutputsForInput,
|
|
[](auto &LHS, auto &RHS) { return LHS.first < RHS.first; });
|
|
|
|
std::optional<ObjectRef> Result;
|
|
// Use a clang compatible result CAS object schema when emiting PCM.
|
|
if (Action == FrontendOptions::ActionType::EmitPCM) {
|
|
clang::cas::CompileJobCacheResult::Builder Builder;
|
|
|
|
for (auto &Outs : OutputsForInput) {
|
|
if (Outs.first == file_types::ID::TY_ClangModuleFile)
|
|
Builder.addOutput(
|
|
clang::cas::CompileJobCacheResult::OutputKind::MainOutput,
|
|
Outs.second);
|
|
else if (Outs.first == file_types::ID::TY_CachedDiagnostics)
|
|
Builder.addOutput(clang::cas::CompileJobCacheResult::OutputKind::
|
|
SerializedDiagnostics,
|
|
Outs.second);
|
|
else if (Outs.first == file_types::ID::TY_Dependencies)
|
|
Builder.addOutput(
|
|
clang::cas::CompileJobCacheResult::OutputKind::Dependencies,
|
|
Outs.second);
|
|
else
|
|
llvm_unreachable("Unexpected output when compiling clang module");
|
|
}
|
|
|
|
if (auto Err = Builder.build(CAS).moveInto(Result))
|
|
return Err;
|
|
|
|
} else {
|
|
swift::cas::CompileJobCacheResult::Builder Builder;
|
|
|
|
for (auto &Outs : OutputsForInput)
|
|
Builder.addOutput(Outs.first, Outs.second);
|
|
|
|
if (auto Err = Builder.build(CAS).moveInto(Result))
|
|
return Err;
|
|
}
|
|
|
|
auto CacheKey = createCompileJobCacheKeyForOutput(CAS, BaseKey, InputIndex);
|
|
if (!CacheKey)
|
|
return CacheKey.takeError();
|
|
|
|
LLVM_DEBUG(llvm::dbgs() << "DEBUG: writing cache entry for input \'"
|
|
<< InputIndex << "\': \'"
|
|
<< CAS.getID(*CacheKey).toString() << "\' => \'"
|
|
<< CAS.getID(*Result).toString() << "\'\n";);
|
|
|
|
if (auto E = Cache.put(CAS.getID(*CacheKey), CAS.getID(*Result))) {
|
|
// If `SWIFT_STRICT_CAS_ERRORS` environment is set, do not attempt to
|
|
// recover from error.
|
|
if (::getenv("SWIFT_STRICT_CAS_ERRORS"))
|
|
return E;
|
|
// Failed to update the action cache, this can happen when there is a
|
|
// non-deterministic output from compiler. Check that the cache entry is
|
|
// not missing, if so, then assume this is a cache poisoning that might
|
|
// be benign and the build might continue without problem.
|
|
auto Check = Cache.get(CAS.getID(*CacheKey));
|
|
if (!Check) {
|
|
// A real CAS error, return the original error.
|
|
consumeError(Check.takeError());
|
|
return E;
|
|
}
|
|
if (!*Check)
|
|
return E;
|
|
// Emit an error message to stderr but don't fail the job. Ideally this
|
|
// should be sent to a DiagnosticEngine but CASBackend lives longer than
|
|
// diagnostic engine and needs to capture all diagnostic outputs before
|
|
// sending this request.
|
|
llvm::errs() << "error: failed to update cache: " << toString(std::move(E))
|
|
<< "\n";
|
|
}
|
|
|
|
return Error::success();
|
|
}
|
|
|
|
} // end namespace swift::cas
|