kernel: Add pure kernel bitcoin-chainstate

Re-write the bitcoin-chainstate utility by only using the kernel C++ API
header instead of internal Bitcoin Core code.
This commit is contained in:
TheCharlatan
2024-06-14 18:33:08 +02:00
parent 36ec9a3ea2
commit 7990463b10
4 changed files with 197 additions and 260 deletions

View File

@@ -202,7 +202,7 @@ jobs:
job-type: [standard, fuzz]
include:
- job-type: standard
generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DWERROR=ON'
generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DWERROR=ON'
job-name: 'Windows native, VS 2022'
- job-type: fuzz
generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON'
@@ -280,7 +280,7 @@ jobs:
$exeName = $_.Name
# Skip as they currently do not have manifests
if ($exeName -eq "fuzz.exe" -or $exeName -eq "bench_bitcoin.exe" -or $exeName -eq "test_bitcoin-qt.exe" -or $exeName -eq "test_kernel.exe") {
if ($exeName -eq "fuzz.exe" -or $exeName -eq "bench_bitcoin.exe" -or $exeName -eq "test_bitcoin-qt.exe" -or $exeName -eq "test_kernel.exe" -or $exeName -eq "bitcoin-chainstate.exe") {
Write-Host "Skipping $exeName (no manifest present)"
return
}
@@ -307,6 +307,7 @@ jobs:
BITCOINTX: '${{ github.workspace }}\build\bin\Release\bitcoin-tx.exe'
BITCOINUTIL: '${{ github.workspace }}\build\bin\Release\bitcoin-util.exe'
BITCOINWALLET: '${{ github.workspace }}\build\bin\Release\bitcoin-wallet.exe'
BITCOINCHAINSTATE: '${{ github.workspace }}\build\bin\Release\bitcoin-chainstate.exe'
TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
run: py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix="${RUNNER_TEMP}" --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA}

View File

@@ -407,28 +407,27 @@ if(BUILD_KERNEL_LIB)
if (BUILD_KERNEL_TEST)
add_subdirectory(test/kernel)
endif()
endif()
if(BUILD_UTIL_CHAINSTATE)
add_executable(bitcoin-chainstate
bitcoin-chainstate.cpp
)
add_windows_application_manifest(bitcoin-chainstate)
# TODO: The `SKIP_BUILD_RPATH` property setting can be deleted
# in the future after reordering Guix script commands to
# perform binary checks after the installation step.
# Relevant discussions:
# - https://github.com/hebasto/bitcoin/pull/236#issuecomment-2183120953
# - https://github.com/bitcoin/bitcoin/pull/30312#issuecomment-2191235833
set_target_properties(bitcoin-chainstate PROPERTIES
SKIP_BUILD_RPATH OFF
)
target_link_libraries(bitcoin-chainstate
PRIVATE
core_interface
bitcoinkernel
)
install_binary_component(bitcoin-chainstate INTERNAL)
if(BUILD_UTIL_CHAINSTATE)
add_executable(bitcoin-chainstate
bitcoin-chainstate.cpp
)
add_windows_application_manifest(bitcoin-chainstate)
# TODO: The `SKIP_BUILD_RPATH` property setting can be deleted
# in the future after reordering Guix script commands to
# perform binary checks after the installation step.
# Relevant discussions:
# - https://github.com/hebasto/bitcoin/pull/236#issuecomment-2183120953
# - https://github.com/bitcoin/bitcoin/pull/30312#issuecomment-2191235833
set_target_properties(bitcoin-chainstate PROPERTIES
SKIP_BUILD_RPATH OFF
)
target_link_libraries(bitcoin-chainstate
PRIVATE
core_interface
bitcoinkernel
)
install_binary_component(bitcoin-chainstate INTERNAL)
endif()
endif()

View File

@@ -1,53 +1,135 @@
// Copyright (c) 2022 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
//
// The bitcoin-chainstate executable serves to surface the dependencies required
// by a program wishing to use Bitcoin Core's consensus engine as it is right
// now.
//
// DEVELOPER NOTE: Since this is a "demo-only", experimental, etc. executable,
// it may diverge from Bitcoin Core's coding style.
//
// It is part of the libbitcoinkernel project.
#include <kernel/chainparams.h>
#include <kernel/chainstatemanager_opts.h>
#include <kernel/checks.h>
#include <kernel/context.h>
#include <kernel/warning.h>
#include <consensus/validation.h>
#include <core_io.h>
#include <kernel/caches.h>
#include <logging.h>
#include <node/blockstorage.h>
#include <node/chainstate.h>
#include <random.h>
#include <script/sigcache.h>
#include <util/chaintype.h>
#include <util/fs.h>
#include <util/signalinterrupt.h>
#include <util/task_runner.h>
#include <util/translation.h>
#include <validation.h>
#include <validationinterface.h>
#include <kernel/bitcoinkernel_wrapper.h>
#include <cassert>
#include <cstdint>
#include <functional>
#include <iosfwd>
#include <memory>
#include <charconv>
#include <filesystem>
#include <iostream>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
using namespace btck;
std::vector<std::byte> hex_string_to_byte_vec(std::string_view hex)
{
std::vector<std::byte> bytes;
bytes.reserve(hex.length() / 2);
for (size_t i{0}; i < hex.length(); i += 2) {
uint8_t byte_value;
auto [ptr, ec] = std::from_chars(hex.data() + i, hex.data() + i + 2, byte_value, 16);
if (ec != std::errc{} || ptr != hex.data() + i + 2) {
throw std::invalid_argument("Invalid hex character");
}
bytes.push_back(static_cast<std::byte>(byte_value));
}
return bytes;
}
class KernelLog
{
public:
void LogMessage(std::string_view message)
{
std::cout << "kernel: " << message;
}
};
class TestValidationInterface : public ValidationInterface
{
public:
TestValidationInterface() = default;
std::optional<std::string> m_expected_valid_block = std::nullopt;
void BlockChecked(const Block block, const BlockValidationState state) override
{
auto mode{state.GetValidationMode()};
switch (mode) {
case ValidationMode::VALID: {
std::cout << "Valid block" << std::endl;
return;
}
case ValidationMode::INVALID: {
std::cout << "Invalid block: ";
auto result{state.GetBlockValidationResult()};
switch (result) {
case BlockValidationResult::UNSET:
std::cout << "initial value. Block has not yet been rejected" << std::endl;
break;
case BlockValidationResult::HEADER_LOW_WORK:
std::cout << "the block header may be on a too-little-work chain" << std::endl;
break;
case BlockValidationResult::CONSENSUS:
std::cout << "invalid by consensus rules" << std::endl;
break;
case BlockValidationResult::CACHED_INVALID:
std::cout << "this block was cached as being invalid and we didn't store the reason why" << std::endl;
break;
case BlockValidationResult::INVALID_HEADER:
std::cout << "invalid proof of work or time too old" << std::endl;
break;
case BlockValidationResult::MUTATED:
std::cout << "the block's data didn't match the data committed to by the PoW" << std::endl;
break;
case BlockValidationResult::MISSING_PREV:
std::cout << "We don't have the previous block the checked one is built on" << std::endl;
break;
case BlockValidationResult::INVALID_PREV:
std::cout << "A block this one builds on is invalid" << std::endl;
break;
case BlockValidationResult::TIME_FUTURE:
std::cout << "block timestamp was > 2 hours in the future (or our clock is bad)" << std::endl;
break;
}
return;
}
case ValidationMode::INTERNAL_ERROR: {
std::cout << "Internal error" << std::endl;
return;
}
}
}
};
class TestKernelNotifications : public KernelNotifications
{
public:
void BlockTipHandler(SynchronizationState, const BlockTreeEntry, double) override
{
std::cout << "Block tip changed" << std::endl;
}
void ProgressHandler(std::string_view title, int progress_percent, bool resume_possible) override
{
std::cout << "Made progress: " << title << " " << progress_percent << "%" << std::endl;
}
void WarningSetHandler(Warning warning, std::string_view message) override
{
std::cout << message << std::endl;
}
void WarningUnsetHandler(Warning warning) override
{
std::cout << "Warning unset: " << static_cast<std::underlying_type_t<Warning>>(warning) << std::endl;
}
void FlushErrorHandler(std::string_view error) override
{
std::cout << error << std::endl;
}
void FatalErrorHandler(std::string_view error) override
{
std::cout << error << std::endl;
}
};
int main(int argc, char* argv[])
{
// We do not enable logging for this app, so explicitly disable it.
// To enable logging instead, replace with:
// LogInstance().m_print_to_console = true;
// LogInstance().StartLogging();
LogInstance().DisableLogging();
// SETUP: Argument parsing and handling
if (argc != 2) {
std::cerr
@@ -58,213 +140,67 @@ int main(int argc, char* argv[])
<< " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl;
return 1;
}
fs::path abs_datadir{fs::absolute(argv[1])};
fs::create_directories(abs_datadir);
std::filesystem::path abs_datadir{std::filesystem::absolute(argv[1])};
std::filesystem::create_directories(abs_datadir);
// SETUP: Context
kernel::Context kernel_context{};
// We can't use a goto here, but we can use an assert since none of the
// things instantiated so far requires running the epilogue to be torn down
// properly
assert(kernel::SanityChecks(kernel_context));
ValidationSignals validation_signals{std::make_unique<util::ImmediateTaskRunner>()};
class KernelNotifications : public kernel::Notifications
{
public:
kernel::InterruptResult blockTip(SynchronizationState, const CBlockIndex&, double) override
{
std::cout << "Block tip changed" << std::endl;
return {};
}
void headerTip(SynchronizationState, int64_t height, int64_t timestamp, bool presync) override
{
std::cout << "Header tip changed: " << height << ", " << timestamp << ", " << presync << std::endl;
}
void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override
{
std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl;
}
void warningSet(kernel::Warning id, const bilingual_str& message) override
{
std::cout << "Warning " << static_cast<int>(id) << " set: " << message.original << std::endl;
}
void warningUnset(kernel::Warning id) override
{
std::cout << "Warning " << static_cast<int>(id) << " unset" << std::endl;
}
void flushError(const bilingual_str& message) override
{
std::cerr << "Error flushing block data to disk: " << message.original << std::endl;
}
void fatalError(const bilingual_str& message) override
{
std::cerr << "Error: " << message.original << std::endl;
}
btck_LoggingOptions logging_options = {
.log_timestamps = true,
.log_time_micros = false,
.log_threadnames = false,
.log_sourcelocations = false,
.always_print_category_levels = true,
};
auto notifications = std::make_unique<KernelNotifications>();
kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
logging_set_options(logging_options);
// SETUP: Chainstate
auto chainparams = CChainParams::Main();
const ChainstateManager::Options chainman_opts{
.chainparams = *chainparams,
.datadir = abs_datadir,
.notifications = *notifications,
.signals = &validation_signals,
};
const node::BlockManager::Options blockman_opts{
.chainparams = chainman_opts.chainparams,
.blocks_dir = abs_datadir / "blocks",
.notifications = chainman_opts.notifications,
.block_tree_db_params = DBParams{
.path = abs_datadir / "blocks" / "index",
.cache_bytes = cache_sizes.block_tree_db,
},
};
util::SignalInterrupt interrupt;
ChainstateManager chainman{interrupt, chainman_opts, blockman_opts};
Logger logger{std::make_unique<KernelLog>()};
node::ChainstateLoadOptions options;
auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options);
if (status != node::ChainstateLoadStatus::SUCCESS) {
std::cerr << "Failed to load Chain state from your datadir." << std::endl;
goto epilogue;
} else {
std::tie(status, error) = node::VerifyLoadedChainstate(chainman, options);
if (status != node::ChainstateLoadStatus::SUCCESS) {
std::cerr << "Failed to verify loaded Chain state from your datadir." << std::endl;
goto epilogue;
}
ContextOptions options{};
ChainParams params{ChainType::MAINNET};
options.SetChainParams(params);
options.SetNotifications(std::make_shared<TestKernelNotifications>());
options.SetValidationInterface(std::make_shared<TestValidationInterface>());
Context context{options};
ChainstateManagerOptions chainman_opts{context, abs_datadir.string(), (abs_datadir / "blocks").string()};
chainman_opts.SetWorkerThreads(4);
std::unique_ptr<ChainMan> chainman;
try {
chainman = std::make_unique<ChainMan>(context, chainman_opts);
} catch (std::exception&) {
std::cerr << "Failed to instantiate ChainMan, exiting" << std::endl;
return 1;
}
for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
BlockValidationState state;
if (!chainstate->ActivateBestChain(state, nullptr)) {
std::cerr << "Failed to connect best block (" << state.ToString() << ")" << std::endl;
goto epilogue;
}
}
// Main program logic starts here
std::cout
<< "Hello! I'm going to print out some information about your datadir." << std::endl
<< "\t"
<< "Path: " << abs_datadir << std::endl;
{
LOCK(chainman.GetMutex());
std::cout
<< "\t" << "Blockfiles Indexed: " << std::boolalpha << chainman.m_blockman.m_blockfiles_indexed.load() << std::noboolalpha << std::endl
<< "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl
<< "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl
<< "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl;
CBlockIndex* tip = chainman.ActiveTip();
if (tip) {
std::cout << "\t" << tip->ToString() << std::endl;
}
}
std::cout << "Enter the block you want to validate on the next line:" << std::endl;
for (std::string line; std::getline(std::cin, line);) {
if (line.empty()) {
std::cerr << "Empty line found" << std::endl;
break;
std::cerr << "Empty line found, try again:" << std::endl;
continue;
}
std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>();
CBlock& block = *blockptr;
if (!DecodeHexBlk(block, line)) {
std::cerr << "Block decode failed" << std::endl;
break;
auto raw_block{hex_string_to_byte_vec(line)};
std::unique_ptr<Block> block;
try {
block = std::make_unique<Block>(raw_block);
} catch (std::exception&) {
std::cerr << "Block decode failed, try again:" << std::endl;
continue;
}
{
LOCK(cs_main);
const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock);
if (pindex) {
chainman.UpdateUncommittedBlockStructures(block, pindex);
}
bool new_block = false;
bool accepted = chainman->ProcessBlock(*block, &new_block);
if (accepted) {
std::cerr << "Block has not yet been rejected" << std::endl;
} else {
std::cerr << "Block was not accepted" << std::endl;
}
// Adapted from rpc/mining.cpp
class submitblock_StateCatcher final : public CValidationInterface
{
public:
uint256 hash;
bool found;
BlockValidationState state;
explicit submitblock_StateCatcher(const uint256& hashIn) : hash(hashIn), found(false), state() {}
protected:
void BlockChecked(const std::shared_ptr<const CBlock>& block, const BlockValidationState& stateIn) override
{
if (block->GetHash() != hash)
return;
found = true;
state = stateIn;
}
};
bool new_block;
auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash());
validation_signals.RegisterSharedValidationInterface(sc);
bool accepted = chainman.ProcessNewBlock(blockptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/&new_block);
validation_signals.UnregisterSharedValidationInterface(sc);
if (!new_block && accepted) {
std::cerr << "duplicate" << std::endl;
break;
}
if (!sc->found) {
std::cerr << "inconclusive" << std::endl;
break;
}
std::cout << sc->state.ToString() << std::endl;
switch (sc->state.GetResult()) {
case BlockValidationResult::BLOCK_RESULT_UNSET:
std::cerr << "initial value. Block has not yet been rejected" << std::endl;
break;
case BlockValidationResult::BLOCK_HEADER_LOW_WORK:
std::cerr << "the block header may be on a too-little-work chain" << std::endl;
break;
case BlockValidationResult::BLOCK_CONSENSUS:
std::cerr << "invalid by consensus rules (excluding any below reasons)" << std::endl;
break;
case BlockValidationResult::BLOCK_CACHED_INVALID:
std::cerr << "this block was cached as being invalid and we didn't store the reason why" << std::endl;
break;
case BlockValidationResult::BLOCK_INVALID_HEADER:
std::cerr << "invalid proof of work or time too old" << std::endl;
break;
case BlockValidationResult::BLOCK_MUTATED:
std::cerr << "the block's data didn't match the data committed to by the PoW" << std::endl;
break;
case BlockValidationResult::BLOCK_MISSING_PREV:
std::cerr << "We don't have the previous block the checked one is built on" << std::endl;
break;
case BlockValidationResult::BLOCK_INVALID_PREV:
std::cerr << "A block this one builds on is invalid" << std::endl;
break;
case BlockValidationResult::BLOCK_TIME_FUTURE:
std::cerr << "block timestamp was > 2 hours in the future (or our clock is bad)" << std::endl;
break;
}
}
epilogue:
// Without this precise shutdown sequence, there will be a lot of nullptr
// dereferencing and UB.
validation_signals.FlushBackgroundCallbacks();
{
LOCK(cs_main);
for (Chainstate* chainstate : chainman.GetAll()) {
if (chainstate->CanFlushToDisk()) {
chainstate->ForceFlushStateToDisk();
chainstate->ResetCoinsViews();
}
if (!new_block) {
std::cerr << "Block is a duplicate" << std::endl;
}
}
}

View File

@@ -374,6 +374,7 @@ fn lint_std_filesystem() -> LintResult {
":(exclude)src/ipc/libmultiprocess/",
":(exclude)src/util/fs.h",
":(exclude)src/test/kernel/test_kernel.cpp",
":(exclude)src/bitcoin-chainstate.cpp",
])
.status()
.expect("command error")