Merge bitcoin/bitcoin#32414: validation: periodically flush dbcache during reindex-chainstate

c1e554d3e5 refactor: consolidate 3 separate locks into one block (Andrew Toth)
41479ed1d2 test: add test for periodic flush inside ActivateBestChain (Andrew Toth)
84820561dc validation: periodically flush dbcache during reindex-chainstate (Andrew Toth)

Pull request description:

  After #30611 we periodically do a non-erasing flush of the dbcache to disk roughly every hour during IBD.
  The intention was to also do this periodic flush during reindex-chainstate, so we would not risk losing progress during a system failure when reindexing with a high dbcache value.

  It was discovered that reindex-chainstate does not perform a PERIODIC flush until it has already reached the tip. Since reindexing to tip usually happens within 24 hours, this behaviour was unnoticed with the previous periodic flush interval. Note that reindex-chainstate still does IF_NEEDED flushes during `ConnectBlock`, so this also would not be noticed when running with a lower dbcache value.

  This patch moves the PERIODIC flush from after the outer loop in `ActivateBestChain` to inside the outer loop after we release `cs_main`. This will periodically flush during IBD, reindex-chainstate, and steady state.

ACKs for top commit:
  l0rinc:
    ACK c1e554d3e5
  achow101:
    ACK c1e554d3e5
  sipa:
    utACK c1e554d3e5

Tree-SHA512: c447ad03e16c9978b8ed2c285b38e1b4c56e7778ab93b6f64435116f47b8931017f5f56ab53eb61656693146aaced776f666af573a41ab28e8f2b6d8657fa756

src/test/chainstate_write_tests.cpp

@@ -8,6 +8,10 @@
 
 #include <boost/test/unit_test.hpp>
 
+// Taken from validation.cpp
+static constexpr auto DATABASE_WRITE_INTERVAL_MIN{50min};
+static constexpr auto DATABASE_WRITE_INTERVAL_MAX{70min};
+
 BOOST_AUTO_TEST_SUITE(chainstate_write_tests)
 
 BOOST_FIXTURE_TEST_CASE(chainstate_write_interval, TestingSetup)
@@ -31,15 +35,68 @@ BOOST_FIXTURE_TEST_CASE(chainstate_write_interval, TestingSetup)
     BOOST_CHECK(!sub->m_did_flush);
 
     // The periodic flush interval is between 50 and 70 minutes (inclusive)
-    SetMockTime(GetTime<std::chrono::minutes>() + 49min);
+    SetMockTime(GetTime<std::chrono::minutes>() + DATABASE_WRITE_INTERVAL_MIN - 1min);
     chainstate.FlushStateToDisk(state_dummy, FlushStateMode::PERIODIC);
     m_node.validation_signals->SyncWithValidationInterfaceQueue();
     BOOST_CHECK(!sub->m_did_flush);
 
-    SetMockTime(GetTime<std::chrono::minutes>() + 70min);
+    SetMockTime(GetTime<std::chrono::minutes>() + DATABASE_WRITE_INTERVAL_MAX);
     chainstate.FlushStateToDisk(state_dummy, FlushStateMode::PERIODIC);
     m_node.validation_signals->SyncWithValidationInterfaceQueue();
     BOOST_CHECK(sub->m_did_flush);
 }
 
+// Test that we do PERIODIC flushes inside ActivateBestChain.
+// This is necessary for reindex-chainstate to be able to periodically flush
+// before reaching chain tip.
+BOOST_FIXTURE_TEST_CASE(write_during_multiblock_activation, TestChain100Setup)
+{
+    struct TestSubscriber final : CValidationInterface
+    {
+        const CBlockIndex* m_tip{nullptr};
+        const CBlockIndex* m_flushed_at_block{nullptr};
+        void ChainStateFlushed(ChainstateRole, const CBlockLocator&) override
+        {
+            m_flushed_at_block = m_tip;
+        }
+        void UpdatedBlockTip(const CBlockIndex* block_index, const CBlockIndex*, bool) override {
+            m_tip = block_index;
+        }
+    };
+
+    auto& chainstate{Assert(m_node.chainman)->ActiveChainstate()};
+    BlockValidationState state_dummy{};
+
+    // Pop two blocks from the tip
+    const CBlockIndex* tip{chainstate.m_chain.Tip()};
+    CBlockIndex* second_from_tip{tip->pprev};
+
+    {
+        LOCK2(m_node.chainman->GetMutex(), chainstate.MempoolMutex());
+        chainstate.DisconnectTip(state_dummy, nullptr);
+        chainstate.DisconnectTip(state_dummy, nullptr);
+    }
+
+    BOOST_CHECK_EQUAL(second_from_tip->pprev, chainstate.m_chain.Tip());
+
+    // Set m_next_write to current time
+    chainstate.FlushStateToDisk(state_dummy, FlushStateMode::ALWAYS);
+    m_node.validation_signals->SyncWithValidationInterfaceQueue();
+    // The periodic flush interval is between 50 and 70 minutes (inclusive)
+    // The next call to a PERIODIC write will flush
+    SetMockTime(GetMockTime() + DATABASE_WRITE_INTERVAL_MAX);
+
+    const auto sub{std::make_shared<TestSubscriber>()};
+    m_node.validation_signals->RegisterSharedValidationInterface(sub);
+
+    // ActivateBestChain back to tip
+    chainstate.ActivateBestChain(state_dummy, nullptr);
+    BOOST_CHECK_EQUAL(tip, chainstate.m_chain.Tip());
+    // Check that we flushed inside ActivateBestChain while we were at the
+    // second block from tip, since FlushStateToDisk is called with PERIODIC
+    // inside the outer loop.
+    m_node.validation_signals->SyncWithValidationInterfaceQueue();
+    BOOST_CHECK_EQUAL(sub->m_flushed_at_block, second_from_tip);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/validation.cpp

@@ -3486,14 +3486,24 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr<
         } // release cs_main
         // When we reach this point, we switched to a new tip (stored in pindexNewTip).
 
-        if (exited_ibd) {
-            // If a background chainstate is in use, we may need to rebalance our
-            // allocation of caches once a chainstate exits initial block download.
-            LOCK(::cs_main);
-            m_chainman.MaybeRebalanceCaches();
+        bool disabled{false};
+        {
+            LOCK(m_chainman.GetMutex());
+            if (exited_ibd) {
+                // If a background chainstate is in use, we may need to rebalance our
+                // allocation of caches once a chainstate exits initial block download.
+                m_chainman.MaybeRebalanceCaches();
+            }
+
+            // Write changes periodically to disk, after relay.
+            if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) {
+                return false;
+            }
+
+            disabled = m_disabled;
         }
 
-        if (WITH_LOCK(::cs_main, return m_disabled)) {
+        if (disabled) {
             // Background chainstate has reached the snapshot base block, so exit.
 
             // Restart indexes to resume indexing for all blocks unique to the snapshot
@@ -3517,11 +3527,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr<
 
     m_chainman.CheckBlockIndex();
 
-    // Write changes periodically to disk, after relay.
-    if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) {
-        return false;
-    }
-
     return true;
 }