backport: bitcoin#27460, #27585, #28168, #28200, #28218, #28284, #28289, #28369, #28637

.github/ISSUE_TEMPLATE/good_first_issue.yml

@@ -29,8 +29,10 @@ body:
     attributes:
       label: Useful Skills
       description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Dash masternodes and the Dash Core RPC interface”.
-    validations:
-      required: false
+      value: |
+        * Compiling Dash Core from source
+        * Running the C++ unit tests and the Python functional tests
+        * ...
   - type: textarea
     attributes:
       label: Guidance for new contributors

doc/release-notes-27460.md

@@ -0,0 +1,7 @@
+- A new `importmempool` RPC has been added. It loads a valid `mempool.dat` file and attempts to
+  add its contents to the mempool. This can be useful to import mempool data from another node
+  without having to modify the datadir contents and without having to restart the node. (#8071)
+    - Warning: Importing untrusted files is dangerous, especially if metadata from the file is taken over.
+    - If you want to apply fee deltas, it is recommended to use the `getprioritisedtransactions` and
+      `prioritisetransaction` RPCs instead of the `apply_fee_delta_priority` option to avoid
+      double-prioritising any already-prioritised transactions in the mempool.

src/Makefile.am

@@ -311,6 +311,7 @@ BITCOIN_CORE_H = \
   kernel/mempool_limits.h \
   kernel/mempool_options.h \
   kernel/mempool_persist.h \
+  kernel/mempool_removal_reason.h \
   kernel/notifications_interface.h \
   kernel/validation_cache_sizes.h \
   key.h \
@@ -618,6 +619,7 @@ libbitcoin_node_a_SOURCES = \
   kernel/cs_main.cpp \
   kernel/disconnected_transactions.cpp \
   kernel/mempool_persist.cpp \
+  kernel/mempool_removal_reason.cpp \
   llmq/blockprocessor.cpp \
   llmq/commitment.cpp \
   llmq/context.cpp \
@@ -1342,6 +1344,7 @@ libdashkernel_la_SOURCES = \
   kernel/cs_main.cpp \
   kernel/disconnected_transactions.cpp \
   kernel/mempool_persist.cpp \
+  kernel/mempool_removal_reason.cpp \
   key.cpp \
   key_io.cpp \
   llmq/blockprocessor.cpp \

src/Makefile.test.include

@@ -490,8 +490,9 @@ endif
 %.json.h: %.json
 	@$(MKDIR_P) $(@D)
 	$(AM_V_GEN) { \
+	 echo "#include <string>" && \
 	 echo "namespace json_tests{" && \
-	 echo "static unsigned const char $(*F)[] = {" && \
-	 $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x  ,//g' && \
+	 echo "static const std::string $(*F){" && \
+	 $(HEXDUMP) -v -e '8/1 "char(0x%02x), "' -e '"\n"' $< | $(SED) -e 's/char(0x  ),//g' && \
 	 echo "};};"; \
 	} > "$@.new" && mv -f "$@.new" "$@"

src/bench/net_processing.cpp

@@ -15,8 +15,8 @@ using namespace std::literals;
 static void InventoryBatch(benchmark::Bench& bench, uint32_t count)
 {
     const auto setup = MakeNoLogFileContext<TestingSetup>(ChainType::REGTEST);
-    auto& chainstate = *static_cast<TestChainState*>(&setup->m_node.chainman->ActiveChainstate());
-    chainstate.JumpOutOfIbd();
+    auto& chainman = static_cast<TestChainstateManager&>(*setup->m_node.chainman);
+    chainman.JumpOutOfIbd();
     auto& peerman = *setup->m_node.peerman;
     const auto& connman = *static_cast<ConnmanTestMsg*>(setup->m_node.connman.get());
     auto peer{MakeTestPeer(/*id=*/0)};
@@ -44,7 +44,7 @@ static void InventoryBatch(benchmark::Bench& bench, uint32_t count)
     });
 
     peerman.FinalizeNode(*peer);
-    chainstate.ResetIbd();
+    chainman.ResetIbd();
     SetMockTime(0s);
 }
 

src/bitcoin-chainstate.cpp

@@ -182,7 +182,7 @@ int main(int argc, char* argv[])
         << "\t" << "Reindexing: " << std::boolalpha << node::fReindex.load() << std::noboolalpha << std::endl
         << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl
         << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl
-        << "\t" << "Active IBD: " << std::boolalpha << chainman.ActiveChainstate().IsInitialBlockDownload() << std::noboolalpha << std::endl;
+        << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl;
         CBlockIndex* tip = chainman.ActiveTip();
         if (tip) {
             std::cout << "\t" << tip->ToString() << std::endl;

src/blockfilter.cpp

@@ -9,9 +9,11 @@
 #include <crypto/siphash.h>
 #include <evo/specialtx_filter.h>
 #include <hash.h>
+#include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <script/script.h>
 #include <streams.h>
+#include <undo.h>
 #include <util/golombrice.h>
 #include <util/string.h>
 

src/blockfilter.h

@@ -5,19 +5,22 @@
 #ifndef BITCOIN_BLOCKFILTER_H
 #define BITCOIN_BLOCKFILTER_H
 
-#include <stdint.h>
-#include <string>
+#include <cstddef>
+#include <cstdint>
+#include <ios>
 #include <set>
+#include <string>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include <attributes.h>
-#include <primitives/block.h>
-#include <serialize.h>
 #include <uint256.h>
-#include <undo.h>
 #include <util/bytevectorhash.h>
 
+class CBlock;
+class CBlockUndo;
+
 /**
  * This implements a Golomb-coded set as defined in BIP 158. It is a
  * compact, probabilistic data structure for testing set membership.

src/index/blockfilterindex.cpp

@@ -11,6 +11,7 @@
 #include <logging.h>
 #include <node/blockstorage.h>
 #include <serialize.h>
+#include <undo.h>
 #include <util/fs_helpers.h>
 #include <validation.h>
 

src/index/blockfilterindex.h

@@ -12,6 +12,8 @@
 #include <index/base.h>
 #include <util/hasher.h>
 
+#include <unordered_map>
+
 class AutoFile;
 
 static const char* const DEFAULT_BLOCKFILTERINDEX = "0";

src/init.cpp

@@ -171,6 +171,7 @@
 using kernel::CoinStatsHashType;
 using kernel::DEFAULT_STOPAFTERBLOCKIMPORT;
 using kernel::DumpMempool;
+using kernel::LoadMempool;
 using kernel::ValidationCacheSizes;
 
 using node::ApplyArgsManOptions;
@@ -2489,11 +2490,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &args, &chainman, &node] {
         // ThreadImport can switch fReindex from true to false, fetch its original state here to use later
         bool skip_evodb_repair_on_reindex = fReindex || fReindexChainState;
-        ThreadImport(chainman, vImportFiles, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{});
+        ThreadImport(chainman, vImportFiles);
+
+        // Load mempool from disk
+        if (auto* pool{chainman.ActiveChainstate().GetMempool()}) {
+            LoadMempool(*pool, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}, chainman.ActiveChainstate(), {});
+            pool->SetLoadTried(!ShutdownRequested());
+        }
 
         {
             const CBlockIndex* tip = WITH_LOCK(::cs_main, return chainman.ActiveTip());
-            const bool ibd = chainman.ActiveChainstate().IsInitialBlockDownload();
+            const bool ibd = chainman.IsInitialBlockDownload();
             if (node.active_ctx) {
                 // On masternodes, defer the full broadcast until after
                 // nodeman->Init() so that GetProTxHash() is available
@@ -2584,7 +2591,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
             // directly because the full broadcast (NetQuorum,
             // ActiveContext) depends on a valid proTxHash.
             const CBlockIndex* tip = WITH_LOCK(::cs_main, return chainman.ActiveTip());
-            const bool ibd = chainman.ActiveChainstate().IsInitialBlockDownload();
+            const bool ibd = chainman.IsInitialBlockDownload();
             GetMainSignals().InitializeCurrentBlockTip(tip, ibd);
         }
     });

src/kernel/mempool_persist.cpp

@@ -19,7 +19,6 @@
 #include <util/time.h>
 #include <validation.h>
 
-#include <chrono>
 #include <cstdint>
 #include <cstdio>
 #include <exception>
@@ -37,11 +36,11 @@ namespace kernel {
 
 static const uint64_t MEMPOOL_DUMP_VERSION = 1;
 
-bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, FopenFn mockable_fopen_function)
+bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts)
 {
     if (load_path.empty()) return false;
 
-    FILE* filestr{mockable_fopen_function(load_path, "rb")};
+    FILE* filestr{opts.mockable_fopen_function(load_path, "rb")};
     AutoFile file(filestr);
     if (file.IsNull()) {
         LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n");
@@ -53,7 +52,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
     int64_t failed = 0;
     int64_t already_there = 0;
     int64_t unbroadcast = 0;
-    auto now = NodeClock::now();
+    const auto now{NodeClock::now()};
 
     try {
         uint64_t version;
@@ -82,8 +81,12 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
             file >> nTime;
             file >> nFeeDelta;
 
+            if (opts.use_current_time) {
+                nTime = TicksSinceEpoch<std::chrono::seconds>(now);
+            }
+
             CAmount amountdelta = nFeeDelta;
-            if (amountdelta) {
+            if (amountdelta && opts.apply_fee_delta_priority) {
                 pool.PrioritiseTransaction(tx->GetHash(), amountdelta);
             }
             if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) {
@@ -111,17 +114,21 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
         std::map<uint256, CAmount> mapDeltas;
         file >> mapDeltas;
 
-        for (const auto& i : mapDeltas) {
-            pool.PrioritiseTransaction(i.first, i.second);
+        if (opts.apply_fee_delta_priority) {
+            for (const auto& i : mapDeltas) {
+                pool.PrioritiseTransaction(i.first, i.second);
+            }
         }
 
         std::set<uint256> unbroadcast_txids;
         file >> unbroadcast_txids;
-        unbroadcast = unbroadcast_txids.size();
-        for (const auto& txid : unbroadcast_txids) {
-            // Ensure transactions were accepted to mempool then add to
-            // unbroadcast set.
-            if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid);
+        if (opts.apply_unbroadcast_set) {
+            unbroadcast = unbroadcast_txids.size();
+            for (const auto& txid : unbroadcast_txids) {
+                // Ensure transactions were accepted to mempool then add to
+                // unbroadcast set.
+                if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid);
+            }
         }
     } catch (const std::exception& e) {
         LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what());

src/kernel/mempool_persist.h

@@ -12,15 +12,21 @@ class CTxMemPool;
 
 namespace kernel {
 
-/** Dump the mempool to disk. */
+/** Dump the mempool to a file. */
 bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path,
                  fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen,
                  bool skip_file_commit = false);
 
-/** Load the mempool from disk. */
+struct ImportMempoolOptions {
+    fsbridge::FopenFn mockable_fopen_function{fsbridge::fopen};
+    bool use_current_time{false};
+    bool apply_fee_delta_priority{true};
+    bool apply_unbroadcast_set{true};
+};
+/** Import the file and attempt to add its contents to the mempool. */
 bool LoadMempool(CTxMemPool& pool, const fs::path& load_path,
                  Chainstate& active_chainstate,
-                 fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen);
+                 ImportMempoolOptions&& opts);
 
 } // namespace kernel
 

src/kernel/mempool_removal_reason.cpp

@@ -0,0 +1,21 @@
+// Copyright (c) 2016-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://opensource.org/license/mit/.
+
+#include <kernel/mempool_removal_reason.h>
+
+#include <cassert>
+#include <string>
+
+std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept
+{
+    switch (r) {
+        case MemPoolRemovalReason::EXPIRY: return "expiry";
+        case MemPoolRemovalReason::SIZELIMIT: return "sizelimit";
+        case MemPoolRemovalReason::REORG: return "reorg";
+        case MemPoolRemovalReason::BLOCK: return "block";
+        case MemPoolRemovalReason::CONFLICT: return "conflict";
+        case MemPoolRemovalReason::MANUAL: return "manual";
+    }
+    assert(false);
+}

src/kernel/mempool_removal_reason.h

@@ -0,0 +1,24 @@
+// Copyright (c) 2016-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://opensource.org/license/mit/.
+
+#ifndef BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H
+#define BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H
+
+#include <string>
+
+/** Reason why a transaction was removed from the mempool,
+ * this is passed to the notification signal.
+ */
+enum class MemPoolRemovalReason {
+    EXPIRY,      //!< Expired from mempool
+    SIZELIMIT,   //!< Removed in size limiting
+    REORG,       //!< Removed for reorganization
+    BLOCK,       //!< Removed for block
+    CONFLICT,    //!< Removed for conflict with in-block transaction
+    MANUAL       //!< Removed manually
+};
+
+std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept;
+
+#endif // BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H

src/net_processing.cpp

@@ -2400,7 +2400,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta
     //    the tip yet so we have no way to check this directly here. Instead we
     //    just check that there are currently no other blocks in flight.
     else if (state.IsValid() &&
-             !m_chainman.ActiveChainstate().IsInitialBlockDownload() &&
+             !m_chainman.IsInitialBlockDownload() &&
              mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) {
         if (it != mapBlockSource.end()) {
             MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first);
@@ -3359,7 +3359,7 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, Peer& peer
 
     // If we're in IBD, we want outbound peers that will serve us a useful
     // chain. Disconnect peers that are on chains with insufficient work.
-    if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) {
+    if (m_chainman.IsInitialBlockDownload() && !may_have_more_headers) {
         // If the peer has no more headers to give us, then we know we have
         // their tip.
         if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) {
@@ -4674,7 +4674,7 @@ void PeerManagerImpl::ProcessMessage(
                         return;
                     }
                     bool allowWhileInIBD = allowWhileInIBDObjs.count(inv.type);
-                    if (allowWhileInIBD || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) {
+                    if (allowWhileInIBD || !m_chainman.IsInitialBlockDownload()) {
                         AddObjectAnnouncement(pfrom, inv, current_time);
                     }
                 }
@@ -4964,7 +4964,7 @@ void PeerManagerImpl::ProcessMessage(
         // Stop processing the transaction early if we are still in IBD since we don't
         // have enough information to validate it yet. Sending unsolicited transactions
         // is not considered a protocol violation, so don't punish the peer.
-        if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) return;
+        if (m_chainman.IsInitialBlockDownload()) return;
 
         CTransactionRef ptx;
         CCoinJoinBroadcastTx dstx;
@@ -5167,7 +5167,7 @@ void PeerManagerImpl::ProcessMessage(
 
         if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) {
             // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers
-            if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) {
+            if (!m_chainman.IsInitialBlockDownload()) {
                 std::string ret_val = UsesCompressedHeaders(*peer) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
                 MaybeSendGetHeaders(pfrom, ret_val, GetLocator(m_chainman.m_best_header), *peer);
             }
@@ -6314,7 +6314,7 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros
 
     LOCK(peer.m_addr_send_times_mutex);
     // Periodically advertise our local address to the peer.
-    if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload() &&
+    if (fListen && !m_chainman.IsInitialBlockDownload() &&
         peer.m_next_local_addr_send < current_time) {
         // If we've sent before, clear the bloom filter for the peer, so that our
         // self-announcement will actually go out.
@@ -6940,7 +6940,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto)
         //
         // Message: getdata (blocks)
         //
-        if (CanServeBlocks(*peer) && pto->CanRelay() && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) {
+        if (CanServeBlocks(*peer) && pto->CanRelay() && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) {
             std::vector<const CBlockIndex*> vToDownload;
             NodeId staller = -1;
             FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller);