mining: add getMemoryLoad() and track template non-mempool memory footprint

src/interfaces/mining.h

@@ -91,6 +91,12 @@ class BlockTemplate
     virtual void interruptWait() = 0;
 };
 
+//! Tracks memory usage for template-bound transactions.
+struct MemoryLoad
+{
+    uint64_t usage{0};
+};
+
 //! Interface giving clients (RPC, Stratum v2 Template Provider in the future)
 //! ability to create block templates.
 class Mining
@@ -154,6 +160,12 @@ class Mining
      */
     virtual bool checkBlock(const CBlock& block, const node::BlockCheckOptions& options, std::string& reason, std::string& debug) = 0;
 
+    /**
+     * Returns the current memory load for template transactions outside the
+     * mempool.
+     */
+    virtual MemoryLoad getMemoryLoad() = 0;
+
     //! Get internal node context. Useful for RPC and testing,
     //! but not accessible across processes.
     virtual node::NodeContext* context() { return nullptr; }

src/ipc/capnp/mining.capnp

@@ -25,6 +25,7 @@ interface Mining $Proxy.wrap("interfaces::Mining") {
     createNewBlock @4 (context :Proxy.Context, options: BlockCreateOptions, cooldown: Bool = true) -> (result: BlockTemplate);
     checkBlock @5 (context :Proxy.Context, block: Data, options: BlockCheckOptions) -> (reason: Text, debug: Text, result: Bool);
     interrupt @6 () -> ();
+    getMemoryLoad @7 (context :Proxy.Context) -> (result: MemoryLoad);
 }
 
 interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") {
@@ -46,6 +47,10 @@ struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") {
     coinbaseOutputMaxAdditionalSigops @2 :UInt64 = .defaultCoinbaseOutputMaxAdditionalSigops $Proxy.name("coinbase_output_max_additional_sigops");
 }
 
+struct MemoryLoad $Proxy.wrap("interfaces::MemoryLoad") {
+    usage @0 :UInt64;
+}
+
 struct BlockWaitOptions $Proxy.wrap("node::BlockWaitOptions") {
     timeout @0 : Float64 = .maxDouble $Proxy.name("timeout");
     feeThreshold @1 : Int64 = .maxMoney $Proxy.name("fee_threshold");

src/node/context.h

@@ -5,10 +5,13 @@
 #ifndef BITCOIN_NODE_CONTEXT_H
 #define BITCOIN_NODE_CONTEXT_H
 
+#include <node/types.h>
+
 #include <atomic>
 #include <cstdlib>
 #include <functional>
 #include <memory>
+#include <sync.h>
 #include <thread>
 #include <vector>
 
@@ -26,6 +29,7 @@ class ECC_Context;
 class NetGroupManager;
 class PeerManager;
 namespace interfaces {
+class BlockTemplate;
 class Chain;
 class ChainClient;
 class Mining;
@@ -66,6 +70,15 @@ struct NodeContext {
     std::unique_ptr<AddrMan> addrman;
     std::unique_ptr<CConnman> connman;
     std::unique_ptr<CTxMemPool> mempool;
+    Mutex template_state_mutex;
+    //! Track how many templates (which we hold on to on behalf of connected IPC
+    //! clients) are referencing each transaction.
+    TxTemplateMap template_tx_refs GUARDED_BY(template_state_mutex);
+    //! Cache latest getblocktemplate result for BIP 22 long polling. Must be
+    //! cleared before template_tx_refs because the destructor decrements the
+    //! count in template_tx_refs of each transaction in the template and aborts
+    //! if an entry is missing.
+    std::unique_ptr<interfaces::BlockTemplate> gbt_result;
     std::unique_ptr<const NetGroupManager> netgroupman;
     std::unique_ptr<CBlockPolicyEstimator> fee_estimator;
     std::unique_ptr<PeerManager> peerman;

src/node/interfaces.cpp

@@ -67,6 +67,7 @@
 #include <any>
 #include <memory>
 #include <optional>
+#include <ranges>
 #include <stdexcept>
 #include <utility>
 
@@ -79,6 +80,7 @@ using interfaces::Chain;
 using interfaces::FoundBlock;
 using interfaces::Handler;
 using interfaces::MakeSignalHandler;
+using interfaces::MemoryLoad;
 using interfaces::Mining;
 using interfaces::Node;
 using interfaces::WalletLoader;
@@ -871,7 +873,24 @@ class BlockTemplateImpl : public BlockTemplate
                                                     m_block_template(std::move(block_template)),
                                                     m_node(node)
     {
-        assert(m_block_template);
+        // Don't track the dummy coinbase, because it can be modified in-place
+        // by submitSolution()
+        LOCK(m_node.template_state_mutex);
+        for (const CTransactionRef& tx : Assert(m_block_template)->block.vtx | std::views::drop(1)) {
+            m_node.template_tx_refs[tx]++;
+        }
+    }
+
+    ~BlockTemplateImpl()
+    {
+        LOCK(m_node.template_state_mutex);
+        for (const CTransactionRef& tx : m_block_template->block.vtx | std::views::drop(1)) {
+            auto ref_count{m_node.template_tx_refs.find(tx)};
+            if (!Assume(ref_count != m_node.template_tx_refs.end())) break;
+            if (--ref_count->second == 0) {
+                m_node.template_tx_refs.erase(ref_count);
+            }
+        }
     }
 
     CBlockHeader getBlockHeader() override
@@ -1008,6 +1027,13 @@ class MinerImpl : public Mining
         return state.IsValid();
     }
 
+    MemoryLoad getMemoryLoad() override
+    {
+        LOCK(m_node.template_state_mutex);
+        CTxMemPool& mempool{*Assert(m_node.mempool)};
+        return {.usage = GetTemplateMemoryUsage(mempool, m_node.template_tx_refs)};
+    }
+
     NodeContext* context() override { return &m_node; }
     ChainstateManager& chainman() { return *Assert(m_node.chainman); }
     KernelNotifications& notifications() { return *Assert(m_node.notifications); }

src/node/miner.cpp

@@ -16,12 +16,15 @@
 #include <consensus/validation.h>
 #include <deploymentstatus.h>
 #include <logging.h>
+#include <logging/timer.h>
+#include <memusage.h>
 #include <node/context.h>
 #include <node/kernel_notifications.h>
 #include <policy/feerate.h>
 #include <policy/policy.h>
 #include <pow.h>
 #include <primitives/transaction.h>
+#include <util/check.h>
 #include <util/moneystr.h>
 #include <util/signalinterrupt.h>
 #include <util/time.h>
@@ -524,4 +527,24 @@ std::optional<BlockRef> WaitTipChanged(ChainstateManager& chainman, KernelNotifi
     return GetTip(chainman);
 }
 
+size_t GetTemplateMemoryUsage(const CTxMemPool& mempool, const TxTemplateMap& tx_refs)
+{
+    size_t usage_bytes{0};
+    {
+        LOG_TIME_MILLIS_WITH_CATEGORY("Calculate template transaction reference memory footprint", BCLog::BENCH);
+        for (const auto& [tx_ref, _] : tx_refs) {
+            if (!Assume(tx_ref)) continue;
+            // mempool.exists() locks mempool.cs each time, which slows down
+            // our calculation. This is preferable to potentially blocking
+            // the node from processing new transactions while this
+            // (non-urgent) calculation is in progress.
+            //
+            // As a side-effect the result is not an accurate snapshot, because
+            // the mempool may change during the loop.
+            if (mempool.exists(tx_ref->GetWitnessHash())) continue;
+            usage_bytes += RecursiveDynamicUsage(*tx_ref);
+        }
+    }
+    return usage_bytes;
+}
 } // namespace node

src/node/miner.h

@@ -184,6 +184,15 @@ std::optional<BlockRef> WaitTipChanged(ChainstateManager& chainman, KernelNotifi
  * @returns false if interrupted.
  */
 bool CooldownIfHeadersAhead(ChainstateManager& chainman, KernelNotifications& kernel_notifications, const BlockRef& last_tip, bool& interrupt_mining);
+
+/**
+ * Estimate non-mempool memory footprint of transaction data referenced by block
+ * templates.
+ *
+ * Result is not guaranteed to be an accurate snapshot, because it does not
+ * lock mempool.cs while iterating over transaction references.
+ */
+size_t GetTemplateMemoryUsage(const CTxMemPool& mempool, const TxTemplateMap& tx_refs);
 } // namespace node
 
 #endif // BITCOIN_NODE_MINER_H

src/node/types.h

@@ -20,6 +20,7 @@
 #include <policy/policy.h>
 #include <primitives/transaction.h>
 #include <script/script.h>
+#include <unordered_map>
 #include <uint256.h>
 #include <util/time.h>
 #include <vector>
@@ -172,6 +173,11 @@ enum class TxBroadcast : uint8_t {
     NO_MEMPOOL_PRIVATE_BROADCAST,
 };
 
+/**
+ * Map how many templates refer to each transaction reference.
+ */
+using TxTemplateMap = std::unordered_map<CTransactionRef, size_t>;
+
 } // namespace node
 
 #endif // BITCOIN_NODE_TYPES_H

src/rpc/mining.cpp

@@ -858,7 +858,7 @@ static RPCHelpMan getblocktemplate()
     // Update block
     static CBlockIndex* pindexPrev;
     static int64_t time_start;
-    static std::unique_ptr<BlockTemplate> block_template;
+    std::unique_ptr<BlockTemplate>& block_template{node.gbt_result};
     if (!pindexPrev || pindexPrev->GetBlockHash() != tip ||
         (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5))
     {

test/functional/interface_ipc_mining.py

@@ -11,6 +11,7 @@
 from test_framework.blocktools import NULL_OUTPOINT
 from test_framework.messages import (
     MAX_BLOCK_WEIGHT,
+    MAX_MONEY,
     CBlockHeader,
     CTransaction,
     CTxIn,
@@ -29,6 +30,7 @@
 from test_framework.util import (
     assert_equal,
     assert_greater_than_or_equal,
+    assert_greater_than,
     assert_not_equal
 )
 from test_framework.wallet import MiniWallet
@@ -227,12 +229,15 @@ async def create_block():
                 txsigops = await template.getTxSigops(ctx)
                 assert_equal(len(txsigops.result), 0)
 
-                self.log.debug("Wait for a new template")
+                self.log.debug("Wait for a new template, get one after the tip updates")
                 waitoptions = self.capnp_modules['mining'].BlockWaitOptions()
                 waitoptions.timeout = timeout
-                waitoptions.feeThreshold = 1
+                # Ignore fee increases, wait only for the tip update
+                waitoptions.feeThreshold = MAX_MONEY
+                self.miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0])
                 template2 = await wait_and_do(
                     mining_wait_next_template(template, stack, ctx, waitoptions),
+                    # This mines the transaction, so it won't be in the next template
                     lambda: self.generate(self.nodes[0], 1))
                 assert template2 is not None
                 block2 = await mining_get_block(template2, ctx)
@@ -243,6 +248,7 @@ async def create_block():
                 assert template3 is None
 
                 self.log.debug("Wait for another, get one after increase in fees in the mempool")
+                waitoptions.feeThreshold = 1
                 template4 = await wait_and_do(
                     mining_wait_next_template(template2, stack, ctx, waitoptions),
                     lambda: self.miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]))
@@ -266,6 +272,11 @@ async def create_block():
                 block4 = await mining_get_block(template6, ctx)
                 assert_equal(len(block4.vtx), 3)
 
+                self.log.debug("Memory load should be zero because there was no mempool churn")
+                with self.nodes[0].assert_debug_log(["Calculate template transaction reference memory footprint"]):
+                    memory_load = await mining.getMemoryLoad(ctx)
+                assert_equal(memory_load.result.usage, 0)
+
                 self.log.debug("Wait for another, but time out, since the fee threshold is set now")
                 template7 = await mining_wait_next_template(template6, stack, ctx, waitoptions)
                 assert template7 is None
@@ -279,6 +290,13 @@ async def wait_for_block():
                     assert template7 is None
                 await wait_and_do(wait_for_block(), template6.interruptWait())
 
+                self.log.debug("Template objects go out of scope")
+                # Since multiple templates have common transaction references,
+                # a regression in BlockTemplateImpl / ~BlockTemplateImpl reference
+                # counting should result in a crash here.
+
+            self.log.debug("Template objects are out of scope")
+
         asyncio.run(capnp.run(async_routine()))
 
     def run_ipc_option_override_test(self):
@@ -379,6 +397,18 @@ async def async_routine():
                 submitted = (await template.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())).result
                 assert_equal(submitted, True)
 
+                self.log.debug("Reported memory load should be > 0")
+                # Clients are expected to drop references to stale block templates
+                # briefly after the tip updates. In practice we mainly care about
+                # the memory footprint caused by mempool churn, but this scenario
+                # is easier to test.
+                assert_greater_than((await mining.getMemoryLoad(ctx)).result.usage, 0)
+
+                # Manually release the template:
+                await template.destroy(ctx)
+                self.log.debug("Reported memory load should be 0")
+                assert_equal((await mining.getMemoryLoad(ctx)).result.usage, 0)
+
             self.log.debug("Block should propagate")
             # Check that the IPC node actually updates its own chain
             assert_equal(self.nodes[0].getchaintips()[0]["height"], current_block_height + 1)