From 0253438503c156ecfa8f4074a67c8c0c2ade6a62 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Thu, 19 Sep 2024 00:55:14 +0700
Subject: [PATCH 01/69] feat: new fork WITHDRAWALS introduced

---
 src/chainparams.cpp               | 36 +++++++++++++++++++++++++++++++
 src/consensus/params.h            |  1 +
 src/deploymentinfo.cpp            |  4 ++++
 src/rpc/blockchain.cpp            |  1 +
 test/functional/rpc_blockchain.py | 11 ++++++++++
 5 files changed, 53 insertions(+)

diff --git a/src/chainparams.cpp b/src/chainparams.cpp
index e041149cd1..d551dfeea0 100644
--- a/src/chainparams.cpp
+++ b/src/chainparams.cpp
@@ -209,6 +209,15 @@ class CMainParams : public CChainParams {
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].min_activation_height = 0; // No activation delay
 
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].bit = 11;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nStartTime = 1728864000;   // October 14, 2024
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nTimeout = 1760400000; // October 14, 2025
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nWindowSize = 4032;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdStart = 3226;     // 80% of 4032
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdMin = 2420;       // 60% of 4032
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nFalloffCoeff = 5;          // this corresponds to 10 periods
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].useEHF = true;
+
         // The best chain should have at least this much work.
         consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000000000988117deadb0db9cd5b8"); // 2109672
 
@@ -395,6 +404,15 @@ class CTestNetParams : public CChainParams {
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].min_activation_height = 0; // No activation delay
 
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].bit = 11;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nStartTime = 1728864000;   // October 14, 2024
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nWindowSize = 100;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdStart = 80;     // 80% of 100
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdMin = 60;       // 60% of 100
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nFalloffCoeff = 5;          // this corresponds to 10 periods
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].useEHF = true;
+
         // The best chain should have at least this much work.
         consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000031779704a0f54b4"); // 1069875
 
@@ -556,6 +574,15 @@ class CDevNetParams : public CChainParams {
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].min_activation_height = 0; // No activation delay
 
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].bit = 11;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nStartTime = 1704067200;   // January 1, 2024
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nWindowSize = 120;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdStart = 80;     // 80% of 100
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdMin = 60;       // 60% of 100
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nFalloffCoeff = 5;          // this corresponds to 10 periods
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].useEHF = true;
+
         // The best chain should have at least this much work.
         consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000000000000000000");
 
@@ -782,6 +809,15 @@ class CRegTestParams : public CChainParams {
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
         consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].min_activation_height = 0; // No activation delay
 
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].bit = 11;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nStartTime = 0;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nWindowSize = 300;
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdStart = 300 / 5 * 4;     // 80% of 12
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nThresholdMin = 300 / 5 * 3;       // 60% of 7
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].nFalloffCoeff = 5;          // this corresponds to 10 periods
+        consensus.vDeployments[Consensus::DEPLOYMENT_WITHDRAWALS].useEHF = true;
+
         // The best chain should have at least this much work.
         consensus.nMinimumChainWork = uint256S("0x00");
 
diff --git a/src/consensus/params.h b/src/consensus/params.h
index 95918aa6db..508d477c54 100644
--- a/src/consensus/params.h
+++ b/src/consensus/params.h
@@ -39,6 +39,7 @@ constexpr bool ValidDeployment(BuriedDeployment dep) { return dep <= DEPLOYMENT_
 
 enum DeploymentPos : uint16_t {
     DEPLOYMENT_TESTDUMMY,
+    DEPLOYMENT_WITHDRAWALS, // Deployment of Fix for quorum selection for withdrawals
     // NOTE: Also add new deployments to VersionBitsDeploymentInfo in deploymentinfo.cpp
     MAX_VERSION_BITS_DEPLOYMENTS
 };
diff --git a/src/deploymentinfo.cpp b/src/deploymentinfo.cpp
index 98ba9b3652..6ea50a3d47 100644
--- a/src/deploymentinfo.cpp
+++ b/src/deploymentinfo.cpp
@@ -11,6 +11,10 @@ const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_B
         /*.name =*/ "testdummy",
         /*.gbt_force =*/ true,
     },
+    {
+        /*.name =*/"withdrawals",
+        /*.gbt_force =*/true,
+    },
 };
 
 std::string DeploymentName(Consensus::BuriedDeployment dep)
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 6bf41d7ba8..defe3fb34e 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1841,6 +1841,7 @@ RPCHelpMan getblockchaininfo()
     SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_V19);
     SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_V20);
     SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_MN_RR);
+    SoftForkDescPushBack(tip, ehfSignals, softforks, consensusParams, Consensus::DEPLOYMENT_WITHDRAWALS);
     SoftForkDescPushBack(tip, ehfSignals, softforks, consensusParams, Consensus::DEPLOYMENT_TESTDUMMY);
 
     obj.pushKV("softforks", softforks);
diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py
index a36e881c68..a327a630aa 100755
--- a/test/functional/rpc_blockchain.py
+++ b/test/functional/rpc_blockchain.py
@@ -157,6 +157,17 @@ def _test_getblockchaininfo(self):
             'v19': { 'type': 'buried', 'active': False, 'height': 900},
             'v20': { 'type': 'buried', 'active': False, 'height': 900},
             'mn_rr': { 'type': 'buried', 'active': False, 'height': 900},
+            'withdrawals': {
+                'type': 'bip9',
+                'bip9': {
+                    'status': 'defined',
+                    'start_time': 0,
+                    'timeout': 9223372036854775807,  # testdummy does not have a timeout so is set to the max int64 value
+                    'since': 0,
+                    'min_activation_height': 0,
+                    'ehf': True
+                },
+                'active': False},
             'testdummy': {
                 'type': 'bip9',
                 'bip9': {

From 27040030e9b7c78ca2197bc8b653d483d7589ced Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Wed, 18 Sep 2024 19:46:45 +0700
Subject: [PATCH 02/69] fix: using proper quorums for asset unlock validation

---
 src/evo/assetlocktx.cpp | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/evo/assetlocktx.cpp b/src/evo/assetlocktx.cpp
index ccbb0042e0..64ee2b32c4 100644
--- a/src/evo/assetlocktx.cpp
+++ b/src/evo/assetlocktx.cpp
@@ -12,6 +12,7 @@
 #include <chainparams.h>
 #include <consensus/params.h>
 #include <consensus/validation.h>
+#include <deploymentstatus.h>
 #include <logging.h>
 #include <tinyformat.h>
 #include <util/ranges_set.h>
@@ -113,13 +114,18 @@ bool CAssetUnlockPayload::VerifySig(const llmq::CQuorumManager& qman, const uint
     // and at the quorumHash must be active in either the current or previous quorum cycle
     // and the sig must validate against that specific quorumHash.
 
+
     Consensus::LLMQType llmqType = Params().GetConsensus().llmqTypePlatform;
 
-    // We check at most 2 quorums
-    const auto quorums = qman.ScanQuorums(llmqType, pindexTip, 2);
+    const auto& llmq_params_opt = Params().GetLLMQ(llmqType);
+    assert(llmq_params_opt.has_value());
+
+    // after deployment WITHDRAWALS activated we check not to quorum, but all active quorums + 1 the latest inactive
+    const int quorums_to_scan = DeploymentActiveAt(*pindexTip, Params().GetConsensus(), Consensus::DEPLOYMENT_WITHDRAWALS) ? (llmq_params_opt->signingActiveQuorumCount + 1) : 2;
+    const auto quorums = qman.ScanQuorums(llmqType, pindexTip, quorums_to_scan);
 
     if (bool isActive = std::any_of(quorums.begin(), quorums.end(), [&](const auto &q) { return q->qc->quorumHash == quorumHash; }); !isActive) {
-        return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-assetunlock-not-active-quorum");
+        return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-assetunlock-too-old-quorum");
     }
 
     if (static_cast<uint32_t>(pindexTip->nHeight) < requestedHeight || pindexTip->nHeight >= getHeightToExpiry()) {

From 5fd7b07ddc24fe5fbb2bebbf8e38d6b92f1ddff8 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Thu, 19 Sep 2024 03:50:44 +0700
Subject: [PATCH 03/69] chore: test new validation of asset unlocks tests after
 fork 'withdrawals'

---
 test/functional/feature_asset_locks.py | 35 ++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py
index 91480c0007..c49903f996 100755
--- a/test/functional/feature_asset_locks.py
+++ b/test/functional/feature_asset_locks.py
@@ -38,6 +38,7 @@
     assert_equal,
     assert_greater_than,
     assert_greater_than_or_equal,
+    softfork_active,
 )
 from test_framework.wallet_util import bytes_to_wif
 
@@ -271,6 +272,7 @@ def run_test(self):
         self.test_asset_unlocks(node_wallet, node, pubkey)
         self.test_withdrawal_limits(node_wallet, node, pubkey)
         self.test_mn_rr(node_wallet, node, pubkey)
+        self.test_withdrawal_fork(node_wallet, node, pubkey)
 
 
     def test_asset_locks(self, node_wallet, node, pubkey):
@@ -430,8 +432,9 @@ def test_asset_unlocks(self, node_wallet, node, pubkey):
         self.log.info("Checking that two quorums later it is too late because quorum is not active...")
         self.mine_quorum_2_nodes(llmq_type_name='llmq_test_platform', llmq_type=106)
         self.log.info("Expecting new reject-reason...")
+        assert not softfork_active(self.nodes[0], 'withdrawals')
         self.check_mempool_result(tx=asset_unlock_tx_too_late,
-                result_expected={'allowed': False, 'reject-reason' : 'bad-assetunlock-not-active-quorum'})
+                result_expected={'allowed': False, 'reject-reason' : 'bad-assetunlock-too-old-quorum'})
 
         block_to_reconsider = node.getbestblockhash()
         self.log.info("Test block invalidation with asset unlock tx...")
@@ -445,7 +448,8 @@ def test_asset_unlocks(self, node_wallet, node, pubkey):
         self.validate_credit_pool_balance(locked - 2 * COIN)
 
         self.log.info("Forcibly mining asset_unlock_tx_too_late and ensure block is invalid")
-        self.create_and_check_block([asset_unlock_tx_too_late], expected_error = "bad-assetunlock-not-active-quorum")
+        assert not softfork_active(self.nodes[0], 'withdrawals')
+        self.create_and_check_block([asset_unlock_tx_too_late], expected_error = "bad-assetunlock-too-old-quorum")
 
         self.generate(node, 1)
 
@@ -636,6 +640,33 @@ def test_mn_rr(self, node_wallet, node, pubkey):
         self.generate(node, 1)
         assert_equal(locked, self.get_credit_pool_balance())
 
+    def test_withdrawal_fork(self, node_wallet, node, pubkey):
+        self.log.info("Testing asset unlock after 'withdrawal' activation...")
+
+        assert softfork_active(self.nodes[0], 'withdrawals')
+        self.log.info("Generating several txes by same quorum....")
+
+        asset_unlock_tx = self.create_assetunlock(501, COIN, pubkey)
+        asset_unlock_tx_payload = CAssetUnlockTx()
+        asset_unlock_tx_payload.deserialize(BytesIO(asset_unlock_tx.vExtraPayload))
+
+        self.log.info("Check that new Asset Unlock is valid for current quorum")
+        self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+
+        while asset_unlock_tx_payload.quorumHash in node.quorum('list')['llmq_test_platform']:
+            self.log.info(f"Generate one more quorum until signing quorum becomes not available")
+            self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
+
+            self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+
+        self.log.info(f"Generate one more quorum after which asset unlock meant to be still valid")
+        self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
+        self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+        self.log.info(f"Generate one more quorum after which asset unlock meant to be expired")
+        self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
+        self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': False, 'reject-reason': 'bad-assetunlock-too-old-quorum'})
+
+
 
 if __name__ == '__main__':
     AssetLocksTest().main()

From e4774b9dad2fa7cbc59cdd57a83eed81e84d75f1 Mon Sep 17 00:00:00 2001
From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com>
Date: Thu, 26 Aug 2021 22:49:27 +0300
Subject: [PATCH 04/69] Merge bitcoin-core/gui#384: Add copy IP/Netmask action
 for banned peer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ab1461d5d36b70fd4982679ac6143c25e7617dbf qt: Add copy IP/Netmask action for banned peer (Shashwat)

Pull request description:

  This PR adds a Copy IP/Netmask context menu action to the Banned Peers Table.

  This feature is helpful if a node using GUI might want to alert its peer about a particular malicious user. So it can copy that user’s IP/Netmask and broadcast it to its peers so they can ban it instantly using the setban command in the console.

  | Master        | PR               |
  | ----------- | ----------- |
  | ![Screenshot_from_2021-07-21_00-01-331](https://user-images.githubusercontent.com/23396902/126377808-bd23bb19-3f47-4f1b-8371-39baa9747bbe.png) | ![Screenshot from 2021-08-20 20-13-28(1)(1)](https://user-images.githubusercontent.com/85434418/130251441-a8d0f816-a2e9-4e63-a22d-94885c5cec98.png) |

ACKs for top commit:
  jarolrod:
    re-ACK ab1461d
  hebasto:
    re-ACK ab1461d5d36b70fd4982679ac6143c25e7617dbf, tested on Linux Mint 20.2 (Qt 5.12.8).

Tree-SHA512: a528f089bd4cb5b51fec987550d21c2587459ad80f854b55850bc62c776c21f3fa31052a17e2b0e9e9d0b3468799c8070ed306543730fb7b324f283847151e17
---
 src/qt/rpcconsole.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp
index b76b05bced..ab15df60c9 100644
--- a/src/qt/rpcconsole.cpp
+++ b/src/qt/rpcconsole.cpp
@@ -744,6 +744,13 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_
 
         // create ban table context menu
         banTableContextMenu = new QMenu(this);
+        /*: Context menu action to copy the IP/Netmask of a banned peer.
+            IP/Netmask is the combination of a peer's IP address and its Netmask.
+            For IP address see: https://en.wikipedia.org/wiki/IP_address */
+        banTableContextMenu->addAction(tr("&Copy IP/Netmask"), [this] {
+            GUIUtil::copyEntryData(ui->banlistWidget, BanTableModel::Address, Qt::DisplayRole);
+        });
+        banTableContextMenu->addSeparator();
         banTableContextMenu->addAction(tr("&Unban"), this, &RPCConsole::unbanSelectedNode);
         connect(ui->banlistWidget, &QTableView::customContextMenuRequested, this, &RPCConsole::showBanTableContextMenu);
 

From 02b5fce94266f1bd3caf337b43e749a32dab9517 Mon Sep 17 00:00:00 2001
From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com>
Date: Sun, 12 Sep 2021 18:58:21 +0300
Subject: [PATCH 05/69] Merge bitcoin-core/gui#318: Add `Copy address` Peers
 Tab Context Menu Action

3ec061d9da0c8742bd9dec94ffeb82a11d000aba qt: Add "Copy address" item to the context menu in the Peers table (Hennadii Stepanov)

Pull request description:

  Picking up #264

  This adds a `Copy Address` context menu action to the `Peers Tab`.

  Based on the first commit of PR #317 so that we can use `Qt::DisplayRole` in the `copyEntryData` function.

  | Master        | PR               |
  | ----------- | ----------- |
  |  ![Screen Shot 2021-05-05 at 4 51 11 AM](https://user-images.githubusercontent.com/23396902/117117822-fb067400-ad5d-11eb-9466-228456108e52.png) | ![Screen Shot 2021-05-05 at 4 49 15 AM](https://user-images.githubusercontent.com/23396902/117117835-fe99fb00-ad5d-11eb-8de0-f6a9acdbf40e.png) |

ACKs for top commit:
  shaavan:
    tACK 3ec061d9da0c8742bd9dec94ffeb82a11d000aba
  luke-jr:
    utACK 3ec061d9da0c8742bd9dec94ffeb82a11d000aba
  hebasto:
    ACK 3ec061d9da0c8742bd9dec94ffeb82a11d000aba, tested on Linux Mint 20.2 (Qt 5.12.8):

Tree-SHA512: be0d7324592aae3928fa3cc522294f17226419fe8cbe3587df12a36bd4fa9c81bead377b13051e950b9a3fcd290b273861e70d6c76b75cdf76eaf58224b834cd
---
 src/qt/rpcconsole.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp
index ab15df60c9..084a65ff68 100644
--- a/src/qt/rpcconsole.cpp
+++ b/src/qt/rpcconsole.cpp
@@ -718,6 +718,11 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_
 
         // create peer table context menu
         peersTableContextMenu = new QMenu(this);
+        //: Context menu action to copy the address of a peer
+        peersTableContextMenu->addAction(tr("&Copy address"), [this] {
+            GUIUtil::copyEntryData(ui->peerWidget, PeerTableModel::Address, Qt::DisplayRole);
+        });
+        peersTableContextMenu->addSeparator();
         peersTableContextMenu->addAction(tr("&Disconnect"), this, &RPCConsole::disconnectSelectedNode);
         peersTableContextMenu->addAction(ts.ban_for + " " + tr("1 &hour"), [this] { banSelectedNode(60 * 60); });
         peersTableContextMenu->addAction(ts.ban_for + " " + tr("1 d&ay"), [this] { banSelectedNode(60 * 60 * 24); });

From 95aeb6a08d20c20bae46a2279df27a6bc9d9dc2b Mon Sep 17 00:00:00 2001
From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com>
Date: Wed, 29 Sep 2021 17:18:33 +0300
Subject: [PATCH 06/69] Merge bitcoin-core/gui#436: Include vout when copying
 transaction ID from coin selection

10c6929d55ba9bc203bbadfb834537445dbd67ce Include vout when copying transaction ID from coin selection (Samuel Dobson)

Pull request description:

  Fixes #432

  I think it makes sense to just add the vout to the existing function because I can't imagine a situation where a user in the coin selection dialog would want just the transaction ID rather than the specific outpoint, and they can just delete it from the end anyway.

ACKs for top commit:
  kristapsk:
    ACK 10c6929d55ba9bc203bbadfb834537445dbd67ce
  hebasto:
    ACK 10c6929d55ba9bc203bbadfb834537445dbd67ce, tested on Linux Mint 20.2 (Qt 5.12.8).
  shaavan:
    ACK 10c6929

Tree-SHA512: df4d132b6c2fd0b590594e91cf54f82c6c0f77ee9ca06296fb726bc3c52b9ae459ca3b50c48b2bf303ccafe832b6b4dba692a812f439991ca6d807ea0d8df934
---
 src/qt/coincontroldialog.cpp | 16 ++++++++++------
 src/qt/coincontroldialog.h   |  4 ++--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp
index 8a88c6161b..4ada490a7a 100644
--- a/src/qt/coincontroldialog.cpp
+++ b/src/qt/coincontroldialog.cpp
@@ -67,7 +67,7 @@ CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _m
     contextMenu->addAction(tr("&Copy address"), this, &CoinControlDialog::copyAddress);
     contextMenu->addAction(tr("Copy &label"), this, &CoinControlDialog::copyLabel);
     contextMenu->addAction(tr("Copy &amount"), this, &CoinControlDialog::copyAmount);
-    copyTransactionHashAction = contextMenu->addAction(tr("Copy transaction &ID"), this, &CoinControlDialog::copyTransactionHash);
+    m_copy_transaction_outpoint_action = contextMenu->addAction(tr("Copy transaction &ID and output index"), this, &CoinControlDialog::copyTransactionOutpoint);
     contextMenu->addSeparator();
     lockAction = contextMenu->addAction(tr("L&ock unspent"), this, &CoinControlDialog::lockCoin);
     unlockAction = contextMenu->addAction(tr("&Unlock unspent"), this, &CoinControlDialog::unlockCoin);
@@ -235,7 +235,7 @@ void CoinControlDialog::showMenu(const QPoint &point)
         // disable some items (like Copy Transaction ID, lock, unlock) for tree roots in context menu
         if (item->data(COLUMN_ADDRESS, TxHashRole).toString().length() == 64) // transaction hash is 64 characters (this means it is a child node, so it is not a parent node in tree mode)
         {
-            copyTransactionHashAction->setEnabled(true);
+            m_copy_transaction_outpoint_action->setEnabled(true);
             if (model->wallet().isLockedCoin(COutPoint(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt())))
             {
                 lockAction->setEnabled(false);
@@ -249,7 +249,7 @@ void CoinControlDialog::showMenu(const QPoint &point)
         }
         else // this means click on parent node in tree mode -> disable all
         {
-            copyTransactionHashAction->setEnabled(false);
+            m_copy_transaction_outpoint_action->setEnabled(false);
             lockAction->setEnabled(false);
             unlockAction->setEnabled(false);
         }
@@ -283,10 +283,14 @@ void CoinControlDialog::copyAddress()
         GUIUtil::setClipboard(contextMenuItem->text(COLUMN_ADDRESS));
 }
 
-// context menu action: copy transaction id
-void CoinControlDialog::copyTransactionHash()
+// context menu action: copy transaction id and vout index
+void CoinControlDialog::copyTransactionOutpoint()
 {
-    GUIUtil::setClipboard(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString());
+    const QString address = contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString();
+    const QString vout = contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toString();
+    const QString outpoint = QString("%1:%2").arg(address).arg(vout);
+
+    GUIUtil::setClipboard(outpoint);
 }
 
 // context menu action: lock coin
diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h
index 3336f2c396..601b4bcb5b 100644
--- a/src/qt/coincontroldialog.h
+++ b/src/qt/coincontroldialog.h
@@ -59,7 +59,7 @@ class CoinControlDialog : public QDialog
 
     QMenu *contextMenu;
     QTreeWidgetItem *contextMenuItem;
-    QAction *copyTransactionHashAction;
+    QAction* m_copy_transaction_outpoint_action;
     QAction *lockAction;
     QAction *unlockAction;
 
@@ -92,7 +92,7 @@ private Q_SLOTS:
     void copyAmount();
     void copyLabel();
     void copyAddress();
-    void copyTransactionHash();
+    void copyTransactionOutpoint();
     void lockCoin();
     void unlockCoin();
     void clipboardQuantity();

From 042e8a4101d8776b5eaf75f559ac705404245205 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Mon, 7 Oct 2024 00:21:21 +0700
Subject: [PATCH 07/69] fix: drop unneded lock assert in net.cpp for
 m_nodes_mutex

---
 src/net.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/net.cpp b/src/net.cpp
index 474fbec3e7..49e9ca2c82 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -643,8 +643,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo
 
 void CNode::CloseSocketDisconnect(CConnman* connman)
 {
-    AssertLockHeld(connman->m_nodes_mutex);
-
     fDisconnect = true;
     LOCK2(connman->cs_mapSocketToNode, m_sock_mutex);
     if (!m_sock) {

From a219a8be154be6d31873de2985dcf4509e19b62b Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Thu, 26 Sep 2024 00:47:28 +0700
Subject: [PATCH 08/69] partial Merge #29040: refactor: Remove pre-C++20 code,
 fs::path cleanup

It fixes return reference to const variable under mutex which is not a good idea
---
 src/util/system.cpp | 4 ++--
 src/util/system.h   | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/util/system.cpp b/src/util/system.cpp
index b0af7eb30b..9ecadeabce 100644
--- a/src/util/system.cpp
+++ b/src/util/system.cpp
@@ -420,7 +420,7 @@ std::optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) co
     return std::nullopt;
 }
 
-const fs::path& ArgsManager::GetBlocksDirPath() const
+const fs::path ArgsManager::GetBlocksDirPath() const
 {
     LOCK(cs_args);
     fs::path& path = m_cached_blocks_path;
@@ -445,7 +445,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const
     return path;
 }
 
-const fs::path& ArgsManager::GetDataDir(bool net_specific) const
+const fs::path ArgsManager::GetDataDir(bool net_specific) const
 {
     LOCK(cs_args);
     fs::path& path = net_specific ? m_cached_network_datadir_path : m_cached_datadir_path;
diff --git a/src/util/system.h b/src/util/system.h
index 1ae230d4f8..4ab1905459 100644
--- a/src/util/system.h
+++ b/src/util/system.h
@@ -288,7 +288,7 @@ class ArgsManager
      *
      * @return Blocks path which is network specific
      */
-    const fs::path& GetBlocksDirPath() const;
+    const fs::path GetBlocksDirPath() const;
 
     /**
      * Get data directory path
@@ -296,7 +296,7 @@ class ArgsManager
      * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned
      * @post Returned directory path is created unless it is empty
      */
-    const fs::path& GetDataDirBase() const { return GetDataDir(false); }
+    const fs::path GetDataDirBase() const { return GetDataDir(false); }
 
     /**
      * Get data directory path with appended network identifier
@@ -304,7 +304,7 @@ class ArgsManager
      * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned
      * @post Returned directory path is created unless it is empty
      */
-    const fs::path& GetDataDirNet() const { return GetDataDir(true); }
+    const fs::path GetDataDirNet() const { return GetDataDir(true); }
 
     fs::path GetBackupsDirPath();
 
@@ -483,7 +483,7 @@ class ArgsManager
      * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned
      * @post Returned directory path is created unless it is empty
      */
-    const fs::path& GetDataDir(bool net_specific) const;
+    const fs::path GetDataDir(bool net_specific) const;
 
     // Helper function for LogArgs().
     void logArgsPrefix(

From 502d6ae8efd024827afbb93db5bb10f50f485505 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Thu, 26 Sep 2024 00:20:01 +0700
Subject: [PATCH 09/69] fix: add multiple missing annotation about locks for
 dash specific code

---
 src/coinjoin/client.cpp      |  9 +++++++--
 src/coinjoin/client.h        | 13 +++++++++----
 src/coinjoin/coinjoin.h      |  4 ++--
 src/evo/deterministicmns.cpp | 12 +++++++-----
 src/evo/simplifiedmns.h      |  7 ++++---
 src/evo/specialtxman.cpp     |  7 ++++---
 src/governance/governance.h  |  9 +++++----
 src/governance/object.h      |  3 ++-
 src/llmq/snapshot.h          |  2 +-
 src/masternode/meta.h        |  6 +++++-
 src/net_processing.cpp       |  2 +-
 src/rpc/evo.cpp              |  5 +++--
 src/txmempool.cpp            |  4 ++--
 src/wallet/wallet.cpp        |  2 +-
 src/wallet/wallet.h          | 18 +++++++++++-------
 15 files changed, 64 insertions(+), 39 deletions(-)

diff --git a/src/coinjoin/client.cpp b/src/coinjoin/client.cpp
index 9739727243..4c15f98f4a 100644
--- a/src/coinjoin/client.cpp
+++ b/src/coinjoin/client.cpp
@@ -1448,11 +1448,16 @@ bool CCoinJoinClientSession::MakeCollateralAmounts()
     });
 
     // First try to use only non-denominated funds
-    if (std::any_of(vecTally.begin(), vecTally.end(), [&](const auto& item) { return MakeCollateralAmounts(item, false); })) {
+    if (ranges::any_of(vecTally, [&](const auto& item) EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet) {
+            return MakeCollateralAmounts(item, false);
+        })) {
         return true;
     }
+
     // There should be at least some denominated funds we should be able to break in pieces to continue mixing
-    if (std::any_of(vecTally.begin(), vecTally.end(), [&](const auto& item) { return MakeCollateralAmounts(item, true); })) {
+    if (ranges::any_of(vecTally, [&](const auto& item) EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet) {
+            return MakeCollateralAmounts(item, true);
+        })) {
         return true;
     }
 
diff --git a/src/coinjoin/client.h b/src/coinjoin/client.h
index a0deebdbd2..c4e22b7123 100644
--- a/src/coinjoin/client.h
+++ b/src/coinjoin/client.h
@@ -161,13 +161,16 @@ class CCoinJoinClientSession : public CCoinJoinBaseSession
 
     /// Create denominations
     bool CreateDenominated(CAmount nBalanceToDenominate);
-    bool CreateDenominated(CAmount nBalanceToDenominate, const CompactTallyItem& tallyItem, bool fCreateMixingCollaterals);
+    bool CreateDenominated(CAmount nBalanceToDenominate, const CompactTallyItem& tallyItem, bool fCreateMixingCollaterals)
+        EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet);
 
     /// Split up large inputs or make fee sized inputs
     bool MakeCollateralAmounts();
-    bool MakeCollateralAmounts(const CompactTallyItem& tallyItem, bool fTryDenominated);
+    bool MakeCollateralAmounts(const CompactTallyItem& tallyItem, bool fTryDenominated)
+        EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet);
 
-    bool CreateCollateralTransaction(CMutableTransaction& txCollateral, std::string& strReason);
+    bool CreateCollateralTransaction(CMutableTransaction& txCollateral, std::string& strReason)
+        EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet);
 
     bool JoinExistingQueue(CAmount nBalanceNeedsAnonymized, CConnman& connman);
     bool StartNewQueue(CAmount nBalanceNeedsAnonymized, CConnman& connman);
@@ -175,7 +178,9 @@ class CCoinJoinClientSession : public CCoinJoinBaseSession
     /// step 0: select denominated inputs and txouts
     bool SelectDenominate(std::string& strErrorRet, std::vector<CTxDSIn>& vecTxDSInRet);
     /// step 1: prepare denominated inputs and outputs
-    bool PrepareDenominate(int nMinRounds, int nMaxRounds, std::string& strErrorRet, const std::vector<CTxDSIn>& vecTxDSIn, std::vector<std::pair<CTxDSIn, CTxOut> >& vecPSInOutPairsRet, bool fDryRun = false);
+    bool PrepareDenominate(int nMinRounds, int nMaxRounds, std::string& strErrorRet, const std::vector<CTxDSIn>& vecTxDSIn,
+                           std::vector<std::pair<CTxDSIn, CTxOut>>& vecPSInOutPairsRet, bool fDryRun = false)
+        EXCLUSIVE_LOCKS_REQUIRED(m_wallet.cs_wallet);
     /// step 2: send denominated inputs and outputs prepared in step 1
     bool SendDenominate(const std::vector<std::pair<CTxDSIn, CTxOut> >& vecPSInOutPairsIn, CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(!cs_coinjoin);
 
diff --git a/src/coinjoin/coinjoin.h b/src/coinjoin/coinjoin.h
index 6bf8e93e11..5680b8abc2 100644
--- a/src/coinjoin/coinjoin.h
+++ b/src/coinjoin/coinjoin.h
@@ -398,8 +398,8 @@ class CDSTXManager
 private:
     void CheckDSTXes(const CBlockIndex* pindex, const llmq::CChainLocksHandler& clhandler)
         EXCLUSIVE_LOCKS_REQUIRED(!cs_mapdstx);
-    void UpdateDSTXConfirmedHeight(const CTransactionRef& tx, std::optional<int> nHeight);
-
+    void UpdateDSTXConfirmedHeight(const CTransactionRef& tx, std::optional<int> nHeight)
+        EXCLUSIVE_LOCKS_REQUIRED(cs_mapdstx);
 };
 
 bool ATMPIfSaneFee(ChainstateManager& chainman, const CTransactionRef& tx, bool test_accept = false)
diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp
index ac167a298b..49912b8880 100644
--- a/src/evo/deterministicmns.cpp
+++ b/src/evo/deterministicmns.cpp
@@ -1067,11 +1067,13 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlockInternal(gsl::not_n
             mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
         } else {
             // keep snapshots for yet alive quorums
-            if (ranges::any_of(Params().GetConsensus().llmqs, [&snapshot, this](const auto& params){
-                AssertLockHeld(cs);
-                return (snapshot.GetHeight() % params.dkgInterval == 0) &&
-                (snapshot.GetHeight() + params.dkgInterval * (params.keepOldConnections + 1) >= tipIndex->nHeight);
-            })) {
+            if (ranges::any_of(Params().GetConsensus().llmqs,
+                               [&snapshot, this](const auto& params) EXCLUSIVE_LOCKS_REQUIRED(cs) {
+                                   AssertLockHeld(cs);
+                                   return (snapshot.GetHeight() % params.dkgInterval == 0) &&
+                                          (snapshot.GetHeight() + params.dkgInterval * (params.keepOldConnections + 1) >=
+                                           tipIndex->nHeight);
+                               })) {
                 mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
             }
         }
diff --git a/src/evo/simplifiedmns.h b/src/evo/simplifiedmns.h
index 9999a5f352..0cfd5603d9 100644
--- a/src/evo/simplifiedmns.h
+++ b/src/evo/simplifiedmns.h
@@ -171,8 +171,9 @@ class CSimplifiedMNListDiff
     [[nodiscard]] UniValue ToJson(bool extended = false) const;
 };
 
-bool BuildSimplifiedMNListDiff(CDeterministicMNManager& dmnman, const ChainstateManager& chainman, const llmq::CQuorumBlockProcessor& qblockman,
-                               const llmq::CQuorumManager& qman, const uint256& baseBlockHash, const uint256& blockHash,
-                               CSimplifiedMNListDiff& mnListDiffRet, std::string& errorRet, bool extended = false);
+bool BuildSimplifiedMNListDiff(CDeterministicMNManager& dmnman, const ChainstateManager& chainman,
+                               const llmq::CQuorumBlockProcessor& qblockman, const llmq::CQuorumManager& qman,
+                               const uint256& baseBlockHash, const uint256& blockHash, CSimplifiedMNListDiff& mnListDiffRet,
+                               std::string& errorRet, bool extended = false) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 
 #endif // BITCOIN_EVO_SIMPLIFIEDMNS_H
diff --git a/src/evo/specialtxman.cpp b/src/evo/specialtxman.cpp
index e20a9e0d4f..c8c84e84be 100644
--- a/src/evo/specialtxman.cpp
+++ b/src/evo/specialtxman.cpp
@@ -19,9 +19,10 @@
 #include <primitives/block.h>
 #include <validation.h>
 
-static bool CheckSpecialTxInner(CDeterministicMNManager& dmnman, const ChainstateManager& chainman, const llmq::CQuorumManager& qman, const CTransaction& tx,
-                                const CBlockIndex* pindexPrev, const CCoinsViewCache& view, const std::optional<CRangesSet>& indexes, bool check_sigs,
-                                TxValidationState& state)
+static bool CheckSpecialTxInner(CDeterministicMNManager& dmnman, const ChainstateManager& chainman,
+                                const llmq::CQuorumManager& qman, const CTransaction& tx, const CBlockIndex* pindexPrev,
+                                const CCoinsViewCache& view, const std::optional<CRangesSet>& indexes, bool check_sigs,
+                                TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
 {
     AssertLockHeld(cs_main);
 
diff --git a/src/governance/governance.h b/src/governance/governance.h
index e5b8e077a8..adc25bb543 100644
--- a/src/governance/governance.h
+++ b/src/governance/governance.h
@@ -347,9 +347,9 @@ class CGovernanceManager : public GovernanceStore
      *   - Track governance objects which are triggers
      *   - After triggers are activated and executed, they can be removed
     */
-    std::vector<std::shared_ptr<CSuperblock>> GetActiveTriggers() const;
-    bool AddNewTrigger(uint256 nHash);
-    void CleanAndRemoveTriggers();
+    std::vector<std::shared_ptr<CSuperblock>> GetActiveTriggers() const EXCLUSIVE_LOCKS_REQUIRED(cs);
+    bool AddNewTrigger(uint256 nHash) EXCLUSIVE_LOCKS_REQUIRED(cs);
+    void CleanAndRemoveTriggers() EXCLUSIVE_LOCKS_REQUIRED(cs);
 
     // Superblocks related:
 
@@ -373,7 +373,8 @@ class CGovernanceManager : public GovernanceStore
 
 private:
     void ExecuteBestSuperblock(const CDeterministicMNList& tip_mn_list, int nBlockHeight);
-    bool GetBestSuperblock(const CDeterministicMNList& tip_mn_list, CSuperblock_sptr& pSuperblockRet, int nBlockHeight);
+    bool GetBestSuperblock(const CDeterministicMNList& tip_mn_list, CSuperblock_sptr& pSuperblockRet, int nBlockHeight)
+        EXCLUSIVE_LOCKS_REQUIRED(cs);
 
     std::optional<const CSuperblock> CreateSuperblockCandidate(int nHeight) const;
     std::optional<const CGovernanceObject> CreateGovernanceTrigger(const std::optional<const CSuperblock>& sb_opt, PeerManager& peerman,
diff --git a/src/governance/object.h b/src/governance/object.h
index e58da6a8a1..8ced4950f9 100644
--- a/src/governance/object.h
+++ b/src/governance/object.h
@@ -235,7 +235,8 @@ class CGovernanceObject
     /// Check the collateral transaction for the budget proposal/finalized budget
     bool IsCollateralValid(const ChainstateManager& chainman, std::string& strError, bool& fMissingConfirmations) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
 
-    void UpdateLocalValidity(const CDeterministicMNList& tip_mn_list, const ChainstateManager& chainman);
+    void UpdateLocalValidity(const CDeterministicMNList& tip_mn_list, const ChainstateManager& chainman)
+        EXCLUSIVE_LOCKS_REQUIRED(cs_main);
 
     void UpdateSentinelVariables(const CDeterministicMNList& tip_mn_list);
 
diff --git a/src/llmq/snapshot.h b/src/llmq/snapshot.h
index b9ca9ac985..1551789f37 100644
--- a/src/llmq/snapshot.h
+++ b/src/llmq/snapshot.h
@@ -209,7 +209,7 @@ class CQuorumRotationInfo
 
 bool BuildQuorumRotationInfo(CDeterministicMNManager& dmnman, const ChainstateManager& chainman, const CQuorumManager& qman,
                              const CQuorumBlockProcessor& qblockman, const CGetQuorumRotationInfo& request,
-                             CQuorumRotationInfo& response, std::string& errorRet);
+                             CQuorumRotationInfo& response, std::string& errorRet) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 uint256 GetLastBaseBlockHash(Span<const CBlockIndex*> baseBlockIndexes, const CBlockIndex* blockIndex);
 
 class CQuorumSnapshotManager
diff --git a/src/masternode/meta.h b/src/masternode/meta.h
index 0388905c44..14870afa63 100644
--- a/src/masternode/meta.h
+++ b/src/masternode/meta.h
@@ -73,7 +73,11 @@ class CMasternodeMetaInfo
     UniValue ToJson() const;
 
 public:
-    const uint256& GetProTxHash() const { LOCK(cs); return proTxHash; }
+    const uint256 GetProTxHash() const
+    {
+        LOCK(cs);
+        return proTxHash;
+    }
     int64_t GetLastDsq() const { return nLastDsq; }
     int GetMixingTxCount() const { return nMixingTxCount; }
 
diff --git a/src/net_processing.cpp b/src/net_processing.cpp
index 9536b8fd07..4537754231 100644
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -5771,7 +5771,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto)
             peer->m_blocks_for_inv_relay.clear();
         }
 
-        auto queueAndMaybePushInv = [this, pto, peer, &vInv, &msgMaker](const CInv& invIn) {
+        auto queueAndMaybePushInv = [this, pto, peer, &vInv, &msgMaker](const CInv& invIn) EXCLUSIVE_LOCKS_REQUIRED(peer->m_tx_relay->m_tx_inventory_mutex) {
             AssertLockHeld(peer->m_tx_relay->m_tx_inventory_mutex);
             peer->m_tx_relay->m_tx_inventory_known_filter.insert(invIn.hash);
             LogPrint(BCLog::NET, "SendMessages -- queued inv: %s  index=%d peer=%d\n", invIn.ToString(), vInv.size(), pto->GetId());
diff --git a/src/rpc/evo.cpp b/src/rpc/evo.cpp
index 30fdab04b7..5fa7506cda 100644
--- a/src/rpc/evo.cpp
+++ b/src/rpc/evo.cpp
@@ -213,8 +213,9 @@ static bool ValidatePlatformPort(const int32_t port)
 
 #ifdef ENABLE_WALLET
 
-template<typename SpecialTxPayload>
-static void FundSpecialTx(CWallet& wallet, CMutableTransaction& tx, const SpecialTxPayload& payload, const CTxDestination& fundDest)
+template <typename SpecialTxPayload>
+static void FundSpecialTx(CWallet& wallet, CMutableTransaction& tx, const SpecialTxPayload& payload,
+                          const CTxDestination& fundDest) EXCLUSIVE_LOCKS_REQUIRED(!wallet.cs_wallet)
 {
     // Make sure the results are valid at least up to the most recent block
     // the user could have gotten from another RPC command prior to now
diff --git a/src/txmempool.cpp b/src/txmempool.cpp
index cb16e2ddfc..291599aea2 100644
--- a/src/txmempool.cpp
+++ b/src/txmempool.cpp
@@ -955,7 +955,7 @@ void CTxMemPool::removeProTxSpentCollateralConflicts(const CTransaction &tx)
     assert(m_dmnman);
 
     // Remove TXs that refer to a MN for which the collateral was spent
-    auto removeSpentCollateralConflict = [&](const uint256& proTxHash) {
+    auto removeSpentCollateralConflict = [&](const uint256& proTxHash) EXCLUSIVE_LOCKS_REQUIRED(cs) {
         // Can't use equal_range here as every call to removeRecursive might invalidate iterators
         AssertLockHeld(cs);
         while (true) {
@@ -1353,7 +1353,7 @@ bool CTxMemPool::existsProviderTxConflict(const CTransaction &tx) const {
 
     LOCK(cs);
 
-    auto hasKeyChangeInMempool = [&](const uint256& proTxHash) {
+    auto hasKeyChangeInMempool = [&](const uint256& proTxHash) EXCLUSIVE_LOCKS_REQUIRED(cs) {
         AssertLockHeld(cs);
         for (auto its = mapProTxRefs.equal_range(proTxHash); its.first != its.second; ++its.first) {
             auto txit = mapTx.find(its.first->second);
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
index edc7b41c9e..b6ac4595a8 100644
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -3681,7 +3681,7 @@ bool CWallet::CreateTransactionInternal(
                     txNew.vin.emplace_back(coin.outpoint, CScript(), CTxIn::SEQUENCE_FINAL - 1);
                 }
 
-                auto calculateFee = [&](CAmount& nFee) -> bool {
+                auto calculateFee = [&](CAmount& nFee) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) -> bool {
                     AssertLockHeld(cs_wallet);
                     nBytes = CalculateMaximumSignedTxSize(CTransaction(txNew), this, coin_control.fAllowWatchOnly);
                     if (nBytes < 0) {
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
index 11824d79ee..411367ac2b 100644
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -516,8 +516,12 @@ class CWalletTx
     CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const;
     CAmount GetChange() const;
 
-    CAmount GetAnonymizedCredit(const CCoinControl* coinControl = nullptr) const;
-    CAmount GetDenominatedCredit(bool unconfirmed, bool fUseCache=true) const;
+    // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct
+    // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The
+    // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid
+    // having to resolve the issue of member access into incomplete type CWallet.
+    CAmount GetAnonymizedCredit(const CCoinControl* coinControl = nullptr) const NO_THREAD_SAFETY_ANALYSIS;
+    CAmount GetDenominatedCredit(bool unconfirmed, bool fUseCache=true) const NO_THREAD_SAFETY_ANALYSIS;
 
     /** Get the marginal bytes if spending the specified output from this transaction */
     int GetSpendSize(unsigned int out, bool use_max_sig = false) const
@@ -569,7 +573,7 @@ class CWalletTx
     int GetDepthInMainChain() const NO_THREAD_SAFETY_ANALYSIS;
     bool IsInMainChain() const { return GetDepthInMainChain() > 0; }
     bool IsLockedByInstantSend() const;
-    bool IsChainLocked() const;
+    bool IsChainLocked() const NO_THREAD_SAFETY_ANALYSIS;
 
     /**
      * @return number of blocks to maturity for this transaction:
@@ -1289,7 +1293,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
     std::set<uint256> GetConflicts(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
 
     //! Check if a given transaction has any of its outputs spent by another transaction in the wallet
-    bool HasWalletSpend(const uint256& txid) const;
+    bool HasWalletSpend(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
 
     //! Flush wallet (bitdb flush)
     void Flush();
@@ -1386,11 +1390,11 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
     void notifyChainLock(const CBlockIndex* pindexChainLock, const std::shared_ptr<const llmq::CChainLockSig>& clsig) override;
 
     /** Load a CGovernanceObject into m_gobjects. */
-    bool LoadGovernanceObject(const Governance::Object& obj);
+    bool LoadGovernanceObject(const Governance::Object& obj) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
     /** Store a CGovernanceObject in the wallet database. This should only be used by governance objects that are created by this wallet via `gobject prepare`. */
-    bool WriteGovernanceObject(const Governance::Object& obj);
+    bool WriteGovernanceObject(const Governance::Object& obj) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
     /** Returns a vector containing pointers to the governance objects in m_gobjects */
-    std::vector<const Governance::Object*> GetGovernanceObjects();
+    std::vector<const Governance::Object*> GetGovernanceObjects() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
 
     /**
      * Blocks until the wallet state is up-to-date to /at least/ the current

From 002580c42d8fc2d7f91ec0ed27876464db0771d2 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Sun, 6 Oct 2024 23:54:32 +0700
Subject: [PATCH 10/69] fix: add ignoring safety-annotation for
 llmq/signing_shares due to template lambdas

---
 src/llmq/signing_shares.cpp | 56 +++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/src/llmq/signing_shares.cpp b/src/llmq/signing_shares.cpp
index 0b157f218a..47a70430fa 100644
--- a/src/llmq/signing_shares.cpp
+++ b/src/llmq/signing_shares.cpp
@@ -565,24 +565,29 @@ void CSigSharesManager::CollectPendingSigSharesToVerify(
         // other nodes would be able to poison us with a large batch with N-1 valid shares and the last one being
         // invalid, making batch verification fail and revert to per-share verification, which in turn would slow down
         // the whole verification process
-
         std::unordered_set<std::pair<NodeId, uint256>, StaticSaltedHasher> uniqueSignHashes;
-        IterateNodesRandom(nodeStates, [&]() {
-            return uniqueSignHashes.size() < maxUniqueSessions;
-        }, [&](NodeId nodeId, CSigSharesNodeState& ns) {
-            if (ns.pendingIncomingSigShares.Empty()) {
-                return false;
-            }
-            const auto& sigShare = *ns.pendingIncomingSigShares.GetFirst();
+        IterateNodesRandom(
+            nodeStates,
+            [&]() {
+                return uniqueSignHashes.size() < maxUniqueSessions;
+                // TODO: remove NO_THREAD_SAFETY_ANALYSIS
+                // using here template IterateNodesRandom makes impossible to use lock annotation
+            },
+            [&](NodeId nodeId, CSigSharesNodeState& ns) NO_THREAD_SAFETY_ANALYSIS {
+                if (ns.pendingIncomingSigShares.Empty()) {
+                    return false;
+                }
+                const auto& sigShare = *ns.pendingIncomingSigShares.GetFirst();
 
-            AssertLockHeld(cs);
-            if (const bool alreadyHave = this->sigShares.Has(sigShare.GetKey()); !alreadyHave) {
-                uniqueSignHashes.emplace(nodeId, sigShare.GetSignHash());
-                retSigShares[nodeId].emplace_back(sigShare);
-            }
-            ns.pendingIncomingSigShares.Erase(sigShare.GetKey());
-            return !ns.pendingIncomingSigShares.Empty();
-        }, rnd);
+                AssertLockHeld(cs);
+                if (const bool alreadyHave = this->sigShares.Has(sigShare.GetKey()); !alreadyHave) {
+                    uniqueSignHashes.emplace(nodeId, sigShare.GetSignHash());
+                    retSigShares[nodeId].emplace_back(sigShare);
+                }
+                ns.pendingIncomingSigShares.Erase(sigShare.GetKey());
+                return !ns.pendingIncomingSigShares.Empty();
+            },
+            rnd);
 
         if (retSigShares.empty()) {
             return;
@@ -1020,7 +1025,10 @@ void CSigSharesManager::CollectSigSharesToAnnounce(std::unordered_map<NodeId, st
 
     std::unordered_map<std::pair<Consensus::LLMQType, uint256>, std::unordered_set<NodeId>, StaticSaltedHasher> quorumNodesMap;
 
-    sigSharesQueuedToAnnounce.ForEach([this, &quorumNodesMap, &sigSharesToAnnounce](const SigShareKey& sigShareKey, bool) {
+    // TODO: remove NO_THREAD_SAFETY_ANALYSIS
+    // using here template ForEach makes impossible to use lock annotation
+    sigSharesQueuedToAnnounce.ForEach([this, &quorumNodesMap, &sigSharesToAnnounce](const SigShareKey& sigShareKey,
+                                                                                    bool) NO_THREAD_SAFETY_ANALYSIS {
         AssertLockHeld(cs);
         const auto& signHash = sigShareKey.first;
         auto quorumMember = sigShareKey.second;
@@ -1076,7 +1084,7 @@ bool CSigSharesManager::SendMessages()
     std::unordered_map<NodeId, std::unordered_map<uint256, CSigSharesInv, StaticSaltedHasher>> sigSharesToAnnounce;
     std::unordered_map<NodeId, std::vector<CSigSesAnn>> sigSessionAnnouncements;
 
-    auto addSigSesAnnIfNeeded = [&](NodeId nodeId, const uint256& signHash) {
+    auto addSigSesAnnIfNeeded = [&](NodeId nodeId, const uint256& signHash) EXCLUSIVE_LOCKS_REQUIRED(cs) {
         AssertLockHeld(cs);
         auto& nodeState = nodeStates[nodeId];
         auto* session = nodeState.GetSessionBySignHash(signHash);
@@ -1357,7 +1365,9 @@ void CSigSharesManager::Cleanup()
             continue;
         }
         // remove global requested state to force a re-request from another node
-        it->second.requestedSigShares.ForEach([this](const SigShareKey& k, bool) {
+        // TODO: remove NO_THREAD_SAFETY_ANALYSIS
+        // using here template ForEach makes impossible to use lock annotation
+        it->second.requestedSigShares.ForEach([this](const SigShareKey& k, bool) NO_THREAD_SAFETY_ANALYSIS {
             AssertLockHeld(cs);
             sigSharesRequested.Erase(k);
         });
@@ -1390,7 +1400,9 @@ void CSigSharesManager::RemoveBannedNodeStates()
     for (auto it = nodeStates.begin(); it != nodeStates.end();) {
         if (Assert(m_peerman)->IsBanned(it->first)) {
             // re-request sigshares from other nodes
-            it->second.requestedSigShares.ForEach([this](const SigShareKey& k, int64_t) {
+            // TODO: remove NO_THREAD_SAFETY_ANALYSIS
+            // using here template ForEach makes impossible to use lock annotation
+            it->second.requestedSigShares.ForEach([this](const SigShareKey& k, int64_t) NO_THREAD_SAFETY_ANALYSIS {
                 AssertLockHeld(cs);
                 sigSharesRequested.Erase(k);
             });
@@ -1419,7 +1431,9 @@ void CSigSharesManager::BanNode(NodeId nodeId)
     auto& nodeState = it->second;
 
     // Whatever we requested from him, let's request it from someone else now
-    nodeState.requestedSigShares.ForEach([this](const SigShareKey& k, int64_t) {
+    // TODO: remove NO_THREAD_SAFETY_ANALYSIS
+    // using here template ForEach makes impossible to use lock annotation
+    nodeState.requestedSigShares.ForEach([this](const SigShareKey& k, int64_t) NO_THREAD_SAFETY_ANALYSIS {
         AssertLockHeld(cs);
         sigSharesRequested.Erase(k);
     });

From 846ebab6e06601105a3eb22eb6b3c6ff68426adb Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Mon, 7 Oct 2024 00:35:30 +0700
Subject: [PATCH 11/69] fix: fixes for orphange's locks and cs_main

---
 src/net_processing.cpp | 12 +++++++-----
 src/txorphanage.cpp    |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/net_processing.cpp b/src/net_processing.cpp
index 4537754231..803f3d51aa 100644
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -1930,12 +1930,14 @@ void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler)
  */
 void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex)
 {
-    LOCK2(::cs_main, g_cs_orphans);
+    {
+        LOCK2(::cs_main, g_cs_orphans);
 
-    auto orphanWorkSet = m_orphanage.GetCandidatesForBlock(*pblock);
-    while (!orphanWorkSet.empty()) {
-        LogPrint(BCLog::MEMPOOL, "Trying to process %d orphans\n", orphanWorkSet.size());
-        ProcessOrphanTx(orphanWorkSet);
+        auto orphanWorkSet = m_orphanage.GetCandidatesForBlock(*pblock);
+        while (!orphanWorkSet.empty()) {
+            LogPrint(BCLog::MEMPOOL, "Trying to process %d orphans\n", orphanWorkSet.size());
+            ProcessOrphanTx(orphanWorkSet);
+        }
     }
 
     m_orphanage.EraseForBlock(*pblock);
diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp
index 84835e0aad..ce9d6435ce 100644
--- a/src/txorphanage.cpp
+++ b/src/txorphanage.cpp
@@ -197,7 +197,7 @@ std::set<uint256> TxOrphanage::GetCandidatesForBlock(const CBlock& block)
 
 void TxOrphanage::EraseForBlock(const CBlock& block)
 {
-    AssertLockHeld(g_cs_orphans);
+    LOCK(g_cs_orphans);
 
     std::vector<uint256> vOrphanErase;
 

From 6d452845dc8caf2c77231218a508676e35180bdf Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Mon, 7 Oct 2024 01:09:14 +0700
Subject: [PATCH 12/69] fix: add missing lock annotation for
 ContextualCheckBlock and related TODO

---
 src/validation.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/validation.cpp b/src/validation.cpp
index 38a030ace1..afd909c993 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -3644,9 +3644,10 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio
  *  in ConnectBlock().
  *  Note that -reindex-chainstate skips the validation that happens here!
  */
-static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev)
+static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
 {
-    AssertLockHeld(cs_main);
+    // TODO: validate - why do we need this cs_main ?
+    AssertLockHeld(::cs_main);
     const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1;
 
     // Enforce BIP113 (Median Time Past).

From 898282d6206381532ae78ade0b47f614afbe7c60 Mon Sep 17 00:00:00 2001
From: Ava Chow <github@achow101.com>
Date: Tue, 23 Jan 2024 14:50:58 -0500
Subject: [PATCH 13/69] Merge bitcoin/bitcoin#28774: wallet: avoid returning a
 reference to vMasterKey after releasing the mutex that guards it

32a9f13cb805ecf6aebb5cf4e5d92b8a47c4548b wallet: avoid returning a reference to vMasterKey after releasing the mutex that guards it (Vasil Dimov)

Pull request description:

  `CWallet::GetEncryptionKey()` would return a reference to the internal
  `CWallet::vMasterKey`, guarded by `CWallet::cs_wallet`, which is unsafe.

  Returning a copy would be a shorter solution, but could have security
  implications of the master key remaining somewhere in the memory even
  after `CWallet::Lock()` (the current calls to
  `CWallet::GetEncryptionKey()` are safe, but that is not future proof).

  So, instead of `EncryptSecret(m_storage.GetEncryptionKey(), ...)`
  change the `GetEncryptionKey()` method to provide the encryption
  key to a given callback:
  `m_storage.WithEncryptionKey([](const CKeyingMaterial& k) { EncryptSecret(k, ...); })`

  This silences the following (clang 18):

  ```
  wallet/wallet.cpp:3520:12: error: returning variable 'vMasterKey' by reference requires holding mutex 'cs_wallet' [-Werror,-Wthread-safety-reference-return]
   3520 |     return vMasterKey;
        |            ^
  ```

  ---
  _Previously this PR modified both ArgsManager and wallet code. But the ArgsManager commit https://github.com/bitcoin/bitcoin/commit/856c88776f8486446602476a1c9e133ac0cff510 was merged in https://github.com/bitcoin/bitcoin/pull/29040 so now this only affects wallet code. The previous PR description was:_

  Avoid this unsafe pattern from `ArgsManager` and `CWallet`:

  ```cpp
  class A
  {
      Mutex mutex;
      Foo member GUARDED_BY(mutex);
      const Foo& Get()
      {
          LOCK(mutex);
          return member;
      } // callers of `Get()` will have access to `member` without owning the mutex.
  ```

ACKs for top commit:
  achow101:
    ACK 32a9f13cb805ecf6aebb5cf4e5d92b8a47c4548b
  ryanofsky:
    Code review ACK 32a9f13cb805ecf6aebb5cf4e5d92b8a47c4548b. This seems like a potentially real race condition, and the fix here is pretty simple.
  furszy:
    ACK 32a9f13c

Tree-SHA512: 133da84691642afc1a73cf14ad004a7266cb4be1a6a3ec634d131dca5dbcdef52522c1d5eb04f5b6c4e06e1fc3e6ac57315f8fe1e207b464ca025c2b4edefdc1
---
 src/wallet/scriptpubkeyman.cpp | 36 ++++++++++++++++++++++++++--------
 src/wallet/scriptpubkeyman.h   |  4 +++-
 src/wallet/wallet.cpp          |  5 +++--
 src/wallet/wallet.h            |  3 ++-
 4 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp
index 9d84a04b6a..52d07ff7fa 100644
--- a/src/wallet/scriptpubkeyman.cpp
+++ b/src/wallet/scriptpubkeyman.cpp
@@ -342,8 +342,11 @@ void LegacyScriptPubKeyMan::UpgradeKeyMetadata()
     CHDChain hdChainCurrent;
     if (!GetHDChain(hdChainCurrent))
         throw std::runtime_error(std::string(__func__) + ": GetHDChain failed");
-    if (!DecryptHDChain(m_storage.GetEncryptionKey(), hdChainCurrent))
+    if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+            return DecryptHDChain(encryption_key, hdChainCurrent);
+        })) {
         throw std::runtime_error(std::string(__func__) + ": DecryptHDChain failed");
+    }
 
     CExtKey masterKey;
     SecureVector vchSeed = hdChainCurrent.GetSeed();
@@ -474,8 +477,11 @@ bool LegacyScriptPubKeyMan::GetDecryptedHDChain(CHDChain& hdChainRet)
         return false;
     }
 
-    if (!DecryptHDChain(m_storage.GetEncryptionKey(), hdChainTmp))
+    if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+            return DecryptHDChain(encryption_key, hdChainTmp);
+        })) {
         return false;
+    }
 
     // make sure seed matches this chain
     if (hdChainTmp.GetID() != hdChainTmp.GetSeedHash())
@@ -911,7 +917,9 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pu
 
     std::vector<unsigned char> vchCryptedSecret;
     CKeyingMaterial vchSecret(key.begin(), key.end());
-    if (!EncryptSecret(m_storage.GetEncryptionKey(), vchSecret, pubkey.GetHash(), vchCryptedSecret)) {
+    if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+            return EncryptSecret(encryption_key, vchSecret, pubkey.GetHash(), vchCryptedSecret);
+        })) {
         return false;
     }
 
@@ -933,7 +941,9 @@ bool LegacyScriptPubKeyMan::GetKeyInner(const CKeyID &address, CKey& keyOut) con
     {
         const CPubKey &vchPubKey = (*mi).second.first;
         const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second;
-        return DecryptKey(m_storage.GetEncryptionKey(), vchCryptedSecret, vchPubKey, keyOut);
+        return m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+            return DecryptKey(encryption_key, vchCryptedSecret, vchPubKey, keyOut);
+        });
     }
     return false;
 }
@@ -1164,8 +1174,11 @@ bool LegacyScriptPubKeyMan::GetKey(const CKeyID &address, CKey& keyOut) const
         CHDChain hdChainCurrent;
         if (!GetHDChain(hdChainCurrent))
             throw std::runtime_error(std::string(__func__) + ": GetHDChain failed");
-        if (!DecryptHDChain(m_storage.GetEncryptionKey(), hdChainCurrent))
+        if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+                return DecryptHDChain(encryption_key, hdChainCurrent);
+            })) {
             throw std::runtime_error(std::string(__func__) + ": DecryptHDChain failed");
+        }
         // make sure seed matches this chain
         if (hdChainCurrent.GetID() != hdChainCurrent.GetSeedHash())
             throw std::runtime_error(std::string(__func__) + ": Wrong HD chain!");
@@ -1276,8 +1289,11 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata&
         throw std::runtime_error(std::string(__func__) + ": GetHDChain failed");
     }
 
-    if (!DecryptHDChain(m_storage.GetEncryptionKey(), hdChainTmp))
+    if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+            return DecryptHDChain(encryption_key, hdChainTmp);
+        })) {
         throw std::runtime_error(std::string(__func__) + ": DecryptHDChain failed");
+    }
     // make sure seed matches this chain
     if (hdChainTmp.GetID() != hdChainTmp.GetSeedHash())
         throw std::runtime_error(std::string(__func__) + ": Wrong HD chain!");
@@ -1897,7 +1913,9 @@ std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const
             const CPubKey& pubkey = key_pair.second.first;
             const std::vector<unsigned char>& crypted_secret = key_pair.second.second;
             CKey key;
-            DecryptKey(m_storage.GetEncryptionKey(), crypted_secret, pubkey, key);
+            m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+                return DecryptKey(encryption_key, crypted_secret, pubkey, key);
+            });
             keys[pubkey.GetID()] = key;
         }
         return keys;
@@ -2010,7 +2028,9 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const
 
         std::vector<unsigned char> crypted_secret;
         CKeyingMaterial secret(key.begin(), key.end());
-        if (!EncryptSecret(m_storage.GetEncryptionKey(), secret, pubkey.GetHash(), crypted_secret)) {
+        if (!m_storage.WithEncryptionKey([&](const CKeyingMaterial& encryption_key) {
+                return EncryptSecret(encryption_key, secret, pubkey.GetHash(), crypted_secret);
+            })) {
             return false;
         }
 
diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h
index 0d6450d671..05f2452d2f 100644
--- a/src/wallet/scriptpubkeyman.h
+++ b/src/wallet/scriptpubkeyman.h
@@ -21,6 +21,7 @@
 
 #include <boost/signals2/signal.hpp>
 
+#include <functional>
 #include <optional>
 
 // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database.
@@ -38,7 +39,8 @@ class WalletStorage
     virtual void UnsetBlankWalletFlag(WalletBatch&) = 0;
     virtual bool CanSupportFeature(enum WalletFeature) const = 0;
     virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr) = 0;
-    virtual const CKeyingMaterial& GetEncryptionKey() const = 0;
+    //! Pass the encryption key to cb().
+    virtual bool WithEncryptionKey(std::function<bool (const CKeyingMaterial&)> cb) const = 0;
     virtual bool HasEncryptionKeys() const = 0;
     virtual bool IsLocked(bool fForMixing) const = 0;
 
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
index b6ac4595a8..00d872582a 100644
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -5743,9 +5743,10 @@ void CWallet::SetupLegacyScriptPubKeyMan()
     m_spk_managers[spk_manager->GetID()] = std::move(spk_manager);
 }
 
-const CKeyingMaterial& CWallet::GetEncryptionKey() const
+bool CWallet::WithEncryptionKey(std::function<bool (const CKeyingMaterial&)> cb) const
 {
-    return vMasterKey;
+    LOCK(cs_wallet);
+    return cb(vMasterKey);
 }
 
 bool CWallet::HasEncryptionKeys() const
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
index 411367ac2b..72c4c747ba 100644
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -1468,7 +1468,8 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
     //! Make a LegacyScriptPubKeyMan and set it for all types, internal, and external.
     void SetupLegacyScriptPubKeyMan();
 
-    const CKeyingMaterial& GetEncryptionKey() const override;
+    bool WithEncryptionKey(std::function<bool (const CKeyingMaterial&)> cb) const override;
+
     bool HasEncryptionKeys() const override;
 
     /** Get last block processed height */

From 55114a682e52ecba0ee27c0cb6f739d09b3f62df Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Tue, 1 Sep 2020 08:18:20 +0200
Subject: [PATCH 14/69] Merge #19668: Do not hide compile-time thread safety
 warnings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ea74e10acf17903e44c85e3678853414653dd4e1 doc: Add best practice for annotating/asserting locks (Hennadii Stepanov)
2ee7743fe723227f2ea1b031eddb14fc6863f4c8 sync.h: Make runtime lock checks require compile-time lock checks (Anthony Towns)
23d71d171e6e22ba5e4a909d597a54595b2a2c1f Do not hide compile-time thread safety warnings (Hennadii Stepanov)
3ddc150857178bfb1c854c05bf9b526777876f56 Add missed thread safety annotations (Hennadii Stepanov)
af9ea55a72c94678b343f5dd98dc78f3a3ac58cb Use LockAssertion utility class instead of AssertLockHeld() (Hennadii Stepanov)

Pull request description:

  On the way of transit from `RecursiveMutex` to `Mutex` (see #19303) it is crucial to have run-time `AssertLockHeld()` assertion that does _not_ hide compile-time Clang Thread Safety Analysis warnings.

  On master (65e4ecabd5b4252154640c7bac38c92a3f3a7018) using `AssertLockHeld()` could hide Clang Thread Safety Analysis warnings, e.g., with the following patch applied:
  ```diff
  --- a/src/txmempool.h
  +++ b/src/txmempool.h
  @@ -607,7 +607,7 @@ public:
       void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main);

       void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs);
  -    void removeForReorg(const CCoinsViewCache* pcoins, unsigned int nMemPoolHeight, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main);
  +    void removeForReorg(const CCoinsViewCache* pcoins, unsigned int nMemPoolHeight, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
       void removeConflicts(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(cs);
       void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(cs);

  ```
  Clang compiles the code without any thread safety warnings.

  See "Add missed thread safety annotations" commit for the actual thread safety warnings that are fixed in this PR.

ACKs for top commit:
  MarcoFalke:
    ACK ea74e10acf 🎙
  jnewbery:
    ACK ea74e10acf17903e44c85e3678853414653dd4e1
  ajtowns:
    ACK ea74e10acf17903e44c85e3678853414653dd4e1

Tree-SHA512: 8cba996e526751a1cb0e613c0cc1b10f027a3e9945fbfb4bd30f6355fd36b9f9c2e1e95ed3183fc254b42df7c30223278e18e5bdb5e1ef85db7fef067595d447
---
 doc/developer-notes.md       | 66 ++++++++++++++++++++++++++++++++++++
 src/sync.cpp                 |  6 +++-
 src/sync.h                   | 10 +++---
 src/wallet/scriptpubkeyman.h |  2 +-
 src/wallet/wallet.h          |  4 +--
 5 files changed, 80 insertions(+), 8 deletions(-)

diff --git a/doc/developer-notes.md b/doc/developer-notes.md
index 248557919a..22785e7349 100644
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@@ -899,6 +899,72 @@ the upper cycle, etc.
 Threads and synchronization
 ----------------------------
 
+- Prefer `Mutex` type to `RecursiveMutex` one
+
+- Consistently use [Clang Thread Safety Analysis](https://clang.llvm.org/docs/ThreadSafetyAnalysis.html) annotations to
+  get compile-time warnings about potential race conditions in code. Combine annotations in function declarations with
+  run-time asserts in function definitions:
+
+```C++
+// txmempool.h
+class CTxMemPool
+{
+public:
+    ...
+    mutable RecursiveMutex cs;
+    ...
+    void UpdateTransactionsFromBlock(...) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, cs);
+    ...
+}
+
+// txmempool.cpp
+void CTxMemPool::UpdateTransactionsFromBlock(...)
+{
+    AssertLockHeld(::cs_main);
+    AssertLockHeld(cs);
+    ...
+}
+```
+
+```C++
+// validation.h
+class ChainstateManager
+{
+public:
+    ...
+    bool ProcessNewBlock(...) EXCLUSIVE_LOCKS_REQUIRED(!::cs_main);
+    ...
+}
+
+// validation.cpp
+bool ChainstateManager::ProcessNewBlock(...)
+{
+    AssertLockNotHeld(::cs_main);
+    ...
+    LOCK(::cs_main);
+    ...
+}
+```
+
+- When Clang Thread Safety Analysis is unable to determine if a mutex is locked, use `LockAssertion` class instances:
+
+```C++
+// net_processing.h
+void RelayTransaction(...) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+// net_processing.cpp
+void RelayTransaction(...)
+{
+    AssertLockHeld(::cs_main);
+
+    connman.ForEachNode([&txid, &wtxid](CNode* pnode) {
+        LockAssertion lock(::cs_main);
+        ...
+    });
+}
+
+```
+
 - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential
   deadlocks are introduced.
 
diff --git a/src/sync.cpp b/src/sync.cpp
index 35d3ffaf12..7540819262 100644
--- a/src/sync.cpp
+++ b/src/sync.cpp
@@ -285,12 +285,16 @@ template void AssertLockHeldInternal(const char*, const char*, int, Mutex*);
 template void AssertLockHeldInternal(const char*, const char*, int, RecursiveMutex*);
 template void AssertLockHeldInternal(const char*, const char*, int, SharedMutex*);
 
-void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs)
+template <typename MutexType>
+void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs)
 {
     if (!LockHeld(cs)) return;
     tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld());
     abort();
 }
+template void AssertLockNotHeldInternal(const char*, const char*, int, Mutex*);
+template void AssertLockNotHeldInternal(const char*, const char*, int, RecursiveMutex*);
+template void AssertLockNotHeldInternal(const char*, const char*, int, SharedMutex*);
 
 void DeleteLock(void* cs)
 {
diff --git a/src/sync.h b/src/sync.h
index 3712bd9824..8d02830726 100644
--- a/src/sync.h
+++ b/src/sync.h
@@ -57,8 +57,9 @@ void LeaveCritical();
 void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line);
 std::string LocksHeld();
 template <typename MutexType>
-void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs);
-void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs);
+void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs);
+template <typename MutexType>
+void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs);
 void DeleteLock(void* cs);
 bool LockStackEmpty();
 
@@ -74,8 +75,9 @@ inline void EnterCritical(const char* pszName, const char* pszFile, int nLine, M
 inline void LeaveCritical() {}
 inline void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {}
 template <typename MutexType>
-inline void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs) {}
-inline void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {}
+inline void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs) {}
+template <typename MutexType>
+void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) {}
 inline void DeleteLock(void* cs) {}
 inline bool LockStackEmpty() { return true; }
 #endif
diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h
index 05f2452d2f..e3f1916272 100644
--- a/src/wallet/scriptpubkeyman.h
+++ b/src/wallet/scriptpubkeyman.h
@@ -528,7 +528,7 @@ class DescriptorScriptPubKeyMan : public ScriptPubKeyMan
     //! keeps track of whether Unlock has run a thorough check before
     bool m_decryption_thoroughly_checked = false;
 
-    bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey);
+    bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man);
 
     KeyMap GetKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man);
 
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
index 72c4c747ba..88ec3db2f1 100644
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -1402,7 +1402,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
      * Obviously holding cs_main/cs_wallet when going into this call may cause
      * deadlock
      */
-    void BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(cs_main, cs_wallet);
+    void BlockUntilSyncedToCurrentChain() const EXCLUSIVE_LOCKS_REQUIRED(!::cs_main, !cs_wallet);
 
     /** set a single wallet flag */
     void SetWalletFlag(uint64_t flags);
@@ -1515,7 +1515,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
     void DeactivateScriptPubKeyMan(uint256 id, bool internal);
 
     //! Create new DescriptorScriptPubKeyMans and add them to the wallet
-    void SetupDescriptorScriptPubKeyMans();
+    void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
 
     //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet
     DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const;

From d0a4198166d0670ce814213d77c39b77377cd373 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Tue, 6 Apr 2021 07:54:07 +0200
Subject: [PATCH 15/69] Merge #21598: refactor: Remove negative lock
 annotations from globals

fa5eabe72117f6e3704858e8d5b2c57a120258ed refactor: Remove negative lock annotations from globals (MarcoFalke)

Pull request description:

  They only make sense for mutexes that are private members. Until cs_main is a private member the negative annotations should be replaced by excluded annotations, which are optional.

ACKs for top commit:
  sipa:
    utACK fa5eabe72117f6e3704858e8d5b2c57a120258ed
  ajtowns:
    ACK fa5eabe72117f6e3704858e8d5b2c57a120258ed
  hebasto:
    ACK fa5eabe72117f6e3704858e8d5b2c57a120258ed
  vasild:
    ACK fa5eabe72117f6e3704858e8d5b2c57a120258ed

Tree-SHA512: 06f8a200304f81533010efcc42d9f59b8c4d0ae355920c0a28efb6fa161a3e3e68f2dfffb0c009afd9c2501e6a293c6e5a419a64d718f1f4e79668ab2ab1fcdc
---
 doc/developer-notes.md | 2 +-
 src/index/base.h       | 2 +-
 src/net_processing.cpp | 2 +-
 src/sync.h             | 4 ++--
 src/txorphanage.h      | 2 +-
 src/wallet/wallet.h    | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/developer-notes.md b/doc/developer-notes.md
index 22785e7349..9e9fb162e9 100644
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@@ -932,7 +932,7 @@ class ChainstateManager
 {
 public:
     ...
-    bool ProcessNewBlock(...) EXCLUSIVE_LOCKS_REQUIRED(!::cs_main);
+    bool ProcessNewBlock(...) LOCKS_EXCLUDED(::cs_main);
     ...
 }
 
diff --git a/src/index/base.h b/src/index/base.h
index 403c8c87b1..e1f2a9b82d 100644
--- a/src/index/base.h
+++ b/src/index/base.h
@@ -118,7 +118,7 @@ class BaseIndex : public CValidationInterface
     /// sync once and only needs to process blocks in the ValidationInterface
     /// queue. If the index is catching up from far behind, this method does
     /// not block and immediately returns false.
-    bool BlockUntilSyncedToCurrentChain() const;
+    bool BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main);
 
     void Interrupt();
 
diff --git a/src/net_processing.cpp b/src/net_processing.cpp
index 803f3d51aa..688a01d19d 100644
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -944,7 +944,7 @@ class PeerManagerImpl final : public PeerManager
     /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */
     CTransactionRef FindTxForGetData(const CNode* peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main);
 
-    void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) EXCLUSIVE_LOCKS_REQUIRED(peer.m_getdata_requests_mutex);
+    void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(peer.m_getdata_requests_mutex) LOCKS_EXCLUDED(::cs_main);
 
     /** Process a new block. Perform any post-processing housekeeping */
     void ProcessBlock(CNode& from, const std::shared_ptr<const CBlock>& pblock, bool force_processing);
diff --git a/src/sync.h b/src/sync.h
index 8d02830726..cab9d559d5 100644
--- a/src/sync.h
+++ b/src/sync.h
@@ -59,7 +59,7 @@ std::string LocksHeld();
 template <typename MutexType>
 void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs);
 template <typename MutexType>
-void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs);
+void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) LOCKS_EXCLUDED(cs);
 void DeleteLock(void* cs);
 bool LockStackEmpty();
 
@@ -77,7 +77,7 @@ inline void CheckLastCritical(void* cs, std::string& lockname, const char* guard
 template <typename MutexType>
 inline void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs) {}
 template <typename MutexType>
-void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) {}
+void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) LOCKS_EXCLUDED(cs) {}
 inline void DeleteLock(void* cs) {}
 inline bool LockStackEmpty() { return true; }
 #endif
diff --git a/src/txorphanage.h b/src/txorphanage.h
index 6fc043eda6..d8ef7a5c44 100644
--- a/src/txorphanage.h
+++ b/src/txorphanage.h
@@ -41,7 +41,7 @@ class TxOrphanage {
     void EraseForPeer(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans);
 
     /** Erase all orphans included in or invalidated by a new block */
-    void EraseForBlock(const CBlock& block) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans);
+    void EraseForBlock(const CBlock& block) LOCKS_EXCLUDED(::g_cs_orphans);
 
     /** Limit the orphanage to the given maximum */
     unsigned int LimitOrphans(unsigned int max_orphans_size) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans);
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
index 88ec3db2f1..af80efb535 100644
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -1402,7 +1402,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati
      * Obviously holding cs_main/cs_wallet when going into this call may cause
      * deadlock
      */
-    void BlockUntilSyncedToCurrentChain() const EXCLUSIVE_LOCKS_REQUIRED(!::cs_main, !cs_wallet);
+    void BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main) EXCLUSIVE_LOCKS_REQUIRED(!cs_wallet);
 
     /** set a single wallet flag */
     void SetWalletFlag(uint64_t flags);

From 1db9e6ac76d895d5e80aba7bf0f0a42d13a904e6 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Wed, 23 Sep 2020 16:36:52 +0200
Subject: [PATCH 16/69] Merge #19979: Replace LockAssertion with
 AssertLockHeld, remove LockAssertion

0bd1184adf6610c0bd14f4e9a25c0a200e65ae25 Remove unused LockAssertion struct (Hennadii Stepanov)
ab2a44297fd0796bf5797ae2a477e8e56d9c3c12 Replace LockAssertion with a proper thread safety annotations (Hennadii Stepanov)
73f71e19965e07534eb47701f2b23c9ed59ef475 refactor: Use explicit function type instead of template (Hennadii Stepanov)

Pull request description:

  This PR replaces `LockAssertion` with `AssertLockHeld`, and removes `LockAssertion`.

  This PR is compared with alternatives in https://github.com/bitcoin-core/bitcoin-devwiki/wiki/AssertLockHeld-PRs

ACKs for top commit:
  MarcoFalke:
    ACK 0bd1184adf6610c0bd14f4e9a25c0a200e65ae25
  ajtowns:
    ACK 0bd1184adf6610c0bd14f4e9a25c0a200e65ae25
  vasild:
    ACK 0bd1184ad

Tree-SHA512: ef7780dd689faf0bb479fdb97c49bc652e2dd10c148234bb95502dfbb676442d8565ee37864d923ca21a25f9dc2a335bf46ee82c095e387b59a664ab05c0ae41
---
 doc/developer-notes.md | 19 -------------------
 src/net.cpp            |  8 ++++----
 src/net.h              | 12 ++++++------
 src/net_processing.cpp | 12 ++++++------
 src/sync.h             | 14 --------------
 5 files changed, 16 insertions(+), 49 deletions(-)

diff --git a/doc/developer-notes.md b/doc/developer-notes.md
index 9e9fb162e9..162547f1dd 100644
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@@ -946,25 +946,6 @@ bool ChainstateManager::ProcessNewBlock(...)
 }
 ```
 
-- When Clang Thread Safety Analysis is unable to determine if a mutex is locked, use `LockAssertion` class instances:
-
-```C++
-// net_processing.h
-void RelayTransaction(...) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
-
-// net_processing.cpp
-void RelayTransaction(...)
-{
-    AssertLockHeld(::cs_main);
-
-    connman.ForEachNode([&txid, &wtxid](CNode* pnode) {
-        LockAssertion lock(::cs_main);
-        ...
-    });
-}
-
-```
-
 - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential
   deadlocks are introduced.
 
diff --git a/src/net.cpp b/src/net.cpp
index 49e9ca2c82..2b6fd4608b 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -3510,8 +3510,8 @@ void CConnman::ThreadOpenMasternodeConnections(CDeterministicMNManager& dmnman,
 
         MasternodeProbeConn isProbe = MasternodeProbeConn::IsNotConnection;
 
-        const auto getPendingQuorumNodes = [&]() {
-            LockAssertion lock(cs_vPendingMasternodes);
+        const auto getPendingQuorumNodes = [&]() EXCLUSIVE_LOCKS_REQUIRED(cs_vPendingMasternodes) {
+            AssertLockHeld(cs_vPendingMasternodes);
             std::vector<CDeterministicMNCPtr> ret;
             for (const auto& group : masternodeQuorumNodes) {
                 for (const auto& proRegTxHash : group.second) {
@@ -3549,8 +3549,8 @@ void CConnman::ThreadOpenMasternodeConnections(CDeterministicMNManager& dmnman,
             return ret;
         };
 
-        const auto getPendingProbes = [&]() {
-            LockAssertion lock(cs_vPendingMasternodes);
+        const auto getPendingProbes = [&]() EXCLUSIVE_LOCKS_REQUIRED(cs_vPendingMasternodes) {
+            AssertLockHeld(cs_vPendingMasternodes);
             std::vector<CDeterministicMNCPtr> ret;
             for (auto it = masternodePendingProbes.begin(); it != masternodePendingProbes.end(); ) {
                 auto dmn = mnList.GetMN(*it);
diff --git a/src/net.h b/src/net.h
index 248a8a6b04..c34c87104a 100644
--- a/src/net.h
+++ b/src/net.h
@@ -1275,6 +1275,8 @@ friend class CNode;
         return ForNode(id, FullyConnectedOnly, func);
     }
 
+    using NodeFn = std::function<void(CNode*)>;
+
     bool IsConnected(const CService& addr, std::function<bool(const CNode* pnode)> cond)
     {
         return ForNode(addr, cond, [](CNode* pnode){
@@ -1331,10 +1333,9 @@ friend class CNode;
         }
     };
 
-    template<typename Callable>
-    void ForEachNode(Callable&& func)
+    void ForEachNode(const NodeFn& fn)
     {
-        ForEachNode(FullyConnectedOnly, func);
+        ForEachNode(FullyConnectedOnly, fn);
     }
 
     template<typename Condition, typename Callable>
@@ -1347,10 +1348,9 @@ friend class CNode;
         }
     };
 
-    template<typename Callable>
-    void ForEachNode(Callable&& func) const
+    void ForEachNode(const NodeFn& fn) const
     {
-        ForEachNode(FullyConnectedOnly, func);
+        ForEachNode(FullyConnectedOnly, fn);
     }
 
     template<typename Condition, typename Callable, typename CallableAfter>
diff --git a/src/net_processing.cpp b/src/net_processing.cpp
index 688a01d19d..d05eb8917e 100644
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -2000,8 +2000,8 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha
         m_most_recent_compact_block = pcmpctblock;
     }
 
-    m_connman.ForEachNode([this, pindex, &lazy_ser, &hashBlock](CNode* pnode) {
-        LockAssertion lock(::cs_main);
+    m_connman.ForEachNode([this, pindex, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) {
+        AssertLockHeld(::cs_main);
         // TODO: Avoid the repeated-serialization here
         if (pnode->fDisconnect)
             return;
@@ -5275,8 +5275,8 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now)
         // We want to prevent recently connected to Onion peers from being disconnected here, protect them as long as
         // there are more non_onion nodes than onion nodes so far
         size_t onion_count = 0;
-        m_connman.ForEachNode([&](CNode* pnode) {
-            LockAssertion lock(::cs_main);
+        m_connman.ForEachNode([&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) {
+            AssertLockHeld(::cs_main);
             if (pnode->addr.IsTor() && ++onion_count <= m_connman.GetMaxOutboundOnionNodeCount()) return;
             // Don't disconnect masternodes just because they were slow in block announcement
             if (pnode->m_masternode_connection) return;
@@ -5293,8 +5293,8 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now)
             }
         });
         if (worst_peer != -1) {
-            bool disconnected = m_connman.ForNode(worst_peer, [&](CNode *pnode) {
-                LockAssertion lock(::cs_main);
+            bool disconnected = m_connman.ForNode(worst_peer, [&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) {
+                AssertLockHeld(::cs_main);
 
                 // Only disconnect a peer that has been connected to us for
                 // some reasonable fraction of our check-frequency, to give
diff --git a/src/sync.h b/src/sync.h
index cab9d559d5..078ee4f6e8 100644
--- a/src/sync.h
+++ b/src/sync.h
@@ -452,18 +452,4 @@ class CSemaphoreGrant
     }
 };
 
-// Utility class for indicating to compiler thread analysis that a mutex is
-// locked (when it couldn't be determined otherwise).
-struct SCOPED_LOCKABLE LockAssertion
-{
-    template <typename Mutex>
-    explicit LockAssertion(Mutex& mutex) EXCLUSIVE_LOCK_FUNCTION(mutex)
-    {
-#ifdef DEBUG_LOCKORDER
-        AssertLockHeld(mutex);
-#endif
-    }
-    ~LockAssertion() UNLOCK_FUNCTION() {}
-};
-
 #endif // BITCOIN_SYNC_H

From ebae59eedfd9abcfbbb3cdb7be97eb730d62e0da Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 15:43:07 +0000
Subject: [PATCH 17/69] fix: make sure we flush our committed best block in
 no-upgrade cases

This was spotted when working on `feature_txindex_compatibility.py`,
attempting to load old databases would fail because `MigrateDBIfNeeded()`
would delete `DB_OLD_BEST_BLOCK`, write `EVODB_BEST_BLOCK`, commit it
but never flush it.

So when `MigrateDBIfNeeded2()` would read it again, it would note the lack
of `DB_OLD_BEST_BLOCK` and conclude it was a failed run and exit. This
is solved by actually flushing our new best block, which would prevent
`MigrateDBIfNeeded2` from raising an objection.
---
 src/evo/deterministicmns.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp
index ac167a298b..495ca8a03b 100644
--- a/src/evo/deterministicmns.cpp
+++ b/src/evo/deterministicmns.cpp
@@ -1238,6 +1238,10 @@ bool CDeterministicMNManager::MigrateDBIfNeeded()
         auto dbTx = m_evoDb.BeginTransaction();
         m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
         dbTx->Commit();
+        if (!m_evoDb.CommitRootTransaction()) {
+            LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
+            return false;
+        }
         return true;
     }
 
@@ -1349,6 +1353,10 @@ bool CDeterministicMNManager::MigrateDBIfNeeded2()
         auto dbTx = m_evoDb.BeginTransaction();
         m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
         dbTx->Commit();
+        if (!m_evoDb.CommitRootTransaction()) {
+            LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
+            return false;
+        }
         return true;
     }
 

From b218f123b7c86da0c483d6e54c25468968505c52 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 15 Oct 2024 06:58:49 +0000
Subject: [PATCH 18/69] merge bitcoin#23046: Add txindex migration test

---
 .../feature_txindex_compatibility.py          | 97 +++++++++++++++++++
 test/functional/test_runner.py                |  1 +
 2 files changed, 98 insertions(+)
 create mode 100755 test/functional/feature_txindex_compatibility.py

diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py
new file mode 100755
index 0000000000..f4a8e7cf55
--- /dev/null
+++ b/test/functional/feature_txindex_compatibility.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test that legacy txindex will be disabled on upgrade.
+
+Previous releases are required by this test, see test/README.md.
+"""
+
+import os
+import shutil
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.wallet import MiniWallet
+
+
+class MempoolCompatibilityTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 3
+        self.extra_args = [
+            ["-reindex", "-txindex"],
+            ["-txindex=0"],
+            ["-txindex=0"],
+        ]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_previous_releases()
+
+    def setup_network(self):
+        self.add_nodes(
+            self.num_nodes,
+            self.extra_args,
+            versions=[
+                170003,   # Last release with legacy txindex
+                None,     # For MiniWallet, without migration code
+                18020200, # Any release with migration code (0.18.x - 21.x)
+                          # We are using v18.2.2 to avoid MigrateDBIfNeeded(2) routines as
+                          # they don't handle a no-upgrade case correctly
+            ],
+        )
+        # Delete v18.2.2 cached datadir to avoid making a legacy version try to
+        # make sense of our current database formats
+        shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain))
+        self.start_nodes()
+        self.connect_nodes(0, 1)
+        self.connect_nodes(1, 2)
+
+    def run_test(self):
+        mini_wallet = MiniWallet(self.nodes[1])
+        mini_wallet.rescan_utxos()
+        spend_utxo = mini_wallet.get_utxo()
+        mini_wallet.send_self_transfer(from_node=self.nodes[1], utxo_to_spend=spend_utxo)
+        self.generate(self.nodes[1], 1)
+
+        self.log.info("Check legacy txindex")
+        self.nodes[0].getrawtransaction(txid=spend_utxo["txid"])  # Requires -txindex
+
+        self.stop_nodes()
+        legacy_chain_dir = os.path.join(self.nodes[0].datadir, self.chain)
+
+        self.log.info("Migrate legacy txindex")
+        migrate_chain_dir = os.path.join(self.nodes[2].datadir, self.chain)
+        shutil.rmtree(migrate_chain_dir)
+        shutil.copytree(legacy_chain_dir, migrate_chain_dir)
+        with self.nodes[2].assert_debug_log([
+                "Upgrading txindex database...",
+                "txindex is enabled at height 200",
+        ]):
+            self.start_node(2, extra_args=["-txindex"])
+        self.nodes[2].getrawtransaction(txid=spend_utxo["txid"])  # Requires -txindex
+
+        self.log.info("Drop legacy txindex")
+        drop_index_chain_dir = os.path.join(self.nodes[1].datadir, self.chain)
+        shutil.rmtree(drop_index_chain_dir)
+        shutil.copytree(legacy_chain_dir, drop_index_chain_dir)
+        self.nodes[1].assert_start_raises_init_error(
+            extra_args=["-txindex"],
+            expected_msg="Error: The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.",
+        )
+        # Build txindex from scratch and check there is no error this time
+        self.start_node(1, extra_args=["-txindex"])
+        self.nodes[2].getrawtransaction(txid=spend_utxo["txid"])  # Requires -txindex
+
+        self.stop_nodes()
+
+        self.log.info("Check migrated txindex can not be read by legacy node")
+        err_msg = f": You need to rebuild the database using -reindex to change -txindex.{os.linesep}Please restart with -reindex or -reindex-chainstate to recover."
+        shutil.rmtree(legacy_chain_dir)
+        shutil.copytree(migrate_chain_dir, legacy_chain_dir)
+        self.nodes[0].assert_start_raises_init_error(extra_args=["-txindex"], expected_msg=err_msg)
+        shutil.rmtree(legacy_chain_dir)
+        shutil.copytree(drop_index_chain_dir, legacy_chain_dir)
+        self.nodes[0].assert_start_raises_init_error(extra_args=["-txindex"], expected_msg=err_msg)
+
+
+if __name__ == "__main__":
+    MempoolCompatibilityTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 5e0cffdf2d..9045045236 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -346,6 +346,7 @@
     'rpc_deriveaddresses.py --usecli',
     'p2p_ping.py',
     'rpc_scantxoutset.py',
+    'feature_txindex_compatibility.py',
     'feature_logging.py',
     'feature_anchors.py',
     'feature_coinstatsindex.py',

From 1caaa85716f526c285c0c1ced91225de46448b6b Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sat, 12 Oct 2024 05:57:17 +0000
Subject: [PATCH 19/69] merge bitcoin#24138: Commit MuHash and best block
 together for coinstatsindex

Continuation of c21f23db from dash#5501
---
 src/test/coinstatsindex_tests.cpp | 42 +++++++++++++++++++++++++++++++
 src/test/util/validation.cpp      |  6 +++++
 src/test/util/validation.h        |  8 ++++++
 src/validationinterface.h         |  1 +
 4 files changed, 57 insertions(+)

diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp
index 6ba80397a3..653790f7d9 100644
--- a/src/test/coinstatsindex_tests.cpp
+++ b/src/test/coinstatsindex_tests.cpp
@@ -2,9 +2,11 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <chainparams.h>
 #include <index/coinstatsindex.h>
 #include <test/util/index.h>
 #include <test/util/setup_common.h>
+#include <test/util/validation.h>
 #include <util/time.h>
 #include <validation.h>
 
@@ -74,4 +76,44 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup)
     coin_stats_index.Stop();
 }
 
+// Test shutdown between BlockConnected and ChainStateFlushed notifications,
+// make sure index is not corrupted and is able to reload.
+BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup)
+{
+    CChainState& chainstate = Assert(m_node.chainman)->ActiveChainstate();
+    const CChainParams& params = Params();
+    {
+        CoinStatsIndex index{1 << 20};
+        BOOST_REQUIRE(index.Start(chainstate));
+        IndexWaitSynced(index);
+        std::shared_ptr<const CBlock> new_block;
+        CBlockIndex* new_block_index = nullptr;
+        {
+            const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG};
+            const CBlock block = this->CreateBlock({}, script_pub_key, chainstate);
+
+            new_block = std::make_shared<CBlock>(block);
+
+            LOCK(cs_main);
+            BlockValidationState state;
+            BOOST_CHECK(CheckBlock(block, state, params.GetConsensus()));
+            BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr));
+            CCoinsViewCache view(&chainstate.CoinsTip());
+            BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view));
+        }
+        // Send block connected notification, then stop the index without
+        // sending a chainstate flushed notification. Prior to #24138, this
+        // would cause the index to be corrupted and fail to reload.
+        ValidationInterfaceTest::BlockConnected(index, new_block, new_block_index);
+        index.Stop();
+    }
+
+    {
+        CoinStatsIndex index{1 << 20};
+        // Make sure the index can be loaded.
+        BOOST_REQUIRE(index.Start(chainstate));
+        index.Stop();
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp
index 1aed492c3c..49535855f9 100644
--- a/src/test/util/validation.cpp
+++ b/src/test/util/validation.cpp
@@ -7,6 +7,7 @@
 #include <util/check.h>
 #include <util/time.h>
 #include <validation.h>
+#include <validationinterface.h>
 
 void TestChainState::ResetIbd()
 {
@@ -20,3 +21,8 @@ void TestChainState::JumpOutOfIbd()
     m_cached_finished_ibd = true;
     Assert(!IsInitialBlockDownload());
 }
+
+void ValidationInterfaceTest::BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
+{
+    obj.BlockConnected(block, pindex);
+}
diff --git a/src/test/util/validation.h b/src/test/util/validation.h
index b13aa0be60..b0bc717b6c 100644
--- a/src/test/util/validation.h
+++ b/src/test/util/validation.h
@@ -7,6 +7,8 @@
 
 #include <validation.h>
 
+class CValidationInterface;
+
 struct TestChainState : public CChainState {
     /** Reset the ibd cache to its initial state */
     void ResetIbd();
@@ -14,4 +16,10 @@ struct TestChainState : public CChainState {
     void JumpOutOfIbd();
 };
 
+class ValidationInterfaceTest
+{
+public:
+    static void BlockConnected(CValidationInterface& obj, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex);
+};
+
 #endif // BITCOIN_TEST_UTIL_VALIDATION_H
diff --git a/src/validationinterface.h b/src/validationinterface.h
index c860f7fb86..a3b45dcf9d 100644
--- a/src/validationinterface.h
+++ b/src/validationinterface.h
@@ -200,6 +200,7 @@ class CValidationInterface {
      * has been received and connected to the headers tree, though not validated yet */
     virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {};
     friend class CMainSignals;
+    friend class ValidationInterfaceTest;
 };
 
 struct MainSignalsInstance;

From 10203560f5367acd9cd96b4857b8b4a223c73550 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sat, 9 Apr 2022 02:14:02 +0200
Subject: [PATCH 20/69] merge bitcoin#24812: Add CHECK_NONFATAL identity
 function and NONFATAL_UNREACHABLE macro

---
 src/rpc/blockchain.cpp      | 25 +++++++----------
 src/rpc/coinjoin.cpp        | 16 ++++-------
 src/rpc/evo.cpp             | 53 +++++++++++--------------------------
 src/rpc/governance.cpp      | 42 +++++++++++------------------
 src/rpc/masternode.cpp      | 25 +++++++----------
 src/rpc/mining.cpp          | 10 +++----
 src/rpc/misc.cpp            |  3 +--
 src/rpc/quorums.cpp         | 15 ++++++-----
 src/rpc/rawtransaction.cpp  |  5 ++--
 src/rpc/util.cpp            | 15 ++++++-----
 src/util/check.h            | 41 +++++++++++++++++++---------
 src/wallet/rpcdump.cpp      |  2 +-
 test/functional/rpc_misc.py |  2 +-
 13 files changed, 110 insertions(+), 144 deletions(-)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 6bf41d7ba8..94426f7da4 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -39,6 +39,7 @@
 #include <sync.h>
 #include <txmempool.h>
 #include <undo.h>
+#include <util/check.h>
 #include <util/strencodings.h>
 #include <util/string.h>
 #include <util/system.h>
@@ -1359,8 +1360,7 @@ static RPCHelpMan pruneblockchain()
     }
 
     PruneBlockFilesManual(active_chainstate, height);
-    const CBlockIndex* block = active_chain.Tip();
-    CHECK_NONFATAL(block);
+    const CBlockIndex* block = CHECK_NONFATAL(active_chain.Tip());
     while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
         block = block->pprev;
     }
@@ -1635,14 +1635,13 @@ static RPCHelpMan verifychain()
     const int check_depth{request.params[1].isNull() ? DEFAULT_CHECKBLOCKS : request.params[1].get_int()};
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
-    CHECK_NONFATAL(node.evodb);
 
     ChainstateManager& chainman = EnsureChainman(node);
     LOCK(cs_main);
 
     CChainState& active_chainstate = chainman.ActiveChainstate();
     return CVerifyDB().VerifyDB(
-        active_chainstate, Params(), active_chainstate.CoinsTip(), *node.evodb, check_level, check_depth);
+        active_chainstate, Params(), active_chainstate.CoinsTip(), *CHECK_NONFATAL(node.evodb), check_level, check_depth);
 },
     };
 }
@@ -1782,13 +1781,11 @@ RPCHelpMan getblockchaininfo()
 
     LOCK(cs_main);
     CChainState& active_chainstate = chainman.ActiveChainstate();
-    const CBlockIndex* tip = active_chainstate.m_chain.Tip();
 
-    CHECK_NONFATAL(tip);
+    const CBlockIndex* tip = CHECK_NONFATAL(active_chainstate.m_chain.Tip());
     const int height = tip->nHeight;
 
-    CHECK_NONFATAL(node.mnhf_manager);
-    const auto ehfSignals = node.mnhf_manager->GetSignalsStage(tip);
+    const auto ehfSignals = CHECK_NONFATAL(node.mnhf_manager)->GetSignalsStage(tip);
 
     UniValue obj(UniValue::VOBJ);
     if (args.IsArgSet("-devnet")) {
@@ -1808,8 +1805,7 @@ RPCHelpMan getblockchaininfo()
     obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage());
     obj.pushKV("pruned", fPruneMode);
     if (fPruneMode) {
-        const CBlockIndex* block = tip;
-        CHECK_NONFATAL(block);
+        const CBlockIndex* block = CHECK_NONFATAL(tip);
         while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
             block = block->pprev;
         }
@@ -2863,10 +2859,8 @@ static RPCHelpMan scantxoutset()
             LOCK(cs_main);
             CChainState& active_chainstate = chainman.ActiveChainstate();
             active_chainstate.ForceFlushStateToDisk();
-            pcursor = active_chainstate.CoinsDB().Cursor();
-            CHECK_NONFATAL(pcursor);
-            tip = active_chainstate.m_chain.Tip();
-            CHECK_NONFATAL(tip);
+            pcursor = CHECK_NONFATAL(active_chainstate.CoinsDB().Cursor());
+            tip = CHECK_NONFATAL(active_chainstate.m_chain.Tip());
         }
         bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point);
         result.pushKV("success", res);
@@ -3061,8 +3055,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil
         }
 
         pcursor = chainstate.CoinsDB().Cursor();
-        tip = chainstate.m_blockman.LookupBlockIndex(stats.hashBlock);
-        CHECK_NONFATAL(tip);
+        tip = CHECK_NONFATAL(chainstate.m_blockman.LookupBlockIndex(stats.hashBlock));
     }
 
     SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
diff --git a/src/rpc/coinjoin.cpp b/src/rpc/coinjoin.cpp
index 0ce1bffff9..8e477ce8ce 100644
--- a/src/rpc/coinjoin.cpp
+++ b/src/rpc/coinjoin.cpp
@@ -9,6 +9,7 @@
 #include <rpc/blockchain.h>
 #include <rpc/server.h>
 #include <rpc/server_util.h>
+#include <util/check.h>
 #include <rpc/util.h>
 #include <util/strencodings.h>
 
@@ -83,11 +84,8 @@ static RPCHelpMan coinjoin_reset()
 
     ValidateCoinJoinArguments();
 
-    CHECK_NONFATAL(node.coinjoin_loader);
-    auto cj_clientman = node.coinjoin_loader->walletman().Get(wallet->GetName());
-
-    CHECK_NONFATAL(cj_clientman);
-    cj_clientman->ResetPool();
+    auto cj_clientman = CHECK_NONFATAL(node.coinjoin_loader)->walletman().Get(wallet->GetName());
+    CHECK_NONFATAL(cj_clientman)->ResetPool();
 
     return "Mixing was reset";
 },
@@ -126,10 +124,7 @@ static RPCHelpMan coinjoin_start()
             throw JSONRPCError(RPC_WALLET_UNLOCK_NEEDED, "Error: Please unlock wallet for mixing with walletpassphrase first.");
     }
 
-    CHECK_NONFATAL(node.coinjoin_loader);
-    auto cj_clientman = node.coinjoin_loader->walletman().Get(wallet->GetName());
-
-    CHECK_NONFATAL(cj_clientman);
+    auto cj_clientman = CHECK_NONFATAL(CHECK_NONFATAL(node.coinjoin_loader)->walletman().Get(wallet->GetName()));
     if (!cj_clientman->StartMixing()) {
         throw JSONRPCError(RPC_INTERNAL_ERROR, "Mixing has been started already.");
     }
@@ -450,8 +445,7 @@ static RPCHelpMan getcoinjoininfo()
         return obj;
     }
 
-    auto manager = node.coinjoin_loader->walletman().Get(wallet->GetName());
-    CHECK_NONFATAL(manager != nullptr);
+    auto* manager = CHECK_NONFATAL(node.coinjoin_loader->walletman().Get(wallet->GetName()));
     manager->GetJsonInfo(obj);
 
     std::string warning_msg{""};
diff --git a/src/rpc/evo.cpp b/src/rpc/evo.cpp
index 30fdab04b7..b584f888ab 100644
--- a/src/rpc/evo.cpp
+++ b/src/rpc/evo.cpp
@@ -26,6 +26,7 @@
 #include <rpc/server.h>
 #include <rpc/server_util.h>
 #include <rpc/util.h>
+#include <util/check.h>
 #include <util/moneystr.h>
 #include <util/translation.h>
 #include <validation.h>
@@ -628,8 +629,7 @@ static UniValue protx_register_common_wrapper(const JSONRPCRequest& request,
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.chain_helper);
-    CChainstateHelper& chain_helper = *node.chain_helper;
+    CChainstateHelper& chain_helper = *CHECK_NONFATAL(node.chain_helper);
 
     const bool isEvoRequested = mnType == MnType::Evo;
 
@@ -855,8 +855,7 @@ static RPCHelpMan protx_register_submit()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.chain_helper);
-    CChainstateHelper& chain_helper = *node.chain_helper;
+    CChainstateHelper& chain_helper = *CHECK_NONFATAL(node.chain_helper);
 
     std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request);
     if (!wallet) return NullUniValue;
@@ -954,11 +953,8 @@ static UniValue protx_update_service_common_wrapper(const JSONRPCRequest& reques
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.chain_helper);
-    CChainstateHelper& chain_helper = *node.chain_helper;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    CChainstateHelper& chain_helper = *CHECK_NONFATAL(node.chain_helper);
 
     const bool isEvoRequested = mnType == MnType::Evo;
     std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request);
@@ -1092,11 +1088,8 @@ static RPCHelpMan protx_update_registrar_wrapper(bool specific_legacy_bls_scheme
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.chain_helper);
-    CChainstateHelper& chain_helper = *node.chain_helper;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    CChainstateHelper& chain_helper = *CHECK_NONFATAL(node.chain_helper);
 
     std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request);
     if (!wallet) return NullUniValue;
@@ -1207,12 +1200,8 @@ static RPCHelpMan protx_revoke()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.chain_helper);
-    CChainstateHelper& chain_helper = *node.chain_helper;
-
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    CChainstateHelper& chain_helper = *CHECK_NONFATAL(node.chain_helper);
 
     std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request);
     if (!pwallet) return NullUniValue;
@@ -1370,11 +1359,8 @@ static RPCHelpMan protx_list()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.mn_metaman);
-    CMasternodeMetaMan& mn_metaman = *node.mn_metaman;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    CMasternodeMetaMan& mn_metaman = *CHECK_NONFATAL(node.mn_metaman);
 
     std::shared_ptr<CWallet> wallet{nullptr};
 #ifdef ENABLE_WALLET
@@ -1486,11 +1472,8 @@ static RPCHelpMan protx_info()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.mn_metaman);
-    CMasternodeMetaMan& mn_metaman = *node.mn_metaman;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    CMasternodeMetaMan& mn_metaman = *CHECK_NONFATAL(node.mn_metaman);
 
     std::shared_ptr<CWallet> wallet{nullptr};
 #ifdef ENABLE_WALLET
@@ -1560,11 +1543,8 @@ static RPCHelpMan protx_diff()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
-
-    CHECK_NONFATAL(node.llmq_ctx);
-    const LLMQContext& llmq_ctx = *node.llmq_ctx;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
+    const LLMQContext& llmq_ctx = *CHECK_NONFATAL(node.llmq_ctx);
 
     LOCK(cs_main);
     uint256 baseBlockHash = ParseBlock(request.params[0], chainman, "baseBlock");
@@ -1621,8 +1601,7 @@ static RPCHelpMan protx_listdiff()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
 
-    CHECK_NONFATAL(node.dmnman);
-    CDeterministicMNManager& dmnman = *node.dmnman;
+    CDeterministicMNManager& dmnman = *CHECK_NONFATAL(node.dmnman);
 
     LOCK(cs_main);
     UniValue ret(UniValue::VOBJ);
diff --git a/src/rpc/governance.cpp b/src/rpc/governance.cpp
index 2137e91a7b..5123945c3f 100644
--- a/src/rpc/governance.cpp
+++ b/src/rpc/governance.cpp
@@ -21,6 +21,7 @@
 #include <rpc/server_util.h>
 #include <rpc/util.h>
 #include <timedata.h>
+#include <util/check.h>
 #include <util/strencodings.h>
 #include <util/system.h>
 #include <validation.h>
@@ -194,12 +195,11 @@ static RPCHelpMan gobject_prepare()
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
 
     {
         LOCK(cs_main);
         std::string strError = "";
-        if (!govobj.IsValidLocally(node.dmnman->GetListAtChainTip(), chainman, strError, false))
+        if (!govobj.IsValidLocally(CHECK_NONFATAL(node.dmnman)->GetListAtChainTip(), chainman, strError, false))
             throw JSONRPCError(RPC_INTERNAL_ERROR, "Governance object is not valid - " + govobj.GetHash().ToString() + " - " + strError);
     }
 
@@ -303,14 +303,12 @@ static RPCHelpMan gobject_submit()
 {
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
-    CHECK_NONFATAL(node.govman);
 
     if(!node.mn_sync->IsBlockchainSynced()) {
         throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Must wait for client to sync with masternode network. Try again in a minute or so.");
     }
 
-    auto mnList = node.dmnman->GetListAtChainTip();
+    auto mnList = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
 
     if (node.mn_activeman) {
         const bool fMnFound = mnList.HasValidMNByCollateral(node.mn_activeman->GetOutPoint());
@@ -373,7 +371,6 @@ static RPCHelpMan gobject_submit()
         }
 
         LOCK2(cs_main, mempool.cs);
-        CHECK_NONFATAL(node.dmnman);
 
         std::string strError;
         if (!govobj.IsValidLocally(node.dmnman->GetListAtChainTip(), chainman, strError, fMissingConfirmations, true) && !fMissingConfirmations) {
@@ -384,7 +381,7 @@ static RPCHelpMan gobject_submit()
 
     // RELAY THIS OBJECT
     // Reject if rate check fails but don't update buffer
-    if (!node.govman->MasternodeRateCheck(govobj)) {
+    if (!CHECK_NONFATAL(node.govman)->MasternodeRateCheck(govobj)) {
         LogPrintf("gobject(submit) -- Object submission rejected because of rate check failure - hash = %s\n", strHash);
         throw JSONRPCError(RPC_INVALID_PARAMETER, "Object creation rate limit exceeded");
     }
@@ -393,9 +390,8 @@ static RPCHelpMan gobject_submit()
 
     PeerManager& peerman = EnsurePeerman(node);
     if (fMissingConfirmations) {
-        CHECK_NONFATAL(node.mn_sync);
         node.govman->AddPostponedObject(govobj);
-        govobj.Relay(peerman, *node.mn_sync);
+        govobj.Relay(peerman, *CHECK_NONFATAL(node.mn_sync));
     } else {
         node.govman->AddGovernanceObject(govobj, peerman);
     }
@@ -452,7 +448,7 @@ static UniValue VoteWithMasternodes(const JSONRPCRequest& request, const CWallet
     int nSuccessful = 0;
     int nFailed = 0;
 
-    auto mnList = node.dmnman->GetListAtChainTip();
+    auto mnList = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
 
     UniValue resultsObj(UniValue::VOBJ);
 
@@ -529,7 +525,6 @@ static RPCHelpMan gobject_vote_many()
     if (!wallet) return NullUniValue;
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
-    CHECK_NONFATAL(node.dmnman);
 
     uint256 hash(ParseHashV(request.params[0], "Object hash"));
     std::string strVoteSignal = request.params[1].get_str();
@@ -551,7 +546,7 @@ static RPCHelpMan gobject_vote_many()
 
     std::map<uint256, CKeyID> votingKeys;
 
-    auto mnList = node.dmnman->GetListAtChainTip();
+    auto mnList = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
     mnList.ForEachMN(true, [&](auto& dmn) {
         const bool is_mine = CheckWalletOwnsKey(*wallet, dmn.pdmnState->keyIDVoting);
         if (is_mine) {
@@ -583,7 +578,6 @@ static RPCHelpMan gobject_vote_alias()
     if (!wallet) return NullUniValue;
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
-    CHECK_NONFATAL(node.dmnman);
 
     uint256 hash(ParseHashV(request.params[0], "Object hash"));
     std::string strVoteSignal = request.params[1].get_str();
@@ -604,7 +598,7 @@ static RPCHelpMan gobject_vote_alias()
     EnsureWalletIsUnlocked(*wallet);
 
     uint256 proTxHash(ParseHashV(request.params[3], "protx-hash"));
-    auto dmn = node.dmnman->GetListAtChainTip().GetValidMN(proTxHash);
+    auto dmn = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip().GetValidMN(proTxHash);
     if (!dmn) {
         throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid or unknown proTxHash");
     }
@@ -718,11 +712,10 @@ static RPCHelpMan gobject_list_helper(const bool make_a_diff)
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
-    CHECK_NONFATAL(node.govman);
 
     const int64_t last_time = make_a_diff ? node.govman->GetLastDiffTime() : 0;
-    return ListObjects(*node.govman, node.dmnman->GetListAtChainTip(), chainman, strCachedSignal, strType, last_time);
+    return ListObjects(*CHECK_NONFATAL(node.govman), CHECK_NONFATAL(node.dmnman)->GetListAtChainTip(), chainman,
+                       strCachedSignal, strType, last_time);
 },
     };
 }
@@ -759,8 +752,8 @@ static RPCHelpMan gobject_get()
     // FIND THE GOVERNANCE OBJECT THE USER IS LOOKING FOR
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman && node.govman);
 
+    CHECK_NONFATAL(node.govman);
     LOCK2(cs_main, node.govman->cs);
     const CGovernanceObject* pGovObj = node.govman->FindConstGovernanceObject(hash);
 
@@ -785,7 +778,7 @@ static RPCHelpMan gobject_get()
     // SHOW (MUCH MORE) INFORMATION ABOUT VOTES FOR GOVERNANCE OBJECT (THAN LIST/DIFF ABOVE)
     // -- FUNDING VOTING RESULTS
 
-    auto tip_mn_list = node.dmnman->GetListAtChainTip();
+    auto tip_mn_list = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
 
     UniValue objFundingResult(UniValue::VOBJ);
     objFundingResult.pushKV("AbsoluteYesCount",  pGovObj->GetAbsoluteYesCount(tip_mn_list, VOTE_SIGNAL_FUNDING));
@@ -858,8 +851,8 @@ static RPCHelpMan gobject_getcurrentvotes()
 
     // FIND OBJECT USER IS LOOKING FOR
     const NodeContext& node = EnsureAnyNodeContext(request.context);
-    CHECK_NONFATAL(node.govman);
 
+    CHECK_NONFATAL(node.govman);
     LOCK(node.govman->cs);
     const CGovernanceObject* pGovObj = node.govman->FindConstGovernanceObject(hash);
 
@@ -872,10 +865,9 @@ static RPCHelpMan gobject_getcurrentvotes()
     UniValue bResult(UniValue::VOBJ);
 
     // GET MATCHING VOTES BY HASH, THEN SHOW USERS VOTE INFORMATION
-    CHECK_NONFATAL(node.dmnman);
     std::vector<CGovernanceVote> vecVotes = node.govman->GetCurrentVotes(hash, mnCollateralOutpoint);
     for (const auto& vote : vecVotes) {
-        bResult.pushKV(vote.GetHash().ToString(), vote.ToString(node.dmnman->GetListAtChainTip()));
+        bResult.pushKV(vote.GetHash().ToString(), vote.ToString(CHECK_NONFATAL(node.dmnman)->GetListAtChainTip()));
     }
 
     return bResult;
@@ -975,8 +967,7 @@ static RPCHelpMan voteraw()
         throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Malformed base64 encoding");
     }
 
-    CHECK_NONFATAL(node.dmnman);
-    const auto tip_mn_list = node.dmnman->GetListAtChainTip();
+    const auto tip_mn_list = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
     auto dmn = tip_mn_list.GetValidMNByCollateral(outpoint);
 
     if (!dmn) {
@@ -1034,7 +1025,6 @@ static RPCHelpMan getgovernanceinfo()
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureAnyChainman(request.context);
-    CHECK_NONFATAL(node.dmnman);
 
     const auto* pindex = WITH_LOCK(cs_main, return chainman.ActiveChain().Tip());
     int nBlockHeight = pindex->nHeight;
@@ -1048,7 +1038,7 @@ static RPCHelpMan getgovernanceinfo()
     obj.pushKV("superblockmaturitywindow", Params().GetConsensus().nSuperblockMaturityWindow);
     obj.pushKV("lastsuperblock", nLastSuperblock);
     obj.pushKV("nextsuperblock", nNextSuperblock);
-    obj.pushKV("fundingthreshold", int(node.dmnman->GetListAtChainTip().GetValidWeightedMNsCount() / 10));
+    obj.pushKV("fundingthreshold", int(CHECK_NONFATAL(node.dmnman)->GetListAtChainTip().GetValidWeightedMNsCount() / 10));
     obj.pushKV("governancebudget", ValueFromAmount(CSuperblock::GetPaymentsLimit(chainman.ActiveChain(), nNextSuperblock)));
 
     return obj;
diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp
index 5b30f0944b..ee5fcbdea0 100644
--- a/src/rpc/masternode.cpp
+++ b/src/rpc/masternode.cpp
@@ -19,6 +19,7 @@
 #include <rpc/server_util.h>
 #include <rpc/util.h>
 #include <univalue.h>
+#include <util/check.h>
 #include <util/strencodings.h>
 #include <validation.h>
 #include <wallet/coincontrol.h>
@@ -138,8 +139,7 @@ static RPCHelpMan masternode_winner()
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
-    return GetNextMasternodeForPayment(chainman.ActiveChain(), *node.dmnman, 10);
+    return GetNextMasternodeForPayment(chainman.ActiveChain(), *CHECK_NONFATAL(node.dmnman), 10);
 },
     };
 }
@@ -159,8 +159,7 @@ static RPCHelpMan masternode_current()
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
-    return GetNextMasternodeForPayment(chainman.ActiveChain(), *node.dmnman, 1);
+    return GetNextMasternodeForPayment(chainman.ActiveChain(), *CHECK_NONFATAL(node.dmnman), 1);
 },
     };
 }
@@ -212,7 +211,7 @@ static RPCHelpMan masternode_status()
         [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
 {
     const NodeContext& node = EnsureAnyNodeContext(request.context);
-    CHECK_NONFATAL(node.dmnman);
+
     if (!node.mn_activeman) {
         throw JSONRPCError(RPC_INTERNAL_ERROR, "This node does not run an active masternode.");
     }
@@ -221,7 +220,7 @@ static RPCHelpMan masternode_status()
     // keep compatibility with legacy status for now (might get deprecated/removed later)
     mnObj.pushKV("outpoint", node.mn_activeman->GetOutPoint().ToStringShort());
     mnObj.pushKV("service", node.mn_activeman->GetService().ToStringAddrPort());
-    CDeterministicMNCPtr dmn = node.dmnman->GetListAtChainTip().GetMN(node.mn_activeman->GetProTxHash());
+    CDeterministicMNCPtr dmn = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip().GetMN(node.mn_activeman->GetProTxHash());
     if (dmn) {
         mnObj.pushKV("proTxHash", dmn->proTxHash.ToString());
         mnObj.pushKV("type", std::string(GetMnType(dmn->nType).description));
@@ -243,12 +242,12 @@ static std::string GetRequiredPaymentsString(CGovernanceManager& govman, const C
     if (payee) {
         CTxDestination dest;
         if (!ExtractDestination(payee->pdmnState->scriptPayout, dest)) {
-            CHECK_NONFATAL(false);
+            NONFATAL_UNREACHABLE();
         }
         strPayments = EncodeDestination(dest);
         if (payee->nOperatorReward != 0 && payee->pdmnState->scriptOperatorPayout != CScript()) {
             if (!ExtractDestination(payee->pdmnState->scriptOperatorPayout, dest)) {
-                CHECK_NONFATAL(false);
+                NONFATAL_UNREACHABLE();
             }
             strPayments += ", " + EncodeDestination(dest);
         }
@@ -310,14 +309,11 @@ static RPCHelpMan masternode_winners()
     int nChainTipHeight = pindexTip->nHeight;
     int nStartHeight = std::max(nChainTipHeight - nCount, 1);
 
-    CHECK_NONFATAL(node.dmnman);
-    CHECK_NONFATAL(node.govman);
-
-    const auto tip_mn_list = node.dmnman->GetListAtChainTip();
+    const auto tip_mn_list = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
     for (int h = nStartHeight; h <= nChainTipHeight; h++) {
         const CBlockIndex* pIndex = pindexTip->GetAncestor(h - 1);
         auto payee = node.dmnman->GetListForBlock(pIndex).GetMNPayee(pIndex);
-        std::string strPayments = GetRequiredPaymentsString(*node.govman, tip_mn_list, h, payee);
+        std::string strPayments = GetRequiredPaymentsString(*CHECK_NONFATAL(node.govman), tip_mn_list, h, payee);
         if (strFilter != "" && strPayments.find(strFilter) == std::string::npos) continue;
         obj.pushKV(strprintf("%d", h), strPayments);
     }
@@ -558,11 +554,10 @@ static RPCHelpMan masternodelist_helper(bool is_composite)
 
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
-    CHECK_NONFATAL(node.dmnman);
 
     UniValue obj(UniValue::VOBJ);
 
-    const auto mnList = node.dmnman->GetListAtChainTip();
+    const auto mnList = CHECK_NONFATAL(node.dmnman)->GetListAtChainTip();
     const auto dmnToStatus = [&](const auto& dmn) {
         if (mnList.IsMNValid(dmn)) {
             return "ENABLED";
diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp
index af51ecc9f8..0444fc5a91 100644
--- a/src/rpc/mining.cpp
+++ b/src/rpc/mining.cpp
@@ -36,6 +36,7 @@
 #include <spork.h>
 #include <txmempool.h>
 #include <univalue.h>
+#include <util/check.h>
 #include <util/fees.h>
 #include <util/strencodings.h>
 #include <util/string.h>
@@ -391,12 +392,11 @@ static RPCHelpMan generateblock()
     block.vtx.insert(block.vtx.end(), txs.begin(), txs.end());
 
     {
-        CHECK_NONFATAL(node.evodb);
-
         LOCK(cs_main);
 
         BlockValidationState state;
-        if (!TestBlockValidity(state, *llmq_ctx.clhandler, *node.evodb, chainparams, active_chainstate, block, chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) {
+        if (!TestBlockValidity(state, *llmq_ctx.clhandler, *CHECK_NONFATAL(node.evodb), chainparams, active_chainstate,
+                               block, chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) {
             throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.GetRejectReason()));
         }
     }
@@ -710,14 +710,14 @@ static RPCHelpMan getblocktemplate()
             }
 
             LLMQContext& llmq_ctx = EnsureLLMQContext(node);
-            CHECK_NONFATAL(node.evodb);
 
             CBlockIndex* const pindexPrev = active_chain.Tip();
             // TestBlockValidity only supports blocks built on the current Tip
             if (block.hashPrevBlock != pindexPrev->GetBlockHash())
                 return "inconclusive-not-best-prevblk";
             BlockValidationState state;
-            TestBlockValidity(state, *llmq_ctx.clhandler, *node.evodb, Params(), active_chainstate, block, pindexPrev, false, true);
+            TestBlockValidity(state, *llmq_ctx.clhandler, *CHECK_NONFATAL(node.evodb), Params(), active_chainstate,
+                              block, pindexPrev, false, true);
             return BIP22ValidationResult(state);
         }
 
diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp
index 0250036d04..40ed64f821 100644
--- a/src/rpc/misc.cpp
+++ b/src/rpc/misc.cpp
@@ -1194,9 +1194,8 @@ static RPCHelpMan mockscheduler()
         throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)");
     }
 
-    auto* node_context = GetContext<NodeContext>(request.context);
+    auto* node_context = CHECK_NONFATAL(GetContext<NodeContext>(request.context));
     // protect against null pointer dereference
-    CHECK_NONFATAL(node_context);
     CHECK_NONFATAL(node_context->scheduler);
     node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds));
 
diff --git a/src/rpc/quorums.cpp b/src/rpc/quorums.cpp
index f6c27f4c73..3aa5de1a5a 100644
--- a/src/rpc/quorums.cpp
+++ b/src/rpc/quorums.cpp
@@ -11,6 +11,7 @@
 #include <rpc/server.h>
 #include <rpc/server_util.h>
 #include <rpc/util.h>
+#include <util/check.h>
 #include <validation.h>
 
 #include <masternode/node.h>
@@ -282,7 +283,6 @@ static RPCHelpMan quorum_dkgstatus()
     const ChainstateManager& chainman = EnsureChainman(node);
     const LLMQContext& llmq_ctx = EnsureLLMQContext(node);
     const CConnman& connman = EnsureConnman(node);
-    CHECK_NONFATAL(node.dmnman);
     CHECK_NONFATAL(node.sporkman);
 
     int detailLevel = 0;
@@ -296,7 +296,7 @@ static RPCHelpMan quorum_dkgstatus()
     llmq::CDKGDebugStatus status;
     llmq_ctx.dkg_debugman->GetLocalDebugStatus(status);
 
-    auto ret = status.ToJson(*node.dmnman, chainman, detailLevel);
+    auto ret = status.ToJson(*CHECK_NONFATAL(node.dmnman), chainman, detailLevel);
 
     CBlockIndex* pindexTip = WITH_LOCK(cs_main, return chainman.ActiveChain().Tip());
     int tipHeight = pindexTip->nHeight;
@@ -385,7 +385,6 @@ static RPCHelpMan quorum_memberof()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
     const LLMQContext& llmq_ctx = EnsureLLMQContext(node);
-    CHECK_NONFATAL(node.dmnman);
 
     uint256 protxHash(ParseHashV(request.params[0], "proTxHash"));
     int scanQuorumsCount = -1;
@@ -397,7 +396,7 @@ static RPCHelpMan quorum_memberof()
     }
 
     const CBlockIndex* pindexTip = WITH_LOCK(cs_main, return chainman.ActiveChain().Tip());
-    auto mnList = node.dmnman->GetListForBlock(pindexTip);
+    auto mnList = CHECK_NONFATAL(node.dmnman)->GetListForBlock(pindexTip);
     auto dmn = mnList.GetMN(protxHash);
     if (!dmn) {
         throw JSONRPCError(RPC_INVALID_PARAMETER, "masternode not found");
@@ -842,7 +841,7 @@ static RPCHelpMan quorum_rotationinfo()
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     const ChainstateManager& chainman = EnsureChainman(node);
     const LLMQContext& llmq_ctx = EnsureLLMQContext(node);
-    CHECK_NONFATAL(node.dmnman);
+    ;
 
     llmq::CGetQuorumRotationInfo cmd;
     llmq::CQuorumRotationInfo quorumRotationInfoRet;
@@ -859,7 +858,8 @@ static RPCHelpMan quorum_rotationinfo()
 
     LOCK(cs_main);
 
-    if (!BuildQuorumRotationInfo(*node.dmnman, chainman, *llmq_ctx.qman, *llmq_ctx.quorum_block_processor, cmd, quorumRotationInfoRet, strError)) {
+    if (!BuildQuorumRotationInfo(*CHECK_NONFATAL(node.dmnman), chainman, *llmq_ctx.qman, *llmq_ctx.quorum_block_processor,
+                                 cmd, quorumRotationInfoRet, strError)) {
         throw JSONRPCError(RPC_INVALID_REQUEST, strError);
     }
 
@@ -1073,7 +1073,8 @@ static RPCHelpMan verifyislock()
     auto llmqType = Params().GetConsensus().llmqTypeDIP0024InstantSend;
     const auto llmq_params_opt = Params().GetLLMQ(llmqType);
     CHECK_NONFATAL(llmq_params_opt.has_value());
-    return VerifyRecoveredSigLatestQuorums(*llmq_params_opt, chainman.ActiveChain(), *llmq_ctx.qman, signHeight, id, txid, sig);
+    return VerifyRecoveredSigLatestQuorums(*llmq_params_opt, chainman.ActiveChain(), *CHECK_NONFATAL(llmq_ctx.qman),
+                                           signHeight, id, txid, sig);
 },
     };
 }
diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp
index 1b01eb3195..e06382711e 100644
--- a/src/rpc/rawtransaction.cpp
+++ b/src/rpc/rawtransaction.cpp
@@ -38,6 +38,7 @@
 #include <txmempool.h>
 #include <uint256.h>
 #include <util/bip32.h>
+#include <util/check.h>
 #include <util/moneystr.h>
 #include <util/strencodings.h>
 #include <util/string.h>
@@ -495,8 +496,6 @@ static RPCHelpMan getassetunlockstatuses()
         throw JSONRPCError(RPC_INTERNAL_ERROR, "No blocks in chain");
     }
 
-    CHECK_NONFATAL(node.cpoolman);
-
     std::optional<CCreditPool> poolCL{std::nullopt};
     std::optional<CCreditPool> poolOnTip{std::nullopt};
     std::optional<int> nSpecificCoreHeight{std::nullopt};
@@ -506,7 +505,7 @@ static RPCHelpMan getassetunlockstatuses()
         if (nSpecificCoreHeight.value() < 0 || nSpecificCoreHeight.value() > chainman.ActiveChain().Height()) {
             throw JSONRPCError(RPC_INVALID_PARAMETER, "Block height out of range");
         }
-        poolCL = std::make_optional(node.cpoolman->GetCreditPool(chainman.ActiveChain()[nSpecificCoreHeight.value()], Params().GetConsensus()));
+        poolCL = std::make_optional(CHECK_NONFATAL(node.cpoolman)->GetCreditPool(chainman.ActiveChain()[nSpecificCoreHeight.value()], Params().GetConsensus()));
     }
     else {
         const auto pBlockIndexBestCL = [&]() -> const CBlockIndex* {
diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp
index 727fa5e694..4a63378b84 100644
--- a/src/rpc/util.cpp
+++ b/src/rpc/util.cpp
@@ -10,6 +10,7 @@
 #include <script/descriptor.h>
 #include <script/signingprovider.h>
 #include <tinyformat.h>
+#include <util/check.h>
 #include <util/system.h>
 #include <util/strencodings.h>
 #include <util/string.h>
@@ -494,7 +495,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP
                 // Null values are accepted in all arguments
                 break;
             default:
-                CHECK_NONFATAL(false);
+                NONFATAL_UNREACHABLE();
                 break;
             }
         }
@@ -743,7 +744,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const
         return;
     }
     case Type::ANY: {
-        CHECK_NONFATAL(false); // Only for testing
+        NONFATAL_UNREACHABLE(); // Only for testing
     }
     case Type::NONE: {
         sections.PushSection({indent + "null" + maybe_separator, Description("json null")});
@@ -807,7 +808,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const
         return;
     }
     } // no default case, so the compiler can warn about missing cases
-    CHECK_NONFATAL(false);
+    NONFATAL_UNREACHABLE();
 }
 
 bool RPCResult::MatchesType(const UniValue& result) const
@@ -843,7 +844,7 @@ bool RPCResult::MatchesType(const UniValue& result) const
         return UniValue::VOBJ == result.getType();
     }
     } // no default case, so the compiler can warn about missing cases
-    CHECK_NONFATAL(false);
+    NONFATAL_UNREACHABLE();
 }
 
 std::string RPCArg::ToStringObj(const bool oneline) const
@@ -878,9 +879,9 @@ std::string RPCArg::ToStringObj(const bool oneline) const
     case Type::OBJ:
     case Type::OBJ_USER_KEYS:
         // Currently unused, so avoid writing dead code
-        CHECK_NONFATAL(false);
+        NONFATAL_UNREACHABLE();
     } // no default case, so the compiler can warn about missing cases
-    CHECK_NONFATAL(false);
+    NONFATAL_UNREACHABLE();
 }
 
 std::string RPCArg::ToString(const bool oneline) const
@@ -915,7 +916,7 @@ std::string RPCArg::ToString(const bool oneline) const
         return "[" + res + "...]";
     }
     } // no default case, so the compiler can warn about missing cases
-    CHECK_NONFATAL(false);
+    NONFATAL_UNREACHABLE();
 }
 
 static std::pair<int64_t, int64_t> ParseRange(const UniValue& value)
diff --git a/src/util/check.h b/src/util/check.h
index cd6663f25c..247d89bdb6 100644
--- a/src/util/check.h
+++ b/src/util/check.h
@@ -18,8 +18,24 @@ class NonFatalCheckError : public std::runtime_error
     using std::runtime_error::runtime_error;
 };
 
+#define format_internal_error(msg, file, line, func, report)                                    \
+    strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\nPlease report this issue here: %s\n", \
+              msg, file, line, func, report)
+
+/** Helper for CHECK_NONFATAL() */
+template <typename T>
+T&& inline_check_non_fatal(T&& val, const char* file, int line, const char* func, const char* assertion)
+{
+    if (!(val)) {
+        throw NonFatalCheckError(
+            format_internal_error(assertion, file, line, func, PACKAGE_BUGREPORT));
+    }
+
+    return std::forward<T>(val);
+}
+
 /**
- * Throw a NonFatalCheckError when the condition evaluates to false
+ * Identity function. Throw a NonFatalCheckError when the condition evaluates to false
  *
  * This should only be used
  * - where the condition is assumed to be true, not for error handling or validating user input
@@ -29,18 +45,8 @@ class NonFatalCheckError : public std::runtime_error
  * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC
  * caller, which can then report the issue to the developers.
  */
-#define CHECK_NONFATAL(condition)                                 \
-    do {                                                          \
-        if (!(condition)) {                                       \
-            throw NonFatalCheckError(                             \
-                strprintf("Internal bug detected: '%s'\n"         \
-                          "%s:%d (%s)\n"                          \
-                          "You may report this issue here: %s\n", \
-                    (#condition),                                 \
-                    __FILE__, __LINE__, __func__,                 \
-                    PACKAGE_BUGREPORT));                          \
-        }                                                         \
-    } while (false)
+#define CHECK_NONFATAL(condition) \
+    inline_check_non_fatal(condition, __FILE__, __LINE__, __func__, #condition)
 
 #if defined(NDEBUG)
 #error "Cannot compile without assertions!"
@@ -80,4 +86,13 @@ T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_u
  */
 #define Assume(val) inline_assertion_check<false>(val, __FILE__, __LINE__, __func__, #val)
 
+/**
+ * NONFATAL_UNREACHABLE() is a macro that is used to mark unreachable code. It throws a NonFatalCheckError.
+ * This is used to mark code that is not yet implemented or is not yet reachable.
+ */
+#define NONFATAL_UNREACHABLE()                                        \
+    throw NonFatalCheckError(                                         \
+        format_internal_error("Unreachable code reached (non-fatal)", \
+                              __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT))
+
 #endif // BITCOIN_UTIL_CHECK_H
diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp
index 4d75d2ae1c..409944bdad 100644
--- a/src/wallet/rpcdump.cpp
+++ b/src/wallet/rpcdump.cpp
@@ -1135,7 +1135,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d
     case TxoutType::NONSTANDARD:
         return "unrecognized script";
     } // no default case, so the compiler can warn about missing cases
-    CHECK_NONFATAL(false);
+    NONFATAL_UNREACHABLE();
 }
 
 static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys)
diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py
index c5aeabd8ca..9e2663349f 100755
--- a/test/functional/rpc_misc.py
+++ b/test/functional/rpc_misc.py
@@ -27,7 +27,7 @@ def run_test(self):
         self.log.info("test CHECK_NONFATAL")
         assert_raises_rpc_error(
             -1,
-            'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'',
+            'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""',
             lambda: node.echo(arg9='trigger_internal_bug'),
         )
 

From bcd24a25e3e107802840cca19b145ed197bfee8e Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 15:42:59 +0000
Subject: [PATCH 21/69] fix: push activation height for forks ahead, fix
 `feature_pruning.py`

We need to push DIP-0001's activation height ahead as post-DIP-0001
consensus rejects transactions greater than `MAX_STANDARD_TX_SIZE`
(`bad-txns-oversize`) and this test generates transactions nearly 9.5x
that.

We also need to push v20 activation ahead to avoid different subsidy
calculation logic (`bad-cb-amount`).
---
 test/functional/feature_pruning.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index 67173dada7..2f62ae0058 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -33,6 +33,13 @@
 # compatible with pruning based on key creation time.
 TIMESTAMP_WINDOW = 2 * 60 * 60
 
+DEPLOYMENT_ARGS = [
+    "-dip3params=2000:2000",
+    "-testactivationheight=dip0001@2000",
+    "-testactivationheight=dip0008@2000",
+    "-testactivationheight=v20@2000",
+]
+
 def mine_large_blocks(node, n):
     # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
     # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
@@ -87,16 +94,16 @@ def set_test_params(self):
 
         # Create nodes 0 and 1 to mine.
         # Create node 2 to test pruning.
-        self.full_node_default_args = ["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-maxreceivebuffer=20000", "-blockmaxsize=999000", "-checkblocks=5"]
+        self.full_node_default_args = ["-maxreceivebuffer=20000", "-blockmaxsize=999000", "-checkblocks=5"] + DEPLOYMENT_ARGS
         # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
         # Create nodes 5 to test wallet in prune mode, but do not connect
         self.extra_args = [
             self.full_node_default_args,
             self.full_node_default_args,
-            ["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-prune=550"],
-            ["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-blockmaxsize=999000"],
-            ["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-blockmaxsize=999000"],
-            ["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance","-txindex=0","-prune=550"],
+            ["-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-prune=550"] + DEPLOYMENT_ARGS,
+            ["-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-blockmaxsize=999000"] + DEPLOYMENT_ARGS,
+            ["-disablegovernance","-txindex=0","-maxreceivebuffer=20000","-blockmaxsize=999000"] + DEPLOYMENT_ARGS,
+            ["-disablegovernance","-txindex=0","-prune=550"] + DEPLOYMENT_ARGS,
         ]
         self.rpc_timeout = 120
 
@@ -213,7 +220,7 @@ def reorg_test(self):
         self.log.info("New best height: %d" % self.nodes[1].getblockcount())
 
         # Mine one block to avoid automatic recovery from forks on restart
-        self.generate(self.nodes[1], 1)
+        self.generate(self.nodes[1], 1, sync_fun=self.no_op)
         # Disconnect node1 and generate the new chain
         self.disconnect_nodes(0, 1)
         self.disconnect_nodes(1, 2)
@@ -283,13 +290,13 @@ def reorg_back(self):
 
     def manual_test(self, node_number, use_timestamp):
         # at this point, node has 995 blocks and has not yet run in prune mode
-        self.start_node(node_number, extra_args=["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance", "-txindex=0"])
+        self.start_node(node_number, extra_args=["-disablegovernance", "-txindex=0"] + DEPLOYMENT_ARGS)
         node = self.nodes[node_number]
         assert_equal(node.getblockcount(), 995)
         assert_raises_rpc_error(-1, "Cannot prune blocks because node is not in prune mode", node.pruneblockchain, 500)
 
         # now re-start in manual pruning mode
-        self.restart_node(node_number, extra_args=["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance", "-txindex=0", "-prune=1"], expected_stderr=EXPECTED_STDERR_NO_GOV)
+        self.restart_node(node_number, extra_args=["-disablegovernance", "-txindex=0", "-prune=1"] + DEPLOYMENT_ARGS, expected_stderr=EXPECTED_STDERR_NO_GOV)
         node = self.nodes[node_number]
         assert_equal(node.getblockcount(), 995)
 
@@ -358,14 +365,14 @@ def has_block(index):
         assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
 
         # stop node, start back up with auto-prune at 550 MiB, make sure still runs
-        self.restart_node(node_number, extra_args=["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance", "-txindex=0", "-prune=550"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+        self.restart_node(node_number, extra_args=["-disablegovernance", "-txindex=0", "-prune=550"] + DEPLOYMENT_ARGS, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
 
         self.log.info("Success")
 
     def wallet_test(self):
         # check that the pruning node's wallet is still in good shape
         self.log.info("Stop and start pruning node to trigger wallet rescan")
-        self.restart_node(2, extra_args=["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance", "-txindex=0", "-prune=550"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+        self.restart_node(2, extra_args=["-disablegovernance", "-txindex=0", "-prune=550"] + DEPLOYMENT_ARGS, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
         self.log.info("Success")
 
         # check that wallet loads successfully when restarting a pruned node after IBD.
@@ -374,7 +381,7 @@ def wallet_test(self):
         self.connect_nodes(0, 5)
         nds = [self.nodes[0], self.nodes[5]]
         self.sync_blocks(nds, wait=5, timeout=300)
-        self.restart_node(5, extra_args=["-dip3params=2000:2000", "-testactivationheight=dip0008@2000", "-disablegovernance", "-txindex=0", "-prune=550"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) # restart to trigger rescan
+        self.restart_node(5, extra_args=["-disablegovernance", "-txindex=0", "-prune=550"] + DEPLOYMENT_ARGS, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE) # restart to trigger rescan
         self.log.info("Success")
 
     def run_test(self):

From c65ec190c562eba578b2c17ec9809bd6e2f0225d Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 21 Mar 2022 13:13:49 +0100
Subject: [PATCH 22/69] merge bitcoin#24626: disallow reindex-chainstate when
 pruning

---
 src/init.cpp                       | 4 +++-
 test/functional/feature_pruning.py | 4 ++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/init.cpp b/src/init.cpp
index 16c7f04860..2887cfc1ea 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -1156,12 +1156,14 @@ bool AppInitParameterInteraction(const ArgsManager& args)
         nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS);
     }
 
-    // if using block pruning, then disallow txindex, coinstatsindex and require disabling governance validation
     if (args.GetArg("-prune", 0)) {
         if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX))
             return InitError(_("Prune mode is incompatible with -txindex."));
         if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX))
             return InitError(_("Prune mode is incompatible with -coinstatsindex."));
+        if (args.GetBoolArg("-reindex-chainstate", false)) {
+            return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead."));
+        }
         if (!args.GetBoolArg("-disablegovernance", !DEFAULT_GOVERNANCE_ENABLE)) {
             return InitError(_("Prune mode is incompatible with -disablegovernance=false."));
         }
diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index 2f62ae0058..e96584f0cb 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -153,6 +153,10 @@ def test_invalid_command_line_options(self):
             expected_msg='Error: Prune mode is incompatible with -coinstatsindex.',
             extra_args=['-prune=550', '-coinstatsindex'],
         )
+        self.nodes[0].assert_start_raises_init_error(
+            expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
+            extra_args=['-prune=550', '-reindex-chainstate'],
+        )
         self.nodes[0].assert_start_raises_init_error(
             expected_msg='Error: Prune mode is incompatible with -disablegovernance=false.',
             extra_args=['-prune=550', '-disablegovernance=false'],

From baf6e26eed4af9d98f1707f292a3c6bdb5e41c56 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 16:53:27 +0000
Subject: [PATCH 23/69] merge bitcoin#21726: Improve Indices on pruned nodes
 via prune blockers

---
 src/index/base.cpp                            |  33 ++--
 src/index/base.h                              |   6 +
 src/index/blockfilterindex.h                  |   2 +
 src/index/coinstatsindex.h                    |   2 +
 src/index/txindex.h                           |   2 +
 src/init.cpp                                  |   4 +-
 src/node/blockstorage.cpp                     |  16 ++
 src/node/blockstorage.h                       |  19 +++
 src/rpc/blockchain.cpp                        |  13 +-
 src/validation.cpp                            |  41 ++++-
 .../feature_blockfilterindex_prune.py         |  80 ---------
 test/functional/feature_index_prune.py        | 161 ++++++++++++++++++
 test/functional/feature_pruning.py            |   4 -
 test/functional/test_runner.py                |   2 +-
 test/lint/lint-circular-dependencies.sh       |   2 -
 15 files changed, 269 insertions(+), 118 deletions(-)
 delete mode 100755 test/functional/feature_blockfilterindex_prune.py
 create mode 100755 test/functional/feature_index_prune.py

diff --git a/src/index/base.cpp b/src/index/base.cpp
index 5390411432..0aa4b8cd58 100644
--- a/src/index/base.cpp
+++ b/src/index/base.cpp
@@ -62,9 +62,9 @@ bool BaseIndex::Init()
     LOCK(cs_main);
     CChain& active_chain = m_chainstate->m_chain;
     if (locator.IsNull()) {
-        m_best_block_index = nullptr;
+        SetBestBlockIndex(nullptr);
     } else {
-        m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator);
+        SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator));
     }
 
     // Note: this will latch to true immediately if the user starts up with an empty
@@ -76,11 +76,7 @@ bool BaseIndex::Init()
         if (!m_best_block_index) {
             // index is not built yet
             // make sure we have all block data back to the genesis
-            const CBlockIndex* block = active_chain.Tip();
-            while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
-                block = block->pprev;
-            }
-            prune_violation = block != active_chain.Genesis();
+            prune_violation = GetFirstStoredBlock(active_chain.Tip()) != active_chain.Genesis();
         }
         // in case the index has a best block set and is not fully synced
         // check if we have the required blocks to continue building the index
@@ -138,7 +134,7 @@ void BaseIndex::ThreadSync()
         std::chrono::steady_clock::time_point last_locator_write_time{0s};
         while (true) {
             if (m_interrupt) {
-                m_best_block_index = pindex;
+                SetBestBlockIndex(pindex);
                 // No need to handle errors in Commit. If it fails, the error will be already be
                 // logged. The best way to recover is to continue, as index cannot be corrupted by
                 // a missed commit to disk for an advanced index state.
@@ -150,7 +146,7 @@ void BaseIndex::ThreadSync()
                 LOCK(cs_main);
                 const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain);
                 if (!pindex_next) {
-                    m_best_block_index = pindex;
+                    SetBestBlockIndex(pindex);
                     m_synced = true;
                     // No need to handle errors in Commit. See rationale above.
                     Commit();
@@ -172,7 +168,7 @@ void BaseIndex::ThreadSync()
             }
 
             if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                m_best_block_index = pindex;
+                SetBestBlockIndex(pindex);
                 last_locator_write_time = current_time;
                 // No need to handle errors in Commit. See rationale above.
                 Commit();
@@ -230,10 +226,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti
     // out of sync may be possible but a users fault.
     // In case we reorg beyond the pruned depth, ReadBlockFromDisk would
     // throw and lead to a graceful shutdown
-    m_best_block_index = new_tip;
+    SetBestBlockIndex(new_tip);
     if (!Commit()) {
         // If commit fails, revert the best block index to avoid corruption.
-        m_best_block_index = current_tip;
+        SetBestBlockIndex(current_tip);
         return false;
     }
 
@@ -274,7 +270,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
     }
 
     if (WriteBlock(*block, pindex)) {
-        m_best_block_index = pindex;
+        SetBestBlockIndex(pindex);
     } else {
         FatalError("%s: Failed to write block %s to index",
                    __func__, pindex->GetBlockHash().ToString());
@@ -381,3 +377,14 @@ IndexSummary BaseIndex::GetSummary() const
     summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0;
     return summary;
 }
+
+void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) {
+    assert(!fPruneMode || AllowPrune());
+
+    m_best_block_index = block;
+    if (AllowPrune() && block) {
+        PruneLockInfo prune_lock;
+        prune_lock.height_first = block->nHeight;
+        WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock));
+    }
+}
diff --git a/src/index/base.h b/src/index/base.h
index 403c8c87b1..323e4e9baf 100644
--- a/src/index/base.h
+++ b/src/index/base.h
@@ -81,6 +81,9 @@ class BaseIndex : public CValidationInterface
     /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
     /// getting corrupted.
     bool Commit();
+
+    virtual bool AllowPrune() const = 0;
+
 protected:
     CChainState* m_chainstate{nullptr};
 
@@ -109,6 +112,9 @@ class BaseIndex : public CValidationInterface
     /// Get the name of the index for display in logs.
     virtual const char* GetName() const = 0;
 
+    /// Update the internal best block index as well as the prune lock.
+    void SetBestBlockIndex(const CBlockIndex* block);
+
 public:
     /// Destructor interrupts sync thread if running and blocks until it exits.
     virtual ~BaseIndex();
diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h
index 760a712349..06b107d111 100644
--- a/src/index/blockfilterindex.h
+++ b/src/index/blockfilterindex.h
@@ -39,6 +39,8 @@ class BlockFilterIndex final : public BaseIndex
     /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */
     std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache);
 
+    bool AllowPrune() const override { return true; }
+
 protected:
     bool Init() override;
 
diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h
index ed0c8673f8..8cfb87ee8b 100644
--- a/src/index/coinstatsindex.h
+++ b/src/index/coinstatsindex.h
@@ -36,6 +36,8 @@ class CoinStatsIndex final : public BaseIndex
 
     bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex);
 
+    bool AllowPrune() const override { return true; }
+
 protected:
     bool Init() override;
 
diff --git a/src/index/txindex.h b/src/index/txindex.h
index 23a1aef1e4..3fe3628f0a 100644
--- a/src/index/txindex.h
+++ b/src/index/txindex.h
@@ -20,6 +20,8 @@ class TxIndex final : public BaseIndex
 private:
     const std::unique_ptr<DB> m_db;
 
+    bool AllowPrune() const override { return false; }
+
 protected:
     bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
 
diff --git a/src/init.cpp b/src/init.cpp
index 2887cfc1ea..c2cf16b1a2 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -538,7 +538,7 @@ void SetupServerArgs(ArgsManager& argsman)
         -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
-    argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex, -rescan and -disablegovernance=false. "
+    argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -rescan and -disablegovernance=false. "
             "Warning: Reverting this setting requires re-downloading the entire blockchain. "
             "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -1159,8 +1159,6 @@ bool AppInitParameterInteraction(const ArgsManager& args)
     if (args.GetArg("-prune", 0)) {
         if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX))
             return InitError(_("Prune mode is incompatible with -txindex."));
-        if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX))
-            return InitError(_("Prune mode is incompatible with -coinstatsindex."));
         if (args.GetBoolArg("-reindex-chainstate", false)) {
             return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead."));
         }
diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp
index d8e59b88ab..960759396f 100644
--- a/src/node/blockstorage.cpp
+++ b/src/node/blockstorage.cpp
@@ -24,6 +24,7 @@
 #include <walletinitinterface.h>
 
 #include <map>
+#include <unordered_map>
 
 std::atomic_bool fImporting(false);
 std::atomic_bool fReindex(false);
@@ -249,6 +250,11 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr
            nLastBlockWeCanPrune, count);
 }
 
+void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
+    AssertLockHeld(::cs_main);
+    m_prune_locks[name] = lock_info;
+}
+
 CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
 {
     AssertLockHeld(cs_main);
@@ -421,6 +427,16 @@ bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex)
     return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
 }
 
+const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) {
+    AssertLockHeld(::cs_main);
+    assert(start_block);
+    const CBlockIndex* last_block = start_block;
+    while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
+        last_block = last_block->pprev;
+    }
+    return last_block;
+}
+
 // If we're using -prune with -reindex, then delete block files that will be ignored by the
 // reindex.  Since reindexing works by starting at block file 0 and looping until a blockfile
 // is missing, do the same here to delete any later block files after a gap.  Also delete all
diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h
index ae7319d5ea..7f671fd5f8 100644
--- a/src/node/blockstorage.h
+++ b/src/node/blockstorage.h
@@ -12,6 +12,7 @@
 #include <txdb.h>
 
 #include <cstdint>
+#include <unordered_map>
 #include <vector>
 
 extern RecursiveMutex cs_main;
@@ -77,6 +78,10 @@ struct CBlockIndexHeightOnlyComparator {
     bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
 };
 
+struct PruneLockInfo {
+    int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
+};
+
 /**
  * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
  * to determine where the most-work tip is.
@@ -137,6 +142,14 @@ class BlockManager
     /** Dirty block file entries. */
     std::set<int> m_dirty_fileinfo;
 
+    /**
+     * Map from external index name to oldest block that must not be pruned.
+     *
+     * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
+     * below will be pruned, but callers should avoid assuming any particular buffer size.
+     */
+    std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
+
 public:
     BlockMap m_block_index GUARDED_BY(cs_main);
     PrevBlockMap m_prev_block_index GUARDED_BY(cs_main);
@@ -185,8 +198,14 @@ class BlockManager
 
     //! Check whether the block associated with this index entry is pruned or not.
     bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
+    //! Create or update a prune lock identified by its name
+    void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 };
 
+//! Find the first block that is not pruned
+const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
 void CleanupBlockRevFiles();
 
 /** Open a block file (blk?????.dat) */
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 94426f7da4..15ae8e26a5 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1361,10 +1361,9 @@ static RPCHelpMan pruneblockchain()
 
     PruneBlockFilesManual(active_chainstate, height);
     const CBlockIndex* block = CHECK_NONFATAL(active_chain.Tip());
-    while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
-        block = block->pprev;
-    }
-    return uint64_t(block->nHeight);
+    const CBlockIndex* last_block = GetFirstStoredBlock(block);
+
+    return static_cast<uint64_t>(last_block->nHeight);
 },
     };
 }
@@ -1806,11 +1805,7 @@ RPCHelpMan getblockchaininfo()
     obj.pushKV("pruned", fPruneMode);
     if (fPruneMode) {
         const CBlockIndex* block = CHECK_NONFATAL(tip);
-        while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
-            block = block->pprev;
-        }
-
-        obj.pushKV("pruneheight",        block->nHeight);
+        obj.pushKV("pruneheight", GetFirstStoredBlock(block)->nHeight);
 
         // if 0, execution bypasses the whole if block.
         bool automatic_pruning{args.GetArg("-prune", 0) != 1};
diff --git a/src/validation.cpp b/src/validation.cpp
index 38a030ace1..efe4e2cfe2 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -19,7 +19,6 @@
 #include <deploymentstatus.h>
 #include <flatfile.h>
 #include <hash.h>
-#include <index/blockfilterindex.h>
 #include <logging.h>
 #include <logging/timer.h>
 #include <node/blockstorage.h>
@@ -93,6 +92,12 @@ const std::vector<std::string> CHECKLEVEL_DOC {
     "level 4 tries to reconnect the blocks",
     "each level includes the checks of the previous levels",
 };
+/** The number of blocks to keep below the deepest prune lock.
+ *  There is nothing special about this number. It is higher than what we
+ *  expect to see in regular mainnet reorgs, but not so high that it would
+ *  noticeably interfere with the pruning mechanism.
+ * */
+static constexpr int PRUNE_LOCK_BUFFER{10};
 
 /**
  * Mutex to guard access to validation specific variables, such as reading
@@ -2368,12 +2373,24 @@ bool CChainState::FlushStateToDisk(
         CoinsCacheSizeState cache_state = GetCoinsCacheSizeState();
         LOCK(m_blockman.cs_LastBlockFile);
         if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) {
-            // make sure we don't prune above the blockfilterindexes bestblocks
+            // make sure we don't prune above any of the prune locks bestblocks
             // pruning is height-based
-            int last_prune = m_chain.Height(); // last height we can prune
-            ForEachBlockFilterIndex([&](BlockFilterIndex& index) {
-               last_prune = std::max(1, std::min(last_prune, index.GetSummary().best_block_height));
-            });
+            int last_prune{m_chain.Height()}; // last height we can prune
+            std::optional<std::string> limiting_lock; // prune lock that actually was the limiting factor, only used for logging
+
+            for (const auto& prune_lock : m_blockman.m_prune_locks) {
+                if (prune_lock.second.height_first == std::numeric_limits<int>::max()) continue;
+                // Remove the buffer and one additional block here to get actual height that is outside of the buffer
+                const int lock_height{prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1};
+                last_prune = std::max(1, std::min(last_prune, lock_height));
+                if (last_prune == lock_height) {
+                    limiting_lock = prune_lock.first;
+                }
+            }
+
+            if (limiting_lock) {
+                LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", limiting_lock.value(), last_prune);
+            }
 
             if (nManualPruneHeight > 0) {
                 LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCHMARK);
@@ -2628,6 +2645,18 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr
         dbTx->Commit();
     }
     LogPrint(BCLog::BENCHMARK, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
+
+    {
+        // Prune locks that began at or after the tip should be moved backward so they get a chance to reorg
+        const int max_height_first{pindexDelete->nHeight - 1};
+        for (auto& prune_lock : m_blockman.m_prune_locks) {
+            if (prune_lock.second.height_first <= max_height_first) continue;
+
+            prune_lock.second.height_first = max_height_first;
+            LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, max_height_first);
+        }
+    }
+
     // Write the chain state to disk, if necessary.
     if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) {
         return false;
diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py
deleted file mode 100755
index 361a9cce01..0000000000
--- a/test/functional/feature_blockfilterindex_prune.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test blockfilterindex in conjunction with prune."""
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import (
-    assert_equal,
-    assert_greater_than,
-    assert_raises_rpc_error,
-)
-from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE
-
-
-class FeatureBlockfilterindexPruneTest(BitcoinTestFramework):
-    def set_test_params(self):
-        self.num_nodes = 1
-        self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1", "-testactivationheight=v20@2000"]]
-
-    def sync_index(self, height):
-        expected = {'basic block filter index': {'synced': True, 'best_block_height': height}}
-        self.wait_until(lambda: self.nodes[0].getindexinfo() == expected)
-
-    def run_test(self):
-        self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned")
-        self.sync_index(height=200)
-        assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0)
-        self.generate(self.nodes[0], 500)
-        self.sync_index(height=700)
-
-        self.log.info("prune some blocks")
-        pruneheight = self.nodes[0].pruneblockchain(400)
-        # the prune heights used here and below are magic numbers that are determined by the
-        # thresholds at which block files wrap, so they depend on disk serialization and default block file size.
-        assert_equal(pruneheight, 366)
-
-        self.log.info("check if we can access the tips blockfilter when we have pruned some blocks")
-        assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0)
-
-        self.log.info("check if we can access the blockfilter of a pruned block")
-        assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0)
-
-        # mine and sync index up to a height that will later be the pruneheight
-        self.generate(self.nodes[0], 298)
-        self.sync_index(height=998)
-
-        self.log.info("start node without blockfilterindex")
-        self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
-
-        self.log.info("make sure accessing the blockfilters throws an error")
-        assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2))
-        self.generate(self.nodes[0], 502)
-
-        self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled")
-        pruneheight_2 = self.nodes[0].pruneblockchain(1000)
-        assert_equal(pruneheight_2, 932)
-        self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
-        self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height")
-        self.sync_index(height=1500)
-
-        self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled")
-        self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
-        self.generate(self.nodes[0], 1000)
-        pruneheight_3 = self.nodes[0].pruneblockchain(2000)
-        assert_greater_than(pruneheight_3, pruneheight_2)
-        self.stop_node(0, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
-
-        self.log.info("make sure we get an init error when starting the node again with block filters")
-        self.nodes[0].assert_start_raises_init_error(
-            extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"],
-            expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}\nError: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)",
-        )
-
-        self.log.info("make sure the node starts again with the -reindex arg")
-        self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"])
-        self.stop_nodes(expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
-
-
-if __name__ == '__main__':
-    FeatureBlockfilterindexPruneTest().main()
diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py
new file mode 100755
index 0000000000..5ec80ced6e
--- /dev/null
+++ b/test/functional/feature_index_prune.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test indices in conjunction with prune."""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_greater_than,
+    assert_raises_rpc_error,
+    p2p_port,
+)
+from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE
+
+DEPLOYMENT_ARG = "-testactivationheight=v20@3000"
+
+class FeatureIndexPruneTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 4
+        self.extra_args = [
+            ["-fastprune", "-prune=1", "-blockfilterindex=1", DEPLOYMENT_ARG],
+            ["-fastprune", "-prune=1", "-coinstatsindex=1", DEPLOYMENT_ARG],
+            ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1", DEPLOYMENT_ARG],
+            [DEPLOYMENT_ARG]
+        ]
+
+    def sync_index(self, height):
+        expected_filter = {
+            'basic block filter index': {'synced': True, 'best_block_height': height},
+        }
+        self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter)
+
+        expected_stats = {
+            'coinstatsindex': {'synced': True, 'best_block_height': height}
+        }
+        self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats)
+
+        expected = {**expected_filter, **expected_stats}
+        self.wait_until(lambda: self.nodes[2].getindexinfo() == expected)
+
+    def reconnect_nodes(self):
+        self.connect_nodes(0,1)
+        self.connect_nodes(0,2)
+        self.connect_nodes(0,3)
+
+    def mine_batches(self, blocks):
+        n = blocks // 250
+        for _ in range(n):
+            self.generate(self.nodes[0], 250)
+        self.generate(self.nodes[0], blocks % 250)
+        self.sync_blocks()
+
+    def restart_without_indices(self):
+        for i in range(3):
+            self.restart_node(i, extra_args=["-fastprune", "-prune=1", DEPLOYMENT_ARG], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+        self.reconnect_nodes()
+
+    def run_test(self):
+        filter_nodes = [self.nodes[0], self.nodes[2]]
+        stats_nodes = [self.nodes[1], self.nodes[2]]
+
+        self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned")
+        self.sync_index(height=200)
+        tip = self.nodes[0].getbestblockhash()
+        for node in filter_nodes:
+            assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
+        for node in stats_nodes:
+            assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash'])
+
+        self.mine_batches(500)
+        self.sync_index(height=700)
+
+        self.log.info("prune some blocks")
+        for node in self.nodes[:2]:
+            with node.assert_debug_log(['limited pruning to height 689']):
+                pruneheight_new = node.pruneblockchain(400)
+                # the prune heights used here and below are magic numbers that are determined by the
+                # thresholds at which block files wrap, so they depend on disk serialization and default block file size.
+                assert_equal(pruneheight_new, 366)
+
+        self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks")
+        tip = self.nodes[0].getbestblockhash()
+        for node in filter_nodes:
+            assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
+        for node in stats_nodes:
+            assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash'])
+
+        self.log.info("check if we can access the blockfilter and coinstats of a pruned block")
+        height_hash = self.nodes[0].getblockhash(2)
+        for node in filter_nodes:
+            assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0)
+        for node in stats_nodes:
+            assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash'])
+
+        # mine and sync index up to a height that will later be the pruneheight
+        self.generate(self.nodes[0], 298)
+        self.sync_index(height=998)
+
+        self.restart_without_indices()
+
+        self.log.info("make sure trying to access the indices throws errors")
+        for node in filter_nodes:
+            msg = "Index is not enabled for filtertype basic"
+            assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash)
+        for node in stats_nodes:
+            msg = "Querying specific block heights requires coinstatsindex"
+            assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash)
+
+        self.mine_batches(502)
+
+        self.log.info("prune exactly up to the indices best blocks while the indices are disabled")
+        for i in range(3):
+            pruneheight_2 = self.nodes[i].pruneblockchain(1000)
+            assert_equal(pruneheight_2, 932)
+            # Restart the nodes again with the indices activated
+            self.restart_node(i, extra_args=self.extra_args[i], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+
+        self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height")
+        self.sync_index(height=1500)
+
+        self.log.info("prune further than the indices best blocks while the indices are disabled")
+        self.restart_without_indices()
+        self.mine_batches(1000)
+
+        for i in range(3):
+            pruneheight_3 = self.nodes[i].pruneblockchain(2000)
+            assert_greater_than(pruneheight_3, pruneheight_2)
+            self.stop_node(i, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+
+        self.log.info("make sure we get an init error when starting the nodes again with the indices")
+        filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
+        stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
+        for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
+            self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}\n{msg}")
+
+        self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
+        ip_port = "127.0.0.1:" + str(p2p_port(3))
+        for i in range(3):
+            # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck
+            restart_args = self.extra_args[i]+["-reindex", f"-connect={ip_port}"]
+            self.restart_node(i, extra_args=restart_args, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+
+        self.sync_blocks(timeout=300)
+
+        for node in self.nodes[:2]:
+            with node.assert_debug_log(['limited pruning to height 2489']):
+                pruneheight_new = node.pruneblockchain(2500)
+                assert_equal(pruneheight_new, 2197)
+
+        self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario")
+        with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']):
+            self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480))
+            self.generate(self.nodes[3], 30)
+            self.sync_blocks()
+
+        for idx in range(self.num_nodes):
+            self.nodes[idx].stop_node(expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE if idx != 3 else "")
+
+
+if __name__ == '__main__':
+    FeatureIndexPruneTest().main()
diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py
index e96584f0cb..6f67fcbcf1 100755
--- a/test/functional/feature_pruning.py
+++ b/test/functional/feature_pruning.py
@@ -149,10 +149,6 @@ def test_invalid_command_line_options(self):
             expected_msg='Error: Prune mode is incompatible with -txindex.',
             extra_args=['-prune=550', '-txindex'],
         )
-        self.nodes[0].assert_start_raises_init_error(
-            expected_msg='Error: Prune mode is incompatible with -coinstatsindex.',
-            extra_args=['-prune=550', '-coinstatsindex'],
-        )
         self.nodes[0].assert_start_raises_init_error(
             expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
             extra_args=['-prune=550', '-reindex-chainstate'],
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 9045045236..8f9a103396 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -87,6 +87,7 @@
     # Longest test should go first, to favor running tests in parallel
     'feature_pruning.py', # NOTE: Prune mode is incompatible with -txindex, should work with governance validation disabled though.
     'feature_dbcrash.py',
+    'feature_index_prune.py',
 ]
 
 BASE_SCRIPTS = [
@@ -365,7 +366,6 @@
     'rpc_help.py',
     'feature_dirsymlinks.py',
     'feature_help.py',
-    'feature_blockfilterindex_prune.py'
     # Don't append tests at the end to avoid merge conflicts
     # Put them in a random line within the section that fits their approximate run-time
 ]
diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh
index 9bbe54a0a0..90eb137688 100755
--- a/test/lint/lint-circular-dependencies.sh
+++ b/test/lint/lint-circular-dependencies.sh
@@ -11,8 +11,6 @@ export LC_ALL=C
 EXPECTED_CIRCULAR_DEPENDENCIES=(
     "chainparamsbase -> util/system -> chainparamsbase"
     "node/blockstorage -> validation -> node/blockstorage"
-    "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex"
-    "index/base -> validation -> index/blockfilterindex -> index/base"
     "index/coinstatsindex -> node/coinstats -> index/coinstatsindex"
     "policy/fees -> txmempool -> policy/fees"
     "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel"

From e6867a35cee0f9e53c8f6c24da3c47ecac2bc553 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Fri, 13 May 2022 00:51:48 +0200
Subject: [PATCH 24/69] merge bitcoin#25123: Fix race condition in index prune
 test

---
 test/functional/feature_index_prune.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py
index 5ec80ced6e..0222de33e3 100755
--- a/test/functional/feature_index_prune.py
+++ b/test/functional/feature_index_prune.py
@@ -8,7 +8,6 @@
     assert_equal,
     assert_greater_than,
     assert_raises_rpc_error,
-    p2p_port,
 )
 from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE
 
@@ -134,11 +133,11 @@ def run_test(self):
             self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}\n{msg}")
 
         self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
-        ip_port = "127.0.0.1:" + str(p2p_port(3))
         for i in range(3):
-            # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck
-            restart_args = self.extra_args[i]+["-reindex", f"-connect={ip_port}"]
+            restart_args = self.extra_args[i]+["-reindex"]
             self.restart_node(i, extra_args=restart_args, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
+            # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck
+            self.connect_nodes(i, 3)
 
         self.sync_blocks(timeout=300)
 

From a86109a017577891382fc1f366e553506e1a0a28 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Fri, 6 May 2022 14:07:51 +0200
Subject: [PATCH 25/69] merge bitcoin#25074: During sync, commit best block
 after indexing

---
 src/index/base.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/index/base.cpp b/src/index/base.cpp
index 0aa4b8cd58..376d0314ca 100644
--- a/src/index/base.cpp
+++ b/src/index/base.cpp
@@ -168,7 +168,7 @@ void BaseIndex::ThreadSync()
             }
 
             if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                SetBestBlockIndex(pindex);
+                SetBestBlockIndex(pindex->pprev);
                 last_locator_write_time = current_time;
                 // No need to handle errors in Commit. See rationale above.
                 Commit();

From e507a51323088541c901d477c6a75df414662b03 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 16:19:57 +0000
Subject: [PATCH 26/69] fix: avoid `mandatory-script-verify-flag-failed` crash
 in bench test

Aberrant behaviour was first introduced in 5d10b413 (dash#6152), changes
meant for SegWit made it in, reverting them fixes the problem.
---
 src/bench/block_assemble.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp
index 56afa0132f..2d2c7c97ac 100644
--- a/src/bench/block_assemble.cpp
+++ b/src/bench/block_assemble.cpp
@@ -32,7 +32,7 @@ static void AssembleBlock(benchmark::Bench& bench)
     std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs;
     for (size_t b{0}; b < NUM_BLOCKS; ++b) {
         CMutableTransaction tx;
-        tx.vin.push_back(MineBlock(test_setup->m_node, P2SH_OP_TRUE));
+        tx.vin.push_back(MineBlock(test_setup->m_node, SCRIPT_PUB));
         tx.vin.back().scriptSig = scriptSig;
         tx.vout.emplace_back(1337, SCRIPT_PUB);
         if (NUM_BLOCKS - b >= COINBASE_MATURITY)
@@ -48,7 +48,7 @@ static void AssembleBlock(benchmark::Bench& bench)
     }
 
     bench.minEpochIterations(700).run([&] {
-        PrepareBlock(test_setup->m_node, P2SH_OP_TRUE);
+        PrepareBlock(test_setup->m_node, SCRIPT_PUB);
     });
 }
 

From 3bd584c845e7aae4fbc94a0f1456dca56a3a5629 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 16:16:07 +0000
Subject: [PATCH 27/69] merge bitcoin#24832: Verify the block filter hash when
 reading the filter from disk

---
 src/bench/gcs_filter.cpp       | 80 ++++++++++++++++++++++++++--------
 src/blockfilter.cpp            |  8 ++--
 src/blockfilter.h              |  6 +--
 src/index/blockfilterindex.cpp | 13 ++++--
 src/index/blockfilterindex.h   |  2 +-
 5 files changed, 81 insertions(+), 28 deletions(-)

diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp
index 3dd53bab1b..7f08cbec68 100644
--- a/src/bench/gcs_filter.cpp
+++ b/src/bench/gcs_filter.cpp
@@ -5,38 +5,84 @@
 #include <bench/bench.h>
 #include <blockfilter.h>
 
-static void ConstructGCSFilter(benchmark::Bench& bench)
+static const GCSFilter::ElementSet GenerateGCSTestElements()
 {
     GCSFilter::ElementSet elements;
-    for (int i = 0; i < 10000; ++i) {
+
+    // Testing the benchmarks with different number of elements show that a filter
+    // with at least 100,000 elements results in benchmarks that have the same
+    // ns/op. This makes it easy to reason about how long (in nanoseconds) a single
+    // filter element takes to process.
+    for (int i = 0; i < 100000; ++i) {
         GCSFilter::Element element(32);
         element[0] = static_cast<unsigned char>(i);
         element[1] = static_cast<unsigned char>(i >> 8);
         elements.insert(std::move(element));
     }
 
+    return elements;
+}
+
+static void GCSBlockFilterGetHash(benchmark::Bench& bench)
+{
+    auto elements = GenerateGCSTestElements();
+
+    GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements);
+    BlockFilter block_filter(BlockFilterType::BASIC_FILTER, {}, filter.GetEncoded(), /*skip_decode_check=*/false);
+
+    bench.run([&] {
+        block_filter.GetHash();
+    });
+}
+
+static void GCSFilterConstruct(benchmark::Bench& bench)
+{
+    auto elements = GenerateGCSTestElements();
+
     uint64_t siphash_k0 = 0;
-    bench.batch(elements.size()).unit("elem").run([&] {
-        GCSFilter filter({siphash_k0, 0, 20, 1 << 20}, elements);
+    bench.run([&]{
+        GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements);
+
         siphash_k0++;
     });
 }
 
-static void MatchGCSFilter(benchmark::Bench& bench)
+static void GCSFilterDecode(benchmark::Bench& bench)
 {
-    GCSFilter::ElementSet elements;
-    for (int i = 0; i < 10000; ++i) {
-        GCSFilter::Element element(32);
-        element[0] = static_cast<unsigned char>(i);
-        element[1] = static_cast<unsigned char>(i >> 8);
-        elements.insert(std::move(element));
-    }
-    GCSFilter filter({0, 0, 20, 1 << 20}, elements);
+    auto elements = GenerateGCSTestElements();
 
-    bench.unit("elem").run([&] {
-        filter.Match(GCSFilter::Element());
+    GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements);
+    auto encoded = filter.GetEncoded();
+
+    bench.run([&] {
+        GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/false);
+    });
+}
+
+static void GCSFilterDecodeSkipCheck(benchmark::Bench& bench)
+{
+    auto elements = GenerateGCSTestElements();
+
+    GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements);
+    auto encoded = filter.GetEncoded();
+
+    bench.run([&] {
+        GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/true);
     });
 }
 
-BENCHMARK(ConstructGCSFilter);
-BENCHMARK(MatchGCSFilter);
+static void GCSFilterMatch(benchmark::Bench& bench)
+{
+    auto elements = GenerateGCSTestElements();
+
+    GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements);
+
+    bench.run([&] {
+        filter.Match(GCSFilter::Element());
+    });
+}
+BENCHMARK(GCSBlockFilterGetHash);
+BENCHMARK(GCSFilterConstruct);
+BENCHMARK(GCSFilterDecode);
+BENCHMARK(GCSFilterDecodeSkipCheck);
+BENCHMARK(GCSFilterMatch);
diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp
index 25ca14c002..e7cc29afda 100644
--- a/src/blockfilter.cpp
+++ b/src/blockfilter.cpp
@@ -47,7 +47,7 @@ GCSFilter::GCSFilter(const Params& params)
     : m_params(params), m_N(0), m_F(0), m_encoded{0}
 {}
 
-GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter)
+GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check)
     : m_params(params), m_encoded(std::move(encoded_filter))
 {
     SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0};
@@ -59,6 +59,8 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi
     }
     m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_params.m_M);
 
+    if (skip_decode_check) return;
+
     // Verify that the encoded filter contains exactly N elements. If it has too much or too little
     // data, a std::ios_base::failure exception will be raised.
     BitStreamReader<SpanReader> bitreader{stream};
@@ -219,14 +221,14 @@ static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
 }
 
 BlockFilter::BlockFilter(BlockFilterType filter_type, const uint256& block_hash,
-                         std::vector<unsigned char> filter)
+                         std::vector<unsigned char> filter, bool skip_decode_check)
     : m_filter_type(filter_type), m_block_hash(block_hash)
 {
     GCSFilter::Params params;
     if (!BuildParams(params)) {
         throw std::invalid_argument("unknown filter_type");
     }
-    m_filter = GCSFilter(params, std::move(filter));
+    m_filter = GCSFilter(params, std::move(filter), skip_decode_check);
 }
 
 BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo)
diff --git a/src/blockfilter.h b/src/blockfilter.h
index 8bdd789dd5..081e35ed9f 100644
--- a/src/blockfilter.h
+++ b/src/blockfilter.h
@@ -60,7 +60,7 @@ class GCSFilter
     explicit GCSFilter(const Params& params = Params());
 
     /** Reconstructs an already-created filter from an encoding. */
-    GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter);
+    GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check);
 
     /** Builds a new filter from the params and set of elements. */
     GCSFilter(const Params& params, const ElementSet& elements);
@@ -123,7 +123,7 @@ class BlockFilter
 
     //! Reconstruct a BlockFilter from parts.
     BlockFilter(BlockFilterType filter_type, const uint256& block_hash,
-                std::vector<unsigned char> filter);
+                std::vector<unsigned char> filter, bool skip_decode_check);
 
     //! Construct a new BlockFilter of the specified type from a block.
     BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo);
@@ -165,7 +165,7 @@ class BlockFilter
         if (!BuildParams(params)) {
             throw std::ios_base::failure("unknown filter_type");
         }
-        m_filter = GCSFilter(params, std::move(encoded_filter));
+        m_filter = GCSFilter(params, std::move(encoded_filter), /*skip_decode_check=*/false);
     }
 };
 
diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp
index 75a190fdb8..2e84934eba 100644
--- a/src/index/blockfilterindex.cpp
+++ b/src/index/blockfilterindex.cpp
@@ -5,6 +5,7 @@
 #include <map>
 
 #include <dbwrapper.h>
+#include <hash.h>
 #include <index/blockfilterindex.h>
 #include <node/blockstorage.h>
 #include <serialize.h>
@@ -146,18 +147,22 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch)
     return BaseIndex::CommitInternal(batch);
 }
 
-bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const
+bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const
 {
     CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
     if (filein.IsNull()) {
         return false;
     }
 
+    // Check that the hash of the encoded_filter matches the one stored in the db.
     uint256 block_hash;
     std::vector<uint8_t> encoded_filter;
     try {
         filein >> block_hash >> encoded_filter;
-        filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
+        uint256 result;
+        CHash256().Write(encoded_filter).Finalize(result);
+        if (result != hash) return error("Checksum mismatch in filter decode.");
+        filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true);
     }
     catch (const std::exception& e) {
         return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
@@ -384,7 +389,7 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter&
         return false;
     }
 
-    return ReadFilterFromDisk(entry.pos, filter_out);
+    return ReadFilterFromDisk(entry.pos, entry.hash, filter_out);
 }
 
 bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out)
@@ -428,7 +433,7 @@ bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* st
     filters_out.resize(entries.size());
     auto filter_pos_it = filters_out.begin();
     for (const auto& entry : entries) {
-        if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
+        if (!ReadFilterFromDisk(entry.pos, entry.hash, *filter_pos_it)) {
             return false;
         }
         ++filter_pos_it;
diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h
index 06b107d111..38754ead04 100644
--- a/src/index/blockfilterindex.h
+++ b/src/index/blockfilterindex.h
@@ -32,7 +32,7 @@ class BlockFilterIndex final : public BaseIndex
     FlatFilePos m_next_filter_pos;
     std::unique_ptr<FlatFileSeq> m_filter_fileseq;
 
-    bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
+    bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const;
     size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
 
     Mutex m_cs_headers_cache;

From 9e75b99c5309767b1e8c7f995f4e8d1976a22f9c Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Fri, 30 Sep 2022 08:44:04 -0400
Subject: [PATCH 28/69] merge bitcoin#26215: Improve
 BaseIndex::BlockUntilSyncedToCurrentChain reliability

---
 src/index/base.cpp | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/index/base.cpp b/src/index/base.cpp
index 376d0314ca..ba403e8002 100644
--- a/src/index/base.cpp
+++ b/src/index/base.cpp
@@ -270,6 +270,10 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
     }
 
     if (WriteBlock(*block, pindex)) {
+        // Setting the best block index is intentionally the last step of this
+        // function, so BlockUntilSyncedToCurrentChain callers waiting for the
+        // best block index to be updated can rely on the block being fully
+        // processed, and the index object being safe to delete.
         SetBestBlockIndex(pindex);
     } else {
         FatalError("%s: Failed to write block %s to index",
@@ -381,10 +385,17 @@ IndexSummary BaseIndex::GetSummary() const
 void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) {
     assert(!fPruneMode || AllowPrune());
 
-    m_best_block_index = block;
     if (AllowPrune() && block) {
         PruneLockInfo prune_lock;
         prune_lock.height_first = block->nHeight;
         WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock));
     }
+
+    // Intentionally set m_best_block_index as the last step in this function,
+    // after updating prune locks above, and after making any other references
+    // to *this, so the BlockUntilSyncedToCurrentChain function (which checks
+    // m_best_block_index as an optimization) can be used to wait for the last
+    // BlockConnected notification and safely assume that prune locks are
+    // updated and that the index object is safe to delete.
+    m_best_block_index = block;
 }

From b04b71a9570912b08ff2e3da9c38e5db737a79e4 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 16:36:33 +0000
Subject: [PATCH 29/69] merge bitcoin#24858: incorrect blk file size
 calculation during reindex results in recoverable blk file corruption

---
 src/Makefile.test.include       |  1 +
 src/node/blockstorage.cpp       | 13 +++++++----
 src/node/blockstorage.h         |  4 ++++
 src/test/blockmanager_tests.cpp | 39 +++++++++++++++++++++++++++++++++
 src/validation.cpp              | 13 ++++++++++-
 5 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 src/test/blockmanager_tests.cpp

diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index 839224e9a1..a9b24c0007 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -89,6 +89,7 @@ BITCOIN_TESTS =\
   test/blockencodings_tests.cpp \
   test/blockfilter_tests.cpp \
   test/blockfilter_index_tests.cpp \
+  test/blockmanager_tests.cpp \
   test/bloom_tests.cpp \
   test/bls_tests.cpp \
   test/bswap_tests.cpp \
diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp
index 960759396f..2c8cacccef 100644
--- a/src/node/blockstorage.cpp
+++ b/src/node/blockstorage.cpp
@@ -790,19 +790,24 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus
     return true;
 }
 
-/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
 FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp)
 {
     unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION);
     FlatFilePos blockPos;
-    if (dbp != nullptr) {
+    const auto position_known {dbp != nullptr};
+    if (position_known) {
         blockPos = *dbp;
+    } else {
+        // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for
+        // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE).
+        // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk.
+        nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
     }
-    if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) {
+    if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) {
         error("%s: FindBlockPos failed", __func__);
         return FlatFilePos();
     }
-    if (dbp == nullptr) {
+    if (!position_known) {
         if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) {
             AbortNode("Failed to write block");
             return FlatFilePos();
diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h
index 7f671fd5f8..1b919df908 100644
--- a/src/node/blockstorage.h
+++ b/src/node/blockstorage.h
@@ -47,6 +47,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
 /** The maximum size of a blk?????.dat file (since 0.8) */
 static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
 
+/** Size of header written by WriteBlockToDisk before a serialized CBlock */
+static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int);
+
 extern std::atomic_bool fImporting;
 extern std::atomic_bool fReindex;
 /** Pruning-related variables and constants */
@@ -185,6 +188,7 @@ class BlockManager
     bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
         EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
 
+    /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */
     FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp);
 
     /** Calculate the amount of disk space the block & undo files currently use */
diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp
new file mode 100644
index 0000000000..a9ce570a3d
--- /dev/null
+++ b/src/test/blockmanager_tests.cpp
@@ -0,0 +1,39 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <chainparams.h>
+#include <node/blockstorage.h>
+#include <node/context.h>
+#include <validation.h>
+
+#include <boost/test/unit_test.hpp>
+#include <test/util/setup_common.h>
+
+// use BasicTestingSetup here for the data directory configuration, setup, and cleanup
+BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
+{
+    const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)};
+    BlockManager blockman {};
+    CChain chain {};
+    // simulate adding a genesis block normally
+    BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
+    // simulate what happens during reindex
+    // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file
+    // the block is found at offset 8 because there is an 8 byte serialization header
+    // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file.
+    FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE};
+    BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
+    // now simulate what happens after reindex for the first new block processed
+    // the actual block contents don't matter, just that it's a block.
+    // verify that the write position is at offset 0x12d.
+    // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur
+    // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293
+    // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301
+    FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)};
+    BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/validation.cpp b/src/validation.cpp
index efe4e2cfe2..844ad7eb6e 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -4705,7 +4705,18 @@ void CChainState::LoadExternalBlockFile(
                     }
                 }
             } catch (const std::exception& e) {
-                LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what());
+                // historical bugs added extra data to the block files that does not deserialize cleanly.
+                // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process.
+                // the code that reads the block files deals with invalid data by simply ignoring it.
+                // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly
+                // and passes all of the other block validation checks dealing with POW and the merkle root, etc...
+                // we merely note with this informational log message when unexpected data is encountered.
+                // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but
+                // less likely scenarios. we don't have enough information to tell a difference here.
+                // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator
+                // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and
+                // perhaps ordered, block files for later reindexing.
+                LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what());
             }
         }
     } catch (const std::runtime_error& e) {

From e531dff5f71aee68a0cbfc029419af463e7fd87d Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Fri, 28 Oct 2022 15:24:22 -0400
Subject: [PATCH 30/69] merge bitcoin#26417: fix intermittent failure in
 feature_index_prune.py

---
 test/functional/feature_index_prune.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py
index 0222de33e3..dcb649362f 100755
--- a/test/functional/feature_index_prune.py
+++ b/test/functional/feature_index_prune.py
@@ -140,6 +140,7 @@ def run_test(self):
             self.connect_nodes(i, 3)
 
         self.sync_blocks(timeout=300)
+        self.sync_index(height=2500)
 
         for node in self.nodes[:2]:
             with node.assert_debug_log(['limited pruning to height 2489']):

From 7d9ff96091312885a5dc5471c55e3b8afdbdb632 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Sun, 13 Oct 2024 16:46:11 +0000
Subject: [PATCH 31/69] merge bitcoin#16981: Improve runtime performance of
 --reindex

---
 src/Makefile.bench.include         |  1 +
 src/bench/load_external.cpp        | 63 ++++++++++++++++++++++++++++
 src/streams.h                      | 48 ++++++++++++++-------
 src/test/streams_tests.cpp         | 67 ++++++++++++++++++++++++++++--
 src/validation.cpp                 | 46 ++++++++++++--------
 test/functional/feature_reindex.py | 50 ++++++++++++++++++++++
 6 files changed, 238 insertions(+), 37 deletions(-)
 create mode 100644 src/bench/load_external.cpp

diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include
index 133bc282ef..6e13fd173b 100644
--- a/src/Makefile.bench.include
+++ b/src/Makefile.bench.include
@@ -35,6 +35,7 @@ bench_bench_dash_SOURCES = \
   bench/ccoins_caching.cpp \
   bench/gcs_filter.cpp \
   bench/hashpadding.cpp \
+  bench/load_external.cpp \
   bench/merkle_root.cpp \
   bench/mempool_eviction.cpp \
   bench/mempool_stress.cpp \
diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp
new file mode 100644
index 0000000000..6bc359a408
--- /dev/null
+++ b/src/bench/load_external.cpp
@@ -0,0 +1,63 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <bench/data.h>
+#include <chainparams.h>
+#include <test/util/setup_common.h>
+#include <validation.h>
+
+/**
+ * The LoadExternalBlockFile() function is used during -reindex and -loadblock.
+ *
+ * Create a test file that's similar to a datadir/blocks/blk?????.dat file,
+ * It contains around 134 copies of the same block (typical size of real block files).
+ * For each block in the file, LoadExternalBlockFile() won't find its parent,
+ * and so will skip the block. (In the real system, it will re-read the block
+ * from disk later when it encounters its parent.)
+ *
+ * This benchmark measures the performance of deserializing the block (or just
+ * its header, beginning with PR 16981).
+ */
+static void LoadExternalBlockFile(benchmark::Bench& bench)
+{
+    const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)};
+
+    // Create a single block as in the blocks files (magic bytes, block size,
+    // block data) as a stream object.
+    const fs::path blkfile{testing_setup.get()->m_path_root / "blk.dat"};
+    CDataStream ss(SER_DISK, 0);
+    auto params{Params()};
+    ss << params.MessageStart();
+    ss << static_cast<uint32_t>(benchmark::data::block813851.size());
+    // We can't use the streaming serialization (ss << benchmark::data::block813851)
+    // because that first writes a compact size.
+    ss.write(MakeByteSpan(benchmark::data::block813851));
+
+    // Create the test file.
+    {
+        // "wb+" is "binary, O_RDWR | O_CREAT | O_TRUNC".
+        FILE* file{fsbridge::fopen(blkfile, "wb+")};
+        // Make the test block file about 128 MB in length.
+        for (size_t i = 0; i < MAX_BLOCKFILE_SIZE / ss.size(); ++i) {
+            if (fwrite(ss.data(), 1, ss.size(), file) != ss.size()) {
+                throw std::runtime_error("write to test file failed\n");
+            }
+        }
+        fclose(file);
+    }
+
+    CChainState& chainstate{testing_setup->m_node.chainman->ActiveChainstate()};
+    std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
+    FlatFilePos pos;
+    bench.run([&] {
+        // "rb" is "binary, O_RDONLY", positioned to the start of the file.
+        // The file will be closed by LoadExternalBlockFile().
+        FILE* file{fsbridge::fopen(blkfile, "rb")};
+        chainstate.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
+    });
+    fs::remove(blkfile);
+}
+
+BENCHMARK(LoadExternalBlockFile);
diff --git a/src/streams.h b/src/streams.h
index ced2b1f76f..76feda569f 100644
--- a/src/streams.h
+++ b/src/streams.h
@@ -641,7 +641,6 @@ class CBufferedFile
     uint64_t nRewind;     //!< how many bytes we guarantee to rewind
     std::vector<std::byte> vchBuf; //!< the buffer
 
-protected:
     //! read data from the source to fill the buffer
     bool Fill() {
         unsigned int pos = nSrcPos % vchBuf.size();
@@ -659,6 +658,28 @@ class CBufferedFile
         return true;
     }
 
+    //! Advance the stream's read pointer (m_read_pos) by up to 'length' bytes,
+    //! filling the buffer from the file so that at least one byte is available.
+    //! Return a pointer to the available buffer data and the number of bytes
+    //! (which may be less than the requested length) that may be accessed
+    //! beginning at that pointer.
+    std::pair<std::byte*, size_t> AdvanceStream(size_t length)
+    {
+        assert(m_read_pos <= nSrcPos);
+        if (m_read_pos + length > nReadLimit) {
+            throw std::ios_base::failure("Attempt to position past buffer limit");
+        }
+        // If there are no bytes available, read from the file.
+        if (m_read_pos == nSrcPos && length > 0) Fill();
+
+        size_t buffer_offset{static_cast<size_t>(m_read_pos % vchBuf.size())};
+        size_t buffer_available{static_cast<size_t>(vchBuf.size() - buffer_offset)};
+        size_t bytes_until_source_pos{static_cast<size_t>(nSrcPos - m_read_pos)};
+        size_t advance{std::min({length, buffer_available, bytes_until_source_pos})};
+        m_read_pos += advance;
+        return std::make_pair(&vchBuf[buffer_offset], advance);
+    }
+
 public:
     CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn)
         : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), m_read_pos(0), nReadLimit(std::numeric_limits<uint64_t>::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0})
@@ -696,24 +717,21 @@ class CBufferedFile
     //! read a number of bytes
     void read(Span<std::byte> dst)
     {
-        if (dst.size() + m_read_pos > nReadLimit) {
-            throw std::ios_base::failure("Read attempted past buffer limit");
-        }
         while (dst.size() > 0) {
-            if (m_read_pos == nSrcPos)
-                Fill();
-            unsigned int pos = m_read_pos % vchBuf.size();
-            size_t nNow = dst.size();
-            if (nNow + pos > vchBuf.size())
-                nNow = vchBuf.size() - pos;
-            if (nNow + m_read_pos > nSrcPos)
-                nNow = nSrcPos - m_read_pos;
-            memcpy(dst.data(), &vchBuf[pos], nNow);
-            m_read_pos += nNow;
-            dst = dst.subspan(nNow);
+            auto [buffer_pointer, length]{AdvanceStream(dst.size())};
+            memcpy(dst.data(), buffer_pointer, length);
+            dst = dst.subspan(length);
         }
     }
 
+    //! Move the read position ahead in the stream to the given position.
+    //! Use SetPos() to back up in the stream, not SkipTo().
+    void SkipTo(const uint64_t file_pos)
+    {
+        assert(file_pos >= m_read_pos);
+        while (m_read_pos < file_pos) AdvanceStream(file_pos - m_read_pos);
+    }
+
     //! return the current reading position
     uint64_t GetPos() const {
         return m_read_pos;
diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp
index 2eae80bf8b..2f8b76d592 100644
--- a/src/test/streams_tests.cpp
+++ b/src/test/streams_tests.cpp
@@ -251,7 +251,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
         BOOST_CHECK(false);
     } catch (const std::exception& e) {
         BOOST_CHECK(strstr(e.what(),
-                        "Read attempted past buffer limit") != nullptr);
+                           "Attempt to position past buffer limit") != nullptr);
     }
     // The default argument removes the limit completely.
     BOOST_CHECK(bf.SetLimit());
@@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
     BOOST_CHECK(!bf.SetPos(0));
     // But we should now be positioned at least as far back as allowed
     // by the rewind window (relative to our farthest read position, 40).
-    BOOST_CHECK(bf.GetPos() <= 30);
+    BOOST_CHECK(bf.GetPos() <= 30U);
 
     // We can explicitly close the file, or the destructor will do it.
     bf.fclose();
@@ -328,6 +328,55 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
     fs::remove("streams_test_tmp");
 }
 
+BOOST_AUTO_TEST_CASE(streams_buffered_file_skip)
+{
+    fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
+    FILE* file = fsbridge::fopen(streams_test_filename, "w+b");
+    // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01).
+    for (uint8_t j = 0; j < 40; ++j) {
+        fwrite(&j, 1, 1, file);
+    }
+    rewind(file);
+
+    // The buffer is 25 bytes, allow rewinding 10 bytes.
+    CBufferedFile bf(file, 25, 10, 222, 333);
+
+    uint8_t i;
+    // This is like bf >> (7-byte-variable), in that it will cause data
+    // to be read from the file into memory, but it's not copied to us.
+    bf.SkipTo(7);
+    BOOST_CHECK_EQUAL(bf.GetPos(), 7U);
+    bf >> i;
+    BOOST_CHECK_EQUAL(i, 7);
+
+    // The bytes in the buffer up to offset 7 are valid and can be read.
+    BOOST_CHECK(bf.SetPos(0));
+    bf >> i;
+    BOOST_CHECK_EQUAL(i, 0);
+    bf >> i;
+    BOOST_CHECK_EQUAL(i, 1);
+
+    bf.SkipTo(11);
+    bf >> i;
+    BOOST_CHECK_EQUAL(i, 11);
+
+    // SkipTo() honors the transfer limit; we can't position beyond the limit.
+    bf.SetLimit(13);
+    try {
+        bf.SkipTo(14);
+        BOOST_CHECK(false);
+    } catch (const std::exception& e) {
+        BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr);
+    }
+
+    // We can position exactly to the transfer limit.
+    bf.SkipTo(13);
+    BOOST_CHECK_EQUAL(bf.GetPos(), 13U);
+
+    bf.fclose();
+    fs::remove(streams_test_filename);
+}
+
 BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
 {
     // Make this test deterministic.
@@ -358,7 +407,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
             // sizes; the boundaries of the objects can interact arbitrarily
             // with the CBufferFile's internal buffer. These first three
             // cases simulate objects of various sizes (1, 2, 5 bytes).
-            switch (InsecureRandRange(5)) {
+            switch (InsecureRandRange(6)) {
             case 0: {
                 uint8_t a[1];
                 if (currentPos + 1 > fileSize)
@@ -396,6 +445,16 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
                 break;
             }
             case 3: {
+                // SkipTo is similar to the "read" cases above, except
+                // we don't receive the data.
+                size_t skip_length{static_cast<size_t>(InsecureRandRange(5))};
+                if (currentPos + skip_length > fileSize) continue;
+                bf.SetLimit(currentPos + skip_length);
+                bf.SkipTo(currentPos + skip_length);
+                currentPos += skip_length;
+                break;
+            }
+            case 4: {
                 // Find a byte value (that is at or ahead of the current position).
                 size_t find = currentPos + InsecureRandRange(8);
                 if (find >= fileSize)
@@ -412,7 +471,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
                 currentPos++;
                 break;
             }
-            case 4: {
+            case 5: {
                 size_t requestPos = InsecureRandRange(maxPos + 4);
                 bool okay = bf.SetPos(requestPos);
                 // The new position may differ from the requested position
diff --git a/src/validation.cpp b/src/validation.cpp
index 844ad7eb6e..0e9ca1b609 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -4603,6 +4603,8 @@ void CChainState::LoadExternalBlockFile(
         unsigned int nMaxBlockSize = MaxBlockSize();
         // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor
         CBufferedFile blkdat(fileIn, 2*nMaxBlockSize, nMaxBlockSize+8, SER_DISK, CLIENT_VERSION);
+        // nRewind indicates where to resume scanning in case something goes wrong,
+        // such as a block fails to deserialize.
         uint64_t nRewind = blkdat.GetPos();
         while (!blkdat.eof()) {
             if (ShutdownRequested()) return;
@@ -4626,28 +4628,30 @@ void CChainState::LoadExternalBlockFile(
                     continue;
             } catch (const std::exception&) {
                 // no valid block header found; don't complain
+                // (this happens at the end of every blk.dat file)
                 break;
             }
             try {
-                // read block
-                uint64_t nBlockPos = blkdat.GetPos();
+                // read block header
+                const uint64_t nBlockPos{blkdat.GetPos()};
                 if (dbp)
                     dbp->nPos = nBlockPos;
                 blkdat.SetLimit(nBlockPos + nSize);
-                std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>();
-                CBlock& block = *pblock;
-                blkdat >> block;
-                nRewind = blkdat.GetPos();
-
-                uint256 hash = block.GetHash();
+                CBlockHeader header;
+                blkdat >> header;
+                const uint256 hash{header.GetHash()};
+                // Skip the rest of this block (this may read from disk into memory); position to the marker before the
+                // next block, but it's still possible to rewind to the start of the current block (without a disk read).
+                nRewind = nBlockPos + nSize;
+                blkdat.SkipTo(nRewind);
                 {
                     LOCK(cs_main);
                     // detect out of order blocks, and store them for later
-                    if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) {
+                    if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(header.hashPrevBlock)) {
                         LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(),
-                                block.hashPrevBlock.ToString());
+                                 header.hashPrevBlock.ToString());
                         if (dbp && blocks_with_unknown_parent) {
-                            blocks_with_unknown_parent->emplace(block.hashPrevBlock, *dbp);
+                            blocks_with_unknown_parent->emplace(header.hashPrevBlock, *dbp);
                         }
                         continue;
                     }
@@ -4655,13 +4659,19 @@ void CChainState::LoadExternalBlockFile(
                     // process in case the block isn't known yet
                     const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash);
                     if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) {
-                      BlockValidationState state;
-                      if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) {
-                          nLoaded++;
-                      }
-                      if (state.IsError()) {
-                          break;
-                      }
+                        // This block can be processed immediately; rewind to its start, read and deserialize it.
+                        blkdat.SetPos(nBlockPos);
+                        std::shared_ptr<CBlock> pblock{std::make_shared<CBlock>()};
+                        blkdat >> *pblock;
+                        nRewind = blkdat.GetPos();
+
+                        BlockValidationState state;
+                        if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) {
+                            nLoaded++;
+                        }
+                        if (state.IsError()) {
+                            break;
+                        }
                     } else if (hash != m_params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) {
                         LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight);
                     }
diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py
index 8c15a74e2f..588d8766ad 100755
--- a/test/functional/feature_reindex.py
+++ b/test/functional/feature_reindex.py
@@ -7,9 +7,12 @@
 - Start a single node and generate 3 blocks.
 - Stop the node and restart it with -reindex. Verify that the node has reindexed up to block 3.
 - Stop the node and restart it with -reindex-chainstate. Verify that the node has reindexed up to block 3.
+- Verify that out-of-order blocks are correctly processed, see LoadExternalBlockFile()
 """
 
+import os
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.p2p import MAGIC_BYTES
 from test_framework.util import assert_equal
 
 
@@ -27,6 +30,50 @@ def reindex(self, justchainstate=False, txindex=0):
         assert_equal(self.nodes[0].getblockcount(), blockcount)  # start_node is blocking on reindex
         self.log.info("Success")
 
+    # Check that blocks can be processed out of order
+    def out_of_order(self):
+        # The previous test created 24 blocks
+        assert_equal(self.nodes[0].getblockcount(), 24)
+        self.stop_nodes()
+
+        # In this test environment, blocks will always be in order (since
+        # we're generating them rather than getting them from peers), so to
+        # test out-of-order handling, swap blocks 1 and 2 on disk.
+        blk0 = os.path.join(self.nodes[0].datadir, self.nodes[0].chain, 'blocks', 'blk00000.dat')
+        with open(blk0, 'r+b') as bf:
+            # Read at least the first few blocks (including genesis)
+            b = bf.read(2000)
+
+            # Find the offsets of blocks 2, 3, and 4 (the first 3 blocks beyond genesis)
+            # by searching for the regtest marker bytes (see pchMessageStart).
+            def find_block(b, start):
+                return b.find(MAGIC_BYTES["regtest"], start)+4
+
+            genesis_start = find_block(b, 0)
+            assert_equal(genesis_start, 4)
+            b2_start = find_block(b, genesis_start)
+            b3_start = find_block(b, b2_start)
+            b4_start = find_block(b, b3_start)
+
+            # Blocks 2 and 3 should be the same size.
+            assert_equal(b3_start-b2_start, b4_start-b3_start)
+
+            # Swap the second and third blocks (don't disturb the genesis block).
+            bf.seek(b2_start)
+            bf.write(b[b3_start:b4_start])
+            bf.write(b[b2_start:b3_start])
+
+        # The reindexing code should detect and accommodate out of order blocks.
+        with self.nodes[0].assert_debug_log([
+            'LoadExternalBlockFile: Out of order block',
+            'LoadExternalBlockFile: Processing out of order child',
+        ]):
+            extra_args = [["-reindex"]]
+            self.start_nodes(extra_args)
+
+        # All blocks should be accepted and processed.
+        assert_equal(self.nodes[0].getblockcount(), 24)
+
     def run_test(self):
         for txindex in [0, 1]:
             self.reindex(False, txindex)
@@ -34,5 +81,8 @@ def run_test(self):
             self.reindex(False, txindex)
             self.reindex(True, txindex)
 
+        self.out_of_order()
+
+
 if __name__ == '__main__':
     ReindexTest().main()

From 6fbd128947e012733c5a02bcda46a93bd20b1aee Mon Sep 17 00:00:00 2001
From: UdjinM6 <UdjinM6@users.noreply.github.com>
Date: Fri, 11 Oct 2024 15:33:52 +0300
Subject: [PATCH 32/69] chore: clang format/reword/typos

---
 src/evo/assetlocktx.cpp           | 7 +++++--
 test/functional/rpc_blockchain.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/evo/assetlocktx.cpp b/src/evo/assetlocktx.cpp
index 64ee2b32c4..27e53e0b24 100644
--- a/src/evo/assetlocktx.cpp
+++ b/src/evo/assetlocktx.cpp
@@ -120,8 +120,11 @@ bool CAssetUnlockPayload::VerifySig(const llmq::CQuorumManager& qman, const uint
     const auto& llmq_params_opt = Params().GetLLMQ(llmqType);
     assert(llmq_params_opt.has_value());
 
-    // after deployment WITHDRAWALS activated we check not to quorum, but all active quorums + 1 the latest inactive
-    const int quorums_to_scan = DeploymentActiveAt(*pindexTip, Params().GetConsensus(), Consensus::DEPLOYMENT_WITHDRAWALS) ? (llmq_params_opt->signingActiveQuorumCount + 1) : 2;
+    // We check two quorums before DEPLOYMENT_WITHDRAWALS activation
+    // and "all active quorums + 1 the latest inactive" after activation.
+    const int quorums_to_scan = DeploymentActiveAt(*pindexTip, Params().GetConsensus(), Consensus::DEPLOYMENT_WITHDRAWALS)
+                                    ? (llmq_params_opt->signingActiveQuorumCount + 1)
+                                    : 2;
     const auto quorums = qman.ScanQuorums(llmqType, pindexTip, quorums_to_scan);
 
     if (bool isActive = std::any_of(quorums.begin(), quorums.end(), [&](const auto &q) { return q->qc->quorumHash == quorumHash; }); !isActive) {
diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py
index a327a630aa..14c4f55f4a 100755
--- a/test/functional/rpc_blockchain.py
+++ b/test/functional/rpc_blockchain.py
@@ -162,7 +162,7 @@ def _test_getblockchaininfo(self):
                 'bip9': {
                     'status': 'defined',
                     'start_time': 0,
-                    'timeout': 9223372036854775807,  # testdummy does not have a timeout so is set to the max int64 value
+                    'timeout': 9223372036854775807,  # "withdrawals" does not have a timeout so is set to the max int64 value
                     'since': 0,
                     'min_activation_height': 0,
                     'ehf': True

From 7455b5557acea71ea9cd20efa2088b59f6186b43 Mon Sep 17 00:00:00 2001
From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com>
Date: Fri, 22 Oct 2021 23:43:28 +0300
Subject: [PATCH 33/69] Merge bitcoin-core/gui#454: Use only Qt translation
 primitives in GUI code

58765a450c40152db8160bca8a6b0f5b754c5858 qt: Use only Qt translation primitives in GUI code (W. J. van der Laan)

Pull request description:

  Use `Object::tr`, `QT_TRANSLATE_NOOP`, and `QCoreApplication::translate` as appropriate instead of using `_()` which doesn't get picked up.

  Replaces bitcoin/bitcoin#22764

  Edit: I checked that the strings end up in the appropriate context in `bitcoin_en.ts` after `make translate`:
  - "Settings file could not be read" "Settings file could not be written" end up in `bitcoin-core`
  - "(press q to shutdown and continue later)" and "press q to shutdown" end up in `SplashScreen`

ACKs for top commit:
  hebasto:
    ACK 58765a450c40152db8160bca8a6b0f5b754c5858

Tree-SHA512: d0cc5901426c47d411d0fe75aac195b9684b51385f74c161da772dbf9f0977f7ad7a0976e17366f49b40718e9b6eb87c3e93306dc845f97adddbc47d00731742
---
 src/qt/bitcoin.cpp      | 14 ++++++++------
 src/qt/splashscreen.cpp |  4 ++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp
index af50ae0129..47b5801c51 100644
--- a/src/qt/bitcoin.cpp
+++ b/src/qt/bitcoin.cpp
@@ -156,10 +156,11 @@ static bool InitSettings()
 
     std::vector<std::string> errors;
     if (!gArgs.ReadSettingsFile(&errors)) {
-        bilingual_str error = _("Settings file could not be read");
-        InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors))));
+        std::string error = QT_TRANSLATE_NOOP("dash-core", "Settings file could not be read");
+        std::string error_translated = QCoreApplication::translate("dash-core", error.c_str()).toStdString();
+        InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors))));
 
-        QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort);
+        QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Reset | QMessageBox::Abort);
         /*: Explanatory text shown on startup when the settings file cannot be read.
             Prompts user to make a choice between resetting or aborting. */
         messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?"));
@@ -178,10 +179,11 @@ static bool InitSettings()
 
     errors.clear();
     if (!gArgs.WriteSettingsFile(&errors)) {
-        bilingual_str error = _("Settings file could not be written");
-        InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors))));
+        std::string error = QT_TRANSLATE_NOOP("dash-core", "Settings file could not be written");
+        std::string error_translated = QCoreApplication::translate("dash-core", error.c_str()).toStdString();
+        InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors))));
 
-        QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok);
+        QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Ok);
         /*: Explanatory text shown on startup when the settings file could not be written.
             Prompts user to check that we have the ability to write to the file.
             Explains that the user has the option of running without a settings file.*/
diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp
index dbe7aa7e65..005c3bcdb4 100644
--- a/src/qt/splashscreen.cpp
+++ b/src/qt/splashscreen.cpp
@@ -189,8 +189,8 @@ static void InitMessage(SplashScreen *splash, const std::string &message)
 static void ShowProgress(SplashScreen *splash, const std::string &title, int nProgress, bool resume_possible)
 {
     InitMessage(splash, title + std::string("\n") +
-            (resume_possible ? _("(press q to shutdown and continue later)").translated
-                                : _("press q to shutdown").translated) +
+            (resume_possible ? SplashScreen::tr("(press q to shutdown and continue later)").toStdString()
+                                : SplashScreen::tr("press q to shutdown").toStdString()) +
             strprintf("\n%d", nProgress) + "%");
 }
 

From c2fd4fe379f6bb123b5127d466e3ee00daf69715 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Mon, 15 Nov 2021 17:19:52 +0100
Subject: [PATCH 34/69] Merge bitcoin/bitcoin#23515: test: Return the largest
 utxo in MiniWallet.get_utxo

fa62207737657e76ba45d5bf826fc0ccac658df6 test: Return the largest utxo in MiniWallet.get_utxo (MarcoFalke)

Pull request description:

  This is for consistency with the `send_self_transfer` method.

  Also, remove the feature that the change of the last transfer can be retrieved via `get_utxo`. This can trivially and clearer be achieved by simply passing the txid of the transfer.

  Also, this fixes the bug in `feature_txindex_compatibility` in current master after a silent merge conflict.

  Fixes #23514

Top commit has no ACKs.

Tree-SHA512: edd066d372aaa72b4e0fc7526f84931c8d1f6d14f53678cb7832bc8e3d211f44b90ec9c59b7d915ef24acc63a36e7d66c8d3b7598355bd490ac637ed3bcc3dff
---
 test/functional/rpc_txoutproof.py        | 2 +-
 test/functional/test_framework/wallet.py | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py
index 34eb48a89e..9fed9cbe15 100755
--- a/test/functional/rpc_txoutproof.py
+++ b/test/functional/rpc_txoutproof.py
@@ -54,7 +54,7 @@ def run_test(self):
         assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2])), txlist)
         assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2], blockhash)), txlist)
 
-        txin_spent = miniwallet.get_utxo()  # Get the change from txid2
+        txin_spent = miniwallet.get_utxo(txid=txid2)  # Get the change from txid2
         tx3 = miniwallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=txin_spent)
         txid3 = tx3['txid']
         self.generate(self.nodes[0], 1)
diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py
index 6d494ee2c6..d0285f3ad9 100644
--- a/test/functional/test_framework/wallet.py
+++ b/test/functional/test_framework/wallet.py
@@ -135,10 +135,9 @@ def get_utxo(self, *, txid: Optional[str]='', mark_as_spent=True):
 
         Args:
         txid: get the first utxo we find from a specific transaction
-
-        Note: Can be used to get the change output immediately after a send_self_transfer
         """
         index = -1  # by default the last utxo
+        self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height']))  # Put the largest utxo last
         if txid:
             utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos))
             index = self._utxos.index(utxo)
@@ -155,8 +154,7 @@ def send_self_transfer(self, **kwargs):
 
     def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0):
         """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed."""
-        self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height']))
-        utxo_to_spend = utxo_to_spend or self._utxos.pop()  # Pick the largest utxo (if none provided) and hope it covers the fee
+        utxo_to_spend = utxo_to_spend or self.get_utxo()
         if self._priv_key is None:
             vsize = Decimal(85)  # anyone-can-spend
         else:

From 73e1861576792f043ebf7c919b49f4f331ded5fb Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Thu, 16 Dec 2021 08:46:06 +0100
Subject: [PATCH 35/69] Merge bitcoin/bitcoin#23750: rpcwallet: mention labels
 are disabled for ranged descriptors

65efbba45d817261f590d043c69a9981e6b637bd rpcwallet: mention labels are deactivated for ranged descriptors (Antoine Poinsot)

Pull request description:

  It was confusing when trying to use it as a blackbox. So mention it so next ones don't have to open the said box :)

  See #23749 for context

ACKs for top commit:
  Sjors:
    utACK 65efbba45d817261f590d043c69a9981e6b637bd
  achow101:
    ACK 65efbba45d817261f590d043c69a9981e6b637bd

Tree-SHA512: d8a3d1f81c16d95855ac2b01e8fd20e83d6dac1721b3da464a9a890e46102992a6882918be87b2a28b929349ee7f1beb1af6c88b22f065fbbb6948275a6d2b8f
---
 src/wallet/rpcdump.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp
index 4d75d2ae1c..801605dd27 100644
--- a/src/wallet/rpcdump.cpp
+++ b/src/wallet/rpcdump.cpp
@@ -1802,7 +1802,7 @@ RPCHelpMan importdescriptors() {
                                         /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"}
                                     },
                                     {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"},
-                                    {"label", RPCArg::Type::STR, RPCArg::Default{""}, "Label to assign to the address, only allowed with internal=false"},
+                                    {"label", RPCArg::Type::STR, RPCArg::Default{""}, "Label to assign to the address, only allowed with internal=false. Disabled for ranged descriptors"},
                                 },
                             },
                         },

From acbf718b57d4365a7dc3b6131b06bb04bfb8b5d7 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Wed, 19 Jan 2022 15:31:52 +0100
Subject: [PATCH 36/69] Merge bitcoin/bitcoin#23976: document and clean up
 MaybeUpdateMempoolForReorg
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

e177fcab3831b6d259da5164cabedcc9e78f6957 Replace `struct update_lock_points` with lambda (glozow)
c7cd98c7176800a51e6a6b3634a26b508aa33ff2 document and clean up MaybeUpdateMempoolForReorg (glozow)

Pull request description:

  followup to #23683, addressing https://github.com/bitcoin/bitcoin/pull/23683#issuecomment-989741186

ACKs for top commit:
  hebasto:
    ACK e177fcab3831b6d259da5164cabedcc9e78f6957, I have reviewed the code and it looks OK, I agree it can be merged.
  instagibbs:
    ACK e177fcab3831b6d259da5164cabedcc9e78f6957
  MarcoFalke:
    Approach ACK e177fcab3831b6d259da5164cabedcc9e78f6957 😶

Tree-SHA512: 8c2709dd5cab73cde41f3e5655d5f237bacfb341f78eac026169be579528695ca628c8777b7d89760d8677a4e6786913293681cfe16ab702b30c909703e1824c
---
 src/txmempool.h    | 22 ++++++++--------------
 src/validation.cpp | 42 ++++++++++++++++++++++++++++--------------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/src/txmempool.h b/src/txmempool.h
index 13f2b964d2..3e06fff3a4 100644
--- a/src/txmempool.h
+++ b/src/txmempool.h
@@ -311,16 +311,6 @@ class CompareTxMemPoolEntryByAncestorFee
     }
 };
 
-struct update_lock_points
-{
-    explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { }
-
-    void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); }
-
-private:
-    const LockPoints& lp;
-};
-
 // Multi_index tag names
 struct descendant_score {};
 struct entry_time {};
@@ -634,10 +624,14 @@ class CTxMemPool
     bool removeSpentIndex(const uint256 txhash);
 
     void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs);
-    /** After reorg, check if mempool entries are now non-final, premature coinbase spends, or have
-     * invalid lockpoints. Update lockpoints and remove entries (and descendants of entries) that
-     * are no longer valid. */
-    void removeForReorg(CChain& chain, std::function<bool(txiter)> check_final_and_mature) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main);
+    /** After reorg, filter the entries that would no longer be valid in the next block, and update
+     * the entries' cached LockPoints if needed.  The mempool does not have any knowledge of
+     * consensus rules. It just appplies the callable function and removes the ones for which it
+     * returns true.
+     * @param[in]   filter_final_and_mature   Predicate that checks the relevant validation rules
+     *                                        and updates an entry's LockPoints.
+     * */
+    void removeForReorg(CChain& chain, std::function<bool(txiter)> filter_final_and_mature) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main);
     void removeConflicts(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(cs);
     void removeProTxPubKeyConflicts(const CTransaction &tx, const CKeyID &keyId) EXCLUSIVE_LOCKS_REQUIRED(cs);
     void removeProTxPubKeyConflicts(const CTransaction &tx, const CBLSLazyPublicKey &pubKey) EXCLUSIVE_LOCKS_REQUIRED(cs);
diff --git a/src/validation.cpp b/src/validation.cpp
index 38a030ace1..a02c1e4b54 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -373,21 +373,37 @@ void CChainState::MaybeUpdateMempoolForReorg(
     // the disconnectpool that were added back and cleans up the mempool state.
     m_mempool->UpdateTransactionsFromBlock(vHashUpdate);
 
-    const auto check_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it)
+    // Predicate to use for filtering transactions in removeForReorg.
+    // Checks whether the transaction is still final and, if it spends a coinbase output, mature.
+    // Also updates valid entries' cached LockPoints if needed.
+    // If false, the tx is still valid and its lockpoints are updated.
+    // If true, the tx would be invalid in the next block; remove this entry and all of its descendants.
+    const auto filter_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it)
         EXCLUSIVE_LOCKS_REQUIRED(m_mempool->cs, ::cs_main) {
-        bool should_remove = false;
         AssertLockHeld(m_mempool->cs);
         AssertLockHeld(::cs_main);
         const CTransaction& tx = it->GetTx();
+
+        // The transaction must be final.
+        if (!CheckFinalTx(m_chain.Tip(), tx, flags)) return true;
         LockPoints lp = it->GetLockPoints();
         const bool validLP{TestLockPointValidity(m_chain, lp)};
         CCoinsViewMemPool view_mempool(&CoinsTip(), *m_mempool);
-        if (!CheckFinalTx(m_chain.Tip(), tx, flags)
-            || !CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) {
-            // Note if CheckSequenceLocks fails the LockPoints may still be invalid
-            // So it's critical that we remove the tx and not depend on the LockPoints.
-            should_remove = true;
-        } else if (it->GetSpendsCoinbase()) {
+        // CheckSequenceLocks checks if the transaction will be final in the next block to be
+        // created on top of the new chain. We use useExistingLockPoints=false so that, instead of
+        // using the information in lp (which might now refer to a block that no longer exists in
+        // the chain), it will update lp to contain LockPoints relevant to the new chain.
+        if (!CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) {
+            // If CheckSequenceLocks fails, remove the tx and don't depend on the LockPoints.
+            return true;
+        } else if (!validLP) {
+            // If CheckSequenceLocks succeeded, it also updated the LockPoints.
+            // Now update the mempool entry lockpoints as well.
+            m_mempool->mapTx.modify(it, [&lp](CTxMemPoolEntry& e) { e.UpdateLockPoints(lp); });
+        }
+
+        // If the transaction spends any coinbase outputs, it must be mature.
+        if (it->GetSpendsCoinbase()) {
             for (const CTxIn& txin : tx.vin) {
                 auto it2 = m_mempool->mapTx.find(txin.prevout.hash);
                 if (it2 != m_mempool->mapTx.end())
@@ -396,18 +412,16 @@ void CChainState::MaybeUpdateMempoolForReorg(
                 assert(!coin.IsSpent());
                 const auto mempool_spend_height{m_chain.Tip()->nHeight + 1};
                 if (coin.IsSpent() || (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY)) {
-                    should_remove = true;
-                    break;
+                    return true;
                 }
             }
         }
-        // CheckSequenceLocks updates lp. Update the mempool entry LockPoints.
-        if (!validLP) m_mempool->mapTx.modify(it, update_lock_points(lp));
-        return should_remove;
+        // Transaction is still valid and cached LockPoints are updated.
+        return false;
     };
 
     // We also need to remove any now-immature transactions
-    m_mempool->removeForReorg(m_chain, check_final_and_mature);
+    m_mempool->removeForReorg(m_chain, filter_final_and_mature);
     // Re-limit mempool size, in case we added any transactions
     LimitMempoolSize(
         *m_mempool,

From a1691c7c2ae8efaa9b1fb223df28fbb5dc08dbdd Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Mon, 24 Jan 2022 12:43:19 +0100
Subject: [PATCH 37/69] Merge bitcoin/bitcoin#24102: mempool: Run coin.IsSpent
 only once in a row

fa2bcc4e42e7fed61727b3de4019e9702d4090ce Run coin.IsSpent only once in a row (MarcoFalke)

Pull request description:

  Follow-up to commit 64e4963c635ec3a73a5fa3f32f6ec08e70609f60 and https://github.com/bitcoin/bitcoin/pull/23976#discussion_r787758193

ACKs for top commit:
  glozow:
    utACK fa2bcc4e42e7fed61727b3de4019e9702d4090ce, agree the assertion is sufficient
  theStack:
    Code-review ACK fa2bcc4e42e7fed61727b3de4019e9702d4090ce
  w0xlt:
    crACK https://github.com/bitcoin/bitcoin/pull/24102/commits/fa2bcc4e42e7fed61727b3de4019e9702d4090ce
  shaavan:
    Code Review ACK fa2bcc4e42e7fed61727b3de4019e9702d4090ce
  brunoerg:
    crACK fa2bcc4e42e7fed61727b3de4019e9702d4090ce

Tree-SHA512: 3be9d6b313bf6bb835f031826c81777b4659118d839001d084e72462391cb64ba81d06a5e07fd21fcfb709a71b08892b23212a98604ce8481da489476b72f072
---
 src/validation.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/validation.cpp b/src/validation.cpp
index a02c1e4b54..ed480eaad3 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -408,10 +408,10 @@ void CChainState::MaybeUpdateMempoolForReorg(
                 auto it2 = m_mempool->mapTx.find(txin.prevout.hash);
                 if (it2 != m_mempool->mapTx.end())
                     continue;
-                const Coin &coin = CoinsTip().AccessCoin(txin.prevout);
+                const Coin& coin{CoinsTip().AccessCoin(txin.prevout)};
                 assert(!coin.IsSpent());
                 const auto mempool_spend_height{m_chain.Tip()->nHeight + 1};
-                if (coin.IsSpent() || (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY)) {
+                if (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY) {
                     return true;
                 }
             }

From 36e9b5fead3e222df445103be6910423f4c6ee60 Mon Sep 17 00:00:00 2001
From: laanwj <126646+laanwj@users.noreply.github.com>
Date: Wed, 23 Feb 2022 15:48:48 +0100
Subject: [PATCH 38/69] Merge bitcoin/bitcoin#24381: test: Run symlink
 regression tests on Windows

fad7ddf9e3710405d727f61d8200d5efed1e705b test: Run symlink regression tests on Windows (MarcoFalke)

Pull request description:

  Seems odd to add tests, but not run them on the platform that needs them most.

ACKs for top commit:
  laanwj:
    Code review ACK fad7ddf9e3710405d727f61d8200d5efed1e705b
  ryanofsky:
    Code review ACK fad7ddf9e3710405d727f61d8200d5efed1e705b, just removing new test. Would be nice if the test could be added later, of course.

Tree-SHA512: 64b235967a38c2eb90657e8d7a0447bcc8ce81d1b75a275b6c48bd42efd9ea7e7939257e484f297ee84598def3738eaeb289561aeba1dd6a99b258d389995139
---
 src/test/fs_tests.cpp                  |  4 ++--
 test/functional/feature_dirsymlinks.py | 24 +++++++++++++-----------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp
index 4a8803a205..726d19a775 100644
--- a/src/test/fs_tests.cpp
+++ b/src/test/fs_tests.cpp
@@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(rename)
     fs::remove(path2);
 }
 
-#ifndef WIN32
+#ifndef __MINGW64__ // no symlinks on mingw
 BOOST_AUTO_TEST_CASE(create_directories)
 {
     // Test fs::create_directories workaround.
@@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE(create_directories)
     fs::remove(symlink);
     fs::remove(dir);
 }
-#endif // WIN32
+#endif // __MINGW64__
 
 BOOST_AUTO_TEST_SUITE_END()
 
diff --git a/test/functional/feature_dirsymlinks.py b/test/functional/feature_dirsymlinks.py
index 85c8e27600..288754c04c 100755
--- a/test/functional/feature_dirsymlinks.py
+++ b/test/functional/feature_dirsymlinks.py
@@ -6,9 +6,8 @@
 """
 
 import os
-import sys
 
-from test_framework.test_framework import BitcoinTestFramework, SkipTest
+from test_framework.test_framework import BitcoinTestFramework
 
 
 def rename_and_link(*, from_name, to_name):
@@ -16,24 +15,27 @@ def rename_and_link(*, from_name, to_name):
     os.symlink(to_name, from_name)
     assert os.path.islink(from_name) and os.path.isdir(from_name)
 
-class SymlinkTest(BitcoinTestFramework):
-    def skip_test_if_missing_module(self):
-        if sys.platform == 'win32':
-            raise SkipTest("Symlinks test skipped on Windows")
 
+class SymlinkTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
 
     def run_test(self):
+        dir_new_blocks = self.nodes[0].chain_path / "new_blocks"
+        dir_new_chainstate = self.nodes[0].chain_path / "new_chainstate"
         self.stop_node(0)
 
-        rename_and_link(from_name=os.path.join(self.nodes[0].datadir, self.chain, "blocks"),
-                        to_name=os.path.join(self.nodes[0].datadir, self.chain, "newblocks"))
-        rename_and_link(from_name=os.path.join(self.nodes[0].datadir, self.chain, "chainstate"),
-                        to_name=os.path.join(self.nodes[0].datadir, self.chain, "newchainstate"))
+        rename_and_link(
+            from_name=self.nodes[0].chain_path / "blocks",
+            to_name=dir_new_blocks,
+        )
+        rename_and_link(
+            from_name=self.nodes[0].chain_path / "chainstate",
+            to_name=dir_new_chainstate,
+        )
 
         self.start_node(0)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     SymlinkTest().main()

From 33b9771ebc45508bf4e112dc6b1571909d5d4636 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Tue, 5 Apr 2022 14:03:41 +0200
Subject: [PATCH 39/69] Merge bitcoin/bitcoin#24749: test: use MiniWallet for
 mempool_unbroadcast.py

d2ba43fec82af84521f1dbe4475d01888ccf4d0d test: use MiniWallet for mempool_unbroadcast.py (Ayush Sharma)

Pull request description:

  This PR enables one of the non-wallet functional tests (mempool_unbroadcast.py) to be run even with the Bitcoin Core wallet disabled by using the MiniWallet instead, as proposed in #20078  .

Top commit has no ACKs.

Tree-SHA512: e4c577899b66855dafca9dab875fa9b9c68b762a8cdb14f3a7547841c4f001e79d62641e6ae202fb56a3f28aeea1779143164c872507ff8da0bd9930a8ed182e
---
 test/functional/mempool_unbroadcast.py | 45 ++++++++++++--------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py
index d200689352..45abcc5a64 100755
--- a/test/functional/mempool_unbroadcast.py
+++ b/test/functional/mempool_unbroadcast.py
@@ -9,21 +9,20 @@
 
 from test_framework.p2p import P2PTxInvStore
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import (
-    assert_equal,
-    create_confirmed_utxos,
-)
+from test_framework.util import assert_equal
+from test_framework.wallet import MiniWallet
 
 MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds
 
 class MempoolUnbroadcastTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
-
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
+        if self.is_wallet_compiled():
+            self.requires_wallet = True
 
     def run_test(self):
+        self.wallet = MiniWallet(self.nodes[0])
+        self.wallet.rescan_utxos()
         self.test_broadcast()
         self.test_txn_removal()
 
@@ -31,30 +30,25 @@ def test_broadcast(self):
         self.log.info("Test that mempool reattempts delivery of locally submitted transaction")
         node = self.nodes[0]
 
-        min_relay_fee = node.getnetworkinfo()["relayfee"]
-        utxos = create_confirmed_utxos(self, min_relay_fee, node, 10)
-
         self.disconnect_nodes(0, 1)
 
         self.log.info("Generate transactions that only node 0 knows about")
 
-        # generate a wallet txn
-        addr = node.getnewaddress()
-        wallet_tx_hsh = node.sendtoaddress(addr, 0.0001)
+        if self.is_wallet_compiled():
+            # generate a wallet txn
+            addr = node.getnewaddress()
+            wallet_tx_hsh = node.sendtoaddress(addr, 0.0001)
 
         # generate a txn using sendrawtransaction
-        us0 = utxos.pop()
-        inputs = [{"txid": us0["txid"], "vout": us0["vout"]}]
-        outputs = {addr: 0.0001}
-        tx = node.createrawtransaction(inputs, outputs)
-        node.settxfee(min_relay_fee)
-        txF = node.fundrawtransaction(tx)
-        txFS = node.signrawtransactionwithwallet(txF["hex"])
+        txFS = self.wallet.create_self_transfer(from_node=node)
         rpc_tx_hsh = node.sendrawtransaction(txFS["hex"])
 
         # check transactions are in unbroadcast using rpc
         mempoolinfo = self.nodes[0].getmempoolinfo()
-        assert_equal(mempoolinfo['unbroadcastcount'], 2)
+        unbroadcast_count = 1
+        if self.is_wallet_compiled():
+            unbroadcast_count += 1
+        assert_equal(mempoolinfo['unbroadcastcount'], unbroadcast_count)
         mempool = self.nodes[0].getrawmempool(True)
         for tx in mempool:
             assert_equal(mempool[tx]['unbroadcast'], True)
@@ -62,7 +56,8 @@ def test_broadcast(self):
         # check that second node doesn't have these two txns
         mempool = self.nodes[1].getrawmempool()
         assert rpc_tx_hsh not in mempool
-        assert wallet_tx_hsh not in mempool
+        if self.is_wallet_compiled():
+            assert wallet_tx_hsh not in mempool
 
         # ensure that unbroadcast txs are persisted to mempool.dat
         self.restart_node(0)
@@ -75,7 +70,8 @@ def test_broadcast(self):
         self.sync_mempools(timeout=30)
         mempool = self.nodes[1].getrawmempool()
         assert rpc_tx_hsh in mempool
-        assert wallet_tx_hsh in mempool
+        if self.is_wallet_compiled():
+            assert wallet_tx_hsh in mempool
 
         # check that transactions are no longer in first node's unbroadcast set
         mempool = self.nodes[0].getrawmempool(True)
@@ -104,8 +100,7 @@ def test_txn_removal(self):
 
         # since the node doesn't have any connections, it will not receive
         # any GETDATAs & thus the transaction will remain in the unbroadcast set.
-        addr = node.getnewaddress()
-        txhsh = node.sendtoaddress(addr, 0.0001)
+        txhsh = self.wallet.send_self_transfer(from_node=node)["txid"]
 
         # check transaction was removed from unbroadcast set due to presence in
         # a block

From 656f52585564365f7fdaf3cb70453e8bc4dc0c89 Mon Sep 17 00:00:00 2001
From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com>
Date: Wed, 13 Apr 2022 00:54:47 +0200
Subject: [PATCH 40/69] Merge bitcoin-core/gui#543: peers-tab: add connection
 duration column to tableview

51708c4516cb9d52e84dc8850d93f556dda1a75b gui: peersWidget - ResizeToContents Age and IP/Netmask columns (randymcmillan)
209301a442512579d57f79c82417dc7c496248b6 gui: add Age column to peers tab (randymcmillan)
127de22c5fb396e1670d2a911faf7a9adc9241e2 gui: add FormatPeerAge() utility helper (Jon Atack)

Pull request description:

  This change adds an "Age" column to the peers table view,
  which displays the duration of each peer's connection.

ACKs for top commit:
  jonatack:
    re-ACK  51708c4516cb9d52e84dc8850d93f556dda1a75b
  Jamewood:
    > re-ACK 51708c4
  shaavan:
    reACK 51708c4516cb9d52e84dc8850d93f556dda1a75b
  hebasto:
    ACK 51708c4516cb9d52e84dc8850d93f556dda1a75b, I have reviewed the code and it looks OK, I agree it can be merged.

Tree-SHA512: 27323f7080ec0d3fcdbf1b190fba1cd2d7406840ab6607c221cf8af950db9134e22721cc5a88f4fc4f390d8b05e98bc4b7521661a31fadad9e2c6c6390e71788
---
 src/qt/guiutil.cpp            | 12 ++++++++++++
 src/qt/guiutil.h              |  3 +++
 src/qt/peertablemodel.cpp     |  3 +++
 src/qt/peertablemodel.h       |  4 ++++
 src/qt/peertablesortproxy.cpp |  2 ++
 src/qt/rpcconsole.cpp         |  2 ++
 6 files changed, 26 insertions(+)

diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp
index 14bcc39eed..40f74f73e7 100644
--- a/src/qt/guiutil.cpp
+++ b/src/qt/guiutil.cpp
@@ -89,6 +89,8 @@
 void ForceActivation();
 #endif
 
+using namespace std::chrono_literals;
+
 namespace GUIUtil {
 
 static RecursiveMutex cs_css;
@@ -1729,6 +1731,16 @@ QString formatDurationStr(std::chrono::seconds dur)
     return str_list.join(" ");
 }
 
+QString FormatPeerAge(std::chrono::seconds time_connected)
+{
+    const auto time_now{GetTime<std::chrono::seconds>()};
+    const auto age{time_now - time_connected};
+    if (age >= 24h) return QObject::tr("%1 d").arg(age / 24h);
+    if (age >= 1h) return QObject::tr("%1 h").arg(age / 1h);
+    if (age >= 1min) return QObject::tr("%1 m").arg(age / 1min);
+    return QObject::tr("%1 s").arg(age / 1s);
+}
+
 QString formatServicesStr(quint64 mask)
 {
     QStringList strList;
diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h
index 8f3cb7df71..3507043262 100644
--- a/src/qt/guiutil.h
+++ b/src/qt/guiutil.h
@@ -420,6 +420,9 @@ namespace GUIUtil
     /** Convert seconds into a QString with days, hours, mins, secs */
     QString formatDurationStr(std::chrono::seconds dur);
 
+    /** Convert peer connection time to a QString denominated in the most relevant unit. */
+    QString FormatPeerAge(std::chrono::seconds time_connected);
+
     /** Format CNodeStats.nServices bitmask into a user-readable string */
     QString formatServicesStr(quint64 mask);
 
diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp
index ecd305a77d..182ff17b55 100644
--- a/src/qt/peertablemodel.cpp
+++ b/src/qt/peertablemodel.cpp
@@ -71,6 +71,8 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const
         switch (column) {
         case NetNodeId:
             return (qint64)rec->nodeStats.nodeid;
+        case Age:
+            return GUIUtil::FormatPeerAge(rec->nodeStats.m_connected);
         case Address:
             // prepend to peer address down-arrow symbol for inbound connection and up-arrow for outbound connection
             return QString::fromStdString((rec->nodeStats.fInbound ? "↓ " : "↑ ") + rec->nodeStats.m_addr_name);
@@ -91,6 +93,7 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const
     } else if (role == Qt::TextAlignmentRole) {
         switch (column) {
         case NetNodeId:
+        case Age:
             return QVariant(Qt::AlignRight | Qt::AlignVCenter);
         case Address:
             return {};
diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h
index 0ff1b5dba7..cf101149fd 100644
--- a/src/qt/peertablemodel.h
+++ b/src/qt/peertablemodel.h
@@ -47,6 +47,7 @@ class PeerTableModel : public QAbstractTableModel
 
     enum ColumnIndex {
         NetNodeId = 0,
+        Age,
         Address,
         ConnectionType,
         Network,
@@ -84,6 +85,9 @@ public Q_SLOTS:
         /*: Title of Peers Table column which contains a
             unique number used to identify a connection. */
         tr("Peer"),
+        /*: Title of Peers Table column which indicates the duration (length of time)
+            since the peer connection started. */
+        tr("Age"),
         /*: Title of Peers Table column which contains the
             IP/Onion/I2P address of the connected peer. */
         tr("Address"),
diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp
index ad966fbcab..6d716366fa 100644
--- a/src/qt/peertablesortproxy.cpp
+++ b/src/qt/peertablesortproxy.cpp
@@ -24,6 +24,8 @@ bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelInd
     switch (static_cast<PeerTableModel::ColumnIndex>(left_index.column())) {
     case PeerTableModel::NetNodeId:
         return left_stats.nodeid < right_stats.nodeid;
+    case PeerTableModel::Age:
+        return left_stats.m_connected > right_stats.m_connected;
     case PeerTableModel::Address:
         return left_stats.m_addr_name.compare(right_stats.m_addr_name) < 0;
     case PeerTableModel::ConnectionType:
diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp
index 084a65ff68..cd9f2bddcd 100644
--- a/src/qt/rpcconsole.cpp
+++ b/src/qt/rpcconsole.cpp
@@ -713,6 +713,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_
             ui->peerWidget->setColumnWidth(PeerTableModel::Subversion, SUBVERSION_COLUMN_WIDTH);
             ui->peerWidget->setColumnWidth(PeerTableModel::Ping, PING_COLUMN_WIDTH);
         }
+        ui->peerWidget->horizontalHeader()->setSectionResizeMode(PeerTableModel::Age, QHeaderView::ResizeToContents);
         ui->peerWidget->horizontalHeader()->setStretchLastSection(true);
         ui->peerWidget->setItemDelegateForColumn(PeerTableModel::NetNodeId, new PeerIdViewDelegate(this));
 
@@ -745,6 +746,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_
             ui->banlistWidget->setColumnWidth(BanTableModel::Address, BANSUBNET_COLUMN_WIDTH);
             ui->banlistWidget->setColumnWidth(BanTableModel::Bantime, BANTIME_COLUMN_WIDTH);
         }
+        ui->banlistWidget->horizontalHeader()->setSectionResizeMode(BanTableModel::Address, QHeaderView::ResizeToContents);
         ui->banlistWidget->horizontalHeader()->setStretchLastSection(true);
 
         // create ban table context menu

From 1288494d4a56fac60c870d789499488ca5e6ff4c Mon Sep 17 00:00:00 2001
From: fanquake <fanquake@gmail.com>
Date: Wed, 4 May 2022 18:57:49 +0100
Subject: [PATCH 41/69] Merge bitcoin/bitcoin#24976: netgroup: Follow-up for
 #22910

e5d183151709ab59d2fa6fe9e0243000e8d6abbe [netgroup] Use nStartByte as offset for the last byte of the group (dergoegge)

Pull request description:

  This addresses my review [comments](https://github.com/bitcoin/bitcoin/pull/22910#discussion_r856095896) I left on #22910.

  This has no effect on the current logic as `nStartByte` is only used for internal addresses which only ever add 10 whole bytes to the returned group. However to avoid future bugs, I think we should use `nStartByte` as offset for the last byte as well, in case we ever add a new address type that makes makes use of `nStartByte` and adds fractional bytes to the group.

ACKs for top commit:
  jnewbery:
    Code review ACK e5d183151709ab59d2fa6fe9e0243000e8d6abbe
  theStack:
    Concept and code-review ACK e5d183151709ab59d2fa6fe9e0243000e8d6abbe

Tree-SHA512: 4c08c7d6cb38b553e998798b3e3b790177aaa2141a48e277dfd538e01a7fccadf644329e93c5b0fb5e7e4037494c8dfe061b94eb52c6b31dc21bdf99eb0e311a
---
 src/netgroup.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/netgroup.cpp b/src/netgroup.cpp
index 3dc59725e1..c3162d0b28 100644
--- a/src/netgroup.cpp
+++ b/src/netgroup.cpp
@@ -71,7 +71,7 @@ std::vector<unsigned char> NetGroupManager::GetGroup(const CNetAddr& address) co
     // ...for the last byte, push nBits and for the rest of the byte push 1's
     if (nBits > 0) {
         assert(num_bytes < addr_bytes.size());
-        vchRet.push_back(addr_bytes[num_bytes] | ((1 << (8 - nBits)) - 1));
+        vchRet.push_back(addr_bytes[num_bytes + nStartByte] | ((1 << (8 - nBits)) - 1));
     }
 
     return vchRet;

From e4f8b7097d73c8804cee3a983ec6479a42626170 Mon Sep 17 00:00:00 2001
From: laanwj <126646+laanwj@users.noreply.github.com>
Date: Wed, 4 May 2022 20:19:51 +0200
Subject: [PATCH 42/69] Merge bitcoin/bitcoin#24852: util: optimize HexStr
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

5e61532e72c1021fda9c7b213bd9cf397cb3a802 util: optimizes HexStr (Martin Leitner-Ankerl)
4e2b99f72a90b956f3050095abed4949aff9b516 bench: Adds a benchmark for HexStr (Martin Leitner-Ankerl)
67c8411c37b483caa2fe3f7f4f40b68ed2a9bcf7 test: Adds a test for HexStr that checks all 256 bytes (Martin Leitner-Ankerl)

Pull request description:

  In my benchmark, this rewrite improves runtime 27% (g++) to 46% (clang++) for the benchmark `HexStrBench`:

  g++ 11.2.0
  |             ns/byte |              byte/s |    err% |        ins/byte |        cyc/byte |    IPC |       bra/byte |   miss% |     total | benchmark
  |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
  |                0.94 |    1,061,381,310.36 |    0.7% |           12.00 |            3.01 |  3.990 |           1.00 |    0.0% |      0.01 | `HexStrBench` master
  |                0.68 |    1,465,366,544.25 |    1.7% |            6.00 |            2.16 |  2.778 |           1.00 |    0.0% |      0.01 | `HexStrBench` branch

  clang++ 13.0.1
  |             ns/byte |              byte/s |    err% |        ins/byte |        cyc/byte |    IPC |       bra/byte |   miss% |     total | benchmark
  |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
  |                0.80 |    1,244,713,415.92 |    0.9% |           10.00 |            2.56 |  3.913 |           0.50 |    0.0% |      0.01 | `HexStrBench` master
  |                0.43 |    2,324,188,940.72 |    0.2% |            4.00 |            1.37 |  2.914 |           0.25 |    0.0% |      0.01 | `HexStrBench` branch

  Note that the idea for this change comes from denis2342 in #23364. This is a rewrite so no unaligned accesses occur.

  Also, the lookup table is now calculated at compile time, which hopefully makes the code a bit easier to review.

ACKs for top commit:
  laanwj:
    Code review ACK 5e61532e72c1021fda9c7b213bd9cf397cb3a802
  aureleoules:
    tACK 5e61532e72c1021fda9c7b213bd9cf397cb3a802.
  theStack:
    ACK 5e61532e72c1021fda9c7b213bd9cf397cb3a802 🚤

Tree-SHA512: 40b53d5908332473ef24918d3a80ad1292b60566c02585fa548eb4c3189754971be5a70325f4968fce6d714df898b52d9357aba14d4753a8c70e6ffd273a2319
---
 src/Makefile.bench.include |  1 +
 src/bench/strencodings.cpp | 18 ++++++++++++++++++
 src/test/util_tests.cpp    | 18 ++++++++++++++++++
 src/util/strencodings.cpp  | 33 +++++++++++++++++++++++++++------
 4 files changed, 64 insertions(+), 6 deletions(-)
 create mode 100644 src/bench/strencodings.cpp

diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include
index 133bc282ef..6fde78ddad 100644
--- a/src/Makefile.bench.include
+++ b/src/Makefile.bench.include
@@ -44,6 +44,7 @@ bench_bench_dash_SOURCES = \
   bench/pool.cpp \
   bench/rpc_blockchain.cpp \
   bench/rpc_mempool.cpp \
+  bench/strencodings.cpp \
   bench/util_time.cpp \
   bench/base58.cpp \
   bench/bech32.cpp \
diff --git a/src/bench/strencodings.cpp b/src/bench/strencodings.cpp
new file mode 100644
index 0000000000..4be7cd6d6f
--- /dev/null
+++ b/src/bench/strencodings.cpp
@@ -0,0 +1,18 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <bench/data.h>
+#include <util/strencodings.h>
+
+static void HexStrBench(benchmark::Bench& bench)
+{
+    auto const& data = benchmark::data::block813851;
+    bench.batch(data.size()).unit("byte").run([&] {
+        auto hex = HexStr(data);
+        ankerl::nanobench::doNotOptimizeAway(hex);
+    });
+}
+
+BENCHMARK(HexStrBench);
diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp
index 444714ec78..ae0cf8e060 100644
--- a/src/test/util_tests.cpp
+++ b/src/test/util_tests.cpp
@@ -206,6 +206,24 @@ BOOST_AUTO_TEST_CASE(util_HexStr)
         BOOST_CHECK_EQUAL(HexStr(in_s), out_exp);
         BOOST_CHECK_EQUAL(HexStr(in_b), out_exp);
     }
+
+    {
+        auto input = std::string();
+        for (size_t i=0; i<256; ++i) {
+            input.push_back(static_cast<char>(i));
+        }
+
+        auto hex = HexStr(input);
+        BOOST_TEST_REQUIRE(hex.size() == 512);
+        static constexpr auto hexmap = std::string_view("0123456789abcdef");
+        for (size_t i = 0; i < 256; ++i) {
+            auto upper = hexmap.find(hex[i * 2]);
+            auto lower = hexmap.find(hex[i * 2 + 1]);
+            BOOST_TEST_REQUIRE(upper != std::string_view::npos);
+            BOOST_TEST_REQUIRE(lower != std::string_view::npos);
+            BOOST_TEST_REQUIRE(i == upper*16 + lower);
+        }
+    }
 }
 
 BOOST_AUTO_TEST_CASE(span_write_bytes)
diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp
index 64f1a08425..95050042fb 100644
--- a/src/util/strencodings.cpp
+++ b/src/util/strencodings.cpp
@@ -9,6 +9,7 @@
 #include <tinyformat.h>
 
 #include <algorithm>
+#include <array>
 #include <cstdlib>
 #include <cstring>
 #include <limits>
@@ -493,17 +494,37 @@ std::string Capitalize(std::string str)
     return str;
 }
 
+namespace {
+
+using ByteAsHex = std::array<char, 2>;
+
+constexpr std::array<ByteAsHex, 256> CreateByteToHexMap()
+{
+    constexpr char hexmap[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+
+    std::array<ByteAsHex, 256> byte_to_hex{};
+    for (size_t i = 0; i < byte_to_hex.size(); ++i) {
+        byte_to_hex[i][0] = hexmap[i >> 4];
+        byte_to_hex[i][1] = hexmap[i & 15];
+    }
+    return byte_to_hex;
+}
+
+} // namespace
+
 std::string HexStr(const Span<const uint8_t> s)
 {
     std::string rv(s.size() * 2, '\0');
-    static constexpr char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7',
-                                         '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
-    auto it = rv.begin();
+    static constexpr auto byte_to_hex = CreateByteToHexMap();
+    static_assert(sizeof(byte_to_hex) == 512);
+
+    char* it = rv.data();
     for (uint8_t v : s) {
-        *it++ = hexmap[v >> 4];
-        *it++ = hexmap[v & 15];
+        std::memcpy(it, byte_to_hex[v].data(), 2);
+        it += 2;
     }
-    assert(it == rv.end());
+
+    assert(it == rv.data() + rv.size());
     return rv;
 }
 

From 90744d0d658616532647251f59a87cd9239ea2e4 Mon Sep 17 00:00:00 2001
From: fanquake <fanquake@gmail.com>
Date: Thu, 12 May 2022 21:07:32 +0100
Subject: [PATCH 43/69] Merge bitcoin/bitcoin#25115: scripted-diff: replace
 non-standard fixed width integer types (`u_int`... -> `uint`...)

672d49c863fd6b0f096f166aeb3504f51dfa7d19 scripted-diff: replace non-standard fixed width integer types (`u_int`...` -> `uint`...) (Sebastian Falbesoner)

Pull request description:

  Fixed width integer types prefixed with `u_int` are not part of C++ (see https://en.cppreference.com/w/cpp/types/integer), so it's better to avoid and replace them with their standard-conforming counterparts. (For those interested in history, according to one theory those u_int... types have been introduced by BSD: https://stackoverflow.com/a/5163960, http://lists.freedesktop.org/archives/release-wranglers/2004-August/000923.html).

ACKs for top commit:
  laanwj:
    Code review ACK 672d49c863fd6b0f096f166aeb3504f51dfa7d19
  fanquake:
    ACK 672d49c863fd6b0f096f166aeb3504f51dfa7d19

Tree-SHA512: 68134a0adca0d5c87a7432367cb493491a67288d69a174be2181f8e26efa968d966b9eb1cde94813942405063ee3be2a3437cf2aa5f71375f59205cbdbf501bb
---
 src/validation.cpp | 6 +++---
 src/wallet/bdb.cpp | 4 ++--
 src/wallet/bdb.h   | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/validation.cpp b/src/validation.cpp
index ed480eaad3..1243625f3a 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -2485,9 +2485,9 @@ bool CChainState::FlushStateToDisk(
             full_flush_completed = true;
             TRACE5(utxocache, flush,
                    (int64_t)(GetTimeMicros() - nNow.count()), // in microseconds (µs)
-                   (u_int32_t)mode,
-                   (u_int64_t)coins_count,
-                   (u_int64_t)coins_mem_usage,
+                   (uint32_t)mode,
+                   (uint64_t)coins_count,
+                   (uint64_t)coins_mem_usage,
                    (bool)fFlushForPrune);
         }
     }
diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp
index 110f1ace98..4ffc08335c 100644
--- a/src/wallet/bdb.cpp
+++ b/src/wallet/bdb.cpp
@@ -98,7 +98,7 @@ void BerkeleyEnvironment::Close()
     if (ret != 0)
         LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
     if (!fMockDb)
-        DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
+        DbEnv((uint32_t)0).remove(strPath.c_str(), 0);
 
     if (error_file) fclose(error_file);
 
@@ -246,7 +246,7 @@ const void* BerkeleyBatch::SafeDbt::get_data() const
     return m_dbt.get_data();
 }
 
-u_int32_t BerkeleyBatch::SafeDbt::get_size() const
+uint32_t BerkeleyBatch::SafeDbt::get_size() const
 {
     return m_dbt.get_size();
 }
diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h
index 4ea0f460fe..f09e76047b 100644
--- a/src/wallet/bdb.h
+++ b/src/wallet/bdb.h
@@ -34,7 +34,7 @@ static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100;
 static const bool DEFAULT_WALLET_PRIVDB = true;
 
 struct WalletDatabaseFileId {
-    u_int8_t value[DB_FILE_ID_LEN];
+    uint8_t value[DB_FILE_ID_LEN];
     bool operator==(const WalletDatabaseFileId& rhs) const;
 };
 
@@ -182,7 +182,7 @@ class BerkeleyBatch : public DatabaseBatch
 
         // delegate to Dbt
         const void* get_data() const;
-        u_int32_t get_size() const;
+        uint32_t get_size() const;
 
         // conversion operator to access the underlying Dbt
         operator Dbt*();

From d3345c9ee4cb0445f62ec1f13cf4ae389f3b7ee8 Mon Sep 17 00:00:00 2001
From: UdjinM6 <UdjinM6@users.noreply.github.com>
Date: Wed, 16 Oct 2024 13:16:00 +0300
Subject: [PATCH 44/69] fix: adjust the number of spork defaults

6275 follow-up
---
 src/spork.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/spork.h b/src/spork.h
index e8d5d81d71..a9ed23424c 100644
--- a/src/spork.h
+++ b/src/spork.h
@@ -68,7 +68,7 @@ struct CSporkDef
 };
 
 #define MAKE_SPORK_DEF(name, defaultValue) CSporkDef{name, defaultValue, #name}
-[[maybe_unused]] static constexpr std::array<CSporkDef, 8> sporkDefs = {
+[[maybe_unused]] static constexpr std::array<CSporkDef, 7> sporkDefs = {
     MAKE_SPORK_DEF(SPORK_2_INSTANTSEND_ENABLED,            4070908800ULL), // OFF
     MAKE_SPORK_DEF(SPORK_3_INSTANTSEND_BLOCK_FILTERING,    4070908800ULL), // OFF
     MAKE_SPORK_DEF(SPORK_9_SUPERBLOCKS_ENABLED,            4070908800ULL), // OFF

From 7d933d876d76cfd490b353fb5c57ba5a62be685e Mon Sep 17 00:00:00 2001
From: UdjinM6 <UdjinM6@users.noreply.github.com>
Date: Wed, 16 Oct 2024 13:30:51 +0300
Subject: [PATCH 45/69] test: should have no spork with an empty name

---
 test/functional/feature_sporks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/functional/feature_sporks.py b/test/functional/feature_sporks.py
index 46e96c81e5..b6b5fdf7b8 100755
--- a/test/functional/feature_sporks.py
+++ b/test/functional/feature_sporks.py
@@ -59,5 +59,8 @@ def run_test(self):
         self.connect_nodes(1, 2)
         self.wait_until(lambda: self.get_test_spork_state(self.nodes[2]), timeout=10)
 
+        assert "" not in self.nodes[0].spork('show').keys()
+
+
 if __name__ == '__main__':
     SporkTest().main()

From c709df74cc4b9f99c86bb7ac60069a923d2bac52 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 14 Oct 2024 21:18:17 +0000
Subject: [PATCH 46/69] merge bitcoin#20524: Move MIN_VERSION_SUPPORTED to
 p2p.py

continuation of cbff29a6 (dash#6067)
---
 test/functional/test_framework/test_node.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 6aa1bbd844..df020736b7 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -27,6 +27,7 @@
 from .p2p import P2P_SUBVERSION
 from .util import (
     MAX_NODES,
+    assert_equal,
     append_config,
     delete_cookie_file,
     get_auth_cookie,
@@ -637,6 +638,11 @@ def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
             # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely.
             p2p_conn.sync_with_ping()
 
+            # Consistency check that the Dash Core has received our user agent string. This checks the
+            # node's newest peer. It could be racy if another Dash Core node has connected since we opened
+            # our connection, but we don't expect that to happen.
+            assert_equal(self.getpeerinfo()[-1]['subver'], p2p_conn.strSubVer)
+
         return p2p_conn
 
     def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs):

From b7c0030d3d8d070c4346aa6aaa3daa73578d8675 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 19 Sep 2022 17:11:32 +0300
Subject: [PATCH 47/69] partial bitcoin#23443: Erlay support signaling

contains:
- e56d1d2a (changes to `test_node.py`)
---
 test/functional/test_framework/test_node.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index df020736b7..3d1140e1d9 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -645,7 +645,7 @@ def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
 
         return p2p_conn
 
-    def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs):
+    def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, p2p_idx, connection_type="outbound-full-relay", **kwargs):
         """Add an outbound p2p connection from node. Must be an
         "outbound-full-relay", "block-relay-only", "addr-fetch" or "feeler" connection.
 
@@ -667,8 +667,9 @@ def addconnection_callback(address, port):
             p2p_conn.wait_for_connect()
             self.p2ps.append(p2p_conn)
 
-            p2p_conn.wait_for_verack()
-            p2p_conn.sync_with_ping()
+            if wait_for_verack:
+                p2p_conn.wait_for_verack()
+                p2p_conn.sync_with_ping()
 
         return p2p_conn
 

From 5bf245b4a0d2400f89ade6e15c7c562201e30c36 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 14 Oct 2024 18:30:40 +0000
Subject: [PATCH 48/69] partial bitcoin#26448: fix intermittent failure in
 p2p_sendtxrcncl.py

contains:
- 74d97531 (changes to `test_node.py`)
---
 test/functional/test_framework/test_node.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 3d1140e1d9..12f05b6ec4 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -651,6 +651,10 @@ def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, p2p_idx
 
         This method adds the p2p connection to the self.p2ps list and returns
         the connection to the caller.
+
+        p2p_idx must be different for simultaneously connected peers. When reusing it for the next peer
+        after disconnecting the previous one, it is necessary to wait for the disconnect to finish to avoid
+        a race condition.
         """
 
         def addconnection_callback(address, port):

From 1bf135bbc91d24d5f3cf947148caa37980d6c9e5 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 15 Oct 2024 07:11:47 +0000
Subject: [PATCH 49/69] merge bitcoin#26553: Fix intermittent failure in
 rpc_net.py

---
 test/functional/rpc_net.py                  | 5 +++--
 test/functional/test_framework/test_node.py | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py
index e8d2607763..d7d7d5d0c4 100755
--- a/test/functional/rpc_net.py
+++ b/test/functional/rpc_net.py
@@ -114,7 +114,7 @@ def test_getpeerinfo(self):
         no_version_peer_id = 3
         no_version_peer_conntime = self.mocktime
         with self.nodes[0].assert_debug_log([f"Added connection peer={no_version_peer_id}"]):
-            self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False)
+            no_version_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False)
         peer_info = self.nodes[0].getpeerinfo()[no_version_peer_id]
         peer_info.pop("addr")
         peer_info.pop("addrbind")
@@ -155,7 +155,8 @@ def test_getpeerinfo(self):
                 "version": 0,
             },
         )
-        self.nodes[0].disconnect_p2ps()
+        no_version_peer.peer_disconnect()
+        self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 3)
 
     def test_getnettotals(self):
         self.log.info("Test getnettotals")
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 12f05b6ec4..fb59e6c206 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -682,7 +682,8 @@ def num_test_p2p_connections(self):
         return len([peer for peer in self.getpeerinfo() if P2P_SUBVERSION % "" in peer['subver']])
 
     def disconnect_p2ps(self):
-        """Close all p2p connections to the node."""
+        """Close all p2p connections to the node.
+        Use only after each p2p has sent a version message to ensure the wait works."""
         for p in self.p2ps:
             p.peer_disconnect()
 

From fffe6e716b509e87d5dd30b27cfd2df8ecd5a0e7 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Wed, 28 Jun 2023 10:15:41 +0200
Subject: [PATCH 50/69] merge bitcoin#27986: remove race in the user-agent
 reception check

---
 test/functional/test_framework/test_node.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index fb59e6c206..3da025570f 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -638,10 +638,13 @@ def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
             # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely.
             p2p_conn.sync_with_ping()
 
-            # Consistency check that the Dash Core has received our user agent string. This checks the
-            # node's newest peer. It could be racy if another Dash Core node has connected since we opened
-            # our connection, but we don't expect that to happen.
-            assert_equal(self.getpeerinfo()[-1]['subver'], p2p_conn.strSubVer)
+            # Consistency check that the node received our user agent string.
+            # Find our connection in getpeerinfo by our address:port, as it is unique.
+            sockname = p2p_conn._transport.get_extra_info("socket").getsockname()
+            our_addr_and_port = f"{sockname[0]}:{sockname[1]}"
+            info = [peer for peer in self.getpeerinfo() if peer["addr"] == our_addr_and_port]
+            assert_equal(len(info), 1)
+            assert_equal(info[0]["subver"], p2p_conn.strSubVer)
 
         return p2p_conn
 

From deaee147b7ef4a28a304ee16b92e3ca0712226bc Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 15 Oct 2024 11:19:07 +0000
Subject: [PATCH 51/69] merge bitcoin-core/gui#754: Add BIP324-specific labels
 to peer details

---
 src/qt/forms/debugwindow.ui | 156 ++++++++++++++++++++++++------------
 src/qt/rpcconsole.cpp       |  23 +++++-
 2 files changed, 125 insertions(+), 54 deletions(-)

diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui
index d6ea9fd3eb..c45f6bcaeb 100644
--- a/src/qt/forms/debugwindow.ui
+++ b/src/qt/forms/debugwindow.ui
@@ -977,6 +977,58 @@
                 </widget>
                </item>
                <item row="2" column="0">
+                <widget class="QLabel" name="peerTransportTypeLabel">
+                 <property name="toolTip">
+                  <string>The transport layer version: %1</string>
+                 </property>
+                 <property name="text">
+                  <string>Transport</string>
+                 </property>
+                </widget>
+               </item>
+               <item row="2" column="2">
+                <widget class="QLabel" name="peerTransportType">
+                 <property name="cursor">
+                  <cursorShape>IBeamCursor</cursorShape>
+                 </property>
+                 <property name="text">
+                  <string>N/A</string>
+                 </property>
+                 <property name="textFormat">
+                  <enum>Qt::PlainText</enum>
+                 </property>
+                 <property name="textInteractionFlags">
+                  <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set>
+                 </property>
+                </widget>
+               </item>
+               <item row="3" column="0">
+                <widget class="QLabel" name="peerSessionIdLabel">
+                 <property name="toolTip">
+                  <string>The BIP324 session ID string in hex, if any.</string>
+                 </property>
+                 <property name="text">
+                  <string>Session ID</string>
+                 </property>
+                </widget>
+               </item>
+               <item row="3" column="2">
+                <widget class="QLabel" name="peerSessionId">
+                 <property name="cursor">
+                  <cursorShape>IBeamCursor</cursorShape>
+                 </property>
+                 <property name="text">
+                  <string>N/A</string>
+                 </property>
+                 <property name="textFormat">
+                  <enum>Qt::PlainText</enum>
+                 </property>
+                 <property name="textInteractionFlags">
+                  <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set>
+                 </property>
+                </widget>
+               </item>
+               <item row="4" column="0">
                 <widget class="QLabel" name="peerNetworkLabel">
                  <property name="toolTip">
                   <string>The network protocol this peer is connected through: IPv4, IPv6, Onion, I2P, or CJDNS.</string>
@@ -986,7 +1038,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="2" column="2">
+               <item row="4" column="2">
                 <widget class="QLabel" name="peerNetwork">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1002,14 +1054,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="3" column="0">
+               <item row="5" column="0">
                 <widget class="QLabel" name="label_30">
                  <property name="text">
                   <string>Permissions</string>
                  </property>
                 </widget>
                </item>
-               <item row="3" column="2">
+               <item row="5" column="2">
                 <widget class="QLabel" name="peerPermissions">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1025,7 +1077,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="4" column="0">
+               <item row="6" column="0">
                <widget class="QLabel" name="peerConnectionTypeLabel">
                  <property name="toolTip">
                    <string>The direction and type of peer connection: %1</string>
@@ -1035,7 +1087,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="4" column="2">
+               <item row="6" column="2">
                 <widget class="QLabel" name="peerConnectionType">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1051,14 +1103,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="5" column="0">
+               <item row="7" column="0">
                 <widget class="QLabel" name="label_21">
                  <property name="text">
                   <string>Version</string>
                  </property>
                 </widget>
                </item>
-               <item row="5" column="2">
+               <item row="7" column="2">
                 <widget class="QLabel" name="peerVersion">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1074,14 +1126,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="6" column="0">
+               <item row="8" column="0">
                 <widget class="QLabel" name="label_28">
                  <property name="text">
                   <string>User Agent</string>
                  </property>
                 </widget>
                </item>
-               <item row="6" column="2">
+               <item row="8" column="2">
                 <widget class="QLabel" name="peerSubversion">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1100,14 +1152,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="7" column="0">
+               <item row="9" column="0">
                 <widget class="QLabel" name="label_4">
                  <property name="text">
                   <string>Services</string>
                  </property>
                 </widget>
                </item>
-               <item row="7" column="2">
+               <item row="9" column="2">
                 <widget class="QLabel" name="peerServices">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1123,7 +1175,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="8" column="0">
+               <item row="10" column="0">
                 <widget class="QLabel" name="peerRelayTxesLabel">
                  <property name="toolTip">
                   <string>Whether the peer requested us to relay transactions.</string>
@@ -1133,7 +1185,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="8" column="2">
+               <item row="10" column="2">
                 <widget class="QLabel" name="peerRelayTxes">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1149,7 +1201,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="9" column="0">
+               <item row="11" column="0">
                 <widget class="QLabel" name="peerHighBandwidthLabel">
                  <property name="toolTip">
                   <string>High bandwidth BIP152 compact block relay: %1</string>
@@ -1159,7 +1211,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="9" column="2">
+               <item row="11" column="2">
                 <widget class="QLabel" name="peerHighBandwidth">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1175,14 +1227,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="10" column="0">
+               <item row="12" column="0">
                 <widget class="QLabel" name="label_29">
                  <property name="text">
                   <string>Starting Block</string>
                  </property>
                 </widget>
                </item>
-               <item row="10" column="2">
+               <item row="12" column="2">
                 <widget class="QLabel" name="peerHeight">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1198,14 +1250,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="11" column="0">
+               <item row="13" column="0">
                 <widget class="QLabel" name="label_27">
                  <property name="text">
                   <string>Synced Headers</string>
                  </property>
                 </widget>
                </item>
-               <item row="11" column="2">
+               <item row="13" column="2">
                 <widget class="QLabel" name="peerSyncHeight">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1221,14 +1273,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="12" column="0">
+               <item row="14" column="0">
                 <widget class="QLabel" name="label_25">
                  <property name="text">
                   <string>Synced Blocks</string>
                  </property>
                 </widget>
                </item>
-               <item row="12" column="2">
+               <item row="14" column="2">
                 <widget class="QLabel" name="peerCommonHeight">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1244,14 +1296,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="13" column="0">
+               <item row="15" column="0">
                 <widget class="QLabel" name="label_22">
                  <property name="text">
                   <string>Connection Time</string>
                  </property>
                 </widget>
                </item>
-               <item row="13" column="2">
+               <item row="15" column="2">
                 <widget class="QLabel" name="peerConnTime">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1267,7 +1319,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="14" column="0">
+               <item row="16" column="0">
                 <widget class="QLabel" name="peerLastBlockLabel">
                  <property name="toolTip">
                   <string>Elapsed time since a novel block passing initial validity checks was received from this peer.</string>
@@ -1277,7 +1329,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="14" column="2">
+               <item row="16" column="2">
                 <widget class="QLabel" name="peerLastBlock">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1293,7 +1345,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="15" column="0">
+               <item row="17" column="0">
                 <widget class="QLabel" name="peerLastTxLabel">
                  <property name="toolTip">
                   <string extracomment="Tooltip text for the Last Transaction field in the peer details area.">Elapsed time since a novel transaction accepted into our mempool was received from this peer.</string>
@@ -1303,7 +1355,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="15" column="2">
+               <item row="17" column="2">
                 <widget class="QLabel" name="peerLastTx">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1319,14 +1371,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="16" column="0">
+               <item row="18" column="0">
                 <widget class="QLabel" name="label_15">
                  <property name="text">
                   <string>Last Send</string>
                  </property>
                 </widget>
                </item>
-               <item row="16" column="2">
+               <item row="18" column="2">
                 <widget class="QLabel" name="peerLastSend">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1342,14 +1394,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="17" column="0">
+               <item row="19" column="0">
                 <widget class="QLabel" name="label_19">
                  <property name="text">
                   <string>Last Receive</string>
                  </property>
                 </widget>
                </item>
-               <item row="17" column="2">
+               <item row="19" column="2">
                 <widget class="QLabel" name="peerLastRecv">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1365,14 +1417,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="18" column="0">
+               <item row="20" column="0">
                 <widget class="QLabel" name="label_18">
                  <property name="text">
                   <string>Sent</string>
                  </property>
                 </widget>
                </item>
-               <item row="18" column="2">
+               <item row="20" column="2">
                 <widget class="QLabel" name="peerBytesSent">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1388,14 +1440,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="19" column="0">
+               <item row="21" column="0">
                 <widget class="QLabel" name="label_20">
                  <property name="text">
                   <string>Received</string>
                  </property>
                 </widget>
                </item>
-               <item row="19" column="2">
+               <item row="21" column="2">
                 <widget class="QLabel" name="peerBytesRecv">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1411,14 +1463,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="20" column="0">
+               <item row="22" column="0">
                 <widget class="QLabel" name="label_26">
                  <property name="text">
                   <string>Ping Time</string>
                  </property>
                 </widget>
                </item>
-               <item row="20" column="2">
+               <item row="22" column="2">
                 <widget class="QLabel" name="peerPingTime">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1434,7 +1486,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="21" column="0">
+               <item row="23" column="0">
                 <widget class="QLabel" name="peerPingWaitLabel">
                  <property name="toolTip">
                   <string>The duration of a currently outstanding ping.</string>
@@ -1444,7 +1496,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="21" column="2">
+               <item row="23" column="2">
                 <widget class="QLabel" name="peerPingWait">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1460,14 +1512,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="22" column="0">
+               <item row="24" column="0">
                 <widget class="QLabel" name="peerMinPingLabel">
                  <property name="text">
                   <string>Min Ping</string>
                  </property>
                 </widget>
                </item>
-               <item row="22" column="2">
+               <item row="24" column="2">
                 <widget class="QLabel" name="peerMinPing">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1483,14 +1535,14 @@
                  </property>
                 </widget>
                </item>
-               <item row="23" column="0">
+               <item row="25" column="0">
                 <widget class="QLabel" name="label_timeoffset">
                  <property name="text">
                   <string>Time Offset</string>
                  </property>
                 </widget>
                </item>
-               <item row="23" column="2">
+               <item row="25" column="2">
                 <widget class="QLabel" name="timeoffset">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1506,7 +1558,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="24" column="0">
+               <item row="26" column="0">
                 <widget class="QLabel" name="peerMappedASLabel">
                  <property name="toolTip">
                   <string>The mapped Autonomous System used for diversifying peer selection.</string>
@@ -1516,7 +1568,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="24" column="2">
+               <item row="26" column="2">
                 <widget class="QLabel" name="peerMappedAS">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1532,7 +1584,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="25" column="0">
+               <item row="27" column="0">
                 <widget class="QLabel" name="peerAddrRelayEnabledLabel">
                  <property name="toolTip">
                   <string extracomment="Tooltip text for the Address Relay field in the peer details area.">Whether we relay addresses to this peer.</string>
@@ -1542,7 +1594,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="25" column="2">
+               <item row="27" column="2">
                 <widget class="QLabel" name="peerAddrRelayEnabled">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1558,7 +1610,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="26" column="0">
+               <item row="28" column="0">
                 <widget class="QLabel" name="peerAddrProcessedLabel">
                  <property name="toolTip">
                   <string extracomment="Tooltip text for the Addresses Processed field in the peer details area.">Total number of addresses processed, excluding those dropped due to rate-limiting.</string>
@@ -1568,7 +1620,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="26" column="2">
+               <item row="28" column="2">
                 <widget class="QLabel" name="peerAddrProcessed">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1584,7 +1636,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="27" column="0">
+               <item row="29" column="0">
                 <widget class="QLabel" name="peerAddrRateLimitedLabel">
                  <property name="toolTip">
                   <string extracomment="Tooltip text for the Addresses Rate-Limited field in the peer details area.">Total number of addresses dropped due to rate-limiting.</string>
@@ -1594,7 +1646,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="27" column="2">
+               <item row="29" column="2">
                 <widget class="QLabel" name="peerAddrRateLimited">
                  <property name="cursor">
                   <cursorShape>IBeamCursor</cursorShape>
@@ -1610,7 +1662,7 @@
                  </property>
                 </widget>
                </item>
-               <item row="28" column="0">
+               <item row="30" column="0">
                 <spacer name="verticalSpacer_3">
                  <property name="orientation">
                   <enum>Qt::Vertical</enum>
diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp
index b76b05bced..e5c6c20466 100644
--- a/src/qt/rpcconsole.cpp
+++ b/src/qt/rpcconsole.cpp
@@ -15,6 +15,7 @@
 #include <chainparams.h>
 #include <interfaces/node.h>
 #include <netbase.h>
+#include <node/connection_types.h>
 #include <qt/bantablemodel.h>
 #include <qt/clientmodel.h>
 #include <qt/peertablesortproxy.h>
@@ -533,8 +534,17 @@ RPCConsole::RPCConsole(interfaces::Node& node, QWidget* parent, Qt::WindowFlags
         /*: Explanatory text for a short-lived outbound peer connection that is used
             to request addresses from a peer. */
         tr("Outbound Address Fetch: short-lived, for soliciting addresses")};
-    const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"};
-    ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list));
+    const QString connection_types_list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"};
+    ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(connection_types_list));
+    const std::vector<QString> TRANSPORT_TYPE_DOC{
+        //: Explanatory text for "detecting" transport type.
+        tr("detecting: peer could be v1 or v2"),
+        //: Explanatory text for v1 transport type.
+        tr("v1: unencrypted, plaintext transport protocol"),
+        //: Explanatory text for v2 transport type.
+        tr("v2: BIP324 encrypted transport protocol")};
+    const QString transport_types_list{"<ul><li>" + Join(TRANSPORT_TYPE_DOC, QString("</li><li>")) + "</li></ul>"};
+    ui->peerTransportTypeLabel->setToolTip(ui->peerTransportTypeLabel->toolTip().arg(transport_types_list));
     const QString hb_list{"<ul><li>\""
         + ts.to + "\" – " + tr("we selected the peer for high bandwidth relay") + "</li><li>\""
         + ts.from + "\" – " + tr("the peer selected us for high bandwidth relay") + "</li><li>\""
@@ -1272,6 +1282,15 @@ void RPCConsole::updateDetailWidget()
     ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion));
     ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer));
     ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /* prepend_direction */ true));
+    ui->peerTransportType->setText(QString::fromStdString(TransportTypeAsString(stats->nodeStats.m_transport_type)));
+    if (stats->nodeStats.m_transport_type == TransportProtocolType::V2) {
+        ui->peerSessionIdLabel->setVisible(true);
+        ui->peerSessionId->setVisible(true);
+        ui->peerSessionId->setText(QString::fromStdString(stats->nodeStats.m_session_id));
+    } else {
+        ui->peerSessionIdLabel->setVisible(false);
+        ui->peerSessionId->setVisible(false);
+    }
     ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network));
     if (stats->nodeStats.m_permission_flags == NetPermissionFlags::None) {
         ui->peerPermissions->setText(ts.na);

From 6a4ca62fd135190ecf8126ac15269f3ad6ce07cd Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 15 Oct 2024 09:43:55 +0000
Subject: [PATCH 52/69] merge bitcoin#28645: fix `assert_debug_log` call-site
 bugs, add type checks

---
 test/functional/feature_governance.py       | 2 +-
 test/functional/p2p_v2_transport.py         | 4 ++--
 test/functional/test_framework/test_node.py | 3 +++
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/functional/feature_governance.py b/test/functional/feature_governance.py
index 4e499ffa31..43c49784d2 100755
--- a/test/functional/feature_governance.py
+++ b/test/functional/feature_governance.py
@@ -302,7 +302,7 @@ def sync_gov(node):
 
         self.log.info("Move a few block past the recent superblock height and make sure we have no new votes")
         for _ in range(5):
-            with self.nodes[1].assert_debug_log("", [f"Voting NO-FUNDING for trigger:{winning_trigger_hash} success"]):
+            with self.nodes[1].assert_debug_log(expected_msgs=[""], unexpected_msgs=[f"Voting NO-FUNDING for trigger:{winning_trigger_hash} success"]):
                 self.bump_mocktime(1)
                 self.generate(self.nodes[0], 1, sync_fun=self.sync_blocks())
             # Votes on both triggers should NOT change
diff --git a/test/functional/p2p_v2_transport.py b/test/functional/p2p_v2_transport.py
index a7f208c858..a99d368c70 100755
--- a/test/functional/p2p_v2_transport.py
+++ b/test/functional/p2p_v2_transport.py
@@ -145,7 +145,7 @@ def run_test(self):
         wrong_network_magic_prefix = MAGIC_BYTES["mainnet"] + V1_PREFIX[4:]
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
             s.connect(("127.0.0.1", p2p_port(0)))
-            with self.nodes[0].assert_debug_log("V2 transport error: V1 peer with wrong MessageStart"):
+            with self.nodes[0].assert_debug_log(["V2 transport error: V1 peer with wrong MessageStart"]):
                 s.sendall(wrong_network_magic_prefix + b"somepayload")
 
         # Check detection of missing garbage terminator (hits after fixed amount of data if terminator never matches garbage)
@@ -156,7 +156,7 @@ def run_test(self):
             self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers + 1)
             s.sendall(b'\x00' * (MAX_KEY_GARB_AND_GARBTERM_LEN - 1))
             self.wait_until(lambda: self.nodes[0].getpeerinfo()[-1]["bytesrecv"] == MAX_KEY_GARB_AND_GARBTERM_LEN - 1)
-            with self.nodes[0].assert_debug_log("V2 transport error: missing garbage terminator"):
+            with self.nodes[0].assert_debug_log(["V2 transport error: missing garbage terminator"]):
                 s.sendall(b'\x00')  # send out last byte
                 # should disconnect immediately
                 self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers)
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 3da025570f..91671e1f1f 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -427,6 +427,9 @@ def debug_log_bytes(self) -> int:
     def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2):
         if unexpected_msgs is None:
             unexpected_msgs = []
+        assert_equal(type(expected_msgs), list)
+        assert_equal(type(unexpected_msgs), list)
+
         time_end = time.time() + timeout * self.timeout_factor
         prev_size = self.debug_log_bytes()
 

From 35253cdd15c3b9b0e7d3e327c5b4c0474d4f2290 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 10 Oct 2023 12:11:00 -0400
Subject: [PATCH 53/69] merge bitcoin#28632: make python p2p not send getaddr
 on incoming connections

---
 test/functional/p2p_addr_relay.py           | 5 +++--
 test/functional/test_framework/p2p.py       | 3 ++-
 test/functional/test_framework/test_node.py | 2 ++
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py
index cbc15dcbf5..e89734fcfe 100755
--- a/test/functional/p2p_addr_relay.py
+++ b/test/functional/p2p_addr_relay.py
@@ -257,15 +257,16 @@ def getaddr_tests(self):
         full_outbound_peer.sync_with_ping()
         assert full_outbound_peer.getaddr_received()
 
-        self.log.info('Check that we do not send a getaddr message upon connecting to a block-relay-only peer')
+        self.log.info('Check that we do not send a getaddr message to a block-relay-only or inbound peer')
         block_relay_peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=1, connection_type="block-relay-only")
         block_relay_peer.sync_with_ping()
         assert_equal(block_relay_peer.getaddr_received(), False)
 
-        self.log.info('Check that we answer getaddr messages only from inbound peers')
         inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver(send_getaddr=False))
         inbound_peer.sync_with_ping()
+        assert_equal(inbound_peer.getaddr_received(), False)
 
+        self.log.info('Check that we answer getaddr messages only from inbound peers')
         # Add some addresses to addrman
         for i in range(1000):
             first_octet = i >> 8
diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py
index 43c27a4702..3bdccbd89c 100755
--- a/test/functional/test_framework/p2p.py
+++ b/test/functional/test_framework/p2p.py
@@ -490,7 +490,8 @@ def on_version(self, message):
             self.send_message(msg_sendaddrv2())
         self.send_message(msg_verack())
         self.nServices = message.nServices
-        self.send_message(msg_getaddr())
+        if self.p2p_connected_to_node:
+            self.send_message(msg_getaddr())
 
     # Connection helper methods
 
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 91671e1f1f..fd8df3eec5 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -623,6 +623,7 @@ def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
         if 'dstaddr' not in kwargs:
             kwargs['dstaddr'] = '127.0.0.1'
 
+        p2p_conn.p2p_connected_to_node = True
         p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)()
         self.p2ps.append(p2p_conn)
         p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False)
@@ -667,6 +668,7 @@ def addconnection_callback(address, port):
             self.log.debug("Connecting to %s:%d %s" % (address, port, connection_type))
             self.addconnection('%s:%d' % (address, port), connection_type)
 
+        p2p_conn.p2p_connected_to_node = False
         p2p_conn.peer_accept_connection(connect_cb=addconnection_callback, connect_id=p2p_idx + 1, net=self.chain, timeout_factor=self.timeout_factor, **kwargs)()
 
         if connection_type == "feeler":

From c0b3062215f66cdf5f26daf8af268583be9bf735 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 14 Oct 2024 20:53:38 +0000
Subject: [PATCH 54/69] merge bitcoin#28782: Add missing sync on send_version
 in peer_connect

---
 test/functional/test_framework/p2p.py       | 4 ++--
 test/functional/test_framework/test_node.py | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py
index 3bdccbd89c..2be688ee69 100755
--- a/test/functional/test_framework/p2p.py
+++ b/test/functional/test_framework/p2p.py
@@ -392,8 +392,8 @@ def peer_connect_send_version(self, services):
         vt.strSubVer = self.strSubVer
         self.on_connection_send_msg = vt  # Will be sent soon after connection_made
 
-    def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs):
-        create_conn = super().peer_connect(*args, **kwargs)
+    def peer_connect(self, *, services=P2P_SERVICES, send_version, **kwargs):
+        create_conn = super().peer_connect(**kwargs)
 
         if send_version:
             self.peer_connect_send_version(services)
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index fd8df3eec5..8c1a5b1717 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -613,7 +613,7 @@ def assert_start_raises_init_error(self, extra_args=None, expected_msg=None, mat
                     assert_msg += "with expected error " + expected_msg
                 self._raise_assertion_error(assert_msg)
 
-    def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
+    def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, **kwargs):
         """Add an inbound p2p connection to the node.
 
         This method adds the p2p connection to the self.p2ps list and also
@@ -624,9 +624,11 @@ def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs):
             kwargs['dstaddr'] = '127.0.0.1'
 
         p2p_conn.p2p_connected_to_node = True
-        p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)()
+        p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor)()
         self.p2ps.append(p2p_conn)
         p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False)
+        if send_version:
+            p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg)
         if wait_for_verack:
             # Wait for the node to send us the version and verack
             p2p_conn.wait_for_verack()

From d0804d4bf08a4764905dd0f52e0448defbddaf91 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Mon, 14 Oct 2024 21:26:07 +0000
Subject: [PATCH 55/69] merge bitcoin#28822: Add missing wait for version to be
 sent in add_outbound_p2p_connection

---
 test/functional/test_framework/test_node.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index 8c1a5b1717..ded022473e 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -681,6 +681,7 @@ def addconnection_callback(address, port):
             p2p_conn.wait_for_connect()
             self.p2ps.append(p2p_conn)
 
+            p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg)
             if wait_for_verack:
                 p2p_conn.wait_for_verack()
                 p2p_conn.sync_with_ping()

From 1a293c7cc57fb6555806b12e5c5ffda43a214fe2 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Tue, 15 Oct 2024 10:36:47 +0000
Subject: [PATCH 56/69] merge bitcoin#29006: fix v2 transport intermittent test
 failure

---
 test/functional/p2p_v2_transport.py         | 16 ++++++++--------
 test/functional/test_framework/test_node.py | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/test/functional/p2p_v2_transport.py b/test/functional/p2p_v2_transport.py
index a99d368c70..4c566df1a9 100755
--- a/test/functional/p2p_v2_transport.py
+++ b/test/functional/p2p_v2_transport.py
@@ -133,9 +133,8 @@ def run_test(self):
         V1_PREFIX = MAGIC_BYTES[self.chain] + b"version\x00\x00\x00\x00\x00"
         assert_equal(len(V1_PREFIX), 16)
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            num_peers = len(self.nodes[0].getpeerinfo())
-            s.connect(("127.0.0.1", p2p_port(0)))
-            self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers + 1)
+            with self.nodes[0].wait_for_new_peer():
+                s.connect(("127.0.0.1", p2p_port(0)))
             s.sendall(V1_PREFIX[:-1])
             assert_equal(self.nodes[0].getpeerinfo()[-1]["transport_protocol_type"], "detecting")
             s.sendall(bytes([V1_PREFIX[-1]]))  # send out last prefix byte
@@ -144,22 +143,23 @@ def run_test(self):
         # Check wrong network prefix detection (hits if the next 12 bytes correspond to a v1 version message)
         wrong_network_magic_prefix = MAGIC_BYTES["mainnet"] + V1_PREFIX[4:]
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            s.connect(("127.0.0.1", p2p_port(0)))
+            with self.nodes[0].wait_for_new_peer():
+                s.connect(("127.0.0.1", p2p_port(0)))
             with self.nodes[0].assert_debug_log(["V2 transport error: V1 peer with wrong MessageStart"]):
                 s.sendall(wrong_network_magic_prefix + b"somepayload")
 
         # Check detection of missing garbage terminator (hits after fixed amount of data if terminator never matches garbage)
         MAX_KEY_GARB_AND_GARBTERM_LEN = 64 + 4095 + 16
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            num_peers = len(self.nodes[0].getpeerinfo())
-            s.connect(("127.0.0.1", p2p_port(0)))
-            self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers + 1)
+            with self.nodes[0].wait_for_new_peer():
+                s.connect(("127.0.0.1", p2p_port(0)))
             s.sendall(b'\x00' * (MAX_KEY_GARB_AND_GARBTERM_LEN - 1))
             self.wait_until(lambda: self.nodes[0].getpeerinfo()[-1]["bytesrecv"] == MAX_KEY_GARB_AND_GARBTERM_LEN - 1)
             with self.nodes[0].assert_debug_log(["V2 transport error: missing garbage terminator"]):
+                peer_id = self.nodes[0].getpeerinfo()[-1]["id"]
                 s.sendall(b'\x00')  # send out last byte
                 # should disconnect immediately
-                self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == num_peers)
+                self.wait_until(lambda: not peer_id in [p["id"] for p in self.nodes[0].getpeerinfo()])
 
 
 if __name__ == '__main__':
diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py
index ded022473e..7080149b5d 100755
--- a/test/functional/test_framework/test_node.py
+++ b/test/functional/test_framework/test_node.py
@@ -490,6 +490,24 @@ def wait_for_debug_log(self, expected_msgs, timeout=60):
             'Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(
                 str(expected_msgs), print_log))
 
+    @contextlib.contextmanager
+    def wait_for_new_peer(self, timeout=5):
+        """
+        Wait until the node is connected to at least one new peer. We detect this
+        by watching for an increased highest peer id, using the `getpeerinfo` RPC call.
+        Note that the simpler approach of only accounting for the number of peers
+        suffers from race conditions, as disconnects from unrelated previous peers
+        could happen anytime in-between.
+        """
+        def get_highest_peer_id():
+            peer_info = self.getpeerinfo()
+            return peer_info[-1]["id"] if peer_info else -1
+
+        initial_peer_id = get_highest_peer_id()
+        yield
+        wait_until_helper(lambda: get_highest_peer_id() > initial_peer_id,
+                          timeout=timeout, timeout_factor=self.timeout_factor)
+
     @contextlib.contextmanager
     def profile_with_perf(self, profile_name: str):
         """

From aa5311d0fc0868e9d3dcb386a13d24f79cdfa29f Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com>
Date: Wed, 16 Oct 2024 10:32:29 +0000
Subject: [PATCH 57/69] merge bitcoin#29212: Fix -netinfo backward compat with
 getpeerinfo pre-v26

---
 src/bitcoin-cli.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp
index 11f1a144f5..cd691e151c 100644
--- a/src/bitcoin-cli.cpp
+++ b/src/bitcoin-cli.cpp
@@ -518,7 +518,7 @@ class NetinfoRequestHandler : public BaseRequestHandler
                 const std::string addr{peer["addr"].get_str()};
                 const std::string age{conn_time == 0 ? "" : ToString((m_time_now - conn_time) / 60)};
                 const std::string sub_version{peer["subver"].get_str()};
-                const std::string transport{peer["transport_protocol_type"].get_str()};
+                const std::string transport{peer["transport_protocol_type"].isNull() ? "v1" : peer["transport_protocol_type"].get_str()};
                 const bool is_addr_relay_enabled{peer["addr_relay_enabled"].isNull() ? false : peer["addr_relay_enabled"].get_bool()};
                 const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()};
                 const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()};
@@ -538,7 +538,7 @@ class NetinfoRequestHandler : public BaseRequestHandler
         // Report detailed peer connections list sorted by direction and minimum ping time.
         if (DetailsRequested() && !m_peers.empty()) {
             std::sort(m_peers.begin(), m_peers.end());
-            result += strprintf("<->   type   net tp  mping   ping send recv  txn  blk  hb %*s%*s%*s ",
+            result += strprintf("<->   type   net  v  mping   ping send recv  txn  blk  hb %*s%*s%*s ",
                                 m_max_addr_processed_length, "addrp",
                                 m_max_addr_rate_limited_length, "addrl",
                                 m_max_age_length, "age");
@@ -551,7 +551,7 @@ class NetinfoRequestHandler : public BaseRequestHandler
                     peer.is_outbound ? "out" : "in",
                     ConnectionTypeForNetinfo(peer.conn_type),
                     peer.network,
-                    peer.transport_protocol_type == "detecting" ? "*" : peer.transport_protocol_type,
+                    peer.transport_protocol_type.rfind('v', 0) == 0 ? peer.transport_protocol_type[1] : ' ',
                     PingTimeToString(peer.min_ping),
                     PingTimeToString(peer.ping),
                     peer.last_send ? ToString(m_time_now - peer.last_send) : "",
@@ -660,7 +660,7 @@ class NetinfoRequestHandler : public BaseRequestHandler
         "           \"feeler\" - short-lived connection for testing addresses\n"
         "           \"addr\"   - address fetch; short-lived connection for requesting addresses\n"
         "  net      Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", \"cjdns\", or \"npr\" (not publicly routable))\n"
-        "  tp       Transport protocol used for the connection (\"v1\", \"v2\" or \"*\" if detecting)\n"
+        "  v        Version of transport protocol used for the connection\n"
         "  mping    Minimum observed ping time, in milliseconds (ms)\n"
         "  ping     Last observed ping time, in milliseconds (ms)\n"
         "  send     Time since last message sent to the peer, in seconds\n"

From ecd0a96f63bf20659add1c0901d594bfa12af641 Mon Sep 17 00:00:00 2001
From: UdjinM6 <UdjinM6@users.noreply.github.com>
Date: Wed, 16 Oct 2024 00:01:39 +0300
Subject: [PATCH 58/69] test: fix test_withdrawal_fork

---
 test/functional/feature_asset_locks.py | 49 ++++++++++++++++----------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py
index c49903f996..89bbd6caa8 100755
--- a/test/functional/feature_asset_locks.py
+++ b/test/functional/feature_asset_locks.py
@@ -45,6 +45,7 @@
 llmq_type_test = 106 # LLMQType::LLMQ_TEST_PLATFORM
 tiny_amount = int(Decimal("0.0007") * COIN)
 blocks_in_one_day = 576
+HEIGHT_DIFF_EXPIRING = 48
 
 class AssetLocksTest(DashTestFramework):
     def set_test_params(self):
@@ -272,7 +273,7 @@ def run_test(self):
         self.test_asset_unlocks(node_wallet, node, pubkey)
         self.test_withdrawal_limits(node_wallet, node, pubkey)
         self.test_mn_rr(node_wallet, node, pubkey)
-        self.test_withdrawal_fork(node_wallet, node, pubkey)
+        self.test_withdrawal_fork(node_wallet, pubkey)
 
 
     def test_asset_locks(self, node_wallet, node, pubkey):
@@ -347,7 +348,7 @@ def test_asset_unlocks(self, node_wallet, node, pubkey):
         asset_unlock_tx_duplicate_index = copy.deepcopy(asset_unlock_tx)
         # modify this tx with duplicated index to make a hash of tx different, otherwise tx would be refused too early
         asset_unlock_tx_duplicate_index.vout[0].nValue += COIN
-        too_late_height = node.getblockcount() + 48
+        too_late_height = node.getblockcount() + HEIGHT_DIFF_EXPIRING
 
         self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
         self.check_mempool_result(tx=asset_unlock_tx_too_big_fee,
@@ -640,33 +641,45 @@ def test_mn_rr(self, node_wallet, node, pubkey):
         self.generate(node, 1)
         assert_equal(locked, self.get_credit_pool_balance())
 
-    def test_withdrawal_fork(self, node_wallet, node, pubkey):
+    def test_withdrawal_fork(self, node_wallet, pubkey):
         self.log.info("Testing asset unlock after 'withdrawal' activation...")
+        assert softfork_active(node_wallet, 'withdrawals')
 
-        assert softfork_active(self.nodes[0], 'withdrawals')
-        self.log.info("Generating several txes by same quorum....")
+        index = 501
+        while True:
+            self.log.info(f"Generating new Asset Unlock tx, index={index}...")
+            asset_unlock_tx = self.create_assetunlock(index, COIN, pubkey)
+            asset_unlock_tx_payload = CAssetUnlockTx()
+            asset_unlock_tx_payload.deserialize(BytesIO(asset_unlock_tx.vExtraPayload))
 
-        asset_unlock_tx = self.create_assetunlock(501, COIN, pubkey)
-        asset_unlock_tx_payload = CAssetUnlockTx()
-        asset_unlock_tx_payload.deserialize(BytesIO(asset_unlock_tx.vExtraPayload))
+            self.log.info("Check that Asset Unlock tx is valid for current quorum")
+            self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
 
-        self.log.info("Check that new Asset Unlock is valid for current quorum")
-        self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+            quorumHash_str = format(asset_unlock_tx_payload.quorumHash, '064x')
+            assert quorumHash_str in node_wallet.quorum('list')['llmq_test_platform']
+
+            while quorumHash_str != node_wallet.quorum('list')['llmq_test_platform'][-1]:
+                self.log.info("Generate one more quorum until signing quorum becomes the last one in the list")
+                self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
+                self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
 
-        while asset_unlock_tx_payload.quorumHash in node.quorum('list')['llmq_test_platform']:
-            self.log.info(f"Generate one more quorum until signing quorum becomes not available")
+            self.log.info("Generate one more quorum after which signing quorum is gone but Asset Unlock tx is still valid")
+            assert quorumHash_str in node_wallet.quorum('list')['llmq_test_platform']
             self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
+            assert quorumHash_str not in node_wallet.quorum('list')['llmq_test_platform']
 
-            self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+            if asset_unlock_tx_payload.requestedHeight + HEIGHT_DIFF_EXPIRING > node_wallet.getblockcount():
+                self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
+                break
+            else:
+                self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': False, 'reject-reason' : 'bad-assetunlock-too-late'})
+                self.log.info("Asset Unlock tx expired, let's try again...")
+                index += 1
 
-        self.log.info(f"Generate one more quorum after which asset unlock meant to be still valid")
-        self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
-        self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': True, 'fees': {'base': Decimal(str(tiny_amount / COIN))}})
-        self.log.info(f"Generate one more quorum after which asset unlock meant to be expired")
+        self.log.info("Generate one more quorum after which signing quorum becomes too old")
         self.mine_quorum_2_nodes(llmq_type_name="llmq_test_platform", llmq_type=106)
         self.check_mempool_result(tx=asset_unlock_tx, result_expected={'allowed': False, 'reject-reason': 'bad-assetunlock-too-old-quorum'})
 
 
-
 if __name__ == '__main__':
     AssetLocksTest().main()

From f5f69ae9824fc1ada8cc7d2f3df83c54ebd79192 Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Wed, 1 Dec 2021 08:44:09 +0100
Subject: [PATCH 59/69] Merge bitcoin/bitcoin#23590: Crash debug builds when
 mempool ConsensusScriptChecks fails

faad05c6d2ad1840c26f6fdea7d358bf9ba6efea Crash debug builds when mempool ConsensusScriptChecks fails (MarcoFalke)

Pull request description:

  Currently a bug in the function might sneak around our testing infrastructure.

  Fix that by turning bugs into crashes during tests.

ACKs for top commit:
  glozow:
    utACK faad05c6d2ad1840c26f6fdea7d358bf9ba6efea, there's something seriously wrong with the code if this returns false, good to throw in debug mode

Tree-SHA512: dfea1cd9ce3f1c303f49cca1417cd5c77c6ed12849aaff7b6ab1b6060f2f0c9cf5d4689017355d11f66639bab35823f65f848e6979042fa875181509dfd5d3d7
---
 src/validation.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/validation.cpp b/src/validation.cpp
index 6521a3e611..c4f58393ba 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -825,8 +825,8 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, P
     // transactions into the mempool can be exploited as a DoS attack.
     unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus());
     if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata, m_active_chainstate.CoinsTip())) {
-        return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s",
-                __func__, hash.ToString(), state.ToString());
+        LogPrintf("BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s\n", hash.ToString(), state.ToString());
+        return Assume(false);
     }
 
     return true;

From ed7ad7af74ece54f92cea5ce04a06239d542b658 Mon Sep 17 00:00:00 2001
From: merge-script <falke.marco@gmail.com>
Date: Mon, 27 Sep 2021 10:12:14 +0200
Subject: [PATCH 60/69] Merge bitcoin/bitcoin#23061: Fix (inverse) meaning of
 -persistmempool

faa9c19a4b27e7fabf7c5deae1b4c4ca612ed01a doc: Add 23061 release notes (MarcoFalke)
faff17bbde6dcb1482a6210bc48b3192603a446f Fix (inverse) meaning of -persistmempool (MarcoFalke)

Pull request description:

  Passing `-persistmempool` is currently treated as `-nopersistmempool`

ACKs for top commit:
  jnewbery:
    reACK faa9c19a4b27e7fabf7c5deae1b4c4ca612ed01a
  hebasto:
    ACK faa9c19a4b27e7fabf7c5deae1b4c4ca612ed01a, I have reviewed the code and it looks OK, I agree it can be merged.

Tree-SHA512: f34a89a07745dabe340eb845b2a348b79c093e9056f7a21c17e1ba2e278177c9b4cf30e8095791fd645a7f90eb34850b2eee0c869b4f6ec02bf749c73b0e52ee
---
 test/functional/mempool_persist.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py
index a40c7f4e72..bc9a6d949c 100755
--- a/test/functional/mempool_persist.py
+++ b/test/functional/mempool_persist.py
@@ -154,7 +154,7 @@ def run_test(self):
         self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions")
         os.rename(mempooldat0, mempooldat1)
         self.stop_nodes()
-        self.start_node(1, extra_args=[])
+        self.start_node(1, extra_args=["-persistmempool"])
         assert self.nodes[1].getmempoolinfo()["loaded"]
         assert_equal(len(self.nodes[1].getrawmempool()), 6)
 

From 7d170bd1ccf68796c0d1a4d99547f139f23ddbea Mon Sep 17 00:00:00 2001
From: MarcoFalke <falke.marco@gmail.com>
Date: Tue, 7 Dec 2021 18:44:57 +0100
Subject: [PATCH 61/69] Merge bitcoin/bitcoin#23694: doc: Add missing optional
 to MempoolEntryDescription

fa1571b156dfa757d0ce0fddd53c25256196522b doc: Add missing optional to MempoolEntryDescription (MarcoFalke)

Pull request description:

  Needed for https://github.com/bitcoin/bitcoin/pull/23083.

  Can be reviewed with `--word-diff-regex=.`.

ACKs for top commit:
  josibake:
    ACK https://github.com/bitcoin/bitcoin/pull/23694/commits/fa1571b156dfa757d0ce0fddd53c25256196522b
  shaavan:
    ACK fa1571b156dfa757d0ce0fddd53c25256196522b

Tree-SHA512: b4370003d2aeadce438778e15bd9a0d6a7fef4711acbe8471a50a9d72bbf74e1705fecbaae6f7eb367ece7c795a816c4b8b6583ed6c8f91b35621ca30fd95c18
---
 src/rpc/blockchain.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 6bf41d7ba8..e3cf7f3717 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -458,16 +458,16 @@ static RPCHelpMan getdifficulty()
 
 static std::vector<RPCResult> MempoolEntryDescription() { return {
     RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size. This can be different from actual serialized size for high-sigop transactions."},
-    RPCResult{RPCResult::Type::STR_AMOUNT, "fee", "transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)"},
-    RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", "transaction fee with fee deltas used for mining priority (DEPRECATED)"},
+    RPCResult{RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, "transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)"},
+    RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", /*optional=*/true, "transaction fee with fee deltas used for mining priority (DEPRECATED)"},
     RPCResult{RPCResult::Type::NUM_TIME, "time", "local time transaction entered pool in " + UNIX_EPOCH_TIME},
     RPCResult{RPCResult::Type::NUM, "height", "block height when transaction entered pool"},
     RPCResult{RPCResult::Type::NUM, "descendantcount", "number of in-mempool descendant transactions (including this one)"},
     RPCResult{RPCResult::Type::NUM, "descendantsize", "size of in-mempool descendants (including this one)"},
-    RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees", "modified fees (see above) of in-mempool descendants (including this one) (DEPRECATED)"},
+    RPCResult{RPCResult::Type::STR_AMOUNT, "descendantfees",  /*optional=*/true, "modified fees (see above) of in-mempool descendants (including this one) (DEPRECATED)"},
     RPCResult{RPCResult::Type::NUM, "ancestorcount", "number of in-mempool ancestor transactions (including this one)"},
     RPCResult{RPCResult::Type::NUM, "ancestorsize", "size of in-mempool ancestors (including this one)"},
-    RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", "modified fees (see above) of in-mempool ancestors (including this one) (DEPRECATED)"},
+    RPCResult{RPCResult::Type::STR_AMOUNT, "ancestorfees", /*optional=*/true, "modified fees (see above) of in-mempool ancestors (including this one) (DEPRECATED)"},
     RPCResult{RPCResult::Type::OBJ, "fees", "",
     {
         RPCResult{RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT},

From d69030951389d5d2c160e13024c86ca815a5a5e4 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Thu, 17 Oct 2024 03:03:09 +0700
Subject: [PATCH 62/69] fix: limit amount of attempts for test
 `test_withdrawal_fork`

---
 test/functional/feature_asset_locks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py
index 89bbd6caa8..f802edd056 100755
--- a/test/functional/feature_asset_locks.py
+++ b/test/functional/feature_asset_locks.py
@@ -646,7 +646,7 @@ def test_withdrawal_fork(self, node_wallet, pubkey):
         assert softfork_active(node_wallet, 'withdrawals')
 
         index = 501
-        while True:
+        while index < 511:
             self.log.info(f"Generating new Asset Unlock tx, index={index}...")
             asset_unlock_tx = self.create_assetunlock(index, COIN, pubkey)
             asset_unlock_tx_payload = CAssetUnlockTx()

From 96c9b469cad439f2e2489a2a15d30d7a02e195cd Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 18:56:14 +0700
Subject: [PATCH 63/69] fix: getblock for withdrawal transaction if verbosity
 level is 2

---
 src/core_write.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/core_write.cpp b/src/core_write.cpp
index ffad7897ff..8999f9406f 100644
--- a/src/core_write.cpp
+++ b/src/core_write.cpp
@@ -334,7 +334,10 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add
     }
 
     if (calculate_fee) {
-        const CAmount fee = amt_total_in - amt_total_out;
+        CAmount fee = amt_total_in - amt_total_out;
+        if (tx.nType == TRANSACTION_ASSET_UNLOCK) {
+            fee = CHECK_NONFATAL(GetTxPayload<CAssetUnlockPayload>(tx))->getFee();
+        }
         CHECK_NONFATAL(MoneyRange(fee));
         entry.pushKV("fee", ValueFromAmount(fee));
     }

From ab7172bc8f0197d278ab1e4d94aa0dc66ef18f3a Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 19:20:49 +0700
Subject: [PATCH 64/69] fix: getblockstats rpc to work with withdrawal
 transactions

---
 src/rpc/blockchain.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 411d6cb75d..fe44b5f2c3 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -49,6 +49,7 @@
 #include <versionbits.h>
 #include <warnings.h>
 
+#include <evo/assetlocktx.h>
 #include <evo/cbtx.h>
 #include <evo/evodb.h>
 #include <evo/mnhftx.h>
@@ -2450,6 +2451,11 @@ static RPCHelpMan getblockstats()
             }
 
             CAmount txfee = tx_total_in - tx_total_out;
+
+            if (tx->nType == TRANSACTION_ASSET_UNLOCK) {
+                txfee = CHECK_NONFATAL(GetTxPayload<CAssetUnlockPayload>(*tx))->getFee();
+            }
+
             CHECK_NONFATAL(MoneyRange(txfee));
             if (do_medianfee) {
                 fee_array.push_back(txfee);

From b0d06f0b5fc8f23625dcc08aed839b58f4fac559 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 18:58:07 +0700
Subject: [PATCH 65/69] feat: add regression test for `getblock` and
 `getblockstats` for withdrawal fee calculation failure

---
 test/functional/feature_asset_locks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py
index f802edd056..45a1608958 100755
--- a/test/functional/feature_asset_locks.py
+++ b/test/functional/feature_asset_locks.py
@@ -407,6 +407,9 @@ def test_asset_unlocks(self, node_wallet, node, pubkey):
         self.mempool_size -= 2
         self.check_mempool_size()
         block_asset_unlock = node.getrawtransaction(asset_unlock_tx.rehash(), 1)['blockhash']
+        self.log.info("Checking rpc `getblock` and `getblockstats` succeeds as they use own fee calculation mechanism")
+        assert_equal(node.getblockstats(node.getblockcount())['maxfee'], tiny_amount)
+        node.getblock(block_asset_unlock, 2)
 
         self.send_tx(asset_unlock_tx,
             expected_error = "Transaction already in block chain",

From e498378eb79332f576931c1ad06d3f42f3dc5c54 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 20:16:22 +0700
Subject: [PATCH 66/69] feat: add test for fee in getmempoolentry

---
 test/functional/feature_asset_locks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/functional/feature_asset_locks.py b/test/functional/feature_asset_locks.py
index 45a1608958..ff6ac61289 100755
--- a/test/functional/feature_asset_locks.py
+++ b/test/functional/feature_asset_locks.py
@@ -370,6 +370,7 @@ def test_asset_unlocks(self, node_wallet, node, pubkey):
         self.wait_for_sporks_same()
 
         txid = self.send_tx(asset_unlock_tx)
+        assert_equal(node.getmempoolentry(txid)['fee'], Decimal("0.0007"))
         is_id = node_wallet.sendtoaddress(node_wallet.getnewaddress(), 1)
         for node in self.nodes:
             self.wait_for_instantlock(is_id, node)

From f6169fade4303284aa97aba7f3869c761e0ab242 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 20:40:50 +0700
Subject: [PATCH 67/69] fix: make composite rpc 'masternode payments' to work
 with withdrawals

---
 src/rpc/masternode.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/rpc/masternode.cpp b/src/rpc/masternode.cpp
index ee5fcbdea0..cecaa6cbfb 100644
--- a/src/rpc/masternode.cpp
+++ b/src/rpc/masternode.cpp
@@ -3,6 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <chainparams.h>
+#include <evo/assetlocktx.h>
 #include <evo/chainhelper.h>
 #include <evo/deterministicmns.h>
 #include <governance/classes.h>
@@ -406,6 +407,11 @@ static RPCHelpMan masternode_payments()
             if (tx->IsCoinBase()) {
                 continue;
             }
+            if (tx->IsPlatformTransfer()) {
+                nBlockFees += CHECK_NONFATAL(GetTxPayload<CAssetUnlockPayload>(*tx))->getFee();
+                continue;
+            }
+
             CAmount nValueIn{0};
             for (const auto& txin : tx->vin) {
                 uint256 blockHashTmp;

From b9a46f6d2c14a07803cc26cf79b3a0d97fe8e486 Mon Sep 17 00:00:00 2001
From: Konstantin Akimov <knstqq@gmail.com>
Date: Tue, 22 Oct 2024 21:48:05 +0700
Subject: [PATCH 68/69] refactor: use IsPlatformTransfer in core_write and
 rpc/blockchain

---
 src/core_write.cpp     | 2 +-
 src/rpc/blockchain.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core_write.cpp b/src/core_write.cpp
index 8999f9406f..2edee784bb 100644
--- a/src/core_write.cpp
+++ b/src/core_write.cpp
@@ -335,7 +335,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add
 
     if (calculate_fee) {
         CAmount fee = amt_total_in - amt_total_out;
-        if (tx.nType == TRANSACTION_ASSET_UNLOCK) {
+        if (tx.IsPlatformTransfer()) {
             fee = CHECK_NONFATAL(GetTxPayload<CAssetUnlockPayload>(tx))->getFee();
         }
         CHECK_NONFATAL(MoneyRange(fee));
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index fe44b5f2c3..409d24c879 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -2452,7 +2452,7 @@ static RPCHelpMan getblockstats()
 
             CAmount txfee = tx_total_in - tx_total_out;
 
-            if (tx->nType == TRANSACTION_ASSET_UNLOCK) {
+            if (tx->IsPlatformTransfer()) {
                 txfee = CHECK_NONFATAL(GetTxPayload<CAssetUnlockPayload>(*tx))->getFee();
             }
 

From cd712e86b7ea11fe3c5ce13107beec089514911c Mon Sep 17 00:00:00 2001
From: pasta <pasta@dashboost.org>
Date: Tue, 22 Oct 2024 09:57:23 -0500
Subject: [PATCH 69/69] ci: attest results of guix builds

---
 .github/workflows/guix-build.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/guix-build.yml b/.github/workflows/guix-build.yml
index 1bd3d2a2eb..1f7722abef 100644
--- a/.github/workflows/guix-build.yml
+++ b/.github/workflows/guix-build.yml
@@ -2,6 +2,8 @@ name: Guix Build
 
 permissions:
   packages: write
+  id-token: write
+  attestations: write
 
 on:
   pull_request_target:
@@ -127,3 +129,7 @@ jobs:
           path: |
             ${{ github.workspace }}/dash/guix-build*/output/${{ matrix.build_target }}/
 
+      - name: Attest build provenance
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: ${{ github.workspace }}/dash/guix-build*/output/${{ matrix.build_target }}/*