feat!: add new header field (#6686)

Description
---
Add new header field, block output mmr which only tracks outputs in
current block

Motivation and Context
---
This is needed in order to validate that all outputs included in the
block is correctly captured in the pow of the header. The current
strategy is to only use the output smt, which commits to all outputs on
the blockchain. This is required for horizon sync and prune mode to
verify the blockchain as we sync.
But someone else wants to verify the block outputs, they have to have
the entire outputs set aka entire blockchain in order to verify that the
outputs are indeed locked behind pow.
This addition allows any person to take only the block and know that the
outputs contained in that block, is correctly locked behind the pow of
the header.

The output field is constructed as two merkle mountain ranges.
the primary one used in the header field is constructed as
[coinbase_1, coinbase_2, .., coinbase_x,
non_coinbase_ouputs_merkle_root]
with the non_coinbase_ouputs_merkle_root constructed as
[output_1, output_2, .., output_x]

This double mountain range approach allows you to prove the output are
in the block header, as well as create a proof for a specific output.
But separating them allows p2pool to only need to store a single hash +
list of all outputs in order to verify all coin bases are correctly
included in the header. Rather than all outputs or before this pr, the
entire block.

Breaking change
---
Need to resync database

applications/minotari_app_grpc/proto/block.proto

@@ -38,9 +38,10 @@ message BlockHeader {
     bytes prev_hash = 4;
     // Timestamp at which the block was built.
     uint64 timestamp = 5;
-    // This is the UTXO merkle root of the outputs
-    // This is calculated as Hash (txo MMR root  || roaring bitmap hash of UTXO indices)
+    // This is the UTXO merkle root of the outputs in the blockchain
     bytes output_mr = 6;
+    // This is the merkle root of all outputs in this block
+    bytes block_output_mr = 7;
     // This is the MMR root of the kernels
     bytes kernel_mr = 8;
     // This is the Merkle root of the inputs in this block

applications/minotari_app_grpc/src/conversions/block_header.rs

@@ -39,6 +39,7 @@ impl From<BlockHeader> for grpc::BlockHeader {
             timestamp: h.timestamp.as_u64(),
             input_mr: h.input_mr.to_vec(),
             output_mr: h.output_mr.to_vec(),
+            block_output_mr: h.block_output_mr.to_vec(),
             output_mmr_size: h.output_smt_size,
             kernel_mr: h.kernel_mr.to_vec(),
             kernel_mmr_size: h.kernel_mmr_size,
@@ -76,6 +77,7 @@ impl TryFrom<grpc::BlockHeader> for BlockHeader {
             timestamp: EpochTime::from(header.timestamp),
             input_mr: FixedHash::try_from(header.input_mr).map_err(|err| err.to_string())?,
             output_mr: FixedHash::try_from(header.output_mr).map_err(|err| err.to_string())?,
+            block_output_mr: FixedHash::try_from(header.block_output_mr).unwrap_or_default(),
             output_smt_size: header.output_mmr_size,
             kernel_mr: FixedHash::try_from(header.kernel_mr).map_err(|err| err.to_string())?,
             kernel_mmr_size: header.kernel_mmr_size,

base_layer/core/src/base_node/sync/header_sync/validator.rs

@@ -245,13 +245,18 @@ mod test {
         test_helpers::blockchain::{create_new_blockchain, TempDatabase},
     };
 
-    fn setup() -> (BlockHeaderSyncValidator<TempDatabase>, AsyncBlockchainDb<TempDatabase>) {
+    fn setup() -> (
+        BlockHeaderSyncValidator<TempDatabase>,
+        AsyncBlockchainDb<TempDatabase>,
+        ConsensusManager,
+    ) {
         let rules = ConsensusManager::builder(Network::LocalNet).build().unwrap();
         let randomx_factory = RandomXFactory::default();
         let db = create_new_blockchain();
         (
-            BlockHeaderSyncValidator::new(db.clone().into(), rules, randomx_factory),
+            BlockHeaderSyncValidator::new(db.clone().into(), rules.clone(), randomx_factory),
             db.into(),
+            rules,
         )
     }
 
@@ -262,10 +267,11 @@ mod test {
         AsyncBlockchainDb<TempDatabase>,
         ChainHeader,
     ) {
-        let (validator, db) = setup();
+        let (validator, db, cm) = setup();
         let mut tip = db.fetch_tip_header().await.unwrap();
         for _ in 0..n {
             let mut header = BlockHeader::from_previous(tip.header());
+            header.version = cm.consensus_constants(header.height).blockchain_version();
             // Needed to have unique keys for the blockchain db mmr count indexes (MDB_KEY_EXIST error)
             header.kernel_mmr_size += 1;
             header.output_smt_size += 1;
@@ -301,7 +307,7 @@ mod test {
 
         #[tokio::test]
         async fn it_errors_if_hash_does_not_exist() {
-            let (mut validator, _) = setup();
+            let (mut validator, _, _cm) = setup();
             let start_hash = vec![0; 32];
             let err = validator
                 .initialize_state(&start_hash.clone().try_into().unwrap())

base_layer/core/src/blocks/block_header.rs

@@ -98,9 +98,10 @@ pub struct BlockHeader {
     pub timestamp: EpochTime,
     /// This is the Merkle root of the inputs in this block
     pub input_mr: FixedHash,
-    /// This is the UTXO merkle root of the outputs
-    /// This is calculated as Hash (txo MMR root  || roaring bitmap hash of UTXO indices)
+    /// This is the UTXO merkle root of the outputs on the blockchain
     pub output_mr: FixedHash,
+    /// This is the block_output_mr
+    pub block_output_mr: FixedHash,
     /// The size (number  of leaves) of the output and range proof MMRs at the time of this header
     pub output_smt_size: u64,
     /// This is the MMR root of the kernels
@@ -130,6 +131,7 @@ impl BlockHeader {
             prev_hash: FixedHash::zero(),
             timestamp: EpochTime::now(),
             output_mr: FixedHash::zero(),
+            block_output_mr: FixedHash::zero(),
             output_smt_size: 0,
             kernel_mr: FixedHash::zero(),
             kernel_mmr_size: 0,
@@ -164,6 +166,7 @@ impl BlockHeader {
             timestamp: EpochTime::now(),
             output_mr: FixedHash::zero(),
             output_smt_size: prev.output_smt_size,
+            block_output_mr: FixedHash::zero(),
             kernel_mr: FixedHash::zero(),
             kernel_mmr_size: prev.kernel_mmr_size,
             input_mr: FixedHash::zero(),
@@ -222,7 +225,7 @@ impl BlockHeader {
     /// Provides a mining hash of the header, used for the mining.
     /// This differs from the normal hash by not hashing the nonce and kernel pow.
     pub fn mining_hash(&self) -> FixedHash {
-        DomainSeparatedConsensusHasher::<BlocksHashDomain, Blake2b<U32>>::new("block_header")
+        let incomplete = DomainSeparatedConsensusHasher::<BlocksHashDomain, Blake2b<U32>>::new("block_header")
             .chain(&self.version)
             .chain(&self.height)
             .chain(&self.prev_hash)
@@ -235,9 +238,12 @@ impl BlockHeader {
             .chain(&self.total_kernel_offset)
             .chain(&self.total_script_offset)
             .chain(&self.validator_node_mr)
-            .chain(&self.validator_node_size)
-            .finalize()
-            .into()
+            .chain(&self.validator_node_size);
+
+        match self.version {
+            0 => incomplete.finalize().into(),
+            _ => incomplete.chain(&self.block_output_mr).finalize().into(),
+        }
     }
 
     pub fn merge_mining_hash(&self) -> FixedHash {
@@ -273,6 +279,7 @@ impl From<NewBlockHeaderTemplate> for BlockHeader {
             prev_hash: header_template.prev_hash,
             timestamp: EpochTime::now(),
             output_mr: FixedHash::zero(),
+            block_output_mr: FixedHash::zero(),
             output_smt_size: 0,
             kernel_mr: FixedHash::zero(),
             kernel_mmr_size: 0,

base_layer/core/src/blocks/genesis_block.rs

@@ -191,6 +191,8 @@ pub fn get_nextnet_genesis_block() -> ChainBlock {
     // TODO: Fix this hack with the next nextnet reset!!
     block.header.input_mr =
         FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();
+    block.header.block_output_mr =
+        FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();
 
     // Add pre-mine utxos - enable/disable as required
     let add_pre_mine_utxos = false;
@@ -270,6 +272,8 @@ pub fn get_mainnet_genesis_block() -> ChainBlock {
             FixedHash::from_hex("b7b38b76f5832b5b63691a8334dfa67d8c762b77b2b4aa4f648c4eb1dfb25c1e").unwrap();
         block.header.output_mr =
             FixedHash::from_hex("a77ecf05b20c426d3d400a63397be6c622843c66d5751ecbe3390c8a4885158e").unwrap();
+        block.header.block_output_mr =
+            FixedHash::from_hex("91e997520b0eee770914334692080f92d18db434d373561f8842c56d70c11b97").unwrap();
         block.header.validator_node_mr =
             FixedHash::from_hex("277da65c40b2cf99db86baedb903a3f0a38540f3a94d40c826eecac7e27d5dfc").unwrap();
     }
@@ -371,6 +375,10 @@ pub fn get_esmeralda_genesis_block() -> ChainBlock {
     // lets get the block
     let mut block = get_esmeralda_genesis_block_raw();
 
+    // TODO: Fix this hack with the next esme reset!!
+    block.header.block_output_mr =
+        FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();
+
     // Add pre-mine utxos - enable/disable as required
     let add_pre_mine_utxos = true;
     if add_pre_mine_utxos {
@@ -482,7 +490,9 @@ fn get_raw_block(genesis_timestamp: &DateTime<FixedOffset>, not_before_proof: &P
             height: 0,
             prev_hash: FixedHash::zero(),
             timestamp: timestamp.into(),
-            output_mr: FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap(),
+            output_mr: FixedHash::zero(),
+            block_output_mr: FixedHash::from_hex("622720a6571c33d6bf6138d9e737d3468c77f1193640698ad459953d24ec0812")
+                .unwrap(),
             output_smt_size: 0,
             kernel_mr: FixedHash::from_hex("c14803066909d6d22abf0d2d2782e8936afc3f713f2af3a4ef5c42e8400c1303").unwrap(),
             kernel_mmr_size: 0,
@@ -518,6 +528,7 @@ mod test {
 
     use super::*;
     use crate::{
+        block_output_mr_hash_from_pruned_mmr,
         chain_storage::calculate_validator_node_mr,
         consensus::ConsensusManager,
         test_helpers::blockchain::create_new_blockchain_with_network,
@@ -527,8 +538,8 @@ mod test {
         },
         validation::{ChainBalanceValidator, FinalHorizonStateValidation},
         KernelMmr,
+        PrunedOutputMmr,
     };
-
     #[test]
     #[serial]
     fn esmeralda_genesis_sanity_check() {
@@ -661,9 +672,15 @@ mod test {
             kernel_mmr.push(k.hash().to_vec()).unwrap();
         }
         let mut output_smt = OutputSmt::new();
-
+        let mut block_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
+        let mut normal_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
         let mut vn_nodes = Vec::new();
         for o in block.block().body.outputs() {
+            if o.features.is_coinbase() {
+                block_output_mmr.push(o.hash().to_vec()).unwrap();
+            } else {
+                normal_output_mmr.push(o.hash().to_vec()).unwrap();
+            }
             let smt_key = NodeKey::try_from(o.commitment.as_bytes()).unwrap();
             let smt_node = ValueHash::try_from(o.smt_hash(block.header().height).as_slice()).unwrap();
             output_smt.insert(smt_key, smt_node).unwrap();
@@ -681,6 +698,11 @@ mod test {
                 ));
             }
         }
+
+        block_output_mmr
+            .push(normal_output_mmr.get_merkle_root().unwrap().to_vec())
+            .unwrap();
+
         for i in block.block().body.inputs() {
             let smt_key = NodeKey::try_from(i.commitment().unwrap().as_bytes()).unwrap();
             output_smt.delete(&smt_key).unwrap();
@@ -718,6 +740,25 @@ mod test {
             output_mr_hash_from_smt(&mut output_smt).unwrap().to_vec().to_hex(),
             block.header().output_mr.to_vec().to_hex(),
         );
+
+        // TODO: Fix this hack with the next nextnet/esme release reset!!
+        if network == Network::NextNet || network == Network::Esmeralda {
+            assert_eq!(
+                FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000")
+                    .unwrap()
+                    .to_vec()
+                    .to_hex(),
+                block.header().block_output_mr.to_vec().to_hex(),
+            );
+        } else {
+            assert_eq!(
+                block_output_mr_hash_from_pruned_mmr(&block_output_mmr)
+                    .unwrap()
+                    .to_vec()
+                    .to_hex(),
+                block.header().block_output_mr.to_vec().to_hex(),
+            );
+        }
         if network == Network::NextNet {
             // TODO: Fix this hack with the next nextnet reset!!
             assert_eq!(

base_layer/core/src/chain_storage/blockchain_database.rs

@@ -49,6 +49,7 @@ use tari_utilities::{epoch_time::EpochTime, hex::Hex, ByteArray};
 
 use super::TemplateRegistrationEntry;
 use crate::{
+    block_output_mr_hash_from_pruned_mmr,
     blocks::{
         Block,
         BlockAccumulatedData,
@@ -105,6 +106,7 @@ use crate::{
     OutputSmt,
     PrunedInputMmr,
     PrunedKernelMmr,
+    PrunedOutputMmr,
     ValidatorNodeBMT,
 };
 
@@ -913,6 +915,7 @@ where B: BlockchainBackend
         block.header.kernel_mmr_size = roots.kernel_mmr_size;
         block.header.input_mr = roots.input_mr;
         block.header.output_mr = roots.output_mr;
+        block.header.block_output_mr = roots.block_output_mr;
         block.header.output_smt_size = roots.output_smt_size;
         block.header.validator_node_mr = roots.validator_node_mr;
         block.header.validator_node_size = roots.validator_node_size;
@@ -1322,6 +1325,7 @@ pub struct MmrRoots {
     pub kernel_mmr_size: u64,
     pub input_mr: FixedHash,
     pub output_mr: FixedHash,
+    pub block_output_mr: FixedHash,
     pub output_smt_size: u64,
     pub validator_node_mr: FixedHash,
     pub validator_node_size: u64,
@@ -1334,6 +1338,7 @@ impl std::fmt::Display for MmrRoots {
         writeln!(f, "Kernel MR       : {}", self.kernel_mr)?;
         writeln!(f, "Kernel MMR Size : {}", self.kernel_mmr_size)?;
         writeln!(f, "Output MR       : {}", self.output_mr)?;
+        writeln!(f, "Block Output MR : {}", self.block_output_mr)?;
         writeln!(f, "Output SMT Size : {}", self.output_smt_size)?;
         writeln!(f, "Validator MR    : {}", self.validator_node_mr)?;
         Ok(())
@@ -1373,13 +1378,20 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
 
     let mut kernel_mmr = PrunedKernelMmr::new(kernels);
     let mut input_mmr = PrunedInputMmr::new(PrunedHashSet::default());
+    let mut block_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
+    let mut normal_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
 
     for kernel in body.kernels() {
         kernel_mmr.push(kernel.hash().to_vec())?;
     }
 
     let mut outputs_to_remove = Vec::new();
     for output in body.outputs() {
+        if output.features.is_coinbase() {
+            block_output_mmr.push(output.hash().to_vec())?;
+        } else {
+            normal_output_mmr.push(output.hash().to_vec())?;
+        }
         if !output.is_burned() {
             let smt_key = NodeKey::try_from(output.commitment.as_bytes())?;
             let smt_node = ValueHash::try_from(output.smt_hash(header.height).as_slice())?;
@@ -1394,6 +1406,7 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
             }
         }
     }
+    block_output_mmr.push(normal_output_mmr.get_merkle_root()?.to_vec())?;
 
     let mut outputs_to_add = Vec::new();
     for input in body.inputs() {
@@ -1427,11 +1440,18 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
         (tip_header.validator_node_mr, 0)
     };
 
+    let block_output_mr = if block.version() > 0 {
+        block_output_mr_hash_from_pruned_mmr(&block_output_mmr)?
+    } else {
+        FixedHash::zero()
+    };
+
     let mmr_roots = MmrRoots {
         kernel_mr: kernel_mr_hash_from_pruned_mmr(&kernel_mmr)?,
         kernel_mmr_size: kernel_mmr.get_leaf_count()? as u64,
         input_mr: input_mr_hash_from_pruned_mmr(&input_mmr)?,
         output_mr: output_mr_hash_from_smt(output_smt)?,
+        block_output_mr,
         output_smt_size: output_smt.size(),
         validator_node_mr,
         validator_node_size: validator_node_size as u64,