feat!: implement command inclusion proof for eviction

Cargo.toml

@@ -27,6 +27,7 @@ members = [
     "common_sqlite",
     "infrastructure/libtor",
     "infrastructure/metrics",
+    "infrastructure/jellyfish",
     "infrastructure/shutdown",
     "infrastructure/storage",
     "infrastructure/tari_script",
@@ -44,6 +45,7 @@ members = [
 ]
 
 [workspace.dependencies]
+tari_jellyfish = { path = "infrastructure/jellyfish" }
 tari_comms = { path = "comms/core" }
 tari_comms_dht = { path = "comms/dht", default-features = false }
 tari_common = { path = "common" }

applications/minotari_app_grpc/proto/sidechain_types.proto

@@ -97,6 +97,7 @@ message CommitProof {
 message CommitProofV1 {
   bytes command = 1;
   SidechainBlockCommitProof commit_proof = 2;
+  bytes encoded_inclusion_proof = 3;
 }
 
 message SidechainBlockCommitProof {

applications/minotari_app_grpc/src/conversions/sidechain_feature.rs

@@ -361,6 +361,9 @@ impl TryFrom<grpc::CommitProofV1> for CommandCommitProofV1<EvictNodeAtom> {
         Ok(CommandCommitProofV1 {
             command: command.try_into()?,
             commit_proof: value.commit_proof.ok_or("commit_proof not provided")?.try_into()?,
+
+            inclusion_proof: borsh::from_slice(&value.encoded_inclusion_proof)
+                .map_err(|e| format!("Failed to decode SparseMerkleProofExt: {e}"))?,
         })
     }
 }
@@ -370,6 +373,10 @@ impl From<&CommandCommitProofV1<EvictNodeAtom>> for grpc::CommitProofV1 {
         Self {
             command: grpc::EvictAtom::from(value.command()).encode_to_vec(),
             commit_proof: Some(value.commit_proof().into()),
+            // Encode since the type is complex
+            // TODO: making this fallible is a pain - we may need to implement the proto for this
+            encoded_inclusion_proof: borsh::to_vec(value.inclusion_proof())
+                .expect("Failed to encode SparseMerkleProofExt"),
         }
     }
 }

base_layer/core/src/proto/sidechain_feature.proto

@@ -80,6 +80,7 @@ message CommitProof {
 message CommitProofV1 {
   bytes command = 1;
   SidechainBlockCommitProof commit_proof = 2;
+  bytes encoded_inclusion_proof = 3;
 }
 
 message SidechainBlockCommitProof {

base_layer/core/src/proto/sidechain_feature.rs

@@ -355,15 +355,22 @@ impl TryFrom<proto::types::CommitProofV1> for CommandCommitProofV1<EvictNodeAtom
         Ok(CommandCommitProofV1 {
             command: command.try_into()?,
             commit_proof: value.commit_proof.ok_or("commit_proof not provided")?.try_into()?,
+            inclusion_proof: borsh::from_slice(&value.encoded_inclusion_proof)
+                .map_err(|e| format!("Failed to decode SparseMerkleProofExt: {e}"))?,
         })
     }
 }
 
 impl From<&CommandCommitProofV1<EvictNodeAtom>> for proto::types::CommitProofV1 {
     fn from(value: &CommandCommitProofV1<EvictNodeAtom>) -> Self {
         Self {
+            // Encode since command is generic
             command: proto::types::EvictAtom::from(value.command()).encode_to_vec(),
             commit_proof: Some(value.commit_proof().into()),
+            // Encode since the type is complex
+            // TODO: making this fallible is a pain - we may need to implement the proto for this
+            encoded_inclusion_proof: borsh::to_vec(value.inclusion_proof())
+                .expect("Failed to encode SparseMerkleProofExt"),
         }
     }
 }

base_layer/sidechain/Cargo.toml

@@ -9,6 +9,7 @@ tari_hashing = { workspace = true }
 tari_crypto = { version = "0.21.0", features = ["borsh"] }
 tari_utilities = "0.8"
 tari_common_types = { workspace = true }
+tari_jellyfish = { workspace = true }
 
 log = "0.4.22"
 thiserror = "2.0"

base_layer/sidechain/src/commit_proof.rs

@@ -12,6 +12,7 @@ use tari_hashing::{
     layer2::{block_hasher, vote_signature_hasher},
     ValidatorNodeHashDomain,
 };
+use tari_jellyfish::{LeafKey, SparseMerkleProofExt, TreeHash};
 
 use super::error::SidechainProofValidationError;
 use crate::{
@@ -29,8 +30,12 @@ pub enum CommandCommitProof<C> {
 }
 
 impl<C: ToCommand> CommandCommitProof<C> {
-    pub fn new(command: C, commit_proof: SidechainBlockCommitProof) -> Self {
-        Self::V1(CommandCommitProofV1 { command, commit_proof })
+    pub fn new(command: C, commit_proof: SidechainBlockCommitProof, inclusion_proof: SparseMerkleProofExt) -> Self {
+        Self::V1(CommandCommitProofV1 {
+            command,
+            commit_proof,
+            inclusion_proof,
+        })
     }
 
     pub fn command(&self) -> &C {
@@ -71,10 +76,9 @@ impl<C: ToCommand> CommandCommitProof<C> {
 
 #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, BorshSerialize, BorshDeserialize)]
 pub struct CommandCommitProofV1<C> {
-    // TODO: Implement MerkleProof
-    // command_merkle_proof: MerkleProof,
     pub command: C,
     pub commit_proof: SidechainBlockCommitProof,
+    pub inclusion_proof: SparseMerkleProofExt,
 }
 
 impl<C: ToCommand> CommandCommitProofV1<C> {
@@ -86,12 +90,26 @@ impl<C: ToCommand> CommandCommitProofV1<C> {
         &self.commit_proof
     }
 
+    pub fn inclusion_proof(&self) -> &SparseMerkleProofExt {
+        &self.inclusion_proof
+    }
+
+    fn validate_inclusion_proof(&self, command: &Command) -> Result<(), SidechainProofValidationError> {
+        let command_hash = TreeHash::new(command.hash().into_array());
+        // Command JMT uses an identity mapping between hashes and keys.
+        let key = LeafKey::new(command_hash);
+        let root_hash = TreeHash::new(self.commit_proof.header.command_merkle_root.into_array());
+        self.inclusion_proof.verify_inclusion(&root_hash, &key, &command_hash)?;
+        Ok(())
+    }
+
     pub fn validate_committed(
         &self,
         quorum_threshold: usize,
         check_vn: &CheckVnFunc<'_>,
     ) -> Result<(), SidechainProofValidationError> {
         let command = self.command.to_command();
+        self.validate_inclusion_proof(&command)?;
         self.commit_proof
             .validate_committed(&command, quorum_threshold, check_vn)
     }

base_layer/sidechain/src/error.rs

@@ -9,6 +9,8 @@ pub enum SidechainProofValidationError {
     InvalidProof { details: String },
     #[error("Internal error: {details}")]
     InternalError { details: String },
+    #[error("Jellyfish proof verification error: {0}")]
+    JmtProofVerifyError(#[from] tari_jellyfish::JmtProofVerifyError),
 }
 
 impl SidechainProofValidationError {

hashing/src/layer2.rs

@@ -19,6 +19,10 @@ pub fn tari_hasher64<M: DomainSeparation>(label: &'static str) -> TariDomainHash
     TariDomainHasher::<M, U64>::new_with_label(label)
 }
 
+pub fn tari_hasher32<M: DomainSeparation>(label: &'static str) -> TariDomainHasher<M, U32> {
+    TariDomainHasher::<M, U32>::new_with_label(label)
+}
+
 fn tari_consensus_hasher(label: &'static str) -> TariConsensusHasher {
     TariConsensusHasher::new_with_label(label)
 }

infrastructure/jellyfish/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "tari_jellyfish"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+
+[dependencies]
+tari_hashing = { workspace = true }
+tari_crypto = "0.21.0"
+
+serde = { version = "1.0", features = ["derive"] }
+borsh = "1.5"
+digest = "0.10"
+thiserror = "2.0"
+indexmap = { version = "2.6", features = ["serde"] }

infrastructure/jellyfish/src/bit_iter.rs

@@ -0,0 +1,54 @@
+//   Copyright 2024 The Tari Project
+//   SPDX-License-Identifier: BSD-3-Clause
+
+use std::ops::Range;
+
+/// An iterator over a hash value that generates one bit for each iteration.
+pub struct BitIterator<'a> {
+    /// The reference to the bytes that represent the `HashValue`.
+    bytes: &'a [u8],
+    pos: Range<usize>,
+    // invariant hash_bytes.len() == HashValue::LENGTH;
+    // invariant pos.end == hash_bytes.len() * 8;
+}
+
+impl<'a> BitIterator<'a> {
+    /// Constructs a new `BitIterator` using given `HashValue`.
+    pub fn new(bytes: &'a [u8]) -> Self {
+        BitIterator {
+            bytes,
+            pos: 0..bytes.len() * 8,
+        }
+    }
+
+    /// Returns the `index`-th bit in the bytes.
+    fn get_bit(&self, index: usize) -> bool {
+        // MIRAI annotations - important?
+        // assume!(index < self.pos.end); // assumed precondition
+        // assume!(self.hash_bytes.len() == 32); // invariant
+        // assume!(self.pos.end == self.hash_bytes.len() * 8); // invariant
+        let pos = index / 8;
+        let bit = 7 - index % 8;
+        (self.bytes[pos] >> bit) & 1 != 0
+    }
+}
+
+impl<'a> Iterator for BitIterator<'a> {
+    type Item = bool;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.pos.next().map(|x| self.get_bit(x))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.pos.size_hint()
+    }
+}
+
+impl<'a> DoubleEndedIterator for BitIterator<'a> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.pos.next_back().map(|x| self.get_bit(x))
+    }
+}
+
+impl<'a> ExactSizeIterator for BitIterator<'a> {}

infrastructure/jellyfish/src/error.rs

@@ -0,0 +1,30 @@
+//   Copyright 2024 The Tari Project
+//   SPDX-License-Identifier: BSD-3-Clause
+
+use crate::{LeafKey, TreeHash};
+
+#[derive(Debug, thiserror::Error)]
+pub enum JmtProofVerifyError {
+    #[error("Sparse Merkle Tree proof has more than 256 ({num_siblings}) siblings.")]
+    TooManySiblings { num_siblings: usize },
+    #[error("Keys do not match. Key in proof: {actual_key}. Expected key: {expected_key}.")]
+    KeyMismatch { actual_key: LeafKey, expected_key: LeafKey },
+    #[error("Value hashes do not match. Value hash in proof: {actual}. Expected value hash: {expected}.")]
+    ValueMismatch { actual: TreeHash, expected: TreeHash },
+    #[error("Expected inclusion proof. Found non-inclusion proof.")]
+    ExpectedInclusionProof,
+    #[error("Expected non-inclusion proof, but key exists in proof.")]
+    ExpectedNonInclusionProof,
+    #[error(
+        "Key would not have ended up in the subtree where the provided key in proof is the only existing  key, if it \
+         existed. So this is not a valid non-inclusion proof."
+    )]
+    InvalidNonInclusionProof,
+    #[error(
+        "Root hashes do not match. Actual root hash: {actual_root_hash}. Expected root hash: {expected_root_hash}."
+    )]
+    RootHashMismatch {
+        actual_root_hash: TreeHash,
+        expected_root_hash: TreeHash,
+    },
+}

infrastructure/jellyfish/src/hash.rs

@@ -0,0 +1,77 @@
+// Copyright 2024 The Tari Project
+// SPDX-License-Identifier: BSD-3-Clause
+
+use std::{
+    fmt::{Display, Formatter},
+    ops::{Deref, DerefMut},
+};
+
+use borsh::{BorshDeserialize, BorshSerialize};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, BorshSerialize, BorshDeserialize, Serialize, Deserialize,
+)]
+pub struct TreeHash([u8; 32]);
+
+impl TreeHash {
+    pub const fn new(bytes: [u8; 32]) -> Self {
+        Self(bytes)
+    }
+
+    pub const fn zero() -> Self {
+        Self([0; 32])
+    }
+
+    pub const fn into_array(self) -> [u8; 32] {
+        self.0
+    }
+
+    pub fn try_from_bytes(bytes: &[u8]) -> Result<Self, TreeHashSizeError> {
+        if bytes.len() != 32 {
+            return Err(TreeHashSizeError);
+        }
+        let mut arr = [0u8; 32];
+        arr.copy_from_slice(bytes);
+        Ok(Self(arr))
+    }
+}
+
+impl From<[u8; 32]> for TreeHash {
+    fn from(bytes: [u8; 32]) -> Self {
+        Self(bytes)
+    }
+}
+
+impl Deref for TreeHash {
+    type Target = [u8; 32];
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for TreeHash {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl<T: AsRef<[u8]>> PartialEq<T> for TreeHash {
+    fn eq(&self, other: &T) -> bool {
+        self.0 == other.as_ref()
+    }
+}
+
+impl Display for TreeHash {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        for b in self.0 {
+            write!(f, "{:02x}", b)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid TreeHash byte size. Must be 32 bytes.")]
+pub struct TreeHashSizeError;