feat: initial draft of inner proofs

nmt.go

@@ -190,13 +190,86 @@ func (n *NamespacedMerkleTree) ProveRange(start, end int) (Proof, error) {
 	if err := n.validateRange(start, end); err != nil {
 		return NewEmptyRangeProof(isMaxNsIgnored), err
 	}
-	proof, err := n.buildRangeProof(start, end)
+	proof, _, err := n.buildRangeProof(start, end)
 	if err != nil {
 		return Proof{}, err
 	}
 	return NewInclusionProof(start, end, proof, isMaxNsIgnored), nil
 }
 
+// Coordinate identifies a tree node using the depth and position
+//
+//	Depth       Position
+//	0              0
+//	              / \
+//	             /   \
+//	1           0     1
+//	           /\     /\
+//	2         0  1   2  3
+//	         /\  /\ /\  /\
+//	3       0 1 2 3 4 5 6 7
+type Coordinate struct {
+	// depth is the typical depth of a tree, 0 being the root
+	depth int
+	// position is the index of a node of a given depth, 0 being the left most
+	// node
+	position int
+}
+
+// ProveInner
+// TODO: range is consecutive
+func (n *NamespacedMerkleTree) ProveInner(coordinates []Coordinate) (InnerProof, error) {
+	isMaxNsIgnored := n.treeHasher.IsMaxNamespaceIDIgnored()
+	start, end := toRange(coordinates, n.Size())
+	proof, coordinates, err := n.buildRangeProof(start, end)
+	if err != nil {
+		return InnerProof{}, err
+	}
+	return NewInnerInclusionProof(proof, coordinates, n.Size(), isMaxNsIgnored), nil
+}
+
+// toRange
+// makes the range consecutive
+func toRange(coordinates []Coordinate, treeSize int) (int, int) {
+	//if err := validateRange(coordinates, treeSize); err != nil {
+	//	return -1, -1, err // TODO is -1 a good return? or 0? or maybe remove this from here and keep it in ProveInner?
+	//}
+	start := 0
+	end := 0
+	maxDepth := maxDepth(treeSize)
+	for _, coord := range coordinates {
+		currentStart := startLeafIndex(coord, maxDepth)
+		currentEnd := endLeafIndex(coord, maxDepth)
+		if currentEnd < start {
+			start = currentStart
+		}
+		if currentEnd > end {
+			end = currentEnd
+		}
+	}
+	return start, end
+}
+
+func maxDepth(treeSize int) int {
+	return bits.Len(uint(treeSize)) - 1
+}
+
+func endLeafIndex(coordinate Coordinate, maxDepth int) int {
+	height := maxDepth - coordinate.depth
+	subtreeSize := 1 << height
+	return (coordinate.position + 1) * subtreeSize
+}
+
+func startLeafIndex(coordinate Coordinate, maxDepth int) int {
+	// since the coordinates are expressed in depth. We need to calculate the height
+	// using ...
+	height := maxDepth - coordinate.depth
+	// In a merkle tree, the tree height grows with every number of leaves multiple of 2.
+	// For example, for all the trees of size 4 to 7, the RFC 6962 tree will have a height of 3.
+	subtreeSize := 1 << height
+	return coordinate.position * subtreeSize
+}
+
 // ProveNamespace returns a range proof for the given NamespaceID.
 //
 // case 1) If the namespace nID is out of the range of the tree's min and max
@@ -265,7 +338,7 @@ func (n *NamespacedMerkleTree) ProveNamespace(nID namespace.ID) (Proof, error) {
 	// the tree or calculated the range it would be in (to generate a proof of
 	// absence and to return the corresponding leaf hashes).
 
-	proof, err := n.buildRangeProof(proofStart, proofEnd)
+	proof, _, err := n.buildRangeProof(proofStart, proofEnd)
 	if err != nil {
 		return Proof{}, err
 	}
@@ -290,13 +363,14 @@ func (n *NamespacedMerkleTree) validateRange(start, end int) error {
 // supplied range i.e., [proofStart, proofEnd) where proofEnd is non-inclusive.
 // The nodes are ordered according to in order traversal of the namespaced tree.
 // Any errors returned by this method are irrecoverable and indicate an illegal state of the tree (n).
-func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, error) {
-	proof := [][]byte{} // it is the list of nodes hashes (as byte slices) with no index
+func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]byte, []Coordinate, error) {
+	var proof [][]byte // it is the list of nodes hashes (as byte slices) with no index
+	var coordinates []Coordinate
 	var recurse func(start, end int, includeNode bool) ([]byte, error)
 
 	// validate the range
 	if err := n.validateRange(proofStart, proofEnd); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// start, end are indices of leaves in the tree hence they should be within
@@ -318,6 +392,10 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by
 			if (start < proofStart || start >= proofEnd) && includeNode {
 				// add the leafHash to the proof
 				proof = append(proof, leafHash)
+				coordinates = append(coordinates, Coordinate{
+					depth:    maxDepth(n.Size()),
+					position: start,
+				})
 			}
 			// if the index of the leaf is within the queried range i.e.,
 			// [proofStart, proofEnd] OR if the leaf is not required as part of
@@ -368,6 +446,7 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by
 		// of the proof but not its left and right subtrees
 		if includeNode && !newIncludeNode {
 			proof = append(proof, hash)
+			coordinates = append(coordinates, ToCoordinate(start, end, n.Size()))
 		}
 
 		return hash, nil
@@ -378,9 +457,19 @@ func (n *NamespacedMerkleTree) buildRangeProof(proofStart, proofEnd int) ([][]by
 		fullTreeSize = 1
 	}
 	if _, err := recurse(0, fullTreeSize, true); err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+	return proof, coordinates, nil
+}
+
+func ToCoordinate(start, end, treeSize int) Coordinate {
+	height := bits.Len(uint(end-start)) - 1
+	maxDepth := maxDepth(treeSize)
+	position := start / (1 << height)
+	return Coordinate{
+		depth:    maxDepth - height,
+		position: position,
 	}
-	return proof, nil
 }
 
 // Get returns leaves for the given namespace.ID.

nmt_test.go

@@ -906,7 +906,7 @@ func Test_buildRangeProof_Err(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := tt.tree.buildRangeProof(tt.proofStart, tt.proofEnd)
+			_, _, err := tt.tree.buildRangeProof(tt.proofStart, tt.proofEnd)
 			assert.Equal(t, tt.wantErr, err != nil)
 			if tt.wantErr {
 				assert.True(t, errors.Is(err, tt.errType))

proof.go

@@ -9,7 +9,7 @@ import (
 	"math/bits"
 
 	"github.com/celestiaorg/nmt/namespace"
-	pb "github.com/celestiaorg/nmt/pb"
+	"github.com/celestiaorg/nmt/pb"
 )
 
 var (
@@ -404,6 +404,130 @@ func (proof Proof) VerifyLeafHashes(nth *NmtHasher, verifyCompleteness bool, nID
 	return bytes.Equal(rootHash, root), nil
 }
 
+type InnerProof struct {
+	nodes                   [][]byte
+	coordinates             []Coordinate
+	treeSize                int
+	isMaxNamespaceIDIgnored bool
+}
+
+// TODO add marshallers and protobuf definitions
+
+// NewInnerInclusionProof constructs a proof that proves that a set of inner is
+// included in an NMT.
+func NewInnerInclusionProof(proofNodes [][]byte, coordinates []Coordinate, treeSize int, ignoreMaxNamespace bool) InnerProof {
+	return InnerProof{
+		nodes:                   proofNodes,
+		coordinates:             coordinates,
+		treeSize:                treeSize,
+		isMaxNamespaceIDIgnored: ignoreMaxNamespace,
+	}
+}
+
+// VerifyInnerNodes
+// coordinates should be in the same order as inner nodes
+func (proof InnerProof) VerifyInnerNodes(nth *NmtHasher, nID namespace.ID, innerNodes [][]byte, coordinates []Coordinate, root []byte) (bool, error) {
+	// validate the inner proof: same number of nodes and coordinates
+
+	// perform some consistency checks:
+	if nID.Size() != nth.NamespaceSize() {
+		return false, fmt.Errorf("namespace ID size (%d) does not match the namespace size of the NMT hasher (%d)", nID.Size(), nth.NamespaceSize())
+	}
+	// check that the root is valid w.r.t the NMT hasher
+	if err := nth.ValidateNodeFormat(root); err != nil {
+		return false, fmt.Errorf("root does not match the NMT hasher's hash format: %w", err)
+	}
+	// check that all the proof.nodes are valid w.r.t the NMT hasher
+	for _, node := range proof.nodes {
+		if err := nth.ValidateNodeFormat(node); err != nil {
+			return false, fmt.Errorf("proof nodes do not match the NMT hasher's hash format: %w", err)
+		}
+	}
+	// check that all the leafHashes are valid w.r.t the NMT hasher
+	for _, leafHash := range innerNodes {
+		if err := nth.ValidateNodeFormat(leafHash); err != nil {
+			return false, fmt.Errorf("leaf hash does not match the NMT hasher's hash format: %w", err)
+		}
+	}
+	// validate that nID is included in the inner nodes
+
+	_, proofEnd := toRange(coordinates, proof.treeSize)
+
+	allNodes := append(proof.nodes, innerNodes...)
+	allCoordinates := append(proof.coordinates, coordinates...)
+
+	var computeRoot func(start, end int) ([]byte, error)
+	// computeRoot can return error iff the HashNode function fails while calculating the root
+	computeRoot = func(start, end int) ([]byte, error) {
+		innerNode, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, end)
+		if found {
+			return innerNode, nil
+		}
+
+		// Recursively get left and right subtree
+		k := getSplitPoint(end - start)
+		left, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start, start+k)
+		var err error
+		if found {
+			// TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation
+		} else {
+			left, err = computeRoot(start, start+k)
+			if err != nil {
+				return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start, start+k, err)
+			}
+		}
+		right, found := getInnerNode(allNodes, allCoordinates, proof.treeSize, start+k, end)
+		if found {
+			// TODO: do we want to remove the node and coordinates from allNodes and allCoordinates? Or it's just a premature optimisation
+		} else {
+			right, err = computeRoot(start+k, end)
+			if err != nil {
+				return nil, fmt.Errorf("failed to compute subtree root [%d, %d): %w", start+k, end, err)
+			}
+		}
+
+		// only right leaf/subtree can be non-existent
+		if right == nil {
+			return left, nil
+		}
+		hash, err := nth.HashNode(left, right)
+		if err != nil {
+			return nil, fmt.Errorf("failed to hash node: %w", err)
+		}
+		return hash, nil
+	}
+
+	// estimate the leaf size of the subtree containing the proof range
+	proofRangeSubtreeEstimate := getSplitPoint(proofEnd) * 2
+	if proofRangeSubtreeEstimate < 1 {
+		proofRangeSubtreeEstimate = 1
+	}
+	rootHash, err := computeRoot(0, proofRangeSubtreeEstimate)
+	if err != nil {
+		return false, fmt.Errorf("failed to compute root [%d, %d): %w", 0, proofRangeSubtreeEstimate, err)
+	}
+	for i := 0; i < len(proof.nodes); i++ {
+		rootHash, err = nth.HashNode(rootHash, proof.nodes[i])
+		if err != nil {
+			return false, fmt.Errorf("failed to hash node: %w", err)
+		}
+	}
+
+	return bytes.Equal(rootHash, root), nil
+}
+
+// getInnerNode
+// expect the number of nodes and coordinates to be the same
+func getInnerNode(nodes [][]byte, coordinates []Coordinate, treeSize int, start int, end int) ([]byte, bool) {
+	for index, coordinate := range coordinates {
+		startLeaf, endLeaf := toRange([]Coordinate{coordinate}, treeSize)
+		if startLeaf == start && endLeaf == end {
+			return nodes[index], true
+		}
+	}
+	return nil, false
+}
+
 // VerifyInclusion checks that the inclusion proof is valid by using leaf data
 // and the provided proof to regenerate and compare the root. Note that the leavesWithoutNamespace data should not contain the prefixed namespace, unlike the tree.Push method,
 // which takes prefixed data. All leaves implicitly have the same namespace ID:

proof_test.go

@@ -122,7 +122,7 @@ func TestProof_VerifyNamespace_False(t *testing.T) {
 		t.Fatalf("invalid test setup: error on ProveNamespace(): %v", err)
 	}
 	// inclusion proof of the leaf index 0
-	incProof0, err := n.buildRangeProof(0, 1)
+	incProof0, _, err := n.buildRangeProof(0, 1)
 	require.NoError(t, err)
 	incompleteFirstNs := NewInclusionProof(0, 1, incProof0, false)
 	type args struct {
@@ -135,13 +135,13 @@ func TestProof_VerifyNamespace_False(t *testing.T) {
 
 	// an invalid absence proof for an existing namespace ID (2) in the constructed tree
 	leafIndex := 3
-	inclusionProofOfLeafIndex, err := n.buildRangeProof(leafIndex, leafIndex+1)
+	inclusionProofOfLeafIndex, _, err := n.buildRangeProof(leafIndex, leafIndex+1)
 	require.NoError(t, err)
 	leafHash := n.leafHashes[leafIndex] // the only data item with namespace ID = 2 in the constructed tree is at index 3
 	invalidAbsenceProof := NewAbsenceProof(leafIndex, leafIndex+1, inclusionProofOfLeafIndex, leafHash, false)
 
 	// inclusion proof of the leaf index 10
-	incProof10, err := n.buildRangeProof(10, 11)
+	incProof10, _, err := n.buildRangeProof(10, 11)
 	require.NoError(t, err)
 
 	// root
@@ -229,6 +229,14 @@ func TestProof_VerifyNamespace_False(t *testing.T) {
 	}
 }
 
+func TestInnerProofs(t *testing.T) {
+	n := exampleNMT(1, true, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
+	proof, err := n.ProveInner([]Coordinate{
+		{1, 0}, {3, 4}, {4, 10},
+	})
+	assert.NoError(t, err)
+	assert.NotNil(t, proof) // this is just to stop the debugger here and see if the proof is valid
+}
 func TestProof_MultipleLeaves(t *testing.T) {
 	n := New(sha256.New())
 	ns := []byte{1, 2, 3, 4, 5, 6, 7, 8}