Skip to content

Commit 7bffdf9

Browse files
robinbryceRobin Bryce
andauthored
Dev/robin/9709 logconfigmer accumulator consistency proofs (#13)
* tests passing * updates for cose receipts algorithms * linter save * fix a long standing bug in one of the tests --------- Co-authored-by: Robin Bryce <[email protected]>
1 parent c5884b7 commit 7bffdf9

File tree

9 files changed

+120
-179
lines changed

9 files changed

+120
-179
lines changed

go.mod

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,30 +3,28 @@ module github.com/datatrails/go-datatrails-logverification
33
go 1.22
44

55
require (
6-
github.com/datatrails/go-datatrails-common v0.16.1
6+
github.com/datatrails/go-datatrails-common v0.18.0
77
github.com/datatrails/go-datatrails-common-api-gen v0.4.5
8-
github.com/datatrails/go-datatrails-merklelog/massifs v0.1.0
9-
github.com/datatrails/go-datatrails-merklelog/mmr v0.0.2
8+
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.0
9+
github.com/datatrails/go-datatrails-merklelog/mmr v0.1.0
1010
github.com/datatrails/go-datatrails-merklelog/mmrtesting v0.1.0
1111
github.com/datatrails/go-datatrails-simplehash v0.0.5
1212
github.com/stretchr/testify v1.9.0
1313
github.com/veraison/go-cose v1.1.0
14-
google.golang.org/protobuf v1.34.1
14+
google.golang.org/protobuf v1.34.2
1515
)
1616

17-
// replace github.com/datatrails/go-datatrails-merklelog/massifs => ../go-datatrails-merklelog/massifs
18-
1917
require (
2018
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
2119
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
2220
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
23-
github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.0 // indirect
21+
github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.7.1 // indirect
2422
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 // indirect
2523
github.com/Azure/go-amqp v1.0.5 // indirect
2624
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
2725
github.com/Azure/go-autorest/autorest v0.11.29 // indirect
28-
github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
29-
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
26+
github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect
27+
github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect
3028
github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
3129
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
3230
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
@@ -37,10 +35,10 @@ require (
3735
github.com/dimchansky/utfbom v1.1.1 // indirect
3836
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
3937
github.com/fxamacker/cbor/v2 v2.6.0 // indirect
40-
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
38+
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
4139
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
4240
github.com/google/uuid v1.6.0 // indirect
43-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
41+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
4442
github.com/ldclabs/cose/go v0.0.0-20221214142927-d22c1cfc2154 // indirect
4543
github.com/mitchellh/go-homedir v1.1.0 // indirect
4644
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
@@ -53,12 +51,12 @@ require (
5351
github.com/zeebo/bencode v1.0.0 // indirect
5452
go.uber.org/multierr v1.11.0 // indirect
5553
go.uber.org/zap v1.27.0 // indirect
56-
golang.org/x/crypto v0.23.0 // indirect
57-
golang.org/x/net v0.25.0 // indirect
58-
golang.org/x/sys v0.20.0 // indirect
59-
golang.org/x/text v0.15.0 // indirect
60-
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect
61-
google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
62-
google.golang.org/grpc v1.64.0 // indirect
54+
golang.org/x/crypto v0.25.0 // indirect
55+
golang.org/x/net v0.27.0 // indirect
56+
golang.org/x/sys v0.22.0 // indirect
57+
golang.org/x/text v0.16.0 // indirect
58+
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
59+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
60+
google.golang.org/grpc v1.65.0 // indirect
6361
gopkg.in/yaml.v3 v3.0.1 // indirect
6462
)

go.sum

Lines changed: 50 additions & 41 deletions
Large diffs are not rendered by default.

integrationsupport/massifseal.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package integrationsupport
33
import (
44
"context"
55
"crypto/ecdsa"
6-
"crypto/sha256"
76
"testing"
87

98
"github.com/datatrails/go-datatrails-common-api-gen/assets/v2/assets"
@@ -20,21 +19,21 @@ import (
2019
// the test context.
2120
func GenerateMassifSeal(t *testing.T, testContext mmrtesting.TestContext, lastEvent *assets.EventResponse, signingKey ecdsa.PrivateKey) {
2221
massifReader := massifs.NewMassifReader(logger.Sugar, testContext.Storer)
23-
hasher := sha256.New()
2422

2523
// Just handle a single massif for now
2624
massifContext, err := massifReader.GetMassif(context.TODO(), mmrtesting.DefaultGeneratorTenantIdentity, 0)
2725
require.Nil(t, err)
2826

2927
mmrSize := massifContext.RangeCount()
30-
root, err := mmr.GetRoot(mmrSize, &massifContext, hasher)
28+
peaks, err := mmr.PeakHashes(&massifContext, mmrSize-1)
3129
require.Nil(t, err)
3230
id, epoch, err := massifs.SplitIDTimestampHex(lastEvent.MerklelogEntry.Commit.Idtimestamp)
3331
require.Nil(t, err)
3432

3533
mmrState := massifs.MMRState{
34+
Version: 1,
3635
MMRSize: mmrSize,
37-
Root: root,
36+
Peaks: peaks,
3837
CommitmentEpoch: uint32(epoch),
3938
IDTimestamp: id,
4039
}

logverification/proof.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ func EventProof(verifiableEvent VerifiableEvent, massif *massifs.MassifContext)
1818
// Get the size of the complete tenant MMR
1919
mmrSize := massif.RangeCount()
2020

21-
hasher := sha256.New()
22-
proof, err := mmr.IndexProof(mmrSize, massif, hasher, verifiableEvent.MerkleLog.Commit.Index)
21+
proof, err := mmr.InclusionProof(massif, mmrSize-1, verifiableEvent.MerkleLog.Commit.Index)
2322
if err != nil {
2423
return nil, err
2524
}
@@ -33,12 +32,7 @@ func VerifyProof(verifiableEvent VerifiableEvent, proof [][]byte, massif *massif
3332
mmrSize := massif.RangeCount()
3433

3534
hasher := sha256.New()
36-
root, err := mmr.GetRoot(mmrSize, massif, hasher)
37-
if err != nil {
38-
return false, err
39-
}
4035

41-
verified := mmr.VerifyInclusion(mmrSize, hasher, verifiableEvent.LeafHash,
42-
verifiableEvent.MerkleLog.Commit.Index, proof, root)
43-
return verified, nil
36+
return mmr.VerifyInclusion(massif, hasher, mmrSize, verifiableEvent.LeafHash,
37+
verifiableEvent.MerkleLog.Commit.Index, proof)
4438
}

logverification/seal.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,12 @@ func SignedLogState(
4343
massifIndex, err)
4444
}
4545

46-
// The log state at time of sealing is the Payload. It included the root, but this is removed
47-
// from the stored log state. This forces a verifier to recompute the merkle root from their view
46+
// The log state at time of sealing is the Payload. It included the peaks, but this is removed
47+
// from the stored log state. This forces a verifier to recompute the merkle peaks from their view
4848
// of the data. If verification succeeds when this computed root is added to signedStateNow, then
4949
// we can be confident that DataTrails signed this state, and that the root matches your data.
50-
logState.Root, err = mmr.GetRoot(logState.MMRSize, &massifContext, hasher)
50+
51+
logState.Peaks, err = mmr.PeakHashes(&massifContext, logState.MMRSize-1)
5152
if err != nil {
5253
return nil, fmt.Errorf("SignedLogState failed: unable to get root for massifContextNow: %w", err)
5354
}

logverification/verifyconsistency.go

Lines changed: 28 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package logverification
22

33
import (
44
"context"
5-
"crypto/ecdsa"
65
"errors"
76
"fmt"
87
"hash"
@@ -17,10 +16,10 @@ import (
1716
// MMRState is an abstraction, but it is assumed that logStateA comes from a local, trusted copy of the data
1817
// rather than a fresh download from DataTrails.
1918
//
19+
// This function assumes the two log states are from the same massif.
20+
//
2021
// NOTE: the log state's signatures are not verified in this function, it is expected that the signature verification
2122
// is done as a separate step to the consistency verification.
22-
//
23-
// NOTE: it is expected that both logStateA and logStateB have had their root recalculated.
2423
func VerifyConsistency(
2524
ctx context.Context,
2625
hasher hash.Hash,
@@ -30,102 +29,44 @@ func VerifyConsistency(
3029
logStateB *massifs.MMRState,
3130
) (bool, error) {
3231

33-
if logStateA.Root == nil || logStateB.Root == nil {
32+
if logStateA.Peaks == nil || logStateB.Peaks == nil {
3433
return false, errors.New("VerifyConsistency failed: the roots for both log state A and log state B need to be set")
3534
}
3635

37-
if len(logStateA.Root) == 0 || len(logStateB.Root) == 0 {
36+
if len(logStateA.Peaks) == 0 || len(logStateB.Peaks) == 0 {
3837
return false, errors.New("VerifyConsistency failed: the roots for both log state A and log state B need to be set")
3938
}
4039

4140
massifReader := massifs.NewMassifReader(logger.Sugar, reader)
4241

43-
// last massif in the merkle log for log state A
44-
massifContextA, err := Massif(logStateA.MMRSize-1, massifReader, tenantID, DefaultMassifHeight)
45-
if err != nil {
46-
return false, fmt.Errorf("VerifyConsistency failed: unable to get the last massif for log state A: %w", err)
47-
}
48-
4942
// last massif in the merkle log for log state B
5043
massifContextB, err := Massif(logStateB.MMRSize-1, massifReader, tenantID, DefaultMassifHeight)
5144
if err != nil {
5245
return false, fmt.Errorf("VerifyConsistency failed: unable to get the last massif for log state B: %w", err)
5346
}
5447

55-
// We construct a proof of consistency between logStateA and logStateB.
56-
// This will be a proof that logStateB derives from logStateA.
57-
consistencyProof, err := mmr.IndexConsistencyProof(logStateA.MMRSize, logStateB.MMRSize, massifContextB, hasher)
58-
if err != nil {
59-
return false, fmt.Errorf("VerifyConsistency failed: unable to generate consistency proof: %w", err)
60-
}
61-
62-
// In order to verify the proof we take the hashes of all of the peaks in logStateA.
63-
// The hash of each of these peaks guarantees the integrity of all of its child nodes, so we
64-
// don't need to check every hash.
65-
66-
// Peaks returned as MMR positions (1-based), not MMR indices (0-based). The location of these
67-
// is deterministic: Given an MMR of a particular size, the peaks will always be in the same place.
68-
logPeaksA := mmr.Peaks(logStateA.MMRSize)
69-
70-
// Get the hashes of all of the peaks.
71-
logPeakHashesA, err := mmr.PeakBagRHS(massifContextA, hasher, 0, logPeaksA)
72-
if err != nil {
73-
return false, errors.New("error")
74-
}
75-
76-
// Lastly, verify the consistency proof using the peak hashes from our backed-up log. If this
77-
// returns true, then we can confidently say that everything in the backed-up log is in the state
78-
// of the log described by this signed state.
79-
verified := mmr.VerifyConsistency(hasher, logPeakHashesA, consistencyProof, logStateA.Root, logStateB.Root)
80-
return verified, nil
81-
}
82-
83-
// VerifyConsistencyFromMassifs takes a massif context providing access to data from the past, and a massif
84-
// context providing access to the current version of the log. It returns whether or not the
85-
// new version of the log is consistent with the previous version (i.e. it contains all of the
86-
// same nodes in the same positions.)
87-
//
88-
// It is assumed that in a production use case, massifContextBefore provides access to a trusted
89-
// local copy of the massif, rather than a fresh download from DataTrails.
90-
func VerifyConsistencyFromMassifs(
91-
ctx context.Context,
92-
verificationKey ecdsa.PublicKey,
93-
hasher hash.Hash,
94-
blobReader azblob.Reader,
95-
massifContextBefore *massifs.MassifContext,
96-
massifContextNow *massifs.MassifContext,
97-
logStateNow *massifs.MMRState,
98-
) (bool, error) {
99-
// Grab some core info about our backed up merkle log, which we'll need to prove consistency
100-
mmrSizeBefore := massifContextBefore.Count()
101-
rootBefore, err := mmr.GetRoot(mmrSizeBefore, massifContextBefore, hasher)
102-
if err != nil {
103-
return false, fmt.Errorf("VerifyConsistency failed: unable to get root for massifContextBefore: %w", err)
104-
}
105-
106-
// We construct a proof of consistency between the backed up MMR log and the head of the log.
107-
consistencyProof, err := mmr.IndexConsistencyProof(mmrSizeBefore, logStateNow.MMRSize, massifContextNow, hasher)
108-
if err != nil {
109-
return false, errors.New("error")
110-
}
111-
112-
// In order to verify the proof we take the hashes of all of the peaks in the backed up log.
113-
// The hash of each of these peaks guarantees the integrity of all of its child nodes, so we
114-
// don't need to check every hash.
115-
116-
// Peaks returned as MMR positions (1-based), not MMR indices (0-based). The location of these
117-
// is deterministic: Given an MMR of a particular size, the peaks will always be in the same place.
118-
backupLogPeaks := mmr.Peaks(mmrSizeBefore)
119-
120-
// Get the hashes of all of the peaks.
121-
backupLogPeakHashes, err := mmr.PeakBagRHS(massifContextNow, hasher, 0, backupLogPeaks)
122-
if err != nil {
123-
return false, errors.New("error")
124-
}
125-
126-
// Lastly, verify the consistency proof using the peak hashes from our backed-up log. If this
127-
// returns true, then we can confidently say that everything in the backed-up log is in the state
128-
// of the log described by this signed state.
129-
verified := mmr.VerifyConsistency(hasher, backupLogPeakHashes, consistencyProof, rootBefore, logStateNow.Root)
130-
return verified, nil
48+
// We check a proof of consistency between logStateA and logStateB.
49+
// This will be a proof that logStateB includes all elements from logStateA,
50+
// and includes them in the same positions.
51+
52+
// In order to verify the proof we verify that the inclusion proofs of each of
53+
// the peaks from the old log matches a peak in the new log.
54+
// Because a proof of inclusion requires that the proof reproduces the peak,
55+
// and because all nodes in the old tree have proofs that pass through the
56+
// old peaks and then reach the new peaks, we know it is not possible for
57+
// the children to verify unless their peaks also verify. So we don't need
58+
// to check every hash.
59+
60+
verified, _ /*peaksB*/, err := mmr.CheckConsistency(massifContextB, hasher, logStateA.MMRSize, logStateB.MMRSize, logStateA.Peaks)
61+
62+
// A tampered node can not be proven unless the entire log is re-built. If
63+
// a log is re-built, any proof held by a relying party will not verify. And
64+
// as it is signed, it is evidence the log was re-built by someone with
65+
// access to our signing key.
66+
// In the case of a tamper (or corruption) without re-build, the proof of inclusion will fail.
67+
// Examining the parent and sibling of an individually tampered node will reveal the tamper.
68+
// This means we are always fail safe in the case of a tampered node - a
69+
// party relying on the log can guarantee the will never use unverifiable
70+
// data.
71+
return verified, err
13172
}

logverification/verifyconsistency_test.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,11 @@ func (b *TestLogHelper) VerifyConsistencyBetween(fromState *massifs.MMRState, to
9090
result, err := VerifyConsistency(
9191
context.Background(), b.hasher, b.tctx.Storer, inTenant, fromState, toState,
9292
)
93-
94-
require.NoError(b.t, err)
93+
// Some callers are testing negative results, so we only ensure that the
94+
// true/false is consistent with the error state here.
95+
if result == true {
96+
require.NoError(b.t, err)
97+
}
9598
return result
9699
}
97100

logverification/verifylist.go

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ func VerifyList(reader azblob.Reader, eventList []VerifiableEvent, options ...Ve
212212

213213
// if the event is OMITTED add the leaf to the omitted list
214214
if eventType == Omitted {
215-
omittedMMRIndices = append(omittedMMRIndices, mmr.TreeIndex(leafIndex))
215+
omittedMMRIndices = append(omittedMMRIndices, mmr.MMRIndex(leafIndex))
216216

217217
// as the event is still the lowest mmrIndex we check this event
218218
// against the next leaf
@@ -240,7 +240,7 @@ func VerifyEventInList(
240240

241241
hasher.Reset()
242242

243-
leafMMRIndex := mmr.TreeIndex(leafIndex)
243+
leafMMRIndex := mmr.MMRIndex(leafIndex)
244244
eventMMRIndex := event.MerkleLog.Commit.Index
245245

246246
// First we check if the event mmrIndex corresponds to a leaf node.
@@ -348,24 +348,20 @@ func VerifyEventInList(
348348
// Now we know that the event is the event stored on the leaf node,
349349
// we can do an inclusion proof of the leaf node on the merkle log.
350350
mmrSize := massifContext.RangeCount()
351-
root, err := mmr.GetRoot(mmrSize, massifContext, hasher)
352-
if err != nil {
353-
return Unknown, err
354-
}
355351

356-
inclusionProof, err := mmr.IndexProof(mmrSize, massifContext, hasher, leafMMRIndex)
352+
inclusionProof, err := mmr.InclusionProof(massifContext, mmrSize-1, leafMMRIndex)
357353
if err != nil {
358354
return Unknown, err
359355
}
360356

361-
verified := mmr.VerifyInclusion(mmrSize, hasher, event.LeafHash, leafMMRIndex, inclusionProof, root)
362-
363-
// if the inclusion proof verification failed, return EXCLUDED.
364-
//
365-
// This means the leaf node is not included on the merklelog.
366-
if !verified {
357+
verified, err := mmr.VerifyInclusion(
358+
massifContext, hasher, mmrSize, event.LeafHash, leafMMRIndex, inclusionProof)
359+
if !verified || errors.Is(err, mmr.ErrVerifyInclusionFailed) {
367360
return Excluded, ErrInclusionProofVerify
368361
}
362+
if err != nil {
363+
return Unknown, err
364+
}
369365

370366
return Included, nil
371367

taskfiles/Taskfile_gotest.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ tasks:
3535
-v \
3636
-coverprofile={{.UNITTEST_DIR}}/main.out \
3737
./... \
38-
2>&1 | go-junit-report -set-exit-code -debug.print-events > {{.UNITTEST_DIR}}/main.xml
38+
2>&1
3939
4040
gocov convert {{.UNITTEST_DIR}}/main.out > {{.UNITTEST_DIR}}/coverage.json
4141
@@ -58,6 +58,6 @@ tasks:
5858
-v \
5959
-coverprofile={{.UNITTEST_DIR}}/main.out \
6060
./... \
61-
2>&1 | go-junit-report -set-exit-code -debug.print-events > {{.UNITTEST_DIR}}/main.xml
61+
2>&1
6262
6363
gocov convert {{.UNITTEST_DIR}}/main.out > {{.UNITTEST_DIR}}/coverage.json

0 commit comments

Comments
 (0)