Skip to content

Commit

Permalink
enable release of veracity ahead of production v1 seals (#35)
Browse files Browse the repository at this point in the history 
* enable release of veracity ahead of production v1 seals

* test: coverage for extending replicas with legacy seals

Includes test showing that the v1 seals will cause an error unless
explicitly enabled

Once the v1 seals hit production we will make v1 accomodation automatic
and silent

AB#10098

* update comment regarding remote v0 seal upgrade for last (open) massif

* code: use proper range expresion

* code: cleaner range expresion

---------

Co-authored-by: Robin Bryce <[email protected]>
  • Loading branch information
@robinbryce
robinbryce and Robin Bryce authored Nov 4, 2024
1 parent 67a9795 commit 6dbec55
Show file tree
Hide file tree
Showing 4 changed files with 242 additions and 57 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/datatrails/go-datatrails-common v0.18.0
github.com/datatrails/go-datatrails-common-api-gen v0.4.6
github.com/datatrails/go-datatrails-logverification v0.2.0
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.1
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.2
github.com/datatrails/go-datatrails-merklelog/mmr v0.1.1
github.com/datatrails/go-datatrails-merklelog/mmrtesting v0.1.0
github.com/datatrails/go-datatrails-simplehash v0.0.5
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ github.com/datatrails/go-datatrails-common-api-gen v0.4.6 h1:yzviWC2jBOC3ItotQQl
github.com/datatrails/go-datatrails-common-api-gen v0.4.6/go.mod h1:OQN91xvlW6xcWTFvwsM2Nn4PZwFAIOE52FG7yRl4QPQ=
github.com/datatrails/go-datatrails-logverification v0.2.0 h1:CzCSGw1Sn1KUd/X32atSk9PTQpl8QBS5BXn+hPwpwmI=
github.com/datatrails/go-datatrails-logverification v0.2.0/go.mod h1:hu+VUZOvkPIHlhp1+qTELNozgsdvlENbXDzt/6Kcoc8=
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.1 h1:yTJECUNlVJSvSrAV6BVOBtyakAzbBBqLU+rhZrKWI7Y=
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.1/go.mod h1:3V08x15NPbzBTSrvjvgzUA0ADkxBRV7m3p5ODElmB2A=
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.2 h1:zxrwrDV8JI7SpBIJxpEJgDERhIilLFr23QSYJak+Zx4=
github.com/datatrails/go-datatrails-merklelog/massifs v0.2.2/go.mod h1:3V08x15NPbzBTSrvjvgzUA0ADkxBRV7m3p5ODElmB2A=
github.com/datatrails/go-datatrails-merklelog/mmr v0.1.1 h1:Ro2fYdDYxGGcPmudYuvPonx78GkdQuKwzrdknLR55cE=
github.com/datatrails/go-datatrails-merklelog/mmr v0.1.1/go.mod h1:B/Kkz4joZTiTz0q/9FFAgHR+Tcn6UxtphMuCzamAc9Q=
github.com/datatrails/go-datatrails-merklelog/mmrtesting v0.1.0 h1:q9RXtAGydXKSJjARnFObNu743cbfIOfERTXiiVa2tF4=
Expand Down
100 changes: 97 additions & 3 deletions replicatelogs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package veracity
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"sync"
Expand All @@ -11,6 +12,7 @@ import (
"github.com/datatrails/go-datatrails-common/cbor"
"github.com/datatrails/go-datatrails-common/logger"
"github.com/datatrails/go-datatrails-merklelog/massifs"
"github.com/datatrails/go-datatrails-merklelog/mmr"
"github.com/gosuri/uiprogress"
"github.com/urfave/cli/v2"
"golang.org/x/exp/rand"
Expand Down Expand Up @@ -45,6 +47,10 @@ func NewReplicateLogsCmd() *cli.Command {
Usage: `verifies the remote log and replicates it locally, ensuring the remote changes are consistent with the trusted local replica.`,
Flags: []cli.Flag{
&cli.BoolFlag{Name: skipUncommittedFlagName, Value: false},
&cli.BoolFlag{
Name: "enable-v1seals", Value: false,
Usage: "Set to enable pre-release suport for the new V1 seal format. Once the new format is in production, this flag will be removed.",
},
&cli.IntFlag{
Name: "massif", Aliases: []string{"m"},
},
Expand Down Expand Up @@ -372,9 +378,46 @@ func (v *VerifiedReplica) ReplicateVerifiedUpdates(
if local == nil {
return opts
}

return append(opts, massifs.WithTrustedBaseState(local.MMRState))
}

// on demand promotion of a v0 state to a v1 state, for compatibility with the consistency check.
trustedBaseState := func(local *massifs.VerifiedContext) (massifs.MMRState, error) {

if local.MMRState.Version > int(massifs.MMRStateVersion0) {
// this will just work, until v1 seals reach production no customer can encounter this.
return local.MMRState, nil
}

if !v.cCtx.Bool("enable-v1seals") {
// The user has not explictly enabled the new seal format, so we return the legacy state unchanged.
// THis will "just work" for production. Use against a pre-release endpoint will fail verification.
return local.MMRState, nil
}

// At this point we have a local seal in v0 format and we expect the
// remote seal to be in v1 format (post v1 seal release for avid + forestrie).
// We need to promote the legacy base state to a V1 state for the
// consistency check. This is a one way operation, and the legacy seal
// root is discarded. Once the seal for the open massif is upgraded,
// this case will never be encountered again for that tenant.

peaks, err := mmr.PeakHashes(local, local.MMRState.MMRSize-1)
if err != nil {
return massifs.MMRState{}, err
}
root := mmr.HashPeaksRHS(sha256.New(), peaks)
if !bytes.Equal(root, local.MMRState.LegacySealRoot) {
return massifs.MMRState{}, fmt.Errorf("legacy seal root does not match the bagged peaks")
}
state := local.MMRState
state.Version = int(massifs.MMRStateVersion1)
state.LegacySealRoot = nil
state.Peaks = peaks
return state, nil
}

if err := v.localReader.EnsureReplicaDirs(tenantIdentity); err != nil {
return err
}
Expand Down Expand Up @@ -432,6 +475,19 @@ func (v *VerifiedReplica) ReplicateVerifiedUpdates(

for i := startMassif; i <= endMassif; i++ {

if local != nil {
// Promote the trusted base state to a V1 state if it is a V0 state.
// All currently incomplete massifs in remote replicas will have their
// last seal upgraded as a result. Historic seals for previously
// completed massifs will remain as V0 seals. Atempting to replicate a
// V0 state will fail with a suitable error. That just implies the
// veracity tool has been run against a legacy endpoint.
local.MMRState, err = trustedBaseState(local)
if err != nil {
return err
}
}

// On the first iteration local is *either* the predecessor to
// startMassif or it is the, as yet, incomplete local replica of it.
// After the first iteration, local is always the predecessor. (If the
Expand Down Expand Up @@ -517,16 +573,54 @@ func (v *VerifiedReplica) replicateVerifiedContext(
}

func verifiedStateEqual(a *massifs.VerifiedContext, b *massifs.VerifiedContext) bool {

var err error

// There is no difference in the log format between the two versions currently supported.
if len(a.Data) != len(b.Data) {
return false
}
if len(a.ConsistentRoots) != len(b.ConsistentRoots) {
fromRoots := a.ConsistentRoots
toRoots := b.ConsistentRoots
// If either state is a V0 state, compare the legacy seal roots
if a.MMRState.Version == int(massifs.MMRStateVersion0) || b.MMRState.Version == int(massifs.MMRStateVersion0) {
rootA := peakBaggedRoot(a.MMRState)
rootB := peakBaggedRoot(b.MMRState)
if !bytes.Equal(rootA, rootB) {
return false
}
if a.MMRState.Version == int(massifs.MMRStateVersion0) {
fromRoots, err = mmr.PeakHashes(a, a.MMRState.MMRSize-1)
if err != nil {
return false
}
}
if b.MMRState.Version == int(massifs.MMRStateVersion0) {
toRoots, err = mmr.PeakHashes(b, b.MMRState.MMRSize-1)
if err != nil {
return false
}
}

}

// If both states are V1 states, compare the peaks
if len(fromRoots) != len(toRoots) {
return false
}
for i := 0; i < len(a.ConsistentRoots); i++ {
if !bytes.Equal(a.ConsistentRoots[i], b.ConsistentRoots[i]) {
for i := range len(fromRoots) {
if !bytes.Equal(fromRoots[i], toRoots[i]) {
return false
}
}
return true
}

// peakBaggedRoot is used to obtain an MMRState V0 bagged root from a V1 accumulator peak list.
// If a v0 state is provided, the root is returned as is.
func peakBaggedRoot(state massifs.MMRState) []byte {
if state.Version < int(massifs.MMRStateVersion1) {
return state.LegacySealRoot
}
return mmr.HashPeaksRHS(sha256.New(), state.Peaks)
}
193 changes: 142 additions & 51 deletions tests/replicatelogs/replicatelogs_azurite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package verifyconsistency

import (
"context"
"crypto/elliptic"
"encoding/json"
"fmt"
"os"
Expand All @@ -17,6 +18,145 @@ import (
"github.com/stretchr/testify/require"
)

// TestV0ToV1ReplicationTransition tests that v0 seal replica can be continued with v1 seals
// In this tests the log starts with v0 seals, is replicated, and the continues with v1 seals.
// This covers the production case where there are previously replicated logs.
func (s *ReplicateLogsCmdSuite) TestV0ToV1ReplicationTransition() {

logger.New("TestV0ToV1ReplicationTransition")
defer logger.OnExit()

tc := massifs.NewLocalMassifReaderTestContext(
s.T(), logger.Sugar, "TestV0ToV1ReplicationTransition")

h8MassifLeaves := mmr.HeightIndexLeafCount(uint64(8 - 1)) // = ((2 << massifHeight) - 1 + 1) >> 1

tests := []struct {
name string
massifHeight uint8
legacyCount uint64
lastLeagacyLeafCount uint64
// if zero, the last legacy massif will be completed. If the last legacy is full and v1Count is zero the test is invalid
v1Count uint64
lastV1LeafCount uint64
}{
// make sure we cover the obvious edge cases
{name: "complete first massif with v0 promoted to v1", massifHeight: 8, legacyCount: 0, lastLeagacyLeafCount: h8MassifLeaves - 3, v1Count: 0, lastV1LeafCount: 3},
}
key := massifs.TestGenerateECKey(s.T(), elliptic.P256())

for _, tt := range tests {

s.Run(tt.name, func() {

// Populate the log with content under legacy seals

require.True(s.T(), tt.legacyCount > 0 || tt.lastLeagacyLeafCount > 0, uint32(0), "invalid test")
require.True(s.T(), tt.v1Count > 0 || tt.lastV1LeafCount > 0, uint32(0), "invalid test")
replicaDir := s.T().TempDir()
tenantId0 := tc.G.NewTenantIdentity()
// leagacyLeafCount = massifLeaves*tt.legacyCount + tt.lastLeagacyLeafCount

// note: CreateLog both creates the massifs *and* populates them
lastMassif := uint32(tt.legacyCount)
if lastMassif > 0 {
lastMassif--
}

// If we skip CreateLog below, we need to delete the blobs
tc.AzuriteContext.DeleteBlobsByPrefix(massifs.TenantMassifPrefix(tenantId0))

if lastMassif > 0 {
tc.CreateLog(
tenantId0, tt.massifHeight, lastMassif,
massifs.TestWithSealKey(&key), massifs.TestWithV0Seals(),
)
}
if tt.lastLeagacyLeafCount > 0 {
tc.AddLeavesToLog(
tenantId0, tt.massifHeight, int(tt.lastLeagacyLeafCount),
massifs.TestWithSealKey(&key), massifs.TestWithV0Seals(),
)
}

// Replicate the log
// note: VERACITY_IKWID is set in main, we need it to enable --envauth so we force it here
app := veracity.NewApp("tests", true)
veracity.AddCommands(app, true)

err := app.Run([]string{
"veracity",
"--envauth", // uses the emulator
"--container", tc.TestConfig.Container,
"--data-url", s.Env.AzuriteVerifiableDataURL,
"--tenant", tenantId0,
"--height", fmt.Sprintf("%d", tt.massifHeight),
"replicate-logs",
// "--ancestors", fmt.Sprintf("%d", tt.ancestors),
"--replicadir", replicaDir,
"--massif", fmt.Sprintf("%d", lastMassif),
})
s.NoError(err)

// Add v1 sealed content
lastMassif = uint32(tt.v1Count)
if lastMassif > 0 {
lastMassif--
}

// Add v1 sealed content
if lastMassif > 0 {
massifLeaves := mmr.HeightIndexLeafCount(uint64(tt.massifHeight - 1)) // = ((2 << massifHeight) - 1 + 1) >> 1
// CreateLog always deleted blobs, so we can only use AddLeavesToLog here
for range tt.v1Count {
tc.AddLeavesToLog(
tenantId0, tt.massifHeight, int(massifLeaves),
massifs.TestWithSealKey(&key), /*, massifs.TestWithV0Seals() V1 seals*/
)
}
}
if tt.lastLeagacyLeafCount > 0 {
tc.AddLeavesToLog(
tenantId0, tt.massifHeight, int(tt.lastV1LeafCount),
massifs.TestWithSealKey(&key), /*, massifs.TestWithV0Seals() V1 seals*/
)
}

// Replicate the v1 content
err = app.Run([]string{
"veracity",
"--envauth", // uses the emulator
"--container", tc.TestConfig.Container,
"--data-url", s.Env.AzuriteVerifiableDataURL,
"--tenant", tenantId0,
"--height", fmt.Sprintf("%d", tt.massifHeight),
"replicate-logs",
"--replicadir", replicaDir,
"--massif", fmt.Sprintf("%d", lastMassif),
})
s.Error(err) // not explicitly enabling seals, replication should fail

err = app.Run([]string{
"veracity",
"--envauth", // uses the emulator
"--container", tc.TestConfig.Container,
"--data-url", s.Env.AzuriteVerifiableDataURL,
"--tenant", tenantId0,
"--height", fmt.Sprintf("%d", tt.massifHeight),
"replicate-logs",

// enable the v1 seals and the replication should succeed
"--enable-v1seals",
"--replicadir", replicaDir,
"--massif", fmt.Sprintf("%d", lastMassif),
})

s.NoError(err)
})
}

}

// TestReplicatingMassifLogsForOneTenant test that by default af full replica is made
func (s *ReplicateLogsCmdSuite) TestReplicatingMassifLogsForOneTenant() {

Expand Down Expand Up @@ -76,10 +216,10 @@ func (s *ReplicateLogsCmdSuite) TestReplicatingMassifLogsForOneTenant() {
}
}

// TestSingleAncestorMassifsForOneTenant tests that the --ancestors option
// TestAncestorMassifsForOneTenant tests that the --ancestors option
// limits the number of historical massifs that are replicated Note that
// --ancestors=0 still requires consistency against local replica of the remote
func (s *ReplicateLogsCmdSuite) TestSingleAncestorMassifLogsForOneTenant() {
func (s *ReplicateLogsCmdSuite) TestAncestorMassifLogsForOneTenant() {

logger.New("Test4AzuriteMassifsForOneTenant")
defer logger.OnExit()
Expand Down Expand Up @@ -169,55 +309,6 @@ func (s *ReplicateLogsCmdSuite) TestSingleAncestorMassifLogsForOneTenant() {
}
}

func (s *ReplicateLogsCmdSuite) TestSingleAncestorMassifsForOneTenantx() {

logger.New("Test4AzuriteSingleAncestorMassifsForOneTenant")
defer logger.OnExit()

tc := massifs.NewLocalMassifReaderTestContext(
s.T(), logger.Sugar, "Test4AzuriteSingleAncestorMassifsForOneTenant")

massifCount := uint32(4)
massifHeight := uint8(8)

tenantId0 := tc.G.NewTenantIdentity()
// note: CreateLog both creates the massifs *and* populates them
tc.CreateLog(tenantId0, massifHeight, massifCount)

replicaDir := s.T().TempDir()

// note: VERACITY_IKWID is set in main, we need it to enable --envauth so we force it here
app := veracity.NewApp("tests", true)
veracity.AddCommands(app, true)

err := app.Run([]string{
"veracity",
"--envauth", // uses the emulator
"--container", tc.TestConfig.Container,
"--data-url", s.Env.AzuriteVerifiableDataURL,
"--tenant", tenantId0,
"--height", fmt.Sprintf("%d", massifHeight),
"replicate-logs",
"--ancestors", "1",
"--replicadir", replicaDir,
"--massif", fmt.Sprintf("%d", massifCount-1),
})
s.NoError(err)

// check the 0'th massifs and seals were _not_ replicated
expectMassifFile := filepath.Join(replicaDir, massifs.ReplicaRelativeMassifPath(tenantId0, 0))
s.NoFileExistsf(expectMassifFile, "the replicated massif should NOT exist")
expectSealFile := filepath.Join(replicaDir, massifs.ReplicaRelativeSealPath(tenantId0, 0))
s.NoFileExistsf(expectSealFile, "the replicated seal should NOT exist")

for i := uint32(2); i < massifCount; i++ {
expectMassifFile := filepath.Join(replicaDir, massifs.ReplicaRelativeMassifPath(tenantId0, i))
s.FileExistsf(expectMassifFile, "the replicated massif should exist")
expectSealFile := filepath.Join(replicaDir, massifs.ReplicaRelativeSealPath(tenantId0, i))
s.FileExistsf(expectSealFile, "the replicated seal should exist")
}
}

// TestSparseReplicaCreatedAfterExtendedOffline tests that the --ancestors
// option limits the number of historical massifs that are replicated, and in
// the case where the verifier has been off line for a long, the resulting
Expand Down

0 comments on commit 6dbec55

Please sign in to comment.