diff --git a/pkg/database/bpt/bpt_savestate.go b/pkg/database/bpt/bpt_savestate.go index 141f1b531..c0a532a59 100644 --- a/pkg/database/bpt/bpt_savestate.go +++ b/pkg/database/bpt/bpt_savestate.go @@ -7,6 +7,7 @@ package bpt import ( + "crypto/sha256" "io" "os" @@ -67,12 +68,19 @@ func SaveSnapshotV1(b *BPT, file io.WriteSeeker, loadState func(key storage.Key, NodeCnt += uint64(len(bptVals)) for _, v := range bptVals { // For all the key values we got (as many as 1000) - kh := v.Key.Hash() // + kh := v.Key.Hash() + var vh [32]byte + if len(v.Value) == 32 { + copy(vh[:], v.Value) + } else { + vh = sha256.Sum256(v.Value) + } + _, e1 := file.Write(kh[:]) // Write the key out - _, e2 := file.Write(v.Value) // Write the hash out + _, e2 := file.Write(vh[:]) // Write the hash out _, e3 := file.Write(common.Uint64FixedBytes(vOffset)) // And the current offset to the next value - value, e4 := loadState(kh, [32]byte(v.Value)) // Get that next value + value, e4 := loadState(kh, vh) // Get that next value vLen := uint64(len(value)) // get the value's length as uint64 _, e5 := values.Write(common.Uint64FixedBytes(vLen)) // Write out the length _, e6 := values.Write(value) // write out the value diff --git a/pkg/database/bpt/marshal.go b/pkg/database/bpt/marshal.go index 2fefa88b6..ce9e75d5e 100644 --- a/pkg/database/bpt/marshal.go +++ b/pkg/database/bpt/marshal.go @@ -65,10 +65,18 @@ func (n *branch) readFrom(rd *bytes.Buffer, o marshalOpts) error { } func (v *leaf) Type() nodeType { - if isExpandedKey(v.Key) { + key := isExpandedKey(v.Key) + value := len(v.Value) != 32 + switch { + case !key && !value: + return nodeTypeLeaf + case key: return nodeTypeLeafWithExpandedKey + case value: + return nodeTypeLeafWithValue + default: + return nodeTypeLeafWithExpandedKeyAndValue } - return nodeTypeLeaf } // isExpandedKey returns true if the key is an expanded key. A compressed key @@ -105,7 +113,11 @@ func (v *leaf) writeTo(wr io.Writer) (err error) { tryWrite(&err, wr, v.Value[:]) } else { - return fmt.Errorf("invalid BPT value length: want 32 bytes, got %d", len(v.Value)) + // Write the length then the value + var buf [10]byte + n := binary.PutUvarint(buf[:], uint64(len(v.Value))) + tryWrite(&err, wr, buf[:n]) + tryWrite(&err, wr, v.Value) } return err } @@ -132,22 +144,38 @@ func (v *leaf) readFrom(rd *bytes.Buffer, o marshalOpts) error { return err } - // Read the hash - v.Value = make([]byte, 32) - _, err = io.ReadFull(rd, v.Value) - return err + } else { + // Read the key hash + var kh [32]byte + _, err := io.ReadFull(rd, kh[:]) + if err != nil { + return err + } + v.Key = record.KeyFromHash(kh) } - // Read leafStateSize bytes - var buf [leafStateSize]byte - _, err := io.ReadFull(rd, buf[:]) - if err != nil { - return err - } + if o.varWidthVal { + // Read the value size + l, err := binary.ReadUvarint(rd) + if err != nil { + return err + } - // Read the fields - v.Key = record.NewKey(*(*record.KeyHash)(buf[:])) - v.Value = buf[32:] + // Read the value + v.Value = make([]byte, l) + _, err = io.ReadFull(rd, v.Value) + if err != nil { + return err + } + + } else { + // Read the value hash + v.Value = make([]byte, 32) + _, err := io.ReadFull(rd, v.Value) + if err != nil { + return err + } + } return nil } @@ -181,6 +209,7 @@ func writeBlock(err *error, wr io.Writer, e node, mask uint64) { type marshalOpts struct { expandedKey bool + varWidthVal bool } // newNodeWithOpts creates a new node for the specified nodeType. @@ -194,6 +223,10 @@ func newNodeWithOpts(typ nodeType) (node, marshalOpts, error) { return new(leaf), marshalOpts{}, nil case nodeTypeLeafWithExpandedKey: return new(leaf), marshalOpts{expandedKey: true}, nil + case nodeTypeLeafWithValue: + return new(leaf), marshalOpts{varWidthVal: true}, nil + case nodeTypeLeafWithExpandedKeyAndValue: + return new(leaf), marshalOpts{expandedKey: true, varWidthVal: true}, nil } return nil, marshalOpts{}, fmt.Errorf("unknown node %v", typ) } diff --git a/pkg/database/bpt/mutate.go b/pkg/database/bpt/mutate.go index b772560b5..43bd601dd 100644 --- a/pkg/database/bpt/mutate.go +++ b/pkg/database/bpt/mutate.go @@ -8,7 +8,6 @@ package bpt import ( "bytes" - "fmt" "gitlab.com/accumulatenetwork/accumulate/pkg/errors" "gitlab.com/accumulatenetwork/accumulate/pkg/types/record" @@ -28,9 +27,6 @@ func (b *BPT) Insert(key *record.Key, value []byte) error { if b.pending == nil { b.pending = map[[32]byte]*mutation{} } - if len(value) != 32 { - return fmt.Errorf("invalid value: want 32 bytes, got %d", len(value)) - } // Copy the value v := make([]byte, len(value)) copy(v, value) diff --git a/pkg/database/bpt/node.go b/pkg/database/bpt/node.go index 560076860..1dbe728b7 100644 --- a/pkg/database/bpt/node.go +++ b/pkg/database/bpt/node.go @@ -101,7 +101,14 @@ func (e *branch) newBranch(key [32]byte) (*branch, error) { func (*emptyNode) getHash() ([32]byte, bool) { return [32]byte{}, false } // getHash returns the leaf's hash. -func (e *leaf) getHash() ([32]byte, bool) { return *(*[32]byte)(e.Value), true } +func (e *leaf) getHash() ([32]byte, bool) { + // This logic is strange, but it is necessary to preserve backwards + // compatibility + if len(e.Value) == 32 { + return *(*[32]byte)(e.Value), true + } + return sha256.Sum256(e.Value), true +} // getHash returns the branch's hash, recalculating it if the branch has been // changed since the last getHash call. diff --git a/pkg/database/bpt/params.go b/pkg/database/bpt/params.go index 205941acc..4435baf23 100644 --- a/pkg/database/bpt/params.go +++ b/pkg/database/bpt/params.go @@ -50,6 +50,24 @@ func (p paramsRecord) Get() (*stateData, error) { return v, nil } +func (p paramsRecord) Put(v *stateData) error { + old, err := p.Value.Get() + switch { + case errors.Is(err, errors.NotFound): + // Parameters have not been set so they can be set to anything + return p.Value.Put(v) + + case err != nil: + return errors.UnknownError.Wrap(err) + } + + if !old.parameters.Equal(&v.parameters) { + return errors.BadRequest.With("cannot change BPT parameters once set") + } + + return p.Value.Put(v) +} + func (r *stateData) MarshalBinary() ([]byte, error) { buf := new(bytes.Buffer) _, _ = buf.WriteString(paramsV2Magic) diff --git a/pkg/database/bpt/schema.yml b/pkg/database/bpt/schema.yml index 0a4a09bdf..008d43965 100644 --- a/pkg/database/bpt/schema.yml +++ b/pkg/database/bpt/schema.yml @@ -23,6 +23,8 @@ stateData: type: uint - name: Mask type: uint + - name: ArbitraryValues + type: boolean node: class: union @@ -49,6 +51,14 @@ node: description: is a leaf node with an expanded key label: leaf+key value: 5 + LeafWithValue: + description: is a leaf node with a non-hash value + label: leaf+value + value: 6 + LeafWithExpandedKeyAndValue: + description: is a leaf node with an expanded key and non-hash value + label: leaf+key+value + value: 7 members: - name: emptyNode diff --git a/pkg/database/bpt/schema_gen.go b/pkg/database/bpt/schema_gen.go index f78878323..ab903f468 100644 --- a/pkg/database/bpt/schema_gen.go +++ b/pkg/database/bpt/schema_gen.go @@ -172,6 +172,18 @@ func init() { Description: "is a leaf node with an expanded key", Label: "leaf+key", }, + "LeafWithExpandedKeyAndValue": { + Name: "LeafWithExpandedKeyAndValue", + Value: 7, + Description: "is a leaf node with an expanded key and non-hash value", + Label: "leaf+key+value", + }, + "LeafWithValue": { + Name: "LeafWithValue", + Value: 6, + Description: "is a leaf node with a non-hash value", + Label: "leaf+value", + }, }, }).SetGoType() @@ -188,6 +200,10 @@ func init() { Name: "Mask", Type: &schema.SimpleType{Type: schema.SimpleTypeUint}, }, + { + Name: "ArbitraryValues", + Type: &schema.SimpleType{Type: schema.SimpleTypeBool}, + }, }, }).SetGoType() diff --git a/pkg/database/bpt/types_gen.go b/pkg/database/bpt/types_gen.go index 136cb6ef8..b0464f86a 100644 --- a/pkg/database/bpt/types_gen.go +++ b/pkg/database/bpt/types_gen.go @@ -118,11 +118,13 @@ func equalNode(a, b node) bool { type nodeType int64 const ( - nodeTypeBoundary nodeType = 4 - nodeTypeBranch nodeType = 2 - nodeTypeEmpty nodeType = 1 - nodeTypeLeaf nodeType = 3 - nodeTypeLeafWithExpandedKey nodeType = 5 + nodeTypeBoundary nodeType = 4 + nodeTypeBranch nodeType = 2 + nodeTypeEmpty nodeType = 1 + nodeTypeLeaf nodeType = 3 + nodeTypeLeafWithExpandedKey nodeType = 5 + nodeTypeLeafWithExpandedKeyAndValue nodeType = 7 + nodeTypeLeafWithValue nodeType = 6 ) var wnodeType = widget.ForEnum[*nodeType](widget.Identity[*nodeType]) @@ -145,13 +147,15 @@ func (v nodeType) String() string { } type parameters struct { - Power uint64 - Mask uint64 + Power uint64 + Mask uint64 + ArbitraryValues bool } var wparameters = widget.ForCompositePtr(widget.Fields[parameters]{ {Name: "power", ID: 1, Widget: widget.ForUint(func(v *parameters) *uint64 { return &v.Power })}, {Name: "mask", ID: 2, Widget: widget.ForUint(func(v *parameters) *uint64 { return &v.Mask })}, + {Name: "arbitraryValues", ID: 3, Widget: widget.ForBool(func(v *parameters) *bool { return &v.ArbitraryValues })}, }, widget.Identity[**parameters]) // Copy returns a copy of the parameters. diff --git a/pkg/database/bpt/values_test.go b/pkg/database/bpt/values_test.go new file mode 100644 index 000000000..92397be22 --- /dev/null +++ b/pkg/database/bpt/values_test.go @@ -0,0 +1,50 @@ +// Copyright 2023 The Accumulate Authors +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +package bpt + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue" + "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory" + "gitlab.com/accumulatenetwork/accumulate/pkg/types/record" +) + +var testValues = []struct { + Key *record.Key + Value []byte +}{ + {record.NewKey(record.KeyHash{0x00}), []byte{1}}, + {record.NewKey(record.KeyHash{0x80}), []byte{2}}, + {record.NewKey(record.KeyHash{0x40}), []byte{3}}, + {record.NewKey(record.KeyHash{0xC0}), []byte{4}}, +} + +func TestNonHashValues(t *testing.T) { + store := memory.New(nil).Begin(nil, true) + model := new(ChangeSet) + model.store = keyvalue.RecordStore{Store: store} + + for _, e := range testValues { + require.NoError(t, model.BPT().Insert(e.Key, e.Value)) + } + require.NoError(t, model.Commit()) + + lup := map[[32]byte][]byte{} + for _, entry := range testValues { + lup[entry.Key.Hash()] = entry.Value + } + + for it := model.BPT().Iterate(100); it.Next(); { + for _, leaf := range it.Value() { + require.Equal(t, lup[leaf.Key.Hash()], leaf.Value) + } + } + + Print(t, model.BPT(), true) +}