diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..00b6c83d1 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,16 @@ +version: 2 +jobs: + test: + docker: + - image: circleci/golang:1.10.3 + working_directory: /go/src/github.com/tendermint/iavl + steps: + - checkout + - run: dep ensure -v + - run: make test + +workflows: + version: 2 + test: + jobs: + - test \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e3f5ca4..bc67fbb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.11.0 (September 7, 2018) + +BREAKING CHANGES + +- Changed internal database key format to store int64 key components in a full 8-byte fixed width ([#107]) +- Removed some architecture dependent methods (e.g., use `Get` instead of `Get64` etc) ([#96]) + +IMPROVEMENTS + +- Database key format avoids use of fmt.Sprintf fmt.Sscanf leading to ~10% speedup in benchmark BenchmarkTreeLoadAndDelete ([#107], thanks to [@silasdavis]) + +[#107]: https://github.com/tendermint/iavl/pull/107 +[@silasdavis]: https://github.com/silasdavis +[#96]: https://github.com/tendermint/iavl/pull/96 + ## 0.10.0 BREAKING CHANGES diff --git a/Makefile b/Makefile index a77c26f78..8d4c774b8 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ -GOTOOLS := github.com/mitchellh/gox \ - github.com/golang/dep/cmd/dep +GOTOOLS := github.com/golang/dep/cmd/dep + PDFFLAGS := -pdf --nodefraction=0.1 all: get_vendor_deps test test: - GOCACHE=off go test -v --race `glide novendor` + GOCACHE=off go test -v --race tools: go get -u -v $(GOTOOLS) diff --git a/basic_test.go b/basic_test.go index 07e4cb9fa..c5ff62709 100644 --- a/basic_test.go +++ b/basic_test.go @@ -32,7 +32,7 @@ func TestBasic(t *testing.T) { // Test 0x00 { - idx, val := tree.Get64([]byte{0x00}) + idx, val := tree.Get([]byte{0x00}) if val != nil { t.Errorf("Expected no value to exist") } @@ -46,7 +46,7 @@ func TestBasic(t *testing.T) { // Test "1" { - idx, val := tree.Get64([]byte("1")) + idx, val := tree.Get([]byte("1")) if val == nil { t.Errorf("Expected value to exist") } @@ -60,7 +60,7 @@ func TestBasic(t *testing.T) { // Test "2" { - idx, val := tree.Get64([]byte("2")) + idx, val := tree.Get([]byte("2")) if val == nil { t.Errorf("Expected value to exist") } @@ -74,7 +74,7 @@ func TestBasic(t *testing.T) { // Test "4" { - idx, val := tree.Get64([]byte("4")) + idx, val := tree.Get([]byte("4")) if val != nil { t.Errorf("Expected no value to exist") } @@ -88,7 +88,7 @@ func TestBasic(t *testing.T) { // Test "6" { - idx, val := tree.Get64([]byte("6")) + idx, val := tree.Get([]byte("6")) if val != nil { t.Errorf("Expected no value to exist") } @@ -237,7 +237,7 @@ func TestIntegration(t *testing.T) { if !updated { t.Error("should have been updated") } - if tree.Size() != i+1 { + if tree.Size() != int64(i+1) { t.Error("size was wrong", tree.Size(), i+1) } } @@ -249,7 +249,7 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - if _, val := tree.Get64([]byte(r.key)); string(val) != string(r.value) { + if _, val := tree.Get([]byte(r.key)); string(val) != string(r.value) { t.Error("wrong value") } } @@ -267,12 +267,12 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - _, val := tree.Get64([]byte(r.key)) + _, val := tree.Get([]byte(r.key)) if string(val) != string(r.value) { t.Error("wrong value") } } - if tree.Size() != len(records)-(i+1) { + if tree.Size() != int64(len(records)-(i+1)) { t.Error("size was wrong", tree.Size(), (len(records) - (i + 1))) } } @@ -382,7 +382,7 @@ func TestPersistence(t *testing.T) { t2 := NewMutableTree(db, 0) t2.Load() for key, value := range records { - _, t2value := t2.Get64([]byte(key)) + _, t2value := t2.Get([]byte(key)) if string(t2value) != value { t.Fatalf("Invalid value. Expected %v, got %v", value, t2value) } diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 2c4b02306..000000000 --- a/circle.yml +++ /dev/null @@ -1,21 +0,0 @@ -machine: - environment: - GOPATH: /home/ubuntu/.go_workspace - REPO: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME - hosts: - circlehost: 127.0.0.1 - localhost: 127.0.0.1 - -checkout: - post: - - rm -rf $REPO - - mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME - - mv $HOME/$CIRCLE_PROJECT_REPONAME $REPO - -dependencies: - override: - - "cd $REPO && make get_vendor_deps" - -test: - override: - - "cd $REPO && make test" diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index b16aa5f68..000000000 --- a/glide.yaml +++ /dev/null @@ -1,20 +0,0 @@ -package: github.com/tendermint/iavl -import: -- package: github.com/pkg/errors - version: ^0.8.0 -- package: github.com/tendermint/go-amino - version: develop -- package: github.com/tendermint/tmlibs - version: develop - subpackages: - - common - - db -- package: golang.org/x/crypto - subpackages: - - ripemd160 -testImport: -- package: github.com/stretchr/testify - version: ^1.2.1 - subpackages: - - assert - - require diff --git a/immutable_tree.go b/immutable_tree.go index e334ef52b..5abc91823 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -39,11 +39,7 @@ func (t *ImmutableTree) String() string { } // Size returns the number of leaf nodes in the tree. -func (t *ImmutableTree) Size() int { - return int(t.Size64()) -} - -func (t *ImmutableTree) Size64() int64 { +func (t *ImmutableTree) Size() int64 { if t.root == nil { return 0 } @@ -51,20 +47,12 @@ func (t *ImmutableTree) Size64() int64 { } // Version returns the version of the tree. -func (t *ImmutableTree) Version() int { - return int(t.Version64()) -} - -func (t *ImmutableTree) Version64() int64 { +func (t *ImmutableTree) Version() int64 { return t.version } // Height returns the height of the tree. -func (t *ImmutableTree) Height() int { - return int(t.Height8()) -} - -func (t *ImmutableTree) Height8() int8 { +func (t *ImmutableTree) Height() int8 { if t.root == nil { return 0 } @@ -98,12 +86,7 @@ func (t *ImmutableTree) hashWithCount() ([]byte, int64) { // Get returns the index and value of the specified key if it exists, or nil // and the next index, if it doesn't. -func (t *ImmutableTree) Get(key []byte) (index int, value []byte) { - index64, value := t.Get64(key) - return int(index64), value -} - -func (t *ImmutableTree) Get64(key []byte) (index int64, value []byte) { +func (t *ImmutableTree) Get(key []byte) (index int64, value []byte) { if t.root == nil { return 0, nil } @@ -111,11 +94,7 @@ func (t *ImmutableTree) Get64(key []byte) (index int64, value []byte) { } // GetByIndex gets the key and value at the specified index. -func (t *ImmutableTree) GetByIndex(index int) (key []byte, value []byte) { - return t.GetByIndex64(int64(index)) -} - -func (t *ImmutableTree) GetByIndex64(index int64) (key []byte, value []byte) { +func (t *ImmutableTree) GetByIndex(index int64) (key []byte, value []byte) { if t.root == nil { return nil, nil } diff --git a/key_format.go b/key_format.go new file mode 100644 index 000000000..b63f9ea7b --- /dev/null +++ b/key_format.go @@ -0,0 +1,144 @@ +package iavl + +import ( + "encoding/binary" + "fmt" +) + +// Provides a fixed-width lexicographically sortable []byte key format +type KeyFormat struct { + prefix byte + layout []int + length int +} + +// Create a []byte key format based on a single byte prefix and fixed width key segments each of whose length is +// specified by by the corresponding element of layout. +// +// For example, to store keys that could index some objects by a version number and their SHA256 hash using the form: +// 'c' then you would define the KeyFormat with: +// +// var keyFormat = NewKeyFormat('c', 8, 32) +// +// Then you can create a key with: +// +// func ObjectKey(version uint64, objectBytes []byte) []byte { +// hasher := sha256.New() +// hasher.Sum(nil) +// return keyFormat.Key(version, hasher.Sum(nil)) +// } +func NewKeyFormat(prefix byte, layout ...int) *KeyFormat { + // For prefix byte + length := 1 + for _, l := range layout { + length += int(l) + } + return &KeyFormat{ + prefix: prefix, + layout: layout, + length: length, + } +} + +// Format the byte segments into the key format - will panic if the segment lengths do not match the layout. +func (kf *KeyFormat) KeyBytes(segments ...[]byte) []byte { + key := make([]byte, kf.length) + key[0] = kf.prefix + n := 1 + for i, s := range segments { + l := kf.layout[i] + if len(s) > l { + panic(fmt.Errorf("length of segment %X provided to KeyFormat.KeyBytes() is longer than the %d bytes "+ + "required by layout for segment %d", s, l, i)) + } + n += l + // Big endian so pad on left if not given the full width for this segment + copy(key[n-len(s):n], s) + } + return key[:n] +} + +// Format the args passed into the key format - will panic if the arguments passed do not match the length +// of the segment to which they correspond. When called with no arguments returns the raw prefix (useful as a start +// element of the entire keys space when sorted lexicographically). +func (kf *KeyFormat) Key(args ...interface{}) []byte { + if len(args) > len(kf.layout) { + panic(fmt.Errorf("KeyFormat.Key() is provided with %d args but format only has %d segments", + len(args), len(kf.layout))) + } + segments := make([][]byte, len(args)) + for i, a := range args { + segments[i] = format(a) + } + return kf.KeyBytes(segments...) +} + +// Reads out the bytes associated with each segment of the key format from key. +func (kf *KeyFormat) ScanBytes(key []byte) [][]byte { + segments := make([][]byte, len(kf.layout)) + n := 1 + for i, l := range kf.layout { + n += l + if n > len(key) { + return segments[:i] + } + segments[i] = key[n-l : n] + } + return segments +} + +// Extracts the segments into the values pointed to by each of args. Each arg must be a pointer to int64, uint64, or +// []byte, and the width of the args must match layout. +func (kf *KeyFormat) Scan(key []byte, args ...interface{}) { + segments := kf.ScanBytes(key) + if len(args) > len(segments) { + panic(fmt.Errorf("KeyFormat.Scan() is provided with %d args but format only has %d segments in key %X", + len(args), len(segments), key)) + } + for i, a := range args { + scan(a, segments[i]) + } +} + +// Return the prefix as a string. +func (kf *KeyFormat) Prefix() string { + return string([]byte{kf.prefix}) +} + +func scan(a interface{}, value []byte) { + switch v := a.(type) { + case *int64: + // Negative values will be mapped correctly when read in as uint64 and then type converted + *v = int64(binary.BigEndian.Uint64(value)) + case *uint64: + *v = binary.BigEndian.Uint64(value) + case *[]byte: + *v = value + default: + panic(fmt.Errorf("KeyFormat scan() does not support scanning value of type %T: %v", a, a)) + } +} + +func format(a interface{}) []byte { + switch v := a.(type) { + case uint64: + return formatUint64(v) + case int64: + return formatUint64(uint64(v)) + // Provide formatting from int,uint as a convenience to avoid casting arguments + case uint: + return formatUint64(uint64(v)) + case int: + return formatUint64(uint64(v)) + case []byte: + return v + default: + panic(fmt.Errorf("KeyFormat format() does not support formatting value of type %T: %v", a, a)) + } +} + +func formatUint64(v uint64) []byte { + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, v) + return bs +} diff --git a/key_format_test.go b/key_format_test.go new file mode 100644 index 000000000..b6ea3010d --- /dev/null +++ b/key_format_test.go @@ -0,0 +1,70 @@ +package iavl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKeyFormatBytes(t *testing.T) { + kf := NewKeyFormat(byte('e'), 8, 8, 8) + assert.Equal(t, []byte{'e', 0, 0, 0, 0, 0, 1, 2, 3}, kf.KeyBytes([]byte{1, 2, 3})) + assert.Equal(t, []byte{'e', 1, 2, 3, 4, 5, 6, 7, 8}, kf.KeyBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8})) + assert.Equal(t, []byte{'e', 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 1, 1, 2, 2, 3, 3}, + kf.KeyBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8}, []byte{1, 2, 3, 4, 5, 6, 7, 8}, []byte{1, 1, 2, 2, 3, 3})) + assert.Equal(t, []byte{'e'}, kf.KeyBytes()) +} + +func TestKeyFormat(t *testing.T) { + kf := NewKeyFormat(byte('e'), 8, 8, 8) + key := []byte{'e', 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 1, 144} + var a, b, c int64 = 100, 200, 400 + assert.Equal(t, key, kf.Key(a, b, c)) + + var ao, bo, co = new(int64), new(int64), new(int64) + kf.Scan(key, ao, bo, co) + assert.Equal(t, a, *ao) + assert.Equal(t, b, *bo) + assert.Equal(t, c, *co) + + bs := new([]byte) + kf.Scan(key, ao, bo, bs) + assert.Equal(t, a, *ao) + assert.Equal(t, b, *bo) + assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 1, 144}, *bs) + + assert.Equal(t, []byte{'e', 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 200}, kf.Key(a, b)) +} + +func TestNegativeKeys(t *testing.T) { + kf := NewKeyFormat(byte('e'), 8, 8) + + var a, b int64 = -100, -200 + // One's complement plus one + key := []byte{'e', + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, byte(0xff + a + 1), + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, byte(0xff + b + 1)} + assert.Equal(t, key, kf.Key(a, b)) + + var ao, bo = new(int64), new(int64) + kf.Scan(key, ao, bo) + assert.Equal(t, a, *ao) + assert.Equal(t, b, *bo) +} + +func TestOverflow(t *testing.T) { + kf := NewKeyFormat(byte('o'), 8, 8) + + var a int64 = 1 << 62 + var b uint64 = 1 << 63 + key := []byte{'o', + 0x40, 0, 0, 0, 0, 0, 0, 0, + 0x80, 0, 0, 0, 0, 0, 0, 0, + } + assert.Equal(t, key, kf.Key(a, b)) + + var ao, bo = new(int64), new(int64) + kf.Scan(key, ao, bo) + assert.Equal(t, a, *ao) + assert.Equal(t, int64(b), *bo) +} diff --git a/mutable_tree.go b/mutable_tree.go index 8ad073345..05b89cc32 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -294,7 +294,7 @@ func (tree *MutableTree) Rollback() { // GetVersioned gets the value at the specified key and version. func (tree *MutableTree) GetVersioned(key []byte, version int64) ( - index int, value []byte, + index int64, value []byte, ) { if tree.versions[version] { t, err := tree.GetImmutable(version) diff --git a/nodedb.go b/nodedb.go index a933b2662..1844ded05 100644 --- a/nodedb.go +++ b/nodedb.go @@ -7,27 +7,29 @@ import ( "sort" "sync" + "github.com/tendermint/tendermint/crypto/tmhash" dbm "github.com/tendermint/tendermint/libs/db" ) +const ( + int64Size = 8 + hashSize = tmhash.Size +) + var ( - // All node keys are prefixed with this. This ensures no collision is - // possible with the other keys, and makes them easier to traverse. - nodePrefix = "n/" - nodeKeyFmt = "n/%X" + // All node keys are prefixed with the byte 'n'. This ensures no collision is + // possible with the other keys, and makes them easier to traverse. They are indexed by the node hash. + nodeKeyFormat = NewKeyFormat('n', hashSize) // n // Orphans are keyed in the database by their expected lifetime. // The first number represents the *last* version at which the orphan needs // to exist, while the second number represents the *earliest* version at // which it is expected to exist - which starts out by being the version // of the node being orphaned. - orphanPrefix = "o/" - orphanPrefixFmt = "o/%010d/" // o// - orphanKeyFmt = "o/%010d/%010d/%X" // o/// + orphanKeyFormat = NewKeyFormat('o', int64Size, int64Size, hashSize) // o - // r/ - rootPrefix = "r/" - rootPrefixFmt = "r/%010d" + // Root nodes are indexed separately by their version + rootKeyFormat = NewKeyFormat('r', int64Size) // r ) type nodeDB struct { @@ -196,7 +198,7 @@ func (ndb *nodeDB) deleteOrphans(version int64) { // See comment on `orphanKeyFmt`. Note that here, `version` and // `toVersion` are always equal. - fmt.Sscanf(string(key), orphanKeyFmt, &toVersion, &fromVersion) + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) // Delete orphan key and reverse-lookup key. ndb.batch.Delete(key) @@ -218,15 +220,15 @@ func (ndb *nodeDB) deleteOrphans(version int64) { } func (ndb *nodeDB) nodeKey(hash []byte) []byte { - return []byte(fmt.Sprintf(nodeKeyFmt, hash)) + return nodeKeyFormat.KeyBytes(hash) } func (ndb *nodeDB) orphanKey(fromVersion, toVersion int64, hash []byte) []byte { - return []byte(fmt.Sprintf(orphanKeyFmt, toVersion, fromVersion, hash)) + return orphanKeyFormat.Key(toVersion, fromVersion, hash) } func (ndb *nodeDB) rootKey(version int64) []byte { - return []byte(fmt.Sprintf(rootPrefixFmt, version)) + return rootKeyFormat.Key(version) } func (ndb *nodeDB) getLatestVersion() int64 { @@ -244,20 +246,16 @@ func (ndb *nodeDB) updateLatestVersion(version int64) { func (ndb *nodeDB) getPreviousVersion(version int64) int64 { itr := ndb.db.ReverseIterator( - []byte(fmt.Sprintf(rootPrefixFmt, version-1)), - []byte(fmt.Sprintf(rootPrefixFmt, 0)), + rootKeyFormat.Key(version-1), + rootKeyFormat.Key(0), ) defer itr.Close() pversion := int64(-1) for ; itr.Valid(); itr.Next() { k := itr.Key() - _, err := fmt.Sscanf(string(k), rootPrefixFmt, &pversion) - if err != nil { - panic(err) - } else { - return pversion - } + rootKeyFormat.Scan(k, &pversion) + return pversion } return 0 @@ -274,13 +272,12 @@ func (ndb *nodeDB) deleteRoot(version int64) { } func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { - ndb.traversePrefix([]byte(orphanPrefix), fn) + ndb.traversePrefix(orphanKeyFormat.Key(), fn) } // Traverse orphans ending at a certain version. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { - prefix := fmt.Sprintf(orphanPrefixFmt, version) - ndb.traversePrefix([]byte(prefix), fn) + ndb.traversePrefix(orphanKeyFormat.Key(version), fn) } // Traverse all keys. @@ -339,9 +336,9 @@ func (ndb *nodeDB) getRoot(version int64) []byte { func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} - ndb.traversePrefix([]byte(rootPrefix), func(k, v []byte) { + ndb.traversePrefix(rootKeyFormat.Key(), func(k, v []byte) { var version int64 - fmt.Sscanf(string(k), rootPrefixFmt, &version) + rootKeyFormat.Scan(k, &version) roots[version] = v }) return roots, nil @@ -426,12 +423,12 @@ func (ndb *nodeDB) size() int { func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node)) { nodes := []*Node{} - ndb.traversePrefix([]byte(nodePrefix), func(key, value []byte) { + ndb.traversePrefix(nodeKeyFormat.Key(), func(key, value []byte) { node, err := MakeNode(value) if err != nil { panic(fmt.Sprintf("Couldn't decode node from database: %v", err)) } - fmt.Sscanf(string(key), nodeKeyFmt, &node.hash) + nodeKeyFormat.Scan(key, &node.hash) nodes = append(nodes, node) }) @@ -448,7 +445,7 @@ func (ndb *nodeDB) String() string { var str string index := 0 - ndb.traversePrefix([]byte(rootPrefix), func(key, value []byte) { + ndb.traversePrefix(rootKeyFormat.Key(), func(key, value []byte) { str += fmt.Sprintf("%s: %x\n", string(key), value) }) str += "\n" @@ -462,11 +459,13 @@ func (ndb *nodeDB) String() string { if len(hash) == 0 { str += fmt.Sprintf("\n") } else if node == nil { - str += fmt.Sprintf("%s%40x: \n", nodePrefix, hash) + str += fmt.Sprintf("%s%40x: \n", nodeKeyFormat.Prefix(), hash) } else if node.value == nil && node.height > 0 { - str += fmt.Sprintf("%s%40x: %s %-16s h=%d version=%d\n", nodePrefix, hash, node.key, "", node.height, node.version) + str += fmt.Sprintf("%s%40x: %s %-16s h=%d version=%d\n", + nodeKeyFormat.Prefix(), hash, node.key, "", node.height, node.version) } else { - str += fmt.Sprintf("%s%40x: %s = %-16s h=%d version=%d\n", nodePrefix, hash, node.key, node.value, node.height, node.version) + str += fmt.Sprintf("%s%40x: %s = %-16s h=%d version=%d\n", + nodeKeyFormat.Prefix(), hash, node.key, node.value, node.height, node.version) } index++ }) diff --git a/nodedb_test.go b/nodedb_test.go new file mode 100644 index 000000000..eee0ce16b --- /dev/null +++ b/nodedb_test.go @@ -0,0 +1,39 @@ +package iavl + +import ( + "encoding/binary" + "math/rand" + "testing" +) + +func BenchmarkNodeKey(b *testing.B) { + ndb := &nodeDB{} + hashes := makeHashes(b, 2432325) + for i := 0; i < b.N; i++ { + ndb.nodeKey(hashes[i]) + } +} + +func BenchmarkOrphanKey(b *testing.B) { + ndb := &nodeDB{} + hashes := makeHashes(b, 2432325) + for i := 0; i < b.N; i++ { + ndb.orphanKey(1234, 1239, hashes[i]) + } +} + +func makeHashes(b *testing.B, seed int64) [][]byte { + b.StopTimer() + rnd := rand.NewSource(seed) + hashes := make([][]byte, b.N) + hashBytes := 8*((hashSize+7)/8) + for i := 0; i < b.N; i++ { + hashes[i] = make([]byte, hashBytes) + for b := 0; b < hashBytes; b += 8 { + binary.BigEndian.PutUint64(hashes[i][b:b+8], uint64(rnd.Int63())) + } + hashes[i] = hashes[i][:hashSize] + } + b.StartTimer() + return hashes +} diff --git a/proof_test.go b/proof_test.go index b408a0d9f..b9e82a2b2 100644 --- a/proof_test.go +++ b/proof_test.go @@ -17,7 +17,7 @@ func TestTreeGetWithProof(t *testing.T) { require := require.New(t) for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} - tree.Set(key, []byte(rand.Str(8))) + tree.Set(key, []byte(random.Str(8))) } root := tree.WorkingHash() diff --git a/testutils_test.go b/testutils_test.go index 1a76d60b9..9f810a79b 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -115,8 +115,9 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.StopTimer() t := NewMutableTree(db, 100000) + value := []byte{} for i := 0; i < 1000000; i++ { - t.Set(i2b(int(cmn.RandInt32())), nil) + t.Set(i2b(int(cmn.RandInt32())), value) if i > 990000 && i%1000 == 999 { t.SaveVersion() } @@ -124,14 +125,12 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.ReportAllocs() t.SaveVersion() - fmt.Println("ok, starting") - runtime.GC() b.StartTimer() for i := 0; i < b.N; i++ { ri := i2b(int(cmn.RandInt32())) - t.Set(ri, nil) + t.Set(ri, value) t.Remove(ri) if i%100 == 99 { t.SaveVersion() diff --git a/tree_test.go b/tree_test.go index c12eb7116..b55355b3e 100644 --- a/tree_test.go +++ b/tree_test.go @@ -3,7 +3,6 @@ package iavl import ( "bytes" "flag" - "fmt" "os" "runtime" "testing" @@ -18,11 +17,11 @@ import ( var testLevelDB bool var testFuzzIterations int -var rand *cmn.Rand +var random *cmn.Rand func init() { - rand = cmn.NewRand() - rand.Seed(0) // for determinism + random = cmn.NewRand() + random.Seed(0) // for determinism flag.BoolVar(&testLevelDB, "test.leveldb", false, "test leveldb backend") flag.IntVar(&testFuzzIterations, "test.fuzz-iterations", 100000, "number of fuzz testing iterations") flag.Parse() @@ -55,8 +54,8 @@ func TestVersionedRandomTree(t *testing.T) { // Create a tree of size 1000 with 100 versions. for i := 1; i <= versions; i++ { for j := 0; j < keysPerVersion; j++ { - k := []byte(rand.Str(8)) - v := []byte(rand.Str(8)) + k := []byte(random.Str(8)) + v := []byte(random.Str(8)) tree.Set(k, v) } tree.SaveVersion() @@ -79,7 +78,7 @@ func TestVersionedRandomTree(t *testing.T) { // After cleaning up all previous versions, we should have as many nodes // in the db as in the current tree version. - require.Len(tree.ndb.leafNodes(), tree.Size()) + require.Len(tree.ndb.leafNodes(), int(tree.Size())) require.Equal(tree.nodeSize(), len(tree.ndb.nodes())) } @@ -97,8 +96,8 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { for i := 1; i <= versions; i++ { for j := 0; j < keysPerVersion; j++ { // Keys of size one are likely to be overwritten. - k := []byte(rand.Str(1)) - v := []byte(rand.Str(8)) + k := []byte(random.Str(1)) + v := []byte(random.Str(8)) tree.Set(k, v) singleVersionTree.Set(k, v) } @@ -113,13 +112,13 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { // After cleaning up all previous versions, we should have as many nodes // in the db as in the current tree version. The simple tree must be equal // too. - require.Len(tree.ndb.leafNodes(), tree.Size()) + require.Len(tree.ndb.leafNodes(), int(tree.Size())) require.Len(tree.ndb.nodes(), tree.nodeSize()) require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) // Try getting random keys. for i := 0; i < keysPerVersion; i++ { - _, val := tree.Get([]byte(rand.Str(1))) + _, val := tree.Get([]byte(random.Str(1))) require.NotNil(val) require.NotEmpty(val) } @@ -138,8 +137,8 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { for i := 1; i <= versions; i++ { for j := 0; j < keysPerVersion; j++ { // Keys of size one are likely to be overwritten. - k := []byte(rand.Str(1)) - v := []byte(rand.Str(8)) + k := []byte(random.Str(1)) + v := []byte(random.Str(8)) tree.Set(k, v) singleVersionTree.Set(k, v) } @@ -147,20 +146,20 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { } singleVersionTree.SaveVersion() - for _, i := range rand.Perm(versions - 1) { + for _, i := range random.Perm(versions - 1) { tree.DeleteVersion(int64(i + 1)) } // After cleaning up all previous versions, we should have as many nodes // in the db as in the current tree version. The simple tree must be equal // too. - require.Len(tree.ndb.leafNodes(), tree.Size()) + require.Len(tree.ndb.leafNodes(), int(tree.Size())) require.Len(tree.ndb.nodes(), tree.nodeSize()) require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) // Try getting random keys. for i := 0; i < keysPerVersion; i++ { - _, val := tree.Get([]byte(rand.Str(1))) + _, val := tree.Get([]byte(random.Str(1))) require.NotNil(val) require.NotEmpty(val) } @@ -628,14 +627,14 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { preHash := tree.Hash() require.NotNil(preHash) - require.Equal(6, tree.Version()) + require.Equal(int64(6), tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. ntree := NewMutableTree(d, 0) ntree.Load() require.False(ntree.IsEmpty()) - require.Equal(6, ntree.Version()) + require.Equal(int64(6), ntree.Version()) postHash := ntree.Hash() require.Equal(preHash, postHash) @@ -651,7 +650,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { ntree.DeleteVersion(3) require.False(ntree.IsEmpty()) - require.Equal(4, ntree.Size()) + require.Equal(int64(4), ntree.Size()) require.Len(ntree.ndb.nodes(), ntree.nodeSize()) } @@ -697,8 +696,8 @@ func TestVersionedCheckpoints(t *testing.T) { for i := 1; i <= versions; i++ { for j := 0; j < keysPerVersion; j++ { - k := []byte(rand.Str(1)) - v := []byte(rand.Str(8)) + k := []byte(random.Str(1)) + v := []byte(random.Str(8)) keys[int64(i)] = append(keys[int64(i)], k) tree.Set(k, v) } @@ -931,7 +930,7 @@ func TestVersionedTreeEfficiency(t *testing.T) { for i := 1; i <= versions; i++ { for j := 0; j < keysPerVersion; j++ { // Keys of size one are likely to be overwritten. - tree.Set([]byte(rand.Str(1)), []byte(rand.Str(8))) + tree.Set([]byte(random.Str(1)), []byte(random.Str(8))) } sizeBefore := len(tree.ndb.nodes()) tree.SaveVersion() @@ -1051,7 +1050,7 @@ func TestOrphans(t *testing.T) { tree.ndb.traverseOrphans(func(k, v []byte) { var fromVersion, toVersion int64 - fmt.Sscanf(string(k), orphanKeyFmt, &toVersion, &fromVersion) + orphanKeyFormat.Scan(k, &toVersion, &fromVersion) require.Equal(fromVersion, int64(1), "fromVersion should be 1") require.Equal(toVersion, int64(1), "toVersion should be 1") }) @@ -1122,7 +1121,7 @@ func TestRollback(t *testing.T) { tree.SaveVersion() - require.Equal(2, tree.Size()) + require.Equal(int64(2), tree.Size()) _, val := tree.Get([]byte("r")) require.Nil(val) @@ -1182,7 +1181,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { tree := NewMutableTree(d, 0) for v := 1; v < numVersions; v++ { for i := 0; i < numKeysPerVersion; i++ { - tree.Set([]byte(rand.Str(16)), rand.Bytes(32)) + tree.Set([]byte(random.Str(16)), random.Bytes(32)) } tree.SaveVersion() } @@ -1203,7 +1202,7 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { // If we can load quickly into a data-structure that allows for // efficient deletes, we are golden. for v := 0; v < numVersions/10; v++ { - version := (rand.Int() % numVersions) + 1 + version := (random.Int() % numVersions) + 1 tree.DeleteVersion(int64(version)) } } diff --git a/version.go b/version.go index f88f00abc..aa993f09b 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package iavl // Version of iavl. -const Version = "0.10.0" +const Version = "0.11.0"