refactor data store & changes struct

diiyw · Jun 22, 2024 · 3a7bb4d · 3a7bb4d
1 parent c04712f
commit 3a7bb4d
Show file tree

Hide file tree

Showing 41 changed files with 3,867 additions and 2,131 deletions.
diff --git a/README.md b/README.md
@@ -94,22 +94,21 @@ package main
 import (
 	"fmt"
 	"github.com/diiyw/nodis"
-	"github.com/diiyw/nodis/pb"
-	"github.com/diiyw/nodis/sync"
+	"github.com/diiyw/nodis/patch"
 	"time"
 )
 
 func main() {
 	var opt = nodis.DefaultOptions
 	n := nodis.Open(opt)
-	opt.Synchronizer = sync.NewWebsocket()
-	n.Watch([]string{"*"}, func(op *pb.Operation) {
-		fmt.Println("Server:", op.Key, string(op.Value))
+	opt.Synchronizer = nodis.NewWebsocket()
+	n.WatchKey([]string{"*"}, func(op patch.Op) {
+		fmt.Println("Server:", op.Data.GetKey(), op.Data.(*patch.OpSet).Value)
 	})
 	go func() {
 		for {
 			time.Sleep(time.Second)
-			n.Set("test", []byte(time.Now().Format("2006-01-02 15:04:05")))
+			n.Set("test", []byte(time.Now().Format("2006-01-02 15:04:05")), false)
 		}
 	}()
 	err := n.Publish("127.0.0.1:6380", []string{"*"})
@@ -132,17 +131,16 @@ import (
 	"fmt"
 	"github.com/diiyw/nodis"
 	"github.com/diiyw/nodis/fs"
-	"github.com/diiyw/nodis/pb"
-	"github.com/diiyw/nodis/sync"
+	"github.com/diiyw/nodis/patch"
 )
 
 func main() {
 	var opt = nodis.DefaultOptions
 	opt.Filesystem = &fs.Memory{}
-	opt.Synchronizer = sync.NewWebsocket()
+	opt.Synchronizer = nodis.NewWebsocket()
 	n := nodis.Open(opt)
-	n.WatchKey([]string{"*"}, func(op *pb.Operation) {
-		fmt.Println("Subscribe: ", op.Key)
+	n.WatchKey([]string{"*"}, func(op patch.Op) {
+		fmt.Println("Subscribe: ", op.Data.GetKey())
 	})
 	err := n.Subscribe("ws://127.0.0.1:6380")
 	if err != nil {
@@ -195,16 +193,27 @@ Windows 11: 12C/32G
 goos: windows
 goarch: amd64
 pkg: github.com/diiyw/nodis/bench
-BenchmarkSet-12         	 1469863	        715.9 ns/op	     543 B/op	       7 allocs/op
-BenchmarkGet-12         	12480278	        96.47 ns/op	       7 B/op	       0 allocs/op
-BenchmarkLPush-12       	 1484466	        786.2 ns/op	     615 B/op	       9 allocs/op
-BenchmarkLPop-12        	77275986	        15.10 ns/op	       0 B/op	       0 allocs/op
-BenchmarkSAdd-12        	 1542252	        831.9 ns/op	     663 B/op	      10 allocs/op
-BenchmarkSMembers-12    	12739020	        95.18 ns/op	       8 B/op	       1 allocs/op
-BenchmarkZAdd-12        	 1000000	        1177 ns/op	     550 B/op	      10 allocs/op
-BenchmarkZRank-12       	11430135	        104.1 ns/op	       7 B/op	       0 allocs/op
-BenchmarkHSet-12        	 1341817	        863.5 ns/op	     743 B/op	      11 allocs/op
-BenchmarkHGet-12        	 9801158	        105.9 ns/op	       7 B/op	       0 allocs/op
+cpu: 12th Gen Intel(R) Core(TM) i5-12490F
+BenchmarkSet
+BenchmarkSet-12         	 2159343	       514.7 ns/op	     302 B/op	       8 allocs/op
+BenchmarkGet
+BenchmarkGet-12         	 6421864	       183.8 ns/op	     166 B/op	       3 allocs/op
+BenchmarkLPush
+BenchmarkLPush-12       	 2166828	       566.3 ns/op	     358 B/op	      10 allocs/op
+BenchmarkLPop
+BenchmarkLPop-12        	13069830	        80.41 ns/op	     159 B/op	       3 allocs/op
+BenchmarkSAdd
+BenchmarkSAdd-12        	 2007924	       592.6 ns/op	     406 B/op	      11 allocs/op
+BenchmarkSMembers
+BenchmarkSMembers-12    	 6303288	       179.8 ns/op	     166 B/op	       3 allocs/op
+BenchmarkZAdd
+BenchmarkZAdd-12        	 1580179	       832.6 ns/op	     302 B/op	      10 allocs/op
+BenchmarkZRank
+BenchmarkZRank-12       	 6011108	       186.7 ns/op	     165 B/op	       3 allocs/op
+BenchmarkHSet
+BenchmarkHSet-12        	 1997553	       654.3 ns/op	     486 B/op	      11 allocs/op
+BenchmarkHGet
+BenchmarkHGet-12        	 5895134	       193.3 ns/op	     165 B/op	       3 allocs/op
 ```
 
 Linux VM: 4C/8GB
@@ -229,17 +238,19 @@ BenchmarkHGet-4       	 4442625	       243.4 ns/op	       7 B/op	       0 allocs
 <details>
 	<summary>Redis benchmark tool</summary>
 
+Windows 11: 12C/32G
+
 ```bash
 redis-benchmark -p 6380 -t set,get,lpush,lpop,sadd,smembers,zadd,zrank,hset,hget -n 100000 -q
 ```
 
 ```
-SET: 89126.56 requests per second
-GET: 90415.91 requests per second
-LPUSH: 91491.30 requests per second
-LPOP: 92165.90 requests per second
-SADD: 91911.76 requests per second
-HSET: 93023.25 requests per second
+SET: 116144.02 requests per second
+GET: 125156.45 requests per second
+LPUSH: 121951.22 requests per second
+LPOP: 126103.41 requests per second
+SADD: 121951.22 requests per second
+HSET: 122850.12 requests per second
 ```
 
 </details>

diff --git a/README_zh-cn.md b/README_zh-cn.md
@@ -94,7 +94,7 @@ package main
 import (
 	"fmt"
 	"github.com/diiyw/nodis"
-	"github.com/diiyw/nodis/pb"
+	"github.com/diiyw/nodis/patch"
 	"github.com/diiyw/nodis/sync"
 	"time"
 )
@@ -132,7 +132,7 @@ import (
 	"fmt"
 	"github.com/diiyw/nodis"
 	"github.com/diiyw/nodis/fs"
-	"github.com/diiyw/nodis/pb"
+	"github.com/diiyw/nodis/patch"
 	"github.com/diiyw/nodis/sync"
 )
 

diff --git a/ds/ds.go b/ds/ds.go
@@ -2,6 +2,7 @@ package ds
 
 type Value interface {
 	Type() ValueType
+	GetValue() []byte
 }
 
 type ValueType uint8

diff --git a/ds/hash/hash.go b/ds/hash/hash.go
@@ -1,20 +1,45 @@
 package hash
 
 import (
+	"encoding/binary"
 	"errors"
 	"path/filepath"
 	"strconv"
 	"unsafe"
 
 	"github.com/diiyw/nodis/ds"
-	"github.com/diiyw/nodis/pb"
 	"github.com/tidwall/btree"
 )
 
 type HashMap struct {
 	data btree.Map[string, []byte]
 }
 
+type keyValuePair struct {
+	key   string
+	value []byte
+}
+
+func (kvPair *keyValuePair) encode() []byte {
+	var kLen = len(kvPair.key)
+	var b = make([]byte, 8+kLen+len(kvPair.value))
+	n := binary.PutVarint(b, int64(kLen))
+	copy(b[n:], kvPair.key)
+	n += kLen
+	n += copy(b[n:], kvPair.value)
+	return b[:n]
+}
+
+func decodeKeyValuePair(b []byte) *keyValuePair {
+	l, n := binary.Varint(b)
+	b = b[n:]
+	key := string(b[:l])
+	return &keyValuePair{
+		key:   key,
+		value: b[l:],
+	}
+}
+
 // NewHashMap creates a new hash
 func NewHashMap() *HashMap {
 	return &HashMap{}
@@ -97,7 +122,7 @@ func (s *HashMap) HIncrBy(key string, value int64) (int64, error) {
 	return i, nil
 }
 
-// HIncByFloat increments the value of a hash
+// HIncrByFloat increments the value of a hash
 func (s *HashMap) HIncrByFloat(key string, value float64) (float64, error) {
 	v, ok := s.data.Get(key)
 	if !ok {
@@ -173,18 +198,37 @@ func (s *HashMap) HStrLen(field string) int64 {
 	return int64(len(*(*string)(unsafe.Pointer(&v))))
 }
 
-func (s *HashMap) GetValue() []*pb.MemberBytes {
-	values := make([]*pb.MemberBytes, 0, s.data.Len())
+func (s *HashMap) GetValue() []byte {
+	values := make([]byte, 0, s.data.Len())
 	s.data.Scan(func(key string, value []byte) bool {
-		values = append(values, &pb.MemberBytes{Member: key, Value: value})
+		kvPair := &keyValuePair{
+			key:   key,
+			value: value,
+		}
+		data := kvPair.encode()
+		dataLen := len(data)
+		var b = make([]byte, 8+dataLen)
+		n := binary.PutVarint(b, int64(dataLen))
+		copy(b[n:], data)
+		values = append(values, b[:n+dataLen]...)
 		return true
 	})
 	return values
 }
 
 // SetValue the set from bytes
-func (s *HashMap) SetValue(values []*pb.MemberBytes) {
-	for _, v := range values {
-		s.data.Set(v.Member, v.Value)
+func (s *HashMap) SetValue(values []byte) {
+	for {
+		if len(values) == 0 {
+			break
+		}
+		dataLen, n := binary.Varint(values)
+		if n <= 0 {
+			break
+		}
+		end := n + int(dataLen)
+		item := decodeKeyValuePair(values[n:end])
+		s.HSet(item.key, item.value)
+		values = values[end:]
 	}
 }
diff --git a/ds/hash/hash_test.go b/ds/hash/hash_test.go
@@ -262,3 +262,34 @@ func BenchmarkHashMap_HMGet(b *testing.B) {
 		hash.HMGet("testKey1", "testKey2", "testKey3")
 	}
 }
+
+func TestGetSetValue(t *testing.T) {
+	h := NewHashMap()
+	h.HSet("testKey", []byte("testValue"))
+	h.HSet("testKey1", []byte("testValue1"))
+	h.HSet("testKey2", []byte("testValue2"))
+	v := h.GetValue()
+	h2 := NewHashMap()
+	h2.SetValue(v)
+	vv := h.HGet("testKey")
+	if vv == nil {
+		t.Errorf("GetSetValue failed, expected %s but got nothing", "testValue")
+	}
+	if string(vv) != "testValue" {
+		t.Errorf("GetSetValue failed, expected %s but got %s", "testValue", vv)
+	}
+	vv = h.HGet("testKey1")
+	if vv == nil {
+		t.Errorf("GetSetValue failed, expected %s but got nothing", "testValue")
+	}
+	if string(vv) != "testValue1" {
+		t.Errorf("GetSetValue failed, expected %s but got %s", "testValue1", vv)
+	}
+	vv = h.HGet("testKey2")
+	if vv == nil {
+		t.Errorf("GetSetValue failed, expected %s but got nothing", "testValue")
+	}
+	if string(vv) != "testValue2" {
+		t.Errorf("GetSetValue failed, expected %s but got %s", "testValue2", vv)
+	}
+}
diff --git a/ds/list/linked_list.go b/ds/list/linked_list.go
@@ -2,6 +2,7 @@ package list
 
 import (
 	"bytes"
+	"encoding/binary"
 
 	"github.com/diiyw/nodis/ds"
 )
@@ -108,10 +109,18 @@ func (l *LinkedList) RPop(count int64) [][]byte {
 // LRange returns a range of elements from the list
 func (l *LinkedList) LRange(start, end int64) [][]byte {
 	var result [][]byte
+	l.forEach(start, end, func(v []byte) {
+		result = append(result, v)
+	})
+	return result
+}
+
+// LRange returns a range of elements from the list
+func (l *LinkedList) forEach(start, end int64, fn func(v []byte)) {
 	currentNode := l.head
 	var index int64 = 0
 	if start != 0 && start >= end {
-		return result
+		return
 	}
 	if start < 0 {
 		start = l.size() + start
@@ -121,15 +130,14 @@ func (l *LinkedList) LRange(start, end int64) [][]byte {
 	}
 	for currentNode != nil {
 		if index >= start && index <= end {
-			result = append(result, currentNode.data)
+			fn(currentNode.data)
 		}
 		if index > end {
 			break
 		}
 		currentNode = currentNode.next
 		index++
 	}
-	return result
 }
 
 func (l *LinkedList) size() int64 {
@@ -336,13 +344,30 @@ func (l *LinkedList) LTrim(start, end int64) {
 }
 
 // GetValue returns the byte slice of the list
-func (l *LinkedList) GetValue() [][]byte {
-	return l.LRange(0, -1)
+func (l *LinkedList) GetValue() []byte {
+	var list []byte
+	l.forEach(0, -1, func(v []byte) {
+		var b = make([]byte, len(v)+8)
+		var vLen = len(v)
+		n := binary.PutVarint(b, int64(vLen))
+		copy(b[n:], v)
+		list = append(list, b[:n+vLen]...)
+	})
+	return list
 }
 
 // SetValue restores the list from the byte slice
-func (l *LinkedList) SetValue(list [][]byte) {
-	for _, item := range list {
-		l.RPush(item)
+func (l *LinkedList) SetValue(list []byte) {
+	for {
+		if len(list) == 0 {
+			break
+		}
+		vLen, n := binary.Varint(list)
+		if n == 0 {
+			break
+		}
+		v := list[n : n+int(vLen)]
+		list = list[n+int(vLen):]
+		l.RPush(v)
 	}
 }
diff --git a/ds/list/list_test.go b/ds/list/list_test.go
@@ -166,3 +166,23 @@ func BenchmarkDoublyLinkedList_LPop(b *testing.B) {
 		l.LPop(1)
 	}
 }
+
+func TestGetSetValue(t *testing.T) {
+	l := NewLinkedList()
+	for i := 0; i < 10; i++ {
+		is := strconv.Itoa(i)
+		l.LPush([]byte(is))
+	}
+	v := l.GetValue()
+	l2 := NewLinkedList()
+	l2.SetValue(v)
+	if l.LLen() != l2.LLen() {
+		t.Errorf("get set value error")
+	}
+	l2Pop := l2.LPop(10)
+	for i, it := range l.LPop(10) {
+		if string(it) != string(l2Pop[i]) {
+			t.Errorf("get set value error")
+		}
+	}
+}