Skip to content

Commit 83d771e

Browse files
HDT3213HDT3213
committed
bug fix: listpack backlen;
optimize listpack parsing speed
1 parent eac5035 commit 83d771e

File tree

5 files changed

+122
-84
lines changed

5 files changed

+122
-84
lines changed

‎core/hash.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package core
33
import (
44
"encoding/binary"
55
"errors"
6+
67
"github.com/hdt3213/rdb/model"
78
)
89

@@ -163,11 +164,11 @@ func (dec *Decoder) readListPackHash() (map[string][]byte, *model.ListpackDetail
163164
size := readListPackLength(buf, &cursor)
164165
m := make(map[string][]byte)
165166
for i := 0; i < size; i += 2 {
166-
key, _, err := dec.readListPackEntry(buf, &cursor)
167+
key, err := dec.readListPackEntryAsString(buf, &cursor)
167168
if err != nil {
168169
return nil, nil, err
169170
}
170-
val, _, err := dec.readListPackEntry(buf, &cursor)
171+
val, err := dec.readListPackEntryAsString(buf, &cursor)
171172
if err != nil {
172173
return nil, nil, err
173174
}

‎core/listpack.go‎

Lines changed: 105 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ func (dec *Decoder) readListPack() ([][]byte, []uint32, error) {
1818
entries := make([][]byte, 0, size)
1919
entrySizes := make([]uint32, 0, size)
2020
for i := 0; i < size; i++ {
21-
entry, length, err := dec.readListPackEntry(buf, &cursor)
21+
str, intval, length, err := dec.readListPackEntry(buf, &cursor)
2222
if err != nil {
2323
return nil, nil, err
2424
}
25-
entries = append(entries, entry)
25+
if str == nil {
26+
str = []byte(strconv.FormatInt(intval, 10))
27+
}
28+
entries = append(entries, str)
2629
entrySizes = append(entrySizes, length)
2730
}
2831
return entries, entrySizes, nil
@@ -37,135 +40,167 @@ func readListPackLength(buf []byte, cursor *int) int {
3740
return size
3841
}
3942

40-
func readVarInt(buf []byte, cursor *int) uint32 {
41-
var v uint32
42-
shift := 0
43-
for *cursor < len(buf) {
44-
x := buf[*cursor]
45-
*cursor++
46-
v |= uint32(x&0x7f) << shift
47-
shift += 7
48-
if x&0x80 == 0 {
49-
break
50-
}
43+
func getBackLen(elementLen uint32) uint32 {
44+
if elementLen <= 127 {
45+
return 1
46+
} else if elementLen < (1<<14)-1 {
47+
return 2
48+
} else if elementLen < (1<<21)-1 {
49+
return 3
50+
} else if elementLen < (1<<28)-1 {
51+
return 4
52+
} else {
53+
return 5
5154
}
52-
return v
5355
}
5456

55-
// readListPackEntry returns: content(string), length, error
56-
func (dec *Decoder) readListPackEntry(buf []byte, cursor *int) ([]byte, uint32, error) {
57+
58+
// readListPackEntry returns: string content, int content, entry length(encoding+content+backlen), error
59+
func (dec *Decoder) readListPackEntry(buf []byte, cursor *int) ([]byte, int64, uint32, error) {
5760
header, err := readByte(buf, cursor)
5861
if err != nil {
59-
return nil, 0, err
62+
return nil, 0, 0, err
6063
}
61-
var result []byte
62-
var length uint32
6364
switch header >> 6 {
64-
case 0, 1: // 0xxx xxxx -> uint7 [0, 127]
65-
result = []byte(strconv.FormatInt(int64(int8(header)), 10))
66-
length = readVarInt(buf, cursor) // read element length
67-
return result, length, nil
68-
case 2: // 10xx xxxx -> str, len<= 63
65+
case 0, 1: // 0xxxxxxx, uint7
66+
result := int64(int8(header))
67+
var contentLen uint32 = 1
68+
backlen := getBackLen(contentLen)
69+
*cursor += int(backlen)
70+
return nil, result, contentLen + backlen, nil
71+
case 2: // 10xxxxxx + content, string(len<=63)
6972
strLen := int(header & 0x3f)
7073
result, err := readBytes(buf, cursor, strLen)
7174
if err != nil {
72-
return nil, 0, err
75+
return nil, 0, 0, err
7376
}
74-
length = readVarInt(buf, cursor) // read element length
75-
return result, length, nil
77+
var contentLen = uint32(1 + strLen)
78+
backlen := getBackLen(contentLen)
79+
*cursor += int(backlen)
80+
return result, 0, contentLen + backlen, nil
7681
}
77-
// assert header == 11xx xxxx
82+
// assert header == 11xxxxxx
7883
switch header >> 4 {
79-
case 12, 13: // 110x xxxx -> int13
84+
case 12, 13: // 110xxxxx yyyyyyyy, int13
8085
// see https://github.com/CN-annotation-team/redis7.0-chinese-annotated/blob/fba43c524524cbdb54955a28af228b513420d78d/src/listpack.c#L586
8186
next, err := readByte(buf, cursor)
8287
if err != nil {
83-
return nil, 0, err
88+
return nil, 0, 0, err
8489
}
8590
val := ((uint(header) & 0x1F) << 8) | uint(next)
8691
if val >= uint(1<<12) {
8792
val = -(8191 - val) - 1 // val is uint, must use -(8191 - val), val - 8191 will cause overflow
8893
}
89-
result = []byte(strconv.FormatInt(int64(val), 10))
90-
length = readVarInt(buf, cursor) // read element length
91-
return result, length, nil
92-
case 14: // 1110 xxxx -> str, type(len) == uint12
94+
result := int64(val)
95+
var contentLen uint32 = 2
96+
backlen := getBackLen(contentLen)
97+
*cursor += int(backlen)
98+
return nil, result, contentLen + backlen, nil
99+
case 14: // 1110xxxx yyyyyyyy + content, string(len < 1<<12)
93100
dec.buffer[0] = header & 0x0f
94101
dec.buffer[1], err = readByte(buf, cursor)
102+
if err != nil {
103+
return nil, 0, 0, err
104+
}
95105
strLen := binary.BigEndian.Uint16(dec.buffer[:2])
96106
result, err := readBytes(buf, cursor, int(strLen))
97107
if err != nil {
98-
return nil, 0, err
108+
return nil, 0, 0, err
99109
}
100-
length = readVarInt(buf, cursor) // read element length
101-
return result, length, nil
110+
var contentLen = uint32(2 + strLen)
111+
backlen := getBackLen(contentLen)
112+
*cursor += int(backlen)
113+
return result, 0, contentLen + backlen, nil
102114
}
103-
// assert header == 1111 xxxx
115+
// assert header == 1111xxxx
104116
switch header & 0x0f {
105-
case 0: // 1111 0000 -> str, 4 bytes len
117+
case 0: // 11110000 aaaaaaaa bbbbbbbb cccccccc dddddddd + content, string(len < 1<<32)
106118
var lenBytes []byte
107119
lenBytes, err = readBytes(buf, cursor, 4)
108120
if err != nil {
109-
return nil, 0, err
121+
return nil, 0, 0, err
110122
}
111123
strLen := int(binary.LittleEndian.Uint32(lenBytes))
112124
result, err := readBytes(buf, cursor, strLen)
113125
if err != nil {
114-
return nil, 0, err
126+
return nil, 0, 0, err
115127
}
116-
length = readVarInt(buf, cursor) // read element length
117-
return result, length, nil
118-
case 1: // 1111 0001 -> int16
128+
var contentLen = uint32(1 + 4 + strLen)
129+
backlen := getBackLen(contentLen)
130+
*cursor += int(backlen)
131+
return result, 0, contentLen + backlen, nil
132+
case 1: // 11110001 aaaaaaaa bbbbbbbb, int16
119133
var bs []byte
120134
bs, err = readBytes(buf, cursor, 2)
121135
if err != nil {
122-
return nil, 0, err
136+
return nil, 0, 0, err
123137
}
124-
result = []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(bs))), 10))
125-
length = readVarInt(buf, cursor)
126-
return result, length, nil
127-
case 2: // 1111 0010 -> int24
138+
result := int64(int16(binary.LittleEndian.Uint16(bs)))
139+
var contentLen uint32 = 3
140+
backlen := getBackLen(contentLen)
141+
*cursor += int(backlen)
142+
return nil, result, contentLen + backlen, nil
143+
case 2: // 11110010 aaaaaaaa bbbbbbbb cccccccc, int24
128144
var bs []byte
129145
bs, err = readBytes(buf, cursor, 3)
130146
if err != nil {
131-
return nil, 0, err
147+
return nil, 0, 0, err
132148
}
133149
bs = append([]byte{0}, bs...)
134-
result = []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(bs))>>8), 10))
135-
length = readVarInt(buf, cursor)
136-
return result, length, nil
150+
result := int64(int32(binary.LittleEndian.Uint32(bs))>>8)
151+
var contentLen uint32 = 4
152+
backlen := getBackLen(contentLen)
153+
*cursor += int(backlen)
154+
return nil, result, contentLen + backlen, nil
137155
case 3: // 1111 0011 -> int32
138156
var bs []byte
139157
bs, err = readBytes(buf, cursor, 4)
140158
if err != nil {
141-
return nil, 0, err
159+
return nil, 0, 0, err
142160
}
143-
result = []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(bs))), 10))
144-
length = readVarInt(buf, cursor)
145-
return result, length, nil
146-
case 4: // 1111 0100 -> int64
161+
result := int64(int32(binary.LittleEndian.Uint32(bs)))
162+
var contentLen uint32 = 5
163+
backlen := getBackLen(contentLen)
164+
*cursor += int(backlen)
165+
return nil, result, contentLen + backlen, nil
166+
case 4: // 11110100 8Byte -> int64
147167
var bs []byte
148168
bs, err = readBytes(buf, cursor, 8)
149169
if err != nil {
150-
return nil, 0, err
170+
return nil, 0, 0, err
151171
}
152-
result = []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(bs)), 10))
153-
length = readVarInt(buf, cursor)
154-
return result, length, nil
155-
case 15: // 1111 1111 -> end
156-
return nil, 0, errors.New("unexpected end")
172+
result := int64(binary.LittleEndian.Uint64(bs))
173+
var contentLen uint32 = 9
174+
backlen := getBackLen(contentLen)
175+
*cursor += int(backlen)
176+
return nil, result, contentLen + backlen, nil
177+
case 15: // 11111111 -> end
178+
return nil, 0, 0, errors.New("unexpected end")
157179
}
158-
return nil, 0, fmt.Errorf("unknown entry header")
180+
return nil, 0, 0, fmt.Errorf("unknown entry header")
181+
}
182+
183+
// readListPackEntryAsString return a string representation of entry
184+
// It means if the entry is a integer, then format it as string
185+
func (dec *Decoder) readListPackEntryAsString(buf []byte, cursor *int) ([]byte, error) {
186+
str, intval, _, err := dec.readListPackEntry(buf, cursor)
187+
if err != nil {
188+
return nil, fmt.Errorf("read from failed: %v", err)
189+
}
190+
if str != nil {
191+
return str, nil
192+
}
193+
str = []byte(strconv.FormatInt(intval, 10))
194+
return str, nil
159195
}
160196

161197
func (dec *Decoder) readListPackEntryAsInt(buf []byte, cursor *int) (int64, error) {
162-
bin, _, err := dec.readListPackEntry(buf, cursor)
198+
str, intval, _, err := dec.readListPackEntry(buf, cursor)
163199
if err != nil {
164200
return 0, fmt.Errorf("read from failed: %v", err)
165201
}
166-
i, err := strconv.ParseInt(string(bin), 10, 64)
167-
if err != nil {
168-
return 0, fmt.Errorf("%s is not a uint", string(bin))
202+
if str != nil {
203+
return 0, fmt.Errorf("%s is not a integer", string(str))
169204
}
170-
return i, nil
205+
return intval, nil
171206
}

‎core/set.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ package core
33
import (
44
"encoding/binary"
55
"fmt"
6-
"github.com/hdt3213/rdb/model"
76
"math"
87
"sort"
98
"strconv"
9+
10+
"github.com/hdt3213/rdb/model"
1011
)
1112

1213
func (dec *Decoder) readSet() ([][]byte, error) {
@@ -73,7 +74,7 @@ func (dec *Decoder) readListPackSet() ([][]byte, *model.ListpackDetail, error) {
7374
size := readListPackLength(buf, &cursor)
7475
values := make([][]byte, 0, size)
7576
for i := 0; i < size; i += 1 {
76-
member, _, err := dec.readListPackEntry(buf, &cursor)
77+
member, err := dec.readListPackEntryAsString(buf, &cursor)
7778
if err != nil {
7879
return nil, nil, err
7980
}

‎core/stream.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,14 +141,14 @@ func (dec *Decoder) readStreamEntryContent(buf []byte, cursor *int, firstId *mod
141141
masterFieldNum := int(fieldNum0)
142142
masterFieldNames := make([]string, masterFieldNum)
143143
for i := 0; i < masterFieldNum; i++ {
144-
name, _, err := dec.readListPackEntry(buf, cursor)
144+
name, err := dec.readListPackEntryAsString(buf, cursor)
145145
if err != nil {
146146
return nil, fmt.Errorf("read field name of stream entry failed: %v", err)
147147
}
148148
masterFieldNames[i] = string(name)
149149
}
150-
// read end flag
151-
if _, _, err = dec.readListPackEntry(buf, cursor); err != nil {
150+
// read lp count of master entry
151+
if _, err = dec.readListPackEntryAsString(buf, cursor); err != nil {
152152
return nil, fmt.Errorf("read fields end flag failed: %v", err)
153153
}
154154

@@ -191,20 +191,20 @@ func (dec *Decoder) readStreamEntryContent(buf []byte, cursor *int, firstId *mod
191191
if flag&StreamItemFlagSameFields > 0 {
192192
fieldName = masterFieldNames[i]
193193
} else {
194-
fieldNameBin, _, err := dec.readListPackEntry(buf, cursor)
194+
fieldNameBin, err := dec.readListPackEntryAsString(buf, cursor)
195195
if err != nil {
196196
return nil, fmt.Errorf("read stream item field name failed: %v", err)
197197
}
198198
fieldName = unsafeBytes2Str(fieldNameBin)
199199
}
200-
fieldValue, _, err := dec.readListPackEntry(buf, cursor)
200+
fieldValue, err := dec.readListPackEntryAsString(buf, cursor)
201201
if err != nil {
202202
return nil, fmt.Errorf("read stream item field value failed: %v", err)
203203
}
204204
msg.Fields[fieldName] = unsafeBytes2Str(fieldValue)
205205
}
206-
// read end flag
207-
if _, _, err = dec.readListPackEntry(buf, cursor); err != nil {
206+
// read lp count
207+
if _, err = dec.readListPackEntryAsString(buf, cursor); err != nil {
208208
return nil, fmt.Errorf("read fields end flag failed: %v", err)
209209
}
210210
msgs = append(msgs, msg)

‎core/zset.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package core
22

33
import (
4-
"github.com/hdt3213/rdb/model"
54
"strconv"
5+
6+
"github.com/hdt3213/rdb/model"
67
)
78

89
func (dec *Decoder) readZSet(zset2 bool) ([]*model.ZSetEntry, error) {
@@ -74,11 +75,11 @@ func (dec *Decoder) readListPackZSet() ([]*model.ZSetEntry, *model.ListpackDetai
7475
size := readListPackLength(buf, &cursor)
7576
entries := make([]*model.ZSetEntry, 0, size)
7677
for i := 0; i < size; i += 2 {
77-
member, _, err := dec.readListPackEntry(buf, &cursor)
78+
member, err := dec.readListPackEntryAsString(buf, &cursor)
7879
if err != nil {
7980
return nil, nil, err
8081
}
81-
scoreLiteral, _, err := dec.readListPackEntry(buf, &cursor)
82+
scoreLiteral, err := dec.readListPackEntryAsString(buf, &cursor)
8283
if err != nil {
8384
return nil, nil, err
8485
}

0 commit comments

Comments
 (0)