From 4d247aa54605aa8e2e8130745bf367ee1b551114 Mon Sep 17 00:00:00 2001 From: "C.C" Date: Sun, 14 Feb 2021 02:33:27 +0800 Subject: [PATCH 1/2] feat(draft-02): implement draft-02, resolve #60,#61,#62,#63 --- LICENSE | 4 +- Makefile | 10 +- README.md | 65 ++++++++----- api.go | 102 ++++++++++++++++++++ api_test.go | 102 ++++++++++++++++++++ pkg/encoding/nvarint.go | 2 +- pkg/encoding/pvarint.go | 4 +- pkg/encoding/varfloat.go | 4 +- pkg/spec/decode.go | 140 +++++++++++++++++++++++++++ pkg/spec/encode.go | 166 ++++++++++++++++++++++++++++++++ pkg/spec/packet.go | 33 +++++++ pkg/spec/tlv_test.go | 134 ++++++++++++++++++++++++++ pkg/spec/typed_value_test.go | 179 +++++++++++++++++++++++++++++++++++ stream_api.go | 130 +++++++++++++++++++++++++ stream_api_test.go | 158 +++++++++++++++++++++++++++++++ 15 files changed, 1202 insertions(+), 31 deletions(-) create mode 100644 api.go create mode 100644 api_test.go create mode 100644 pkg/spec/decode.go create mode 100644 pkg/spec/encode.go create mode 100644 pkg/spec/packet.go create mode 100644 pkg/spec/tlv_test.go create mode 100644 pkg/spec/typed_value_test.go create mode 100644 stream_api.go create mode 100644 stream_api_test.go diff --git a/LICENSE b/LICENSE index 6313812..affb7f4 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2019~2020 北京熹乐科技有限公司 CELLA + Copyright 2019~2020 CELLA, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -198,4 +198,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile index 179c737..9658dcb 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,14 @@ vet: lint: $(GOLINT) $(GOFILES) -test: +v1test: $(GO) test -v ./... cover: - $(GO) test -coverprofile=prof.out && $(GO) tool cover -html=prof.out && rm prof.out \ No newline at end of file + $(GO) test github.com/yomorun/y3-codec-golang/pkg -coverprofile=prof.out && $(GO) tool cover -html=prof.out && rm prof.out + +test: + $(GO) test -v api.go api_test.go stream_api.go stream_api_test.go + +test-spec: + $(GO) test -v github.com/yomorun/y3-codec-golang/pkg/spec diff --git a/README.md b/README.md index ee46b25..6d782b3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ See [YoMo Codec SPEC](https://github.com/yomorun/yomo-codec) ## Test -`go test ./...` +`make test` ## Use @@ -27,6 +27,8 @@ See [YoMo Codec SPEC](https://github.com/yomorun/yomo-codec) ### Encode examples +Encode Key-Value type `{ID: 3}` to Y3: + ```go package main @@ -36,34 +38,47 @@ import ( ) func main() { - // if we want to repesent `var obj = &foo{ID: -1, bar: &bar{Name: "C"}}` - // in YoMo-Codec: + // Key:ID = Tag:0x01 + buffer, _ = y3.EncodeInt(0x01, 3) + + // get whole buf + fmt.Printf("res=%#v", buffer) // res=[]byte{0x01, 0x01, 0x03} +} +``` - // 0x81 -> node - var foo = y3.NewNodePacketEncoder(0x01) +if we want to repesent JSON `foo: {ID: -1, Name: "C"}` in Y3: - // 0x02 -> foo.ID=-11 - var yp1 = y3.NewPrimitivePacketEncoder(0x02) - yp1.SetInt32Value(-1) - foo.AddPrimitivePacket(yp1) +```go +package main - // 0x83 -> &bar{} - var bar = y3.NewNodePacketEncoder(0x03) +import ( + "fmt" + y3 "github.com/yomorun/y3-codec-golang" +) - // 0x04 -> bar.Name="C" - var yp2 = y3.NewPrimitivePacketEncoder(0x04) - yp2.SetStringValue("C") - bar.AddPrimitivePacket(yp2) +func main() { + // Key:ID 0x01 -> -1 + var id, _ = y3.NewPacket(0x01) + id.SetInt32(-1) - // -> foo.bar=&bar - foo.AddNodePacket(bar) - - fmt.Printf("res=%#v", foo.Encode()) // res=[]byte{0x81, 0x08, 0x02, 0x01, 0x7F, 0x83, 0x03, 0x04, 0x01, 0x43} + // Key:Name 0x02 -> "C" + var name, _ = y3.NewPacket(0x02) + name.SetString("C") + + // parent node + var foo, _ = y3.NewPacket(0x10) + foo.AddNode(id) + foo.AddNode(name) + + // get whole buf + fmt.Printf("res=%#v", foo.Encode()) // res=[]byte{0x10, 0x06, 0x01, 0x01, 0x7F, 0x02, 0x01, 0x43} } ``` ### Decode examples 1: decode a primitive packet +Decode `[0x0A, 0x01, 0x7F]` as Int type + ```go package main @@ -75,13 +90,13 @@ import ( func main() { fmt.Println(">> Parsing [0x0A, 0x01, 0x7F], which like Key-Value format = 0x0A: 127") buf := []byte{0x0A, 0x01, 0x7F} - res, _, err := y3.DecodePrimitivePacket(buf) - v1, err := res.ToUInt32() + res, _, err := y3.DecodePacket(buf) + val, err := res.ToInt32() if err != nil { panic(err) } - fmt.Printf("Tag Key=[%#X], Value=%v\n", res.SeqID(), v1) + fmt.Printf("Tag Key=[%#X], Value=%v\n", res.Tag, val) } ``` @@ -130,3 +145,9 @@ More examples in `/examples/` ## License [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fyomorun%2Fy3-codec-golang.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fyomorun%2Fy3-codec-golang?ref=badge_large) + +## CLA + +[Sign CLA](https://cla-assistant.io/yomorun/y3-codec-golang) + +[![CLA assistant](https://cla-assistant.io/readme/badge/yomorun/y3-codec-golang)](https://cla-assistant.io/yomorun/y3-codec-golang) \ No newline at end of file diff --git a/api.go b/api.go new file mode 100644 index 0000000..8be5a86 --- /dev/null +++ b/api.go @@ -0,0 +1,102 @@ +package y3 + +import ( + "errors" + + "github.com/yomorun/y3-codec-golang/pkg/spec" +) + +// EncodeBool encode bool type data +func EncodeBool(tag int, v bool) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetBool(v) + return p.Encode() +} + +// EncodeUInt encode uint type data +func EncodeUInt(tag int, v uint) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetUInt32(uint32(v)) + return p.Encode() +} + +// EncodeInt encode int type data +func EncodeInt(tag int, v int) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetInt32(v) + return p.Encode() +} + +// EncodeUInt64 encode uint64 type data +func EncodeUInt64(tag int, v uint64) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetUInt64(v) + return p.Encode() +} + +// EncodeInt64 encode int type data +func EncodeInt64(tag int, v int64) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetInt64(v) + return p.Encode() +} + +// EncodeFloat32 encode float32 type data +func EncodeFloat32(tag int, v float32) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetFloat32(v) + return p.Encode() +} + +// EncodeFloat64 encode float64 type data +func EncodeFloat64(tag int, v float64) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetFloat64(v) + return p.Encode() +} + +// EncodeString encode UTF-8 string data +func EncodeString(tag int, v string) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.SetUTF8String(v) + return p.Encode() +} + +// EncodeBytes encode raw bytes +func EncodeBytes(tag int, v []byte) ([]byte, error) { + p, err := spec.NewPacket(uint64(tag)) + if err != nil { + return nil, err + } + p.PutBytes(v) + return p.Encode() +} + +// Marshal TODO wip +func Marshal(tag int, obj interface{}) ([]byte, error) { + panic(errors.New("NotImplementedError")) +} diff --git a/api_test.go b/api_test.go new file mode 100644 index 0000000..f676e1e --- /dev/null +++ b/api_test.go @@ -0,0 +1,102 @@ +package y3 + +import ( + "testing" +) + +func TestD2EncodeUInt32(t *testing.T) { + testD2EncodeUInt32(t, 0x03, 6, []byte{0x03, 0x01, 0x06}) + testD2EncodeUInt32(t, 0x06, 127, []byte{0x06, 0x02, 0x80, 0x7F}) +} + +func TestD2EncodeInt32(t *testing.T) { + testD2EncodeInt(t, 0x03, -1, []byte{0x03, 0x01, 0x7F}) + testD2EncodeInt(t, 0x06, -65, []byte{0x06, 0x02, 0xFF, 0x3F}) + testD2EncodeInt(t, 0x09, 255, []byte{0x09, 0x02, 0x81, 0x7F}) +} + +func TestD2EncodeUInt64(t *testing.T) { + testD2EncodeUInt64(t, 0x03, 0, []byte{0x03, 0x01, 0x00}) + testD2EncodeUInt64(t, 0x06, 1, []byte{0x06, 0x01, 0x01}) + testD2EncodeUInt64(t, 0x09, 18446744073709551615, []byte{0x09, 0x01, 0x7F}) +} + +func TestD2EncodeInt64(t *testing.T) { + testD2EncodeInt64(t, 0x03, 0, []byte{0x03, 0x01, 0x00}) + testD2EncodeInt64(t, 0x06, 1, []byte{0x06, 0x01, 0x01}) + testD2EncodeInt64(t, 0x09, -1, []byte{0x09, 0x01, 0x7F}) +} + +func TestD2EncodeFloat32(t *testing.T) { + testD2EncodeFloat32(t, 0x03, -2, []byte{0x03, 0x01, 0xC0}) + testD2EncodeFloat32(t, 0x06, 0.25, []byte{0x06, 0x02, 0x3E, 0x80}) + testD2EncodeFloat32(t, 0x09, 68.123, []byte{0x09, 0x04, 0x42, 0x88, 0x3E, 0xFA}) +} + +func TestD2EncodeFloat64(t *testing.T) { + testD2EncodeFloat64(t, 0x03, 23, []byte{0x03, 0x02, 0x40, 0x37}) + testD2EncodeFloat64(t, 0x06, 2, []byte{0x06, 0x01, 0x40}) + testD2EncodeFloat64(t, 0x09, 0.01171875, []byte{0x09, 0x02, 0x3F, 0x88}) +} + +func TestD2EncodeString(t *testing.T) { + p, _ := EncodeString(0x01, "C") + compareTwoBytes(t, p, []byte{0x01, 0x01, 0x43}) + p, _ = EncodeString(0x01, "CC") + compareTwoBytes(t, p, []byte{0x01, 0x02, 0x43, 0x43}) + p, _ = EncodeString(0x01, "Yona") + compareTwoBytes(t, p, []byte{0x01, 0x04, 0x59, 0x6F, 0x6E, 0x61}) + p, _ = EncodeString(0x01, "https://yomo.run") + compareTwoBytes(t, p, []byte{0x01, 0x10, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x2F, 0x2F, 0x79, 0x6F, 0x6D, 0x6F, 0x2E, 0x72, 0x75, 0x6E}) +} + +func TestD2EncodeBytes(t *testing.T) { + p, _ := EncodeBytes(0x01, []byte{0x03, 0x06, 0x09, 0x0C, 0x0F}) + compareTwoBytes(t, p, []byte{0x01, 0x05, 0x03, 0x06, 0x09, 0x0C, 0x0F}) +} + +func TestD2EncodeBool(t *testing.T) { + p, _ := EncodeBool(0x01, true) + compareTwoBytes(t, p, []byte{0x01, 0x01, 0x01}) + p, _ = EncodeBool(0x01, false) + compareTwoBytes(t, p, []byte{0x01, 0x01, 0x00}) +} + +func testD2EncodeUInt32(t *testing.T, tag int, val uint, expected []byte) { + p, _ := EncodeUInt(tag, val) + compareTwoBytes(t, p, expected) +} + +func testD2EncodeInt(t *testing.T, tag int, val int, expected []byte) { + p, _ := EncodeInt(tag, val) + compareTwoBytes(t, p, expected) +} + +func testD2EncodeUInt64(t *testing.T, tag int, val uint64, expected []byte) { + p, _ := EncodeUInt64(tag, val) + compareTwoBytes(t, p, expected) +} + +func testD2EncodeInt64(t *testing.T, tag int, val int64, expected []byte) { + p, _ := EncodeInt64(tag, val) + compareTwoBytes(t, p, expected) +} + +func testD2EncodeFloat32(t *testing.T, tag int, val float32, expected []byte) { + p, _ := EncodeFloat32(tag, val) + compareTwoBytes(t, p, expected) +} + +func testD2EncodeFloat64(t *testing.T, tag int, val float64, expected []byte) { + p, _ := EncodeFloat64(tag, val) + compareTwoBytes(t, p, expected) +} + +func compareTwoBytes(t *testing.T, result []byte, expected []byte) { + for i, p := range result { + if p != expected[i] { + t.Errorf("\nexpected:[% X]\n actual:[% X]\n", expected, result) + break + } + } +} diff --git a/pkg/encoding/nvarint.go b/pkg/encoding/nvarint.go index 2b3552c..fead216 100644 --- a/pkg/encoding/nvarint.go +++ b/pkg/encoding/nvarint.go @@ -78,7 +78,7 @@ func sizeOfNVarInt(value int64, width int) int { const unit = 8 // bit width of encoding unit var lead = value >> (width - 1) - for size := width / unit - 1; size > 0; size-- { + for size := width/unit - 1; size > 0; size-- { var lookAhead = value >> (size*unit - 1) if lookAhead != lead { return size + 1 diff --git a/pkg/encoding/pvarint.go b/pkg/encoding/pvarint.go index ca1171e..cb19b07 100644 --- a/pkg/encoding/pvarint.go +++ b/pkg/encoding/pvarint.go @@ -123,7 +123,7 @@ func (codec *VarCodec) decodePVarInt(buffer []byte, value *int64) error { return ErrBufferInsufficient } - const unit = 7 // bit width of encoding unit + const unit = 7 // bit width of encoding unit if codec.Size == 0 { // initialize sign bit const flag = 8 - unit // bit width for non-encoding bits *value = int64(int8(buffer[codec.Ptr]) << flag >> unit) @@ -135,7 +135,7 @@ func (codec *VarCodec) decodePVarInt(buffer []byte, value *int64) error { codec.Ptr++ codec.Size++ - *value = (*value << unit) | int64(mask & part) + *value = (*value << unit) | int64(mask&part) if part >= 0 { // it's the last byte return nil diff --git a/pkg/encoding/varfloat.go b/pkg/encoding/varfloat.go index e281980..b1fb36e 100644 --- a/pkg/encoding/varfloat.go +++ b/pkg/encoding/varfloat.go @@ -47,7 +47,7 @@ func sizeOfVarFloat(bits uint64, width int) int { const mask = uint64(0xFF) // mask of encoding unit for s := 0; width > 1; s += unit { - if bits & (mask << s) != 0 { + if bits&(mask<> ((codec.Size & mask + gap) * unit)) + buffer[codec.Ptr] = byte(bits >> ((codec.Size&mask + gap) * unit)) codec.Ptr++ } diff --git a/pkg/spec/decode.go b/pkg/spec/decode.go new file mode 100644 index 0000000..e5ed260 --- /dev/null +++ b/pkg/spec/decode.go @@ -0,0 +1,140 @@ +package spec + +import ( + "errors" + + "github.com/yomorun/y3-codec-golang/pkg/encoding" +) + +// FromBytes read Y3 buffer +func FromBytes(buf []byte) (p *Packet, err error) { + if len(buf) < 2 { + return nil, errors.New("malformed data") + } + + p = &Packet{buffer: buf} + pos := 0 + + // read Tag Buffer, Tag support PVarUInt64 and raw bytes + // if tagbuf is empty, then check idTag value + cursor := readVariantLengthBuffer(buf, pos) + p.tagbuf = buf[pos:cursor] + pos = cursor + + // read Length + var length uint64 + cursor, err = readPVarUInt64(buf, pos, &length) + if err != nil { + return nil, err + } + p.Length = length + p.lenbuf = buf[pos:cursor] + pos = cursor + + // read Value bytes + cursor = pos + int(p.Length) + if cursor > len(buf) { + return nil, errors.New("malformed valbuf") + } + p.valbuf = buf[pos:cursor] + + return p, nil +} + +func readVariantLengthBuffer(buffer []byte, position int) int { + buf := buffer[position:] + // PVarUInt64 type, MSB(0x80) is continuation bit + cursor := 1 + for i, v := range buf { + if v&0x80 != 0x80 { + cursor += i + break + } + } + return cursor +} + +func readPVarUInt64(buffer []byte, position int, val *uint64) (cursor int, err error) { + buf := buffer[position:] + // tag/length is PVarUInt64 type, MSB(0x80) is continuation bit + cursor = 1 + for i, v := range buf { + if v&0x80 != 0x80 { + cursor += i + break + } + } + // generate tag buffer + bytes := buf[:cursor] + // read as PVarInt64 + codec := encoding.VarCodec{} + err = codec.DecodePVarUInt64(bytes, val) + return cursor + position, err +} + +// GetValueAsUInt32 decode value as uint32 +func (p *Packet) GetValueAsUInt32() (uint32, error) { + var val uint32 + codec := encoding.VarCodec{} + err := codec.DecodePVarUInt32(p.valbuf, &val) + return val, err +} + +// GetValueAsInt32 decode value as int32 +func (p *Packet) GetValueAsInt32() (int32, error) { + var val int32 + codec := encoding.VarCodec{} + err := codec.DecodePVarInt32(p.valbuf, &val) + return val, err +} + +// GetValueAsUInt64 decode value as uint64 +func (p *Packet) GetValueAsUInt64() (uint64, error) { + var val uint64 + codec := encoding.VarCodec{} + err := codec.DecodePVarUInt64(p.valbuf, &val) + return val, err +} + +// GetValueAsInt64 decode value as int64 +func (p *Packet) GetValueAsInt64() (int64, error) { + var val int64 + codec := encoding.VarCodec{} + err := codec.DecodePVarInt64(p.valbuf, &val) + return val, err +} + +// GetValueAsFloat32 decode value as uint32 +func (p *Packet) GetValueAsFloat32() (float32, error) { + var val float32 + codec := encoding.VarCodec{Size: int(p.Length)} + err := codec.DecodeVarFloat32(p.valbuf, &val) + return val, err +} + +// GetValueAsBool decode value as bool +func (p *Packet) GetValueAsBool() (bool, error) { + res, err := p.GetValueAsUInt64() + if res == 1 { + return true, err + } + return false, err +} + +// GetValueAsFloat64 decode value as float64 +func (p *Packet) GetValueAsFloat64() (float64, error) { + var val float64 + codec := encoding.VarCodec{Size: int(p.Length)} + err := codec.DecodeVarFloat64(p.valbuf, &val) + return val, err +} + +// GetValueAsUTF8String decode value as float32 +func (p *Packet) GetValueAsUTF8String() string { + return string(p.valbuf) +} + +// GetValueAsRawBytes decode value as float32 +func (p *Packet) GetValueAsRawBytes() []byte { + return p.valbuf +} diff --git a/pkg/spec/encode.go b/pkg/spec/encode.go new file mode 100644 index 0000000..174c9e6 --- /dev/null +++ b/pkg/spec/encode.go @@ -0,0 +1,166 @@ +package spec + +import ( + "github.com/yomorun/y3-codec-golang/pkg/encoding" +) + +// NewPacket create new Packet object +func NewPacket(sid uint64) (*Packet, error) { + var p = &Packet{} + tmp, err := getPVarUInt64Buffer(sid) + p.tagbuf = tmp + return p, err +} + +// NewRawPacket create new Packet object with +func NewRawPacket(rawID []byte) (*Packet, error) { + // TODO, validate rawID + var p = &Packet{} + p.tagbuf = rawID + return p, nil +} + +// SetNil set nil value, means length=0 packet +func (p *Packet) SetNil() { + p.valbuf = make([]byte, 0) +} + +// SetUInt32 set UInt32 value +func (p *Packet) SetUInt32(v uint32) error { + size := encoding.SizeOfPVarUInt32(v) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodePVarUInt32(p.valbuf, v) + if err != nil { + return err + } + return nil +} + +// SetInt32 set Int32 value +func (p *Packet) SetInt32(v int) error { + size := encoding.SizeOfPVarInt32(int32(v)) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodePVarInt32(p.valbuf, int32(v)) + if err != nil { + return err + } + return nil +} + +// SetUInt64 set UInt32 value +func (p *Packet) SetUInt64(v uint64) error { + size := encoding.SizeOfPVarUInt64(v) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodePVarUInt64(p.valbuf, v) + if err != nil { + return err + } + return nil +} + +// SetInt64 set Int32 value +func (p *Packet) SetInt64(v int64) error { + size := encoding.SizeOfPVarInt64(int64(v)) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodePVarInt64(p.valbuf, int64(v)) + if err != nil { + return err + } + return nil +} + +// SetFloat32 set float32 value +func (p *Packet) SetFloat32(v float32) error { + size := encoding.SizeOfVarFloat32(v) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodeVarFloat32(p.valbuf, v) + if err != nil { + return err + } + return nil +} + +// SetFloat64 set float64 value +func (p *Packet) SetFloat64(v float64) error { + size := encoding.SizeOfVarFloat64(v) + codec := encoding.VarCodec{Size: size} + p.valbuf = make([]byte, size) + // set packet valbuf + err := codec.EncodeVarFloat64(p.valbuf, v) + if err != nil { + return err + } + return nil +} + +// SetBool set boolean value +func (p *Packet) SetBool(v bool) error { + var val uint64 = 0 + if v { + val = 1 + } + return p.SetUInt64(val) +} + +// SetUTF8String set string value +func (p *Packet) SetUTF8String(v string) { + p.valbuf = []byte(v) +} + +// PutBytes append bytes value +func (p *Packet) PutBytes(v []byte) { + p.valbuf = append(p.valbuf, v...) +} + +// AddNode add a child Packet +func (p *Packet) AddNode(child *Packet) (*Packet, error) { + childBuffer, err := child.Encode() + if err != nil { + return nil, err + } + p.PutBytes(childBuffer) + return p, nil +} + +// Encode return whole bytes of this packet +func (p *Packet) Encode() ([]byte, error) { + // if tag buffer is none, read from idTag as PVarUint64 type + if len(p.tagbuf) < 1 { + tagbuf, err := getPVarUInt64Buffer(p.idTag) + if err != nil { + return nil, err + } + p.tagbuf = tagbuf + } + + // set length buffer + p.Length = uint64(len(p.valbuf)) + lenbuf, err := getPVarUInt64Buffer(p.Length) + p.lenbuf = lenbuf + if err != nil { + return nil, err + } + // [Tag][Length][Value] + res := append(p.tagbuf, p.lenbuf...) + res = append(res, p.valbuf...) + p.buffer = res + return res, nil +} + +func getPVarUInt64Buffer(val uint64) ([]byte, error) { + size := encoding.SizeOfPVarUInt64(val) + codec := encoding.VarCodec{Size: size} + buf := make([]byte, size) + err := codec.EncodePVarUInt64(buf, val) + return buf, err +} diff --git a/pkg/spec/packet.go b/pkg/spec/packet.go new file mode 100644 index 0000000..4a50dde --- /dev/null +++ b/pkg/spec/packet.go @@ -0,0 +1,33 @@ +package spec + +// Packet is a TLV group +type Packet struct { + Length uint64 + tagbuf []byte + lenbuf []byte + valbuf []byte + buffer []byte + idTag uint64 +} + +// GetTag return Tag as uint64 value +func (p *Packet) GetTag() uint64 { + var tag uint64 + readPVarUInt64(p.tagbuf, 0, &tag) + return tag +} + +// GetTagBuffer return Tag as raw bytes +func (p *Packet) GetTagBuffer() []byte { + return p.tagbuf +} + +// GetLengthBuffer return Tag as raw bytes +func (p *Packet) GetLengthBuffer() []byte { + return p.lenbuf +} + +// GetValueBuffer return Tag as raw bytes +func (p *Packet) GetValueBuffer() []byte { + return p.valbuf +} diff --git a/pkg/spec/tlv_test.go b/pkg/spec/tlv_test.go new file mode 100644 index 0000000..f5c9126 --- /dev/null +++ b/pkg/spec/tlv_test.go @@ -0,0 +1,134 @@ +package spec + +import ( + "testing" +) + +func TestV2Tag(t *testing.T) { + testV2Tags(t, 0x05, []byte{0x05}) + testV2Tags(t, 0x3F, []byte{0x3F}) + testV2Tags(t, 0x7F, []byte{0x80, 0x7F}) + testV2Tags(t, 0xFF, []byte{0x81, 0x7F}) + testV2Tags(t, 0xFFFF, []byte{0x83, 0xFF, 0x7F}) + testV2Tags(t, 0xFFFFFF, []byte{0x87, 0xFF, 0xFF, 0x7F}) +} + +func TestV2AddNode(t *testing.T) { + parent, _ := NewPacket(0x01) + + child1, _ := NewPacket(0x02) + child1.SetUInt32(3) + + parent.AddNode(child1) + compareBytes(t, parent, []byte{0x01, 0x03, 0x02, 0x01, 0x03}) + + child2, _ := NewPacket(0x03) + child2.SetFloat64(0.01171875) + + parent.AddNode(child2) + compareBytes(t, parent, []byte{0x01, 0x07, 0x02, 0x01, 0x03, 0x03, 0x02, 0x3F, 0x88}) + + child3, _ := NewPacket(0x04) + child3.SetFloat32(68.123) + + parent.AddNode(child3) + compareBytes(t, parent, []byte{0x01, 0x0D, 0x02, 0x01, 0x03, 0x03, 0x02, 0x3F, 0x88, 0x04, 0x04, 0x42, 0x88, 0x3E, 0xFA}) +} + +func TestFromBytes(t *testing.T) { + testFromBytes(t, []byte{0x03, 0x01, 0x02}, &Packet{idTag: 3, Length: 1, valbuf: []byte{0x02}}) + testFromBytes(t, []byte{0x81, 0x03, 0x01, 0x06}, &Packet{idTag: 131, Length: 1, valbuf: []byte{0x06}}) + + foo := make([]byte, 129) + bar := []byte{0x81, 0x03, 0x81, 0x01} + buf := append(bar, foo...) + testFromBytes(t, buf, &Packet{idTag: 131, Length: 129, valbuf: foo}) +} + +func TestReadTag(t *testing.T) { + testReadTag(t, []byte{0xFF, 0x7F, 0x00}, 0, 18446744073709551615, 2) + testReadTag(t, []byte{0x03, 0x01, 0x02}, 0, 3, 1) + testReadTag(t, []byte{0xFF, 0x04, 0x01, 0x02}, 1, 4, 2) + testReadTag(t, []byte{0x81, 0x05, 0x01, 0x02}, 0, 133, 2) + testReadTag(t, []byte{0xFF, 0x81, 0x05, 0x01, 0x02}, 1, 133, 3) +} + +func TestReadLength(t *testing.T) { + testReadLength(t, []byte{0xFF, 0x7F, 0x00}, 2, 0, 3) + testReadLength(t, []byte{0x03, 0x01, 0x02}, 1, 1, 2) + testReadLength(t, []byte{0xFF, 0x04, 0x02, 0x00, 0x00}, 2, 2, 3) + testReadLength(t, []byte{0x05, 0x81, 0x05, 0x00}, 1, 133, 3) +} + +func testReadTag(t *testing.T, buffer []byte, position int, expectValue uint64, expectedCursor int) { + var result uint64 + cursor, err := readPVarUInt64(buffer, position, &result) + if err != nil { + t.Error(err) + } + if result != expectValue { + t.Errorf("expect tag=%d, actual=%d", expectValue, result) + } + if cursor != expectedCursor { + t.Errorf("expect cursor=%d, actual=%d", expectedCursor, cursor) + } +} + +func testReadLength(t *testing.T, buffer []byte, position int, expectValue uint64, expectedCursor int) { + var result uint64 + cursor, err := readPVarUInt64(buffer, position, &result) + if err != nil { + t.Errorf(">>> Got err=%s", err.Error()) + } + if result != expectValue { + t.Errorf("expect length=%d, actual=%d", expectValue, result) + } + if cursor != expectedCursor { + t.Errorf("expect cursor=%d, actual=%d", expectedCursor, cursor) + } +} + +func testFromBytes(t *testing.T, buffer []byte, expected *Packet) { + expected.Encode() + res, err := FromBytes(buffer) + // t.Logf("testFromBytes: res=%v, exptected=%v", res, expected) + if err != nil { + t.Error(err) + } else { + if res.GetTag() != expected.GetTag() { + t.Errorf("Tag expected=[% X], actual=[% X]", expected.GetTag(), res.GetTag()) + } + if res.Length != expected.Length { + t.Errorf("Length expected=[% X], actual=[% X]", expected.Length, res.Length) + } + for i := range res.valbuf { + if res.valbuf[i] != expected.valbuf[i] { + t.Errorf("valbuf on [%d] expected=[% X], actual=[% X]", i, expected.valbuf[i], res.valbuf[i]) + } + } + } +} + +func testV2Tags(t *testing.T, id uint64, expected []byte) { + p, err := NewPacket(id) + if err != nil { + t.Errorf("TestV2Tag err=%v", err) + } + p.SetInt32(0) + expected = append(expected, []byte{0x01, 0x00}...) + compareBytes(t, p, expected) +} + +func compareBytes(t *testing.T, p *Packet, expected []byte) []byte { + result, err := p.Encode() + if err != nil { + t.Errorf("compareBytes error=%v", err) + } + for i, p := range result { + if p != expected[i] { + t.Errorf("\nexpected:[% X]\n actual:[% X]\n", expected, result) + break + } + } + return result +} diff --git a/pkg/spec/typed_value_test.go b/pkg/spec/typed_value_test.go new file mode 100644 index 0000000..305dae2 --- /dev/null +++ b/pkg/spec/typed_value_test.go @@ -0,0 +1,179 @@ +package spec + +import ( + "testing" +) + +func TestV2Nil(t *testing.T) { + p, _ := NewPacket(18446744073709551487) + p.SetNil() + + buf := compareBytes(t, p, []byte{0xFE, 0x7F, 0x00}) + + dp, _ := FromBytes(buf) + if dp.GetTag() != 18446744073709551487 { + t.Errorf("result.Tag=% X, expect.Tag=% X", dp.GetTag(), uint64(18446744073709551487)) + } + + if dp.Length != 0 { + t.Errorf("result.Length=%d, expect.Length=%v", dp.Length, 0) + } + + if len(dp.valbuf) != 0 { + t.Errorf("result.len(valbuf)=%d, expect.len(valbuf)=%v", len(dp.valbuf), 0) + } +} + +func TestV2UInt32(t *testing.T) { + testV2UInt32(t, 0x03, 6, []byte{0x03, 0x01, 0x06}) + testV2UInt32(t, 0x06, 127, []byte{0x06, 0x02, 0x80, 0x7F}) +} + +func TestV2Int32(t *testing.T) { + testV2Int32(t, 0x03, -1, []byte{0x03, 0x01, 0x7F}) + testV2Int32(t, 0x06, -65, []byte{0x06, 0x02, 0xFF, 0x3F}) + testV2Int32(t, 0x09, 255, []byte{0x09, 0x02, 0x81, 0x7F}) +} + +func TestV2UInt64(t *testing.T) { + testV2UInt64(t, 0x03, 0, []byte{0x03, 0x01, 0x00}) + testV2UInt64(t, 0x06, 1, []byte{0x06, 0x01, 0x01}) + testV2UInt64(t, 0x09, 18446744073709551615, []byte{0x09, 0x01, 0x7F}) +} + +func TestV2Int64(t *testing.T) { + testV2Int64(t, 0x03, 0, []byte{0x03, 0x01, 0x00}) + testV2Int64(t, 0x06, 1, []byte{0x06, 0x01, 0x01}) + testV2Int64(t, 0x09, -1, []byte{0x09, 0x01, 0x7F}) +} + +func TestV2Float32(t *testing.T) { + testV2Float32(t, 0x03, -2, []byte{0x03, 0x01, 0xC0}) + testV2Float32(t, 0x06, 0.25, []byte{0x06, 0x02, 0x3E, 0x80}) + testV2Float32(t, 0x09, 68.123, []byte{0x09, 0x04, 0x42, 0x88, 0x3E, 0xFA}) +} + +func TestV2Float64(t *testing.T) { + testV2Float64(t, 0x03, 23, []byte{0x03, 0x02, 0x40, 0x37}) + testV2Float64(t, 0x06, 2, []byte{0x06, 0x01, 0x40}) + testV2Float64(t, 0x09, 0.01171875, []byte{0x09, 0x02, 0x3F, 0x88}) +} + +func TestV2String(t *testing.T) { + p, _ := NewPacket(0x01) + p.SetUTF8String("C") + compareBytes(t, p, []byte{0x01, 0x01, 0x43}) + p.SetUTF8String("CC") + compareBytes(t, p, []byte{0x01, 0x02, 0x43, 0x43}) + p.SetUTF8String("Yona") + compareBytes(t, p, []byte{0x01, 0x04, 0x59, 0x6F, 0x6E, 0x61}) + p.SetUTF8String("https://yomo.run") + buf := compareBytes(t, p, []byte{0x01, 0x10, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x2F, 0x2F, 0x79, 0x6F, 0x6D, 0x6F, 0x2E, 0x72, 0x75, 0x6E}) + + dp, _ := FromBytes(buf) + resval := dp.GetValueAsUTF8String() + if resval != "https://yomo.run" { + t.Errorf("result=%s, expect=https://yomo.run", resval) + } +} + +func TestV2Bytes(t *testing.T) { + p, _ := NewPacket(0x01) + p.PutBytes([]byte{0x03, 0x06, 0x09, 0x0C, 0x0F}) + compareBytes(t, p, []byte{0x01, 0x05, 0x03, 0x06, 0x09, 0x0C, 0x0F}) + p.PutBytes([]byte{0x06, 0x01, 0x01}) + compareBytes(t, p, []byte{0x01, 0x08, 0x03, 0x06, 0x09, 0x0C, 0x0F, 0x06, 0x01, 0x01}) +} + +func TestV2Bool(t *testing.T) { + p, _ := NewPacket(0x01) + p.SetBool(true) + compareBytes(t, p, []byte{0x01, 0x01, 0x01}) + + p.SetBool(false) + buf := compareBytes(t, p, []byte{0x01, 0x01, 0x00}) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsBool() + if resval != false { + t.Errorf("result=%v, expect=%v", resval, false) + } +} + +func testV2UInt32(t *testing.T, tag uint64, val uint32, expected []byte) { + p, _ := NewPacket(tag) + p.SetUInt32(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsUInt32() + if resval != val { + t.Errorf("result=%d, expect=%d", resval, val) + } +} + +func testV2Int32(t *testing.T, tag uint64, val int, expected []byte) { + p, _ := NewPacket(tag) + p.SetInt32(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsInt32() + if resval != int32(val) { + t.Errorf("result=%d, expect=%d", resval, val) + } +} + +func testV2UInt64(t *testing.T, tag uint64, val uint64, expected []byte) { + p, _ := NewPacket(tag) + p.SetUInt64(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsUInt64() + if resval != val { + t.Errorf("result=%d, expect=%d", resval, val) + } +} + +func testV2Int64(t *testing.T, tag uint64, val int64, expected []byte) { + p, _ := NewPacket(tag) + p.SetInt64(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsInt64() + if resval != val { + t.Errorf("result=%d, expect=%d", resval, val) + } +} + +func testV2Float32(t *testing.T, tag uint64, val float32, expected []byte) { + p, _ := NewPacket(tag) + p.SetFloat32(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsFloat32() + if resval != val { + t.Errorf("result=%f, expect=%f", resval, val) + } +} + +func testV2Float64(t *testing.T, tag uint64, val float64, expected []byte) { + p, _ := NewPacket(tag) + p.SetFloat64(val) + + buf := compareBytes(t, p, expected) + + dp, _ := FromBytes(buf) + resval, _ := dp.GetValueAsFloat64() + if resval != val { + t.Errorf("result=%f, expect=%f", resval, val) + } +} diff --git a/stream_api.go b/stream_api.go new file mode 100644 index 0000000..3cb7c5c --- /dev/null +++ b/stream_api.go @@ -0,0 +1,130 @@ +package y3 + +import ( + "io" + "log" + + "github.com/yomorun/y3-codec-golang/pkg/encoding" + "github.com/yomorun/y3-codec-golang/pkg/spec" +) + +// StreamDecoder decode Y3 Packet from a io.Reader +type StreamDecoder struct { + errState bool + tagbuf []byte + lenbuf []byte + valbuf []byte + r io.Reader + state string + len int + callback func(*spec.Packet) +} + +// NewStreamDecoder return a stream decoder +func NewStreamDecoder(r io.Reader) *StreamDecoder { + return &StreamDecoder{ + errState: false, + r: r, + state: "Nil", + } +} + +// OnPacket trigger callback once Y3 packet parsed out +func (sd *StreamDecoder) OnPacket(f func(*spec.Packet)) { + sd.callback = f +} + +// Start the parser +func (sd *StreamDecoder) Start() { + // buffer + tmp := make([]byte, 1) + for { + n, err := sd.r.Read(tmp) + if err != nil { + log.Printf("io err: tmp=[% X]", tmp) + sd.reset(err) + break + } + log.Printf("Recieved: n=%d, tmp=[% X]", n, tmp[:n]) + for _, v := range tmp[:n] { + sd.fill(v) + } + } +} + +func (sd *StreamDecoder) fill(b byte) error { + log.Printf("-> fill b=[% X], state=%s", b, sd.state) + switch sd.state { + case "Nil": + sd.state = "TS" + sd.fill(b) + case "TS": + sd.tagbuf = append(sd.tagbuf, b) + if b&0x81 != 0x81 { + // over of tag + sd.state = "LS" + log.Printf("Parsed Out Tag, tagbuf=[% X]", sd.tagbuf) + return nil + } + case "LS": + sd.lenbuf = append(sd.lenbuf, b) + if b&0x81 != 0x81 { + // over of len, start parse as PVarUInt64 value + var len uint64 + codec := encoding.VarCodec{} + err := codec.DecodePVarUInt64(sd.lenbuf, &len) + if err != nil { + sd.errState = true + panic(err) + } else { + sd.len = int(len) + log.Printf("Parsed Out len=%d, lenbuf=[% X]", sd.len, sd.lenbuf) + } + if sd.len == 0 { + // reset state if zero-len packet + log.Printf("[%s] Parsed Out valbuf=EMPTY", sd.state) + // make a Packet object + sd.fullfiled() + sd.state = "Nil" + } + // update state + sd.state = "VS" + return nil + } + case "VS": + sd.valbuf = append(sd.valbuf, b) + if len(sd.valbuf) == sd.len { + log.Printf("[%s] Parsed Out valbuf=[% X]", sd.state, sd.valbuf) + // make a Packet object + sd.fullfiled() + // reset state + sd.state = "Nil" + } + } + return nil +} + +func (sd *StreamDecoder) fullfiled() { + buf := append(sd.tagbuf, sd.lenbuf...) + buf = append(buf, sd.valbuf...) + + p, err := spec.FromBytes(buf) + if err != nil { + panic(err) + } + log.Printf("--> Fullfiled p=%v", p) + sd.callback(p) + + sd.reset(nil) +} + +func (sd *StreamDecoder) reset(err error) { + if err != nil { + log.Printf("[RESET] cause of error: %s", err.Error()) + } + sd.errState = false + sd.tagbuf = make([]byte, 0) + sd.lenbuf = make([]byte, 0) + sd.valbuf = make([]byte, 0) + sd.state = "Nil" +} diff --git a/stream_api_test.go b/stream_api_test.go new file mode 100644 index 0000000..d984591 --- /dev/null +++ b/stream_api_test.go @@ -0,0 +1,158 @@ +package y3 + +import ( + "bytes" + "testing" + + "github.com/yomorun/y3-codec-golang/pkg/spec" +) + +func TestStreamDecode1(t *testing.T) { + // testStreamDecode(t, []byte{0x01, 0x00}, []byte{0x01}, []byte{0x00}, []byte{}) + data := []byte{0x01, 0x01, 0x03} //, []byte{0x01}, []byte{0x01}, []byte{0x03}, flag) + // testStreamDecode(t, []byte{0x01, 0x02, 0x03, 0x04, 0x05}, []byte{0x01}, []byte{0x02}, []byte{0x03, 0x04}) + // testStreamDecode(t, []byte{0x01, 0x03, 0x03, 0x04, 0x05}, []byte{0x01}, []byte{0x03}, []byte{0x03, 0x04, 0x05}) + // testStreamDecode(t, []byte{0x01, 0x01}, []byte{0x02}, []byte{0x01}, []byte{}) + // t.Errorf("---") + + expectTagbuf := []byte{0x01} + expectLenbuf := []byte{0x01} + expectValbuf := []byte{0x03} + + // as reader + r := bytes.NewReader(data) + // create steam decoder + pr := NewStreamDecoder(r) + + // handler + pr.OnPacket(func(p *spec.Packet) { + t.Logf("[CALLBACK] p=%v", p) + compareBytes(t, p.GetTagBuffer(), expectTagbuf, "T") + compareBytes(t, p.GetLengthBuffer(), expectLenbuf, "L") + compareBytes(t, p.GetValueBuffer(), expectValbuf, "V") + }) + pr.Start() +} + +func TestStreamDecode2(t *testing.T) { + data := []byte{0x01, 0x00} + + expectTagbuf := []byte{0x01} + expectLenbuf := []byte{0x00} + expectValbuf := []byte{} + + // as reader + r := bytes.NewReader(data) + // create steam decoder + pr := NewStreamDecoder(r) + + // handler + pr.OnPacket(func(p *spec.Packet) { + t.Logf("[CALLBACK] p=%v", p) + compareBytes(t, p.GetTagBuffer(), expectTagbuf, "T") + compareBytes(t, p.GetLengthBuffer(), expectLenbuf, "L") + compareBytes(t, p.GetValueBuffer(), expectValbuf, "V") + }) + pr.Start() +} + +func TestStreamDecode3(t *testing.T) { + data := []byte{0x01, 0x02, 0x03, 0x04} + + expectTagbuf := []byte{0x01} + expectLenbuf := []byte{0x02} + expectValbuf := []byte{0x03, 0x04} + + // as reader + r := bytes.NewReader(data) + // create steam decoder + pr := NewStreamDecoder(r) + + // handler + pr.OnPacket(func(p *spec.Packet) { + t.Logf("[CALLBACK] p=%v", p) + compareBytes(t, p.GetTagBuffer(), expectTagbuf, "T") + compareBytes(t, p.GetLengthBuffer(), expectLenbuf, "L") + compareBytes(t, p.GetValueBuffer(), expectValbuf, "V") + }) + pr.Start() +} + +func TestStreamDecode4(t *testing.T) { + data := []byte{0x01, 0x02, 0x03} + + // as reader + r := bytes.NewReader(data) + // create steam decoder + pr := NewStreamDecoder(r) + + parsed := false + + // handler + pr.OnPacket(func(p *spec.Packet) { + t.Logf("[CALLBACK] p=%v", p) + if p != nil { + t.Error(p) + } + parsed = true + }) + pr.Start() + + if parsed == true { + t.Errorf("Should not trigger callback") + } +} + +func TestStreamDecode5(t *testing.T) { + data := []byte{0x01, 0x01, 0x01, 0x02, 0x02, 0x01, 0x02, 0x03, 0x00, 0x04, 0x03, 0x01, 0x02, 0x03} + + // as reader + r := bytes.NewReader(data) + // create steam decoder + pr := NewStreamDecoder(r) + + times := 1 + + // handler + pr.OnPacket(func(p *spec.Packet) { + if times == 1 { + compareBytes(t, p.GetTagBuffer(), []byte{0x01}, "T") + compareBytes(t, p.GetLengthBuffer(), []byte{0x01}, "L") + compareBytes(t, p.GetValueBuffer(), []byte{0x01}, "V") + } + if times == 2 { + compareBytes(t, p.GetTagBuffer(), []byte{0x02}, "T") + compareBytes(t, p.GetLengthBuffer(), []byte{0x02}, "L") + compareBytes(t, p.GetValueBuffer(), []byte{0x01, 0x02}, "V") + } + if times == 3 { + compareBytes(t, p.GetTagBuffer(), []byte{0x03}, "T") + compareBytes(t, p.GetLengthBuffer(), []byte{0x00}, "L") + compareBytes(t, p.GetValueBuffer(), []byte{}, "V") + if p.Length != 0 { + t.Errorf("Packet:Tag=[% X] Length should be 0, actual=%d", p.GetTagBuffer(), p.Length) + } + } + if times == 4 { + compareBytes(t, p.GetTagBuffer(), []byte{0x04}, "T") + compareBytes(t, p.GetLengthBuffer(), []byte{0x03}, "L") + compareBytes(t, p.GetValueBuffer(), []byte{0x01, 0x02, 0x03}, "V") + } + times++ + }) + + pr.Start() +} + +func compareBytes(t *testing.T, result []byte, expected []byte, v string) { + if len(result) != len(expected) { + t.Errorf("\n[%s] expected:[% X]\n actual:[% X]\n", v, expected, result) + } + + for i, p := range result { + if p != expected[i] { + t.Errorf("\n[%s] expected:[% X]\n actual:[% X]\n", v, expected, result) + break + } + } +} From 6396282821770401b334f80c81029ee0e9cc8b79 Mon Sep 17 00:00:00 2001 From: "C.C" Date: Mon, 22 Feb 2021 19:13:34 +0800 Subject: [PATCH 2/2] fix: zero-length packet --- stream_api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/stream_api.go b/stream_api.go index 3cb7c5c..f475809 100644 --- a/stream_api.go +++ b/stream_api.go @@ -86,6 +86,7 @@ func (sd *StreamDecoder) fill(b byte) error { // make a Packet object sd.fullfiled() sd.state = "Nil" + return nil } // update state sd.state = "VS"