From b07ec74f356ce7e853bc18664e0bdb42b6b64404 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Sep 2024 01:29:26 +0800 Subject: [PATCH 1/6] feat: Supports gRPC write hints --- config.go | 4 +- examples/README.md | 1 + examples/hint/README.md | 58 +++++++++++++++++++ examples/hint/main.go | 121 ++++++++++++++++++++++++++++++++++++++++ pkg/hint/hint.go | 39 +++++++++++++ request/request.go | 8 ++- 6 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 examples/hint/README.md create mode 100644 examples/hint/main.go create mode 100644 pkg/hint/hint.go diff --git a/config.go b/config.go index 099e3cd..8aa6020 100644 --- a/config.go +++ b/config.go @@ -107,7 +107,7 @@ func (c *Config) WithMetricsEnabled(b bool) *Config { // WithMeterProvider provides a MeterProvider for SDK. // If metrics colleciton is not enabled, then this option has no effect. -// If metrics colleciton is enabled and this option is not provide +// If metrics colleciton is enabled and this option is not provide. // the global MeterProvider will be used. func (c *Config) WithMeterProvider(p metric.MeterProvider) *Config { c.telemetry.Metrics.MeterProvider = p @@ -122,7 +122,7 @@ func (c *Config) WithTracesEnabled(b bool) *Config { // WithTraceProvider provides a TracerProvider for SDK. // If traces colleciton is not enabled, then this option has no effect. -// If traces colleciton is enabled and this option is not provide +// If traces colleciton is enabled and this option is not provide. // the global MeterProvider will be used. func (c *Config) WithTraceProvider(p trace.TracerProvider) *Config { c.telemetry.Traces.TracerProvider = p diff --git a/examples/README.md b/examples/README.md index 3a450de..6634847 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,6 +16,7 @@ docker run --rm -p 4000-4003:4000-4003 \ - [table](table/README.md) - [object](object/README.md) - [healthcheck](healthcheck/README.md) +- [hint](hint/README.md) ## Query diff --git a/examples/hint/README.md b/examples/hint/README.md new file mode 100644 index 0000000..46a2b46 --- /dev/null +++ b/examples/hint/README.md @@ -0,0 +1,58 @@ +# Write Hints + +## Insert + +```go +go run main.go +``` + +Output: + +```log +2024/09/24 01:16:03 create table, name: 'monitor_table_with_hint' +2024/09/24 01:16:03 affected rows: 3 +``` + +## Query + +Your can using [MySQL Client](https://docs.greptime.com/user-guide/protocols/mysql) to query the data from GreptimeDB. + +```shell +$ mysql -h 127.0.0.1 -P 4002 public + +mysql> select * from monitor_table_with_hint; ++------+-------+-------------+----------------------------+ +| id | host | temperature | timestamp | ++------+-------+-------------+----------------------------+ +| 1 | hello | 1.1 | 2024-09-23 17:16:03.476898 | +| 2 | hello | 2.2 | 2024-09-23 17:16:03.476898 | +| 3 | hello | 3.3 | 2024-09-23 17:16:03.476898 | ++------+-------+-------------+----------------------------+ +3 rows in set (0.03 sec) +``` + +You can view hints using `show create table` command: + +```mysql +mysql> show create table monitor_table_with_hint; ++------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| monitor_table_with_hint | CREATE TABLE IF NOT EXISTS `monitor_table_with_hint` ( + `id` BIGINT NULL, + `host` STRING NULL, + `temperature` DOUBLE NULL, + `timestamp` TIMESTAMP(6) NOT NULL, + TIME INDEX (`timestamp`), + PRIMARY KEY (`id`) +) + +ENGINE=mito +WITH( + append_mode = 'false', + merge_mode = 'last_non_null', + ttl = '3day' +) | ++------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.02 sec) +``` diff --git a/examples/hint/main.go b/examples/hint/main.go new file mode 100644 index 0000000..19247bd --- /dev/null +++ b/examples/hint/main.go @@ -0,0 +1,121 @@ +// Copyright 2024 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "time" + + greptime "github.com/GreptimeTeam/greptimedb-ingester-go" + + "github.com/GreptimeTeam/greptimedb-ingester-go/pkg/hint" + "github.com/GreptimeTeam/greptimedb-ingester-go/table" + "github.com/GreptimeTeam/greptimedb-ingester-go/table/types" +) + +const ( + INSERT = 0 +) + +var ( + client *greptime.Client +) + +func init() { + cfg := greptime.NewConfig("127.0.0.1").WithDatabase("public") + + cli_, err := greptime.NewClient(cfg) + if err != nil { + log.Panic(err) + } + client = cli_ +} + +func initData() []*table.Table { + time1 := time.Now() + time2 := time.Now() + time3 := time.Now() + + itbl, err := table.New("monitor_table_with_hint") + if err != nil { + log.Println(err) + return nil + } + // add column at first. This is to define the schema of the table. + if err := itbl.AddTagColumn("id", types.INT64); err != nil { + log.Println(err) + return nil + } + if err := itbl.AddFieldColumn("host", types.STRING); err != nil { + log.Println(err) + return nil + } + if err := itbl.AddFieldColumn("temperature", types.FLOAT); err != nil { + log.Println(err) + return nil + } + if err := itbl.AddTimestampColumn("timestamp", types.TIMESTAMP_MICROSECOND); err != nil { + log.Println(err) + return nil + } + + if err := itbl.AddRow(1, "hello", 1.1, time1); err != nil { + log.Println(err) + return nil + } + if err := itbl.AddRow(2, "hello", 2.2, time2); err != nil { + log.Println(err) + return nil + } + if err := itbl.AddRow(3, "hello", 3.3, time3); err != nil { + log.Println(err) + return nil + } + + return []*table.Table{itbl} +} + +func write(data *table.Table) { + var hints []hint.Hint + hints = append(hints, + hint.Hint{Key: "ttl", Value: "3d"}, + hint.Hint{Key: "merge_mode", Value: "last_non_null"}, + hint.Hint{Key: "append_mode", Value: "false"}, + ) + + ctx := hint.CreateContextWithHints(hints) + ctx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + + resp, err := client.Write(ctx, data) + if err != nil { + log.Println(err) + return + } + tableName, err := data.GetName() + if err != nil { + log.Println(err) + return + } + log.Printf("create table, name: '%s'", tableName) + log.Printf("affected rows: %d\n", resp.GetAffectedRows().GetValue()) +} + +func main() { + data := initData() + + write(data[INSERT]) +} diff --git a/pkg/hint/hint.go b/pkg/hint/hint.go new file mode 100644 index 0000000..bf0e19d --- /dev/null +++ b/pkg/hint/hint.go @@ -0,0 +1,39 @@ +// Copyright 2024 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hint + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +const ( + hintPrefix = "x-greptime-hint-" +) + +type Hint struct { + Key string + Value string +} + +func CreateContextWithHints(hints []Hint) context.Context { + md := metadata.New(nil) + for _, hint := range hints { + md.Append(hintPrefix+hint.Key, hint.Value) + } + ctx := metadata.NewOutgoingContext(context.Background(), md) + return ctx +} diff --git a/request/request.go b/request/request.go index 686b676..1d5e239 100644 --- a/request/request.go +++ b/request/request.go @@ -76,7 +76,9 @@ func (r *Request) Build() (*gpb.GreptimeRequest, error) { insertReqs = append(insertReqs, req) } req := &gpb.GreptimeRequest_RowInserts{ - RowInserts: &gpb.RowInsertRequests{Inserts: insertReqs}, + RowInserts: &gpb.RowInsertRequests{ + Inserts: insertReqs, + }, } return &gpb.GreptimeRequest{ Header: header, @@ -93,7 +95,9 @@ func (r *Request) Build() (*gpb.GreptimeRequest, error) { } req := &gpb.GreptimeRequest_RowDeletes{ - RowDeletes: &gpb.RowDeleteRequests{Deletes: deleteReqs}, + RowDeletes: &gpb.RowDeleteRequests{ + Deletes: deleteReqs, + }, } return &gpb.GreptimeRequest{ Header: header, From a9fe67ce242b963a37a6d6ac1862d65fc1aa2791 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 24 Sep 2024 01:48:41 +0800 Subject: [PATCH 2/6] chore: change table name --- examples/README.md | 2 +- examples/{hint => hints}/README.md | 8 ++++---- examples/{hint => hints}/main.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename examples/{hint => hints}/README.md (93%) rename examples/{hint => hints}/main.go (98%) diff --git a/examples/README.md b/examples/README.md index 6634847..b8438be 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,7 +16,7 @@ docker run --rm -p 4000-4003:4000-4003 \ - [table](table/README.md) - [object](object/README.md) - [healthcheck](healthcheck/README.md) -- [hint](hint/README.md) +- [hints](hints/README.md) ## Query diff --git a/examples/hint/README.md b/examples/hints/README.md similarity index 93% rename from examples/hint/README.md rename to examples/hints/README.md index 46a2b46..42d3698 100644 --- a/examples/hint/README.md +++ b/examples/hints/README.md @@ -9,7 +9,7 @@ go run main.go Output: ```log -2024/09/24 01:16:03 create table, name: 'monitor_table_with_hint' +2024/09/24 01:16:03 create table, name: 'monitor_table_with_hints' 2024/09/24 01:16:03 affected rows: 3 ``` @@ -20,7 +20,7 @@ Your can using [MySQL Client](https://docs.greptime.com/user-guide/protocols/mys ```shell $ mysql -h 127.0.0.1 -P 4002 public -mysql> select * from monitor_table_with_hint; +mysql> select * from monitor_table_with_hints; +------+-------+-------------+----------------------------+ | id | host | temperature | timestamp | +------+-------+-------------+----------------------------+ @@ -34,11 +34,11 @@ mysql> select * from monitor_table_with_hint; You can view hints using `show create table` command: ```mysql -mysql> show create table monitor_table_with_hint; +mysql> show create table monitor_table_with_hints; +------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Table | Create Table | +------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| monitor_table_with_hint | CREATE TABLE IF NOT EXISTS `monitor_table_with_hint` ( +| monitor_table_with_hints | CREATE TABLE IF NOT EXISTS `monitor_table_with_hints` ( `id` BIGINT NULL, `host` STRING NULL, `temperature` DOUBLE NULL, diff --git a/examples/hint/main.go b/examples/hints/main.go similarity index 98% rename from examples/hint/main.go rename to examples/hints/main.go index 19247bd..13a2122 100644 --- a/examples/hint/main.go +++ b/examples/hints/main.go @@ -49,7 +49,7 @@ func initData() []*table.Table { time2 := time.Now() time3 := time.Now() - itbl, err := table.New("monitor_table_with_hint") + itbl, err := table.New("monitor_table_with_hints") if err != nil { log.Println(err) return nil From 6db41b3d4e1de72a251b793e7437f50a82dc757d Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 27 Sep 2024 11:44:53 +0800 Subject: [PATCH 3/6] refactor: move to the context package --- pkg/hints/hints.go => context/context.go | 2 +- examples/hints/main.go | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) rename pkg/hints/hints.go => context/context.go (98%) diff --git a/pkg/hints/hints.go b/context/context.go similarity index 98% rename from pkg/hints/hints.go rename to context/context.go index 43acbcb..757f8b0 100644 --- a/pkg/hints/hints.go +++ b/context/context.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package hints +package context import ( "context" diff --git a/examples/hints/main.go b/examples/hints/main.go index c2864a6..cbc9b64 100644 --- a/examples/hints/main.go +++ b/examples/hints/main.go @@ -21,7 +21,7 @@ import ( greptime "github.com/GreptimeTeam/greptimedb-ingester-go" - "github.com/GreptimeTeam/greptimedb-ingester-go/pkg/hints" + ingestercontext "github.com/GreptimeTeam/greptimedb-ingester-go/context" "github.com/GreptimeTeam/greptimedb-ingester-go/table" "github.com/GreptimeTeam/greptimedb-ingester-go/table/types" ) @@ -91,14 +91,14 @@ func initData() ([]*table.Table, error) { } func (c client) write(data *table.Table) error { - var h []hints.Hint - h = append(h, - hints.Hint{Key: "ttl", Value: "3d"}, - hints.Hint{Key: "merge_mode", Value: "last_non_null"}, - hints.Hint{Key: "append_mode", Value: "false"}, + var hints []ingestercontext.Hint + hints = append(hints, + ingestercontext.Hint{Key: "ttl", Value: "3d"}, + ingestercontext.Hint{Key: "merge_mode", Value: "last_non_null"}, + ingestercontext.Hint{Key: "append_mode", Value: "false"}, ) - ctx := hints.CreateContextWithHints(h) + ctx := ingestercontext.CreateContextWithHints(hints) ctx, cancel := context.WithTimeout(ctx, time.Second*3) defer cancel() @@ -127,8 +127,7 @@ func main() { log.Fatalf("failed to new client: %v:", err) } - err = c.write(data[INSERT]) - if err != nil { + if err = c.write(data[INSERT]); err != nil { log.Fatalf("failed to write data: %v:", err) } } From 810495cb2203785bddf29bfc2aafdfe854fadf53 Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 27 Sep 2024 15:08:52 +0800 Subject: [PATCH 4/6] refactor: refactor the context package --- context/context.go | 21 ++++++++++++++++----- examples/hints/main.go | 30 +++++++++++++++++++----------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/context/context.go b/context/context.go index 757f8b0..fe09124 100644 --- a/context/context.go +++ b/context/context.go @@ -29,11 +29,22 @@ type Hint struct { Value string } -func CreateContextWithHints(hints []Hint) context.Context { - md := metadata.New(nil) - for _, hint := range hints { - md.Append(hintPrefix+hint.Key, hint.Value) +type Option func(ctx context.Context) context.Context + +func New(parent context.Context, opts ...Option) context.Context { + ctx := parent + for _, opt := range opts { + ctx = opt(parent) } - ctx := metadata.NewOutgoingContext(context.Background(), md) return ctx } + +func WithHints(hints []*Hint) Option { + return func(ctx context.Context) context.Context { + md := metadata.New(nil) + for _, hint := range hints { + md.Append(hintPrefix+hint.Key, hint.Value) + } + return metadata.NewOutgoingContext(ctx, md) + } +} diff --git a/examples/hints/main.go b/examples/hints/main.go index cbc9b64..ceb445d 100644 --- a/examples/hints/main.go +++ b/examples/hints/main.go @@ -21,7 +21,7 @@ import ( greptime "github.com/GreptimeTeam/greptimedb-ingester-go" - ingestercontext "github.com/GreptimeTeam/greptimedb-ingester-go/context" + ingesterContext "github.com/GreptimeTeam/greptimedb-ingester-go/context" "github.com/GreptimeTeam/greptimedb-ingester-go/table" "github.com/GreptimeTeam/greptimedb-ingester-go/table/types" ) @@ -91,21 +91,29 @@ func initData() ([]*table.Table, error) { } func (c client) write(data *table.Table) error { - var hints []ingestercontext.Hint - hints = append(hints, - ingestercontext.Hint{Key: "ttl", Value: "3d"}, - ingestercontext.Hint{Key: "merge_mode", Value: "last_non_null"}, - ingestercontext.Hint{Key: "append_mode", Value: "false"}, - ) - - ctx := ingestercontext.CreateContextWithHints(hints) - ctx, cancel := context.WithTimeout(ctx, time.Second*3) + hints := []*ingesterContext.Hint{ + { + Key: "ttl", + Value: "3d", + }, + { + Key: "merge_mode", + Value: "last_non_null", + }, + { + Key: "append_mode", + Value: "false", + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() - resp, err := c.client.Write(ctx, data) + resp, err := c.client.Write(ingesterContext.New(ctx, ingesterContext.WithHints(hints)), data) if err != nil { return err } + tableName, err := data.GetName() if err != nil { return err From dc7355544aaeaca2a0f3d77d94cc0b122f4674f3 Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 27 Sep 2024 15:15:57 +0800 Subject: [PATCH 5/6] refactor: init table --- examples/hints/main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/hints/main.go b/examples/hints/main.go index ceb445d..d2fe3a4 100644 --- a/examples/hints/main.go +++ b/examples/hints/main.go @@ -27,8 +27,6 @@ import ( ) const ( - INSERT = 0 - // The GreptimeDB address. host = "127.0.0.1" @@ -54,7 +52,7 @@ func newClient() (*client, error) { return c, nil } -func initData() ([]*table.Table, error) { +func initData() (*table.Table, error) { time1 := time.Now() time2 := time.Now() time3 := time.Now() @@ -87,7 +85,7 @@ func initData() ([]*table.Table, error) { return nil, err } - return []*table.Table{itbl}, nil + return itbl, nil } func (c client) write(data *table.Table) error { @@ -135,7 +133,7 @@ func main() { log.Fatalf("failed to new client: %v:", err) } - if err = c.write(data[INSERT]); err != nil { + if err = c.write(data); err != nil { log.Fatalf("failed to write data: %v:", err) } } From dcd480278a01962ea1f117c55fdd2a9ff032e879 Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 27 Sep 2024 15:48:24 +0800 Subject: [PATCH 6/6] Update context/context.go Co-authored-by: zyy17 --- context/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/context.go b/context/context.go index fe09124..598c58b 100644 --- a/context/context.go +++ b/context/context.go @@ -34,7 +34,7 @@ type Option func(ctx context.Context) context.Context func New(parent context.Context, opts ...Option) context.Context { ctx := parent for _, opt := range opts { - ctx = opt(parent) + ctx = opt(ctx) } return ctx }