diff --git a/client/client_test.go b/client/client_test.go index a34142c..98042cf 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -36,8 +36,9 @@ import ( ) var ( - timezone = "UTC" - tableName = "test_insert_monitor" + tableName = "" + timezone = "UTC" + database = "public" host = "127.0.0.1" httpPort, grpcPort, mysqlPort = 4000, 4001, 4002 @@ -46,6 +47,43 @@ var ( db *Mysql ) +// this is to scan all datatypes from GreptimeDB +type datatype struct { + INT8 int8 `gorm:"primaryKey;column:int8"` + INT16 int16 `gorm:"column:int16"` + INT32 int32 `gorm:"column:int32"` + INT64 int64 `gorm:"column:int64"` + UINT8 uint8 `gorm:"column:uint8"` + UINT16 uint16 `gorm:"column:uint16"` + UINT32 uint32 `gorm:"column:uint32"` + UINT64 uint64 `gorm:"column:uint64"` + BOOLEAN bool `gorm:"column:boolean"` + FLOAT32 float32 `gorm:"column:float32"` + FLOAT64 float64 `gorm:"column:float64"` + BINARY []byte `gorm:"column:binary"` + STRING string `gorm:"column:string"` + + DATE time.Time `gorm:"column:date"` + DATETIME time.Time `gorm:"column:datetime"` + TIMESTAMP_SECOND time.Time `gorm:"column:timestamp_second"` + TIMESTAMP_MILLISECOND time.Time `gorm:"column:timestamp_millisecond"` + TIMESTAMP_MICROSECOND time.Time `gorm:"column:timestamp_microsecond"` + TIMESTAMP_NANOSECOND time.Time `gorm:"column:timestamp_nanosecond"` + + DATE_INT time.Time `gorm:"column:date_int"` + DATETIME_INT time.Time `gorm:"column:datetime_int"` + TIMESTAMP_SECOND_INT time.Time `gorm:"column:timestamp_second_int"` + TIMESTAMP_MILLISECOND_INT time.Time `gorm:"column:timestamp_millisecond_int"` + TIMESTAMP_MICROSECOND_INT time.Time `gorm:"column:timestamp_microsecond_int"` + TIMESTAMP_NANOSECOND_INT time.Time `gorm:"column:timestamp_nanosecond_int"` + + TS time.Time `gorm:"column:ts"` +} + +func (datatype) TableName() string { + return tableName +} + type monitor struct { ID int64 `gorm:"primaryKey;column:id"` Host string `gorm:"primaryKey;column:host"` @@ -89,12 +127,18 @@ func (m *Mysql) Setup() error { return nil } -func (p *Mysql) AllMonitors() ([]monitor, error) { +func (p *Mysql) Query(sql string) ([]monitor, error) { var monitors []monitor - err := p.DB.Find(&monitors).Error + err := p.DB.Raw(sql).Scan(&monitors).Error return monitors, err } +func (p *Mysql) AllDatatypes() ([]datatype, error) { + var datatypes []datatype + err := p.DB.Find(&datatypes).Error + return datatypes, err +} + func newClient() *Client { options := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), @@ -182,7 +226,9 @@ func init() { db = newMysql() } -func TestInsertMonitor(t *testing.T) { +func TestInsertMonitors(t *testing.T) { + tableName = "test_insert_monitor" + loc, err := time.LoadLocation(timezone) assert.Nil(t, err) ts1 := time.Now().Add(-1 * time.Minute).UnixMilli() @@ -214,13 +260,13 @@ func TestInsertMonitor(t *testing.T) { table, err := tbl.New(tableName) assert.Nil(t, err) - table.AddTagColumn("id", types.INT64) - table.AddTagColumn("host", types.STRING) - table.AddFieldColumn("memory", types.UINT64) - table.AddFieldColumn("cpu", types.FLOAT64) - table.AddFieldColumn("temperature", types.INT64) - table.AddFieldColumn("running", types.BOOLEAN) - table.AddTimestampColumn("ts", types.TIMESTAMP_MILLISECOND) + assert.Nil(t, table.AddTagColumn("id", types.INT64)) + assert.Nil(t, table.AddTagColumn("host", types.STRING)) + assert.Nil(t, table.AddFieldColumn("memory", types.UINT64)) + assert.Nil(t, table.AddFieldColumn("cpu", types.FLOAT64)) + assert.Nil(t, table.AddFieldColumn("temperature", types.INT64)) + assert.Nil(t, table.AddFieldColumn("running", types.BOOLEAN)) + assert.Nil(t, table.AddTimestampColumn("ts", types.TIMESTAMP_MILLISECOND)) for _, monitor := range monitors { err := table.AddRow(monitor.ID, monitor.Host, @@ -235,7 +281,7 @@ func TestInsertMonitor(t *testing.T) { assert.Empty(t, resp.GetHeader().GetStatus().GetErrMsg()) assert.Equal(t, uint32(len(monitors)), resp.GetAffectedRows().GetValue()) - monitors_, err := db.AllMonitors() + monitors_, err := db.Query(fmt.Sprintf("select * from %s where id in (1, 2) order by id asc", tableName)) assert.Nil(t, err) assert.Equal(t, len(monitors), len(monitors_)) @@ -244,3 +290,171 @@ func TestInsertMonitor(t *testing.T) { assert.Equal(t, monitors[i], monitor_) } } + +func TestInsertMonitorWithNilFields(t *testing.T) { + tableName = "test_insert_monitor_with_nil_fields" + + loc, err := time.LoadLocation(timezone) + assert.Nil(t, err) + ts := time.Now().Add(-1 * time.Minute).UnixMilli() + time := time.UnixMilli(ts).In(loc) + monitor := monitor{ + ID: 11, + Host: "127.0.0.1", + Memory: 1, + Cpu: 1.0, + Temperature: -1, + Ts: time, + Running: true, + } + + table, err := tbl.New(tableName) + assert.Nil(t, err) + + assert.Nil(t, table.AddTagColumn("id", types.INT64)) + assert.Nil(t, table.AddTagColumn("host", types.STRING)) + assert.Nil(t, table.AddFieldColumn("memory", types.UINT64)) + assert.Nil(t, table.AddFieldColumn("cpu", types.FLOAT64)) + assert.Nil(t, table.AddFieldColumn("temperature", types.INT64)) + assert.Nil(t, table.AddFieldColumn("running", types.BOOLEAN)) + assert.Nil(t, table.AddTimestampColumn("ts", types.TIMESTAMP_MILLISECOND)) + + // with nil fields + err = table.AddRow(monitor.ID, monitor.Host, nil, nil, nil, monitor.Running, monitor.Ts) + assert.Nil(t, err) + + resp, err := cli.Write(context.Background(), table) + assert.Nil(t, err) + assert.Zero(t, resp.GetHeader().GetStatus().GetStatusCode()) + assert.Empty(t, resp.GetHeader().GetStatus().GetErrMsg()) + + monitors_, err := db.Query(fmt.Sprintf("select * from %s where id = %d", tableName, monitor.ID)) + assert.Nil(t, err) + assert.Equal(t, 1, len(monitors_)) + monitor_ := monitors_[0] + + assert.Equal(t, monitor.ID, monitor_.ID) + assert.Equal(t, monitor.Host, monitor_.Host) + assert.Equal(t, monitor.Running, monitor_.Running) + assert.Equal(t, monitor.Ts, monitor_.Ts) + + assert.Zero(t, monitor_.Memory) + assert.Zero(t, monitor_.Cpu) + assert.Zero(t, monitor_.Temperature) +} + +func TestInsertMonitorWithAllDatatypes(t *testing.T) { + tableName = "test_insert_monitor_with_all_datatypes" + + loc, err := time.LoadLocation(timezone) + assert.Nil(t, err) + + time_ := time.Now().In(loc) + date_int := time_.Unix() / 86400 + datetime_int := time_.UnixMilli() + + INT8 := 1 + INT16 := 2 + INT32 := 3 + INT64 := 4 + UINT8 := 5 + UINT16 := 6 + UINT32 := 7 + UINT64 := 8 + BOOLEAN := true + FLOAT32 := 9.0 + FLOAT64 := 10.0 + BINARY := []byte{1, 2, 3} + STRING := "string" + + table, err := tbl.New(tableName) + assert.Nil(t, err) + + assert.Nil(t, table.AddTagColumn("int8", types.INT8)) + assert.Nil(t, table.AddFieldColumn("int16", types.INT16)) + assert.Nil(t, table.AddFieldColumn("int32", types.INT32)) + assert.Nil(t, table.AddFieldColumn("int64", types.INT64)) + assert.Nil(t, table.AddFieldColumn("uint8", types.UINT8)) + assert.Nil(t, table.AddFieldColumn("uint16", types.UINT16)) + assert.Nil(t, table.AddFieldColumn("uint32", types.UINT32)) + assert.Nil(t, table.AddFieldColumn("uint64", types.UINT64)) + assert.Nil(t, table.AddFieldColumn("boolean", types.BOOLEAN)) + assert.Nil(t, table.AddFieldColumn("float32", types.FLOAT32)) + assert.Nil(t, table.AddFieldColumn("float64", types.FLOAT64)) + assert.Nil(t, table.AddFieldColumn("binary", types.BINARY)) + assert.Nil(t, table.AddFieldColumn("string", types.STRING)) + + assert.Nil(t, table.AddFieldColumn("date", types.DATE)) + assert.Nil(t, table.AddFieldColumn("datetime", types.DATETIME)) + assert.Nil(t, table.AddFieldColumn("timestamp_second", types.TIMESTAMP_SECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_millisecond", types.TIMESTAMP_MILLISECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_microsecond", types.TIMESTAMP_MICROSECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_nanosecond", types.TIMESTAMP_NANOSECOND)) + + assert.Nil(t, table.AddFieldColumn("date_int", types.DATE)) + assert.Nil(t, table.AddFieldColumn("datetime_int", types.DATETIME)) + assert.Nil(t, table.AddFieldColumn("timestamp_second_int", types.TIMESTAMP_SECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_millisecond_int", types.TIMESTAMP_MILLISECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_microsecond_int", types.TIMESTAMP_MICROSECOND)) + assert.Nil(t, table.AddFieldColumn("timestamp_nanosecond_int", types.TIMESTAMP_NANOSECOND)) + + assert.Nil(t, table.AddTimestampColumn("ts", types.TIMESTAMP_MILLISECOND)) + + // with all fields + err = table.AddRow(INT8, INT16, INT32, INT64, + UINT8, UINT16, UINT32, UINT64, + BOOLEAN, FLOAT32, FLOAT64, + BINARY, STRING, + + time_, time_, // date and datetime + time_, time_, time_, time_, // timestamp + + date_int, datetime_int, // date and datetime + time_.Unix(), time_.UnixMilli(), time_.UnixMicro(), time_.UnixNano(), // timestamp + + time_) + assert.Nil(t, err) + + resp, err := cli.Write(context.Background(), table) + assert.Nil(t, err) + assert.Zero(t, resp.GetHeader().GetStatus().GetStatusCode()) + assert.Empty(t, resp.GetHeader().GetStatus().GetErrMsg()) + + datatypes, err := db.AllDatatypes() + assert.Nil(t, err) + assert.Equal(t, 1, len(datatypes)) + result := datatypes[0] + + assert.EqualValues(t, INT8, result.INT8) + assert.EqualValues(t, INT16, result.INT16) + assert.EqualValues(t, INT32, result.INT32) + assert.EqualValues(t, INT64, result.INT64) + assert.EqualValues(t, UINT8, result.UINT8) + assert.EqualValues(t, UINT16, result.UINT16) + assert.EqualValues(t, UINT32, result.UINT32) + assert.EqualValues(t, UINT64, result.UINT64) + assert.EqualValues(t, BOOLEAN, result.BOOLEAN) + assert.EqualValues(t, FLOAT32, result.FLOAT32) + assert.EqualValues(t, FLOAT64, result.FLOAT64) + assert.EqualValues(t, BINARY, result.BINARY) + assert.EqualValues(t, STRING, result.STRING) + + assert.Equal(t, time_.Format("2006-01-02"), result.DATE.Format("2006-01-02")) + assert.Equal(t, time_.Format("2006-01-02 15:04:05"), result.DATETIME.Format("2006-01-02 15:04:05")) + assert.Equal(t, time_.Unix(), result.TIMESTAMP_SECOND.Unix()) + assert.Equal(t, time_.UnixMilli(), result.TIMESTAMP_MILLISECOND.UnixMilli()) + assert.Equal(t, time_.UnixMicro(), result.TIMESTAMP_MICROSECOND.UnixMicro()) + assert.Equal(t, time_.UnixNano(), result.TIMESTAMP_NANOSECOND.UnixNano()) + + assert.Equal(t, time_.Format("2006-01-02"), result.DATE_INT.Format("2006-01-02")) + assert.Equal(t, time_.Format("2006-01-02 15:04:05"), result.DATETIME_INT.Format("2006-01-02 15:04:05")) + assert.Equal(t, time_.Unix(), result.TIMESTAMP_SECOND_INT.Unix()) + assert.Equal(t, time_.UnixMilli(), result.TIMESTAMP_MILLISECOND_INT.UnixMilli()) + assert.Equal(t, time_.UnixMicro(), result.TIMESTAMP_MICROSECOND_INT.UnixMicro()) + assert.Equal(t, time_.UnixNano(), result.TIMESTAMP_NANOSECOND_INT.UnixNano()) +} + +//TODO(yuanbohan): +// unmatched length of columns in rows and columns in schema +// support pointer +// write pojo diff --git a/table/cell/build.go b/table/cell/build.go index 02debbd..0ffe675 100644 --- a/table/cell/build.go +++ b/table/cell/build.go @@ -61,72 +61,73 @@ func BuildBytes(v any) (*gpb.Value, error) { } func getIntPointer(v any) (*int32, *int64, *uint32, *uint64, error) { - var int32Val *int32 - var int64Val *int64 - var uint32Val *uint32 - var uint64Val *uint64 + var int32Pointer *int32 + var int64Pointer *int64 + var uint32Pointer *uint32 + var uint64Pointer *uint64 + switch t := v.(type) { case int64: - int64Val = &t + int64Pointer = &t case int32: - int32Val = &t + int32Pointer = &t case int16: val := int32(t) - int32Val = &val + int32Pointer = &val case int8: val := int32(t) - int32Val = &val + int32Pointer = &val case int: val := int64(t) - int64Val = &val + int64Pointer = &val case uint64: - uint64Val = &t + uint64Pointer = &t case uint32: - uint32Val = &t + uint32Pointer = &t case uint16: val := uint32(t) - uint32Val = &val + uint32Pointer = &val case uint8: val := uint32(t) - uint32Val = &val + uint32Pointer = &val case uint: val := uint64(t) - uint64Val = &val + uint64Pointer = &val case *int64: - int64Val = t + int64Pointer = t case *int32: - int32Val = t + int32Pointer = t case *int16: val := int32(*t) - int32Val = &val + int32Pointer = &val case *int8: val := int32(*t) - int32Val = &val + int32Pointer = &val case *int: val := int64(*t) - int64Val = &val + int64Pointer = &val case *uint64: - uint64Val = t + uint64Pointer = t case *uint32: - uint32Val = t + uint32Pointer = t case *uint16: val := uint32(*t) - uint32Val = &val + uint32Pointer = &val case *uint8: val := uint32(*t) - uint32Val = &val + uint32Pointer = &val case *uint: val := uint64(*t) - uint64Val = &val + uint64Pointer = &val default: return nil, nil, nil, nil, fmt.Errorf(formatter+" Integer", t, v) } - return int32Val, int64Val, uint32Val, uint64Val, nil + return int32Pointer, int64Pointer, uint32Pointer, uint64Pointer, nil } func getInt32Value(int32Pointer *int32, int64Pointer *int64, uint32Pointer *uint32, uint64Pointer *uint64) int32 { diff --git a/table/types/types.go b/table/types/types.go index 212ffbb..97da291 100644 --- a/table/types/types.go +++ b/table/types/types.go @@ -26,37 +26,37 @@ type ColumnType int // THEY WILL KEEP EXACTLY THE SAME ORDER WITH PROTOCOL BUFFER // https://github.com/GreptimeTeam/greptime-proto/blob/main/proto/greptime/v1/common.proto#L78-L110 const ( - BOOLEAN ColumnType = 0 - INT8 ColumnType = 1 - INT16 ColumnType = 2 - INT32 ColumnType = 3 - INT64 ColumnType = 4 - UINT8 ColumnType = 5 - UINT16 ColumnType = 6 - UINT32 ColumnType = 7 - UINT64 ColumnType = 8 - FLOAT32 ColumnType = 9 - FLOAT64 ColumnType = 10 - BINARY ColumnType = 11 - STRING ColumnType = 12 - DATE ColumnType = 13 - DATETIME ColumnType = 14 - TIMESTAMP_SECOND ColumnType = 15 - TIMESTAMP_MILLISECOND ColumnType = 16 - TIMESTAMP_MICROSECOND ColumnType = 17 - TIMESTAMP_NANOSECOND ColumnType = 18 - TIME_SECOND ColumnType = 19 - TIME_MILLISECOND ColumnType = 20 - TIME_MICROSECOND ColumnType = 21 - TIME_NANOSECOND ColumnType = 22 - INTERVAL_YEAR_MONTH ColumnType = 23 - INTERVAL_DAY_TIME ColumnType = 24 - INTERVAL_MONTH_DAY_NANO ColumnType = 25 - DURATION_SECOND ColumnType = 26 - DURATION_MILLISECOND ColumnType = 27 - DURATION_MICROSECOND ColumnType = 28 - DURATION_NANOSECOND ColumnType = 29 - DECIMAL128 ColumnType = 30 + BOOLEAN ColumnType = 0 + INT8 ColumnType = 1 + INT16 ColumnType = 2 + INT32 ColumnType = 3 + INT64 ColumnType = 4 + UINT8 ColumnType = 5 + UINT16 ColumnType = 6 + UINT32 ColumnType = 7 + UINT64 ColumnType = 8 + FLOAT32 ColumnType = 9 + FLOAT64 ColumnType = 10 + BINARY ColumnType = 11 + STRING ColumnType = 12 + DATE ColumnType = 13 + DATETIME ColumnType = 14 + TIMESTAMP_SECOND ColumnType = 15 + TIMESTAMP_MILLISECOND ColumnType = 16 + TIMESTAMP_MICROSECOND ColumnType = 17 + TIMESTAMP_NANOSECOND ColumnType = 18 + // TIME_SECOND ColumnType = 19 + // TIME_MILLISECOND ColumnType = 20 + // TIME_MICROSECOND ColumnType = 21 + // TIME_NANOSECOND ColumnType = 22 + // INTERVAL_YEAR_MONTH ColumnType = 23 + // INTERVAL_DAY_TIME ColumnType = 24 + // INTERVAL_MONTH_DAY_NANO ColumnType = 25 + // DURATION_SECOND ColumnType = 26 + // DURATION_MILLISECOND ColumnType = 27 + // DURATION_MICROSECOND ColumnType = 28 + // DURATION_NANOSECOND ColumnType = 29 + // DECIMAL128 ColumnType = 30 ) func GetColumnType(type_ ColumnType) (gpb.ColumnDataType, error) { @@ -99,32 +99,8 @@ func GetColumnType(type_ ColumnType) (gpb.ColumnDataType, error) { return gpb.ColumnDataType_TIMESTAMP_MICROSECOND, nil case TIMESTAMP_NANOSECOND: return gpb.ColumnDataType_TIMESTAMP_NANOSECOND, nil - case TIME_SECOND: - return gpb.ColumnDataType_TIME_SECOND, nil - case TIME_MILLISECOND: - return gpb.ColumnDataType_TIME_MILLISECOND, nil - case TIME_MICROSECOND: - return gpb.ColumnDataType_TIME_MICROSECOND, nil - case TIME_NANOSECOND: - return gpb.ColumnDataType_TIME_NANOSECOND, nil - case INTERVAL_YEAR_MONTH: - return gpb.ColumnDataType_INTERVAL_YEAR_MONTH, nil - case INTERVAL_DAY_TIME: - return gpb.ColumnDataType_INTERVAL_DAY_TIME, nil - case INTERVAL_MONTH_DAY_NANO: - return gpb.ColumnDataType_INTERVAL_MONTH_DAY_NANO, nil - case DURATION_SECOND: - return gpb.ColumnDataType_DURATION_SECOND, nil - case DURATION_MILLISECOND: - return gpb.ColumnDataType_DURATION_MILLISECOND, nil - case DURATION_MICROSECOND: - return gpb.ColumnDataType_DURATION_MICROSECOND, nil - case DURATION_NANOSECOND: - return gpb.ColumnDataType_DURATION_NANOSECOND, nil - case DECIMAL128: - return gpb.ColumnDataType_DECIMAL128, nil default: - return 0, fmt.Errorf("unknown column type %d", type_) + return 0, fmt.Errorf("unsupported column type %d", type_) } }