Skip to content

Commit

Permalink
feat: querier modify group logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiaochaoren1
xiaochaoren1 committed Dec 20, 2024
1 parent 53c33eb commit 70e10ae
Show file tree
Hide file tree
Showing 12 changed files with 498 additions and 142 deletions.
2 changes: 1 addition & 1 deletion server/querier/engine/clickhouse/clickhouse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1343,7 +1343,7 @@ func (e *CHEngine) TransDerivativeGroupBy(groups sqlparser.GroupBy) error {
colName, ok := group.(*sqlparser.ColName)
if ok {
groupTag := sqlparser.String(colName)
if !strings.Contains(groupTag, "time") {
if !strings.Contains(groupTag, "time") && !strings.Contains(groupTag, "node_type") && !strings.Contains(groupTag, "icon_id") {
groupSlice = append(groupSlice, groupTag)
}
}
Expand Down
78 changes: 39 additions & 39 deletions server/querier/engine/clickhouse/clickhouse_test.go
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion server/querier/engine/clickhouse/filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,6 @@ func (t *WhereTag) Trans(expr sqlparser.Expr, w *Where, e *CHEngine) (view.Node,
checkTag := strings.TrimSuffix(t.Tag, "_id")
if slices.Contains(chCommon.SHOW_TAG_VALUE_MAP[table], checkTag) {
if strings.HasSuffix(t.Tag, "_id") {

if checkTag == strings.TrimSuffix(table, "_map") || checkTag == common.CHOST_HOSTNAME || checkTag == common.CHOST_IP {
tagItem, ok := tag.GetTag("value", db, table, "default")
if ok {
Expand Down
119 changes: 98 additions & 21 deletions server/querier/engine/clickhouse/function.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,66 @@ func GetAggFunc(name string, args []string, alias string, derivativeArgs []strin
}, levelFlag, unit, nil
}

func TransMultiTag(isMulti bool, field string, dbFields []string, withs []view.Node) (bool, []string, []view.Node) {
// name to id
for _, suffix := range []string{"", "_0", "_1"} {
ip4Suffix := "ip4" + suffix
ip6Suffix := "ip6" + suffix
deviceTypeSuffix := "l3_device_type" + suffix
deviceIDSuffix := "l3_device_id" + suffix
// auto
for _, resourceName := range []string{"resource_gl0", "auto_instance", "resource_gl1", "resource_gl2", "auto_service"} {
if field == resourceName+suffix {
isMulti = true
resourceTypeSuffix := "auto_service_type" + suffix
resourceIDSuffix := "auto_service_id" + suffix
ip4Alias := "auto_ip4" + suffix
ip6Alias := "auto_ip6" + suffix
if common.IsValueInSliceString(resourceName, []string{"resource_gl0", "auto_instance"}) {
resourceTypeSuffix = "auto_instance_type" + suffix
resourceIDSuffix = "auto_instance_id" + suffix
}
ip4WithValue := fmt.Sprintf("if(%s IN (0, 255), if(is_ipv4 = 1, %s, NULL), NULL)", resourceTypeSuffix, ip4Suffix)
ip6WithValue := fmt.Sprintf("if(%s IN (0, 255), if(is_ipv4 = 1, %s, NULL), NULL)", resourceTypeSuffix, ip6Suffix)
dbFields = append(dbFields, []string{"is_ipv4", ip4Alias, ip6Alias, resourceTypeSuffix, resourceIDSuffix}...)
withs = append(withs, []view.Node{&view.With{Value: ip4WithValue, Alias: ip4Alias}, &view.With{Value: ip6WithValue, Alias: ip6Alias}}...)
}
}
// ip
if field == "ip"+suffix {
isMulti = true
dbFields = append(dbFields, []string{"is_ipv4", ip4Suffix, ip6Suffix}...)
}
// device
for resourceStr, deviceTypeValue := range tag.DEVICE_MAP {
if resourceStr == "pod_service" {
continue
} else if field == resourceStr+suffix {
isMulti = true
deviceAlias := "device_type_" + field
deviceWithValue := fmt.Sprintf("if(%s = %d, %s, 0)", deviceTypeSuffix, deviceTypeValue, deviceTypeSuffix)
dbFields = append(dbFields, []string{deviceIDSuffix, deviceAlias}...)
withs = append(withs, &view.With{Value: deviceWithValue, Alias: deviceAlias})
}
}
for resource, _ := range tag.HOSTNAME_IP_DEVICE_MAP {
if slices.Contains([]string{common.CHOST_HOSTNAME, common.CHOST_IP}, resource) && field == resource+suffix {
isMulti = true
deviceAlias := "device_type_" + field
deviceWithValue := fmt.Sprintf("if(%s = %d, %s, 0)", deviceTypeSuffix, tag.VIF_DEVICE_TYPE_VM, deviceTypeSuffix)
dbFields = append(dbFields, []string{deviceIDSuffix, deviceAlias}...)
withs = append(withs, &view.With{Value: deviceWithValue, Alias: deviceAlias})
}
}
}
return isMulti, dbFields, withs
}

func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statement, int, string, error) {
db := e.DB
table := e.Table
function, ok := metrics.METRICS_FUNCTIONS_MAP[name]
withs := []view.Node{}
if !ok {
return nil, 0, "", nil
}
Expand All @@ -167,19 +223,18 @@ func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statem
}
levelFlag := view.MODEL_METRICS_LEVEL_FLAG_UNLAY

fieldsLen := len(fields)
dbFields := make([]string, fieldsLen)
conditions := make([]string, 0, fieldsLen)

dbFields := []string{}
conditions := []string{}
var metricStruct *metrics.Metrics
for i, field := range fields {

for _, field := range fields {
field = strings.Trim(field, "`")
metricStruct, ok = metrics.GetAggMetrics(field, e.DB, e.Table, e.ORGID)
if !ok || metricStruct.Type == metrics.METRICS_TYPE_ARRAY {
return nil, 0, "", nil
}
dbFields[i] = metricStruct.DBField
dbFields = append(dbFields, metricStruct.DBField)
// get withs
_, _, withs = TransMultiTag(false, field, dbFields, withs)
condition := metricStruct.Condition

// enum tag
Expand All @@ -204,25 +259,25 @@ func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statem
}
if tagOK {
enumFileName := tagDescription.EnumFile
dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, nameColumn, enumFileName)
dbFields = append(dbFields, fmt.Sprintf(tagDes.TagTranslator, nameColumn, enumFileName))
} else {
dbFields[i] = fmt.Sprintf(tagDes.TagTranslator, nameColumn, tagEnum)
dbFields = append(dbFields, fmt.Sprintf(tagDes.TagTranslator, nameColumn, tagEnum))
}
}

if condition == "" && metricStruct.TagType != "int" {
if metricStruct.TagType == "string" || metricStruct.TagType == "ip" {
condition = dbFields[i] + " != ''"
condition = metricStruct.DBField + " != ''"
} else if metricStruct.TagType == "int" || metricStruct.TagType == "id" {
condition = dbFields[i] + " != 0"
condition = metricStruct.DBField + " != 0"
} else if metricStruct.TagType == "resource" && strings.Contains(metricStruct.DisplayName, "_id") {
if strings.Contains(metricStruct.DisplayName, "epc") {
condition = dbFields[i] + " != -2"
condition = metricStruct.DBField + " != -2"
} else {
condition = dbFields[i] + " != 0"
condition = metricStruct.DBField + " != 0"
}
} else {
condition = dbFields[i] + " != ''"
condition = metricStruct.DBField + " != ''"
}
conditions = append(conditions, condition)
}
Expand All @@ -239,7 +294,7 @@ func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statem
metricStructCopy := *metricStruct
metricStructCopy.DBField = strings.Join(dbFields, ", ")
metricStructCopy.Condition = strings.Join(conditions, " AND ")
if fieldsLen > 1 {
if len(dbFields) > 1 {
metricStructCopy.DBField = "(" + metricStructCopy.DBField + ")"
if metricStructCopy.Condition != "" {
metricStructCopy.Condition = "(" + strings.Join(conditions, " AND ") + ")"
Expand All @@ -253,6 +308,7 @@ func GetTopKTrans(name string, args []string, alias string, e *CHEngine) (Statem
Name: name,
Args: args,
Alias: alias,
Withs: withs,
}, levelFlag, unit, nil
}

Expand All @@ -266,18 +322,25 @@ func GetUniqTrans(name string, args []string, alias string, e *CHEngine) (Statem
}

levelFlag := view.MODEL_METRICS_LEVEL_FLAG_UNLAY
fieldsLen := len(fields)
dbFields := make([]string, fieldsLen)

dbFields := []string{}
withs := []view.Node{}
var metricStruct *metrics.Metrics
for i, field := range fields {
for _, field := range fields {
field = strings.Trim(field, "`")
metricStruct, ok = metrics.GetAggMetrics(field, e.DB, e.Table, e.ORGID)
if !ok || metricStruct.Type == metrics.METRICS_TYPE_ARRAY {
return nil, 0, "", nil
}
dbFields[i] = metricStruct.DBField

isMulti := false
isMulti, dbFields, withs = TransMultiTag(isMulti, field, dbFields, withs)
if !isMulti {
if metricStruct.GroupField != "" {
dbFields = append(dbFields, metricStruct.GroupField)
} else {
dbFields = append(dbFields, metricStruct.DBField)
}
}
// judge whether the operator supports single layer
if levelFlag == view.MODEL_METRICS_LEVEL_FLAG_UNLAY && db != chCommon.DB_NAME_FLOW_LOG {
unlayFuns := metrics.METRICS_TYPE_UNLAY_FUNCTIONS[metricStruct.Type]
Expand All @@ -289,7 +352,7 @@ func GetUniqTrans(name string, args []string, alias string, e *CHEngine) (Statem

metricStructCopy := *metricStruct
metricStructCopy.DBField = strings.Join(dbFields, ", ")
if fieldsLen > 1 {
if len(dbFields) > 1 {
metricStructCopy.DBField = "(" + metricStructCopy.DBField + ")"
}

Expand All @@ -300,6 +363,7 @@ func GetUniqTrans(name string, args []string, alias string, e *CHEngine) (Statem
Name: name,
Args: args,
Alias: alias,
Withs: withs,
}, levelFlag, unit, nil
}

Expand Down Expand Up @@ -390,6 +454,7 @@ type AggFunction struct {
IsDerivative bool
DerivativeArgs []string
DerivativeGroupBy []string
Withs []view.Node
}

func (f *AggFunction) SetAlias(alias string) {
Expand Down Expand Up @@ -613,6 +678,11 @@ func (f *AggFunction) Trans(m *view.Model) view.Node {
outFunc.SetFlag(view.METRICS_FLAG_OUTER)
outFunc.SetTime(m.Time)
outFunc.Init()
// uniq function has withs
defaultFunc, ok := outFunc.(*view.DefaultFunction)
if ok {
defaultFunc.Withs = f.Withs
}
return outFunc
}

Expand Down Expand Up @@ -1038,10 +1108,17 @@ func (f *TagFunction) Format(m *view.Model) {
if f.Name == TAG_FUNCTION_ICON_ID {
for resourceStr := range tag.DEVICE_MAP {
// 以下分别针对单端/双端-0端/双端-1端生成name和ID的Tag定义
// device group add icon_id
for _, suffix := range []string{"", "_0", "_1"} {
resourceNameSuffix := resourceStr + suffix
if f.Args[0] == resourceNameSuffix {
m.AddGroup(&view.Group{Value: fmt.Sprintf("`%s`", strings.Trim(f.Alias, "`"))})
} else {
for resource, _ := range tag.HOSTNAME_IP_DEVICE_MAP {
if slices.Contains([]string{common.CHOST_HOSTNAME, common.CHOST_IP}, resource) && f.Args[0] == resource+suffix {
m.AddGroup(&view.Group{Value: fmt.Sprintf("`%s`", strings.Trim(f.Alias, "`"))})
}
}
}
}
}
Expand Down
96 changes: 65 additions & 31 deletions server/querier/engine/clickhouse/group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,72 @@ func GetGroup(name string, e *CHEngine) ([]Statement, error) {
for _, autoTagKey := range autoTagSlice {
stmts = append(stmts, &GroupTag{Value: "`" + autoTagKey + "`", AsTagMap: asTagMap})
}
} else if tagItem.TagTranslator != "" {
stmts = append(stmts, &GroupTag{Value: tagItem.TagTranslator, Alias: name, AsTagMap: asTagMap})
} else if tagItem.GroupTranslator != "" {
stmts = append(stmts, &GroupTag{Value: tagItem.GroupTranslator, AsTagMap: asTagMap})
} else {
stmts = append(stmts, &GroupTag{Value: name, AsTagMap: asTagMap})
if table == "alert_event" {
stmts = append(stmts, &GroupTag{Value: name, AsTagMap: asTagMap})
} else {
isMulti := false
for _, suffix := range []string{"", "_0", "_1"} {
ip4Suffix := "ip4" + suffix
ip6Suffix := "ip6" + suffix
deviceTypeSuffix := "l3_device_type" + suffix
deviceIDSuffix := "l3_device_id" + suffix
// auto
for _, resourceName := range []string{"resource_gl0", "auto_instance", "resource_gl1", "resource_gl2", "auto_service"} {
if name == resourceName+suffix {
isMulti = true
resourceTypeSuffix := "auto_service_type" + suffix
resourceIDSuffix := "auto_service_id" + suffix
ip4Alias := "auto_ip4" + suffix
ip6Alias := "auto_ip6" + suffix
if common.IsValueInSliceString(resourceName, []string{"resource_gl0", "auto_instance"}) {
resourceTypeSuffix = "auto_instance_type" + suffix
resourceIDSuffix = "auto_instance_id" + suffix
}
ip4WithValue := fmt.Sprintf("if(%s IN (0, 255), if(is_ipv4 = 1, %s, NULL), NULL)", resourceTypeSuffix, ip4Suffix)
ip6WithValue := fmt.Sprintf("if(%s IN (0, 255), if(is_ipv4 = 1, %s, NULL), NULL)", resourceTypeSuffix, ip6Suffix)
stmts = append(stmts, &GroupTag{Value: "is_ipv4"})
stmts = append(stmts, &GroupTag{Value: ip4Alias, Withs: []view.Node{&view.With{Value: ip4WithValue, Alias: ip4Alias}}})
stmts = append(stmts, &GroupTag{Value: ip6Alias, Withs: []view.Node{&view.With{Value: ip6WithValue, Alias: ip6Alias}}})
stmts = append(stmts, &GroupTag{Value: resourceTypeSuffix})
stmts = append(stmts, &GroupTag{Value: resourceIDSuffix})
}
}
// ip
if name == "ip"+suffix {
isMulti = true
stmts = append(stmts, &GroupTag{Value: "is_ipv4"})
stmts = append(stmts, &GroupTag{Value: ip4Suffix})
stmts = append(stmts, &GroupTag{Value: ip6Suffix})
}
// device
for resourceStr, deviceTypeValue := range tag.DEVICE_MAP {
if resourceStr == "pod_service" {
continue
} else if name == resourceStr+suffix {
isMulti = true
deviceAlias := "device_type_" + name
deviceWithValue := fmt.Sprintf("if(%s = %d, %s, 0)", deviceTypeSuffix, deviceTypeValue, deviceTypeSuffix)
stmts = append(stmts, &GroupTag{Value: deviceIDSuffix})
stmts = append(stmts, &GroupTag{Value: deviceAlias, Withs: []view.Node{&view.With{Value: deviceWithValue, Alias: deviceAlias}}})
}
}
for resource, _ := range tag.HOSTNAME_IP_DEVICE_MAP {
if slices.Contains([]string{common.CHOST_HOSTNAME, common.CHOST_IP}, resource) && name == resource+suffix {
isMulti = true
deviceAlias := "device_type_" + name
deviceWithValue := fmt.Sprintf("if(%s = %d, %s, 0)", deviceTypeSuffix, tag.VIF_DEVICE_TYPE_VM, deviceTypeSuffix)
stmts = append(stmts, &GroupTag{Value: deviceIDSuffix})
stmts = append(stmts, &GroupTag{Value: deviceAlias, Withs: []view.Node{&view.With{Value: deviceWithValue, Alias: deviceAlias}}})
}
}
}
if !isMulti {
stmts = append(stmts, &GroupTag{Value: name, AsTagMap: asTagMap})
}
}
}
} else {
if db == chCommon.DB_NAME_PROMETHEUS && strings.Contains(name, "tag.") {
Expand Down Expand Up @@ -300,32 +362,4 @@ func (g *GroupTag) Format(m *view.Model) {
} else {
m.AddGroup(&view.Group{Value: g.Value, Withs: g.Withs})
}
preAsTag, preAsOK := g.AsTagMap[g.Value]
for _, suffix := range []string{"", "_0", "_1"} {
table := m.From.ToString()
if table == "event.`alert_event`" {
break
}
for _, resourceName := range []string{"resource_gl0", "auto_instance", "resource_gl1", "resource_gl2", "auto_service"} {
resourceTypeSuffix := "auto_service_type" + suffix
oldResourceTypeSuffix := resourceName + "_type" + suffix
if common.IsValueInSliceString(resourceName, []string{"resource_gl0", "auto_instance"}) {
resourceTypeSuffix = "auto_instance_type" + suffix
}
preAsTag, preAsOK = g.AsTagMap[g.Value]
if preAsOK {
if preAsTag == resourceName+suffix {
m.AddTag(&view.Tag{Value: resourceTypeSuffix, Alias: oldResourceTypeSuffix})
m.AddGroup(&view.Group{Value: oldResourceTypeSuffix})
}
} else if g.Alias == resourceName+suffix {
if resourceTypeSuffix != oldResourceTypeSuffix {
m.AddTag(&view.Tag{Value: resourceTypeSuffix, Alias: oldResourceTypeSuffix})
} else {
m.AddTag(&view.Tag{Value: resourceTypeSuffix})
}
m.AddGroup(&view.Group{Value: oldResourceTypeSuffix})
}
}
}
}
2 changes: 1 addition & 1 deletion server/querier/engine/clickhouse/metrics/ext_common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func GetExtMetrics(db, table, where, queryCacheTTL, orgID string, useQueryCache
metricName := fmt.Sprintf("metrics.%s", externalTag)
lm := NewMetrics(
i, dbField, metricName, metricName, metricName, "", "", "", METRICS_TYPE_COUNTER,
"metrics", []bool{true, true, true}, "", tableName, "", "", "", "",
"metrics", []bool{true, true, true}, "", tableName, "", "", "", "", "",
)
loadMetrics[fmt.Sprintf("%s-%s", metricName, tableName)] = lm
}
Expand Down
Loading

0 comments on commit 70e10ae

Please sign in to comment.