feat: querier remove auto ip columns

Author: xiaochaoren1

server/querier/engine/clickhouse/clickhouse_test.go

@@ -139,7 +139,7 @@ var (
 		output: []string{"WITH toStartOfInterval(time, toIntervalSecond(60)) + toIntervalSecond(arrayJoin([0]) * 60) AS `_toi` SELECT toUnixTimestamp(`_toi`) AS `toi`, divide(plus(SUM(byte_tx+byte_rx), 100), 60) AS `persecond_max_byte_100` FROM flow_log.`l4_flow_log` GROUP BY `toi` LIMIT 1"},
 	}, {
 		input:  "select auto_instance_0,ip_0 from l7_flow_log where ip_0='1.1.1.1' and auto_instance_0='abc' and auto_instance_0 regexp 'abc' and auto_instance_id_0=2 group by auto_instance_0,ip_0",
-		output: []string{"WITH if(auto_instance_type_0 IN (0, 255), if(is_ipv4 = 1, ip4_0, NULL), NULL) AS `auto_instance_ip4_0`, if(auto_instance_type_0 IN (0, 255), if(is_ipv4 = 0, ip6_0, NULL), NULL) AS `auto_instance_ip6_0` SELECT if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(auto_instance_ip4_0), IPv6NumToString(auto_instance_ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `auto_instance_0`, auto_instance_ip4_0, auto_instance_ip6_0, auto_instance_type_0, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0` FROM flow_log.`l7_flow_log` WHERE (((if(is_ipv4=1, ip4_0 = toIPv4OrNull('1.1.1.1'), ip6_0 = toIPv6OrNull('1.1.1.1'))))) AND (if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) = 'abc',(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE name = 'abc'))) AND (if(auto_instance_type_0 in (0,255),match(if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),'abc'),(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE match(name,'abc')))) AND (if(auto_instance_type_0 in (0,255),subnet_id_0 = 2,auto_instance_id_0 = 2)) GROUP BY `is_ipv4`, `auto_instance_ip4_0`, `auto_instance_ip6_0`, `auto_instance_type_0`, `auto_instance_id_0`, `ip4_0`, `ip6_0` LIMIT 10000"},
+		output: []string{"WITH if(auto_instance_type_0 IN (0, 255), if(is_ipv4 = 1, ip4_0, NULL), NULL) AS `auto_instance_ip4_0`, if(auto_instance_type_0 IN (0, 255), if(is_ipv4 = 0, ip6_0, NULL), NULL) AS `auto_instance_ip6_0` SELECT if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(auto_instance_ip4_0), IPv6NumToString(auto_instance_ip6_0)),dictGet('flow_tag.device_map', 'name', (toUInt64(auto_instance_type_0),toUInt64(auto_instance_id_0)))) AS `auto_instance_0`, auto_instance_type_0, if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) AS `ip_0` FROM flow_log.`l7_flow_log` WHERE (((if(is_ipv4=1, ip4_0 = toIPv4OrNull('1.1.1.1'), ip6_0 = toIPv6OrNull('1.1.1.1'))))) AND (if(auto_instance_type_0 in (0,255),if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)) = 'abc',(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE name = 'abc'))) AND (if(auto_instance_type_0 in (0,255),match(if(is_ipv4=1, IPv4NumToString(ip4_0), IPv6NumToString(ip6_0)),'abc'),(toUInt64(auto_instance_id_0),toUInt64(auto_instance_type_0)) GLOBAL IN (SELECT deviceid,devicetype FROM flow_tag.device_map WHERE match(name,'abc')))) AND (if(auto_instance_type_0 in (0,255),subnet_id_0 = 2,auto_instance_id_0 = 2)) GROUP BY `is_ipv4`, `auto_instance_ip4_0`, `auto_instance_ip6_0`, `auto_instance_type_0`, `auto_instance_id_0`, `ip4_0`, `ip6_0` LIMIT 10000"},
 	}, {
 		input:  "select pod_service_0 from l7_flow_log where pod_service_0 !='xx' group by pod_service_0",
 		output: []string{"SELECT dictGet('flow_tag.device_map', 'name', (toUInt64(11),toUInt64(service_id_0))) AS `pod_service_0` FROM flow_log.`l7_flow_log` WHERE (not(toUInt64(service_id_0) GLOBAL IN (SELECT deviceid FROM flow_tag.device_map WHERE name = 'xx' AND devicetype=11))) AND (service_id_0!=0) GROUP BY `service_id_0` LIMIT 10000"},

server/querier/engine/clickhouse/view/tag.go

@@ -50,11 +50,17 @@ func (s *Tags) Append(t Node) {
 }
 
 func (s *Tags) WriteTo(buf *bytes.Buffer) {
-	for i, tag := range s.tags {
-		tag.WriteTo(buf)
-		if i < len(s.tags)-1 {
+	first := true
+	for _, tag := range s.tags {
+		node, ok := tag.(*Tag)
+		if ok && node.NoReturn {
+			continue
+		}
+		if !first {
 			buf.WriteString(", ")
 		}
+		tag.WriteTo(buf)
+		first = false
 	}
 }
 
@@ -69,10 +75,11 @@ func (s *Tags) GetWiths() []Node {
 }
 
 type Tag struct {
-	Value string
-	Alias string
-	Flag  int
-	Withs []Node
+	Value    string
+	Alias    string
+	Flag     int
+	Withs    []Node
+	NoReturn bool
 	NodeBase
 }
 

server/querier/engine/clickhouse/view/view.go

@@ -209,6 +209,10 @@ func (v *View) trans() {
 				} else {
 					metricTag.Value = node.Value
 				}
+				// remove auto ip tag
+				if strings.HasPrefix(node.Value, "auto_instance_ip") || strings.HasPrefix(node.Value, "auto_service_ip") {
+					metricTag.NoReturn = true
+				}
 				tagsLevelMetrics = append(tagsLevelMetrics, metricTag)
 				tagsAliasInner = append(tagsAliasInner, metricTag.Value)
 			} else if node.Flag == NODE_FLAG_METRICS_INNER {
@@ -281,8 +285,19 @@ func (v *View) trans() {
 	if v.Model.MetricsLevelFlag == MODEL_METRICS_LEVEL_FLAG_UNLAY {
 		// 计算层不拆层
 		// 里层tag+外层metric
+		// remove auto ip tag
+		newTagsInner := []Node{}
+		for _, tagInner := range tagsLevelInner {
+			node, ok := tagInner.(*Tag)
+			if ok {
+				if strings.HasPrefix(node.Value, "auto_instance_ip") || strings.HasPrefix(node.Value, "auto_service_ip") {
+					node.NoReturn = true
+				}
+				newTagsInner = append(newTagsInner, tagInner)
+			}
+		}
 		sv := SubView{
-			Tags:       &Tags{tags: append(tagsLevelInner, metricsLevelMetrics...)},
+			Tags:       &Tags{tags: append(newTagsInner, metricsLevelMetrics...)},
 			Groups:     v.Model.Groups,
 			From:       v.Model.From,
 			Filters:    v.Model.Filters,