From 9ccaef298279a0f2fdac85511120df12a523858d Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Mon, 18 Nov 2024 00:33:40 +0800 Subject: [PATCH 01/12] update: init kube metrics searching with grok expressions --- go.mod | 9 ++-- go.sum | 17 ++++--- pkg/metrics/matcher.go | 12 +++++ pkg/metrics/parser.go | 108 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 10 deletions(-) create mode 100644 pkg/metrics/matcher.go create mode 100644 pkg/metrics/parser.go diff --git a/go.mod b/go.mod index 7a361e2..d393d69 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module github.com/esonhugh/k8spider -go 1.19 +go 1.21.0 + +toolchain go1.23.2 require ( + github.com/elastic/go-grok v0.3.1 github.com/miekg/dns v1.1.58 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 @@ -10,9 +13,9 @@ require ( require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/tools v0.17.0 // indirect diff --git a/go.sum b/go.sum index d3ed1fe..1d5334b 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= +github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -16,16 +20,15 @@ github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/pkg/metrics/matcher.go b/pkg/metrics/matcher.go new file mode 100644 index 0000000..7f16c22 --- /dev/null +++ b/pkg/metrics/matcher.go @@ -0,0 +1,12 @@ +package metrics + +type MatchRules = []MetricMatcher + +func GenerateMatchRules() MatchRules { + return MatchRules{ + { + header: "", + neededLabel: []string{}, + }, + } +} diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go new file mode 100644 index 0000000..8a1c6c3 --- /dev/null +++ b/pkg/metrics/parser.go @@ -0,0 +1,108 @@ +package metrics + +import ( + "errors" + "strings" + + "github.com/elastic/go-grok" + log "github.com/sirupsen/logrus" +) + +var ( + MetricsPatterns = map[string]string{ + "METRIC_HEAD": "^%{WORD:metric_name}{", + "LAST_LABEL": `(%{DATA:last_label_name}="%{DATA:last_label_value}")`, + "NUMBER_SCI": "%{NUMBER}(e%{NUMBER})?", + "METRIC_TAIL": `}%{SPACE}%{NUMBER_SCI:metric_value}$`, + "NORMAL_EXPR": `^%{WORD:metric_name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}=\"%{DATA:last_label_value}\")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$`, + } + COMMON_MATCH_GROK = grok.New() +) + +const COMMON_MATCH = `^%{WORD:name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}="%{DATA:last_label_value}")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$` + +type MetricMatcher struct { + MtName string + header string + neededLabel []string + grok *grok.Grok +} + +func NewMetricMatcher(label_name string) *MetricMatcher { + return &MetricMatcher{ + MtName: label_name, + grok: grok.New(), + neededLabel: make([]string, 0), + } +} + +// example: ^%{WORD:name}{((namespace="%{DATA:ns}")|(%{DATA:last_label_name}="%{DATA:last_label_value}")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$ +// ^%{WORD:name}{ +// ( +// (namespace="%{DATA:ns}") +// | +// (%{DATA:last_label_name}="%{DATA:last_label_value}") +// (,|) +// ) +// * +// }%{SPACE}%{NUMBER}(e%{NUMBER})?$ + +func (mt *MetricMatcher) Compile() error { + if err := mt.grok.AddPatterns(MetricsPatterns); err != nil { + return err + } + header := mt.header + if mt.header == "" { + header = `^` + mt.MtName + `{` // custom head + } + var body []string + for _, label := range mt.neededLabel { + body = append(body, "("+label+`=`+`"%{DATA:`+strings.ToUpper(label)+`}")`) + } + tail := `${METRIC_TAIL}` + pattern := header + "(" + strings.Join(body, "|") + "|%{LAST_LABEL}(,|))*" + tail + log.Debugln("generated pattern: ", pattern) + return mt.grok.Compile(pattern, true) +} + +func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) { + if !mt.grok.MatchString(target) { + if len(target) > 100 { + target = target[:99] + } + log.Debugf("not match: %s", target) + if COMMON_MATCH_GROK.MatchString(target) { + res, err = COMMON_MATCH_GROK.ParseString(target) + if err != nil { + return nil, errors.Join( + errors.New("match failed, in common expr"), + err, + ) + } + return res, errors.Join( + errors.New("match failed, in custom expr, but success in common expr, check ret result to get more detail"), + err, + ) + } + } + return mt.grok.ParseString(target) +} + +func (mt *MetricMatcher) SetHeader(header string) { + mt.header = `^` + header + `{` +} + +func (mt *MetricMatcher) AddLabel(label string) { + mt.neededLabel = append(mt.neededLabel, label) +} + +func init() { + err := COMMON_MATCH_GROK.AddPatterns(MetricsPatterns) + if err != nil { + log.Fatalf("add patterns failed: %v", err) + } + err = COMMON_MATCH_GROK.Compile(COMMON_MATCH, true) + if err != nil { + log.Fatalf("compile pattern failed: %v", err) + } +} From 4620b5633d37eaaee9b5cf6d0c1316235fa1129a Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Thu, 21 Nov 2024 00:02:33 +0800 Subject: [PATCH 02/12] update: fix bug of parsing error --- pkg/metrics/matcher.go | 4 +- pkg/metrics/metrics_test.go | 42 ++++++++++++++++++ pkg/metrics/parser.go | 87 +++++++++++++++++++++++++++---------- 3 files changed, 107 insertions(+), 26 deletions(-) create mode 100644 pkg/metrics/metrics_test.go diff --git a/pkg/metrics/matcher.go b/pkg/metrics/matcher.go index 7f16c22..de1b9c7 100644 --- a/pkg/metrics/matcher.go +++ b/pkg/metrics/matcher.go @@ -5,8 +5,8 @@ type MatchRules = []MetricMatcher func GenerateMatchRules() MatchRules { return MatchRules{ { - header: "", - neededLabel: []string{}, + Header: "", + Labels: []Label{}, }, } } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 0000000..5214aa6 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,42 @@ +package metrics + +import ( + "bufio" + "os" + "testing" +) + +var rule = []*MetricMatcher{ + NewMetricMatcher("configmap").AddLabel("namespace").AddLabel("configmap"), + NewMetricMatcher("pod").AddLabel("namespace").AddLabel("pod").AddLabel("host_ip").AddLabel("pod_ip"), +} + +func TestMetrics(t *testing.T) { + f, err := os.Open("./metrics_output.txt") + if err != nil { + t.Fatalf("open file failed: %v", err) + t.Fail() + } + defer f.Close() + scanner := bufio.NewScanner(f) + for i := range rule { + err := rule[i].Compile() + if err != nil { + t.Fatalf("compile rule failed: %v", err) + t.Fail() + } + } + + for scanner.Scan() { + line := scanner.Text() + for _, r := range rule { + res, e := r.Match(line) + _ = res + if e != nil { + continue + } else { + t.Log(r.DumpString()) + } + } + } +} diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 8a1c6c3..af9f4f5 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -1,6 +1,7 @@ package metrics import ( + "encoding/json" "errors" "strings" @@ -11,9 +12,9 @@ import ( var ( MetricsPatterns = map[string]string{ "METRIC_HEAD": "^%{WORD:metric_name}{", - "LAST_LABEL": `(%{DATA:last_label_name}="%{DATA:last_label_value}")`, + "LAST_LABEL": `(%{DATA:last_label_name}="%{DATA:last_label_value}",?)`, "NUMBER_SCI": "%{NUMBER}(e%{NUMBER})?", - "METRIC_TAIL": `}%{SPACE}%{NUMBER_SCI:metric_value}$`, + "METRIC_TAIL": `%{SPACE}%{NUMBER_SCI:metric_value}$`, "NORMAL_EXPR": `^%{WORD:metric_name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}=\"%{DATA:last_label_value}\")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$`, } COMMON_MATCH_GROK = grok.New() @@ -21,18 +22,24 @@ var ( const COMMON_MATCH = `^%{WORD:name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}="%{DATA:last_label_value}")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$` +type Label struct { + Key string `json:"key"` + Value string `json:"value""` +} + type MetricMatcher struct { - MtName string - header string - neededLabel []string - grok *grok.Grok + Name string `json:"name"` + Header string `json:"-"` + Labels []Label `json:"labels"` + grok *grok.Grok `json:"-"` + finalPattern string } func NewMetricMatcher(label_name string) *MetricMatcher { return &MetricMatcher{ - MtName: label_name, - grok: grok.New(), - neededLabel: make([]string, 0), + Name: label_name, + grok: grok.New(), + Labels: make([]Label, 0), } } @@ -51,25 +58,23 @@ func (mt *MetricMatcher) Compile() error { if err := mt.grok.AddPatterns(MetricsPatterns); err != nil { return err } - header := mt.header - if mt.header == "" { - header = `^` + mt.MtName + `{` // custom head + if mt.Header == "" { + mt.Header = `kube_` + mt.Name + `_info` // custom head } + header := mt.Header var body []string - for _, label := range mt.neededLabel { - body = append(body, "("+label+`=`+`"%{DATA:`+strings.ToUpper(label)+`}")`) + for _, label := range mt.Labels { + body = append(body, "("+label.Key+`=`+`"%{DATA:`+label.Key+`}",?)`) } - tail := `${METRIC_TAIL}` - pattern := header + "(" + strings.Join(body, "|") + "|%{LAST_LABEL}(,|))*" + tail + tail := `%{METRIC_TAIL}` + pattern := header + "{" + "(" + strings.Join(body, "|") + "|%{LAST_LABEL})*" + "}" + tail + mt.finalPattern = pattern log.Debugln("generated pattern: ", pattern) return mt.grok.Compile(pattern, true) } func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) { - if !mt.grok.MatchString(target) { - if len(target) > 100 { - target = target[:99] - } + if !strings.HasPrefix(target, mt.Header) { log.Debugf("not match: %s", target) if COMMON_MATCH_GROK.MatchString(target) { res, err = COMMON_MATCH_GROK.ParseString(target) @@ -84,16 +89,50 @@ func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) err, ) } + return nil, errors.New("can't match") + } else { + res, err = mt.grok.ParseString(target) + if res != nil && len(res) != 0 { + mt.setResult(res) + } + return res, err } - return mt.grok.ParseString(target) } func (mt *MetricMatcher) SetHeader(header string) { - mt.header = `^` + header + `{` + mt.Header = `^` + header + `{` +} + +func (mt *MetricMatcher) AddLabel(label string) *MetricMatcher { + mt.Labels = append(mt.Labels, Label{ + Key: label, + Value: "", + }) + return mt +} + +func (mt *MetricMatcher) setResult(res map[string]string) { + for i, label := range mt.Labels { + if v, ok := res[label.Key]; ok { + mt.Labels[i].Value = v + } else { + log.Debugf("label %s not found in result", label.Key) // keep it empty + } + } +} + +func (mt *MetricMatcher) FindLabel(Key string) string { + for _, label := range mt.Labels { + if label.Key == Key { + return label.Value + } + } + return "" } -func (mt *MetricMatcher) AddLabel(label string) { - mt.neededLabel = append(mt.neededLabel, label) +func (mt *MetricMatcher) DumpString() string { + b, _ := json.Marshal(mt) + return string(b) } func init() { From 6ebc1ae46c8878ddf0416583c0503d67a1fda751 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Thu, 21 Nov 2024 22:30:52 +0800 Subject: [PATCH 03/12] update: dns utils --- Makefile | 8 ++++++-- cmd/all/all.go | 5 +++++ cmd/dnsutils/dns.go | 46 +++++++++++++++++++++++++++++++++++++++++++++ main.go | 1 + pkg/query_utils.go | 30 ++++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 cmd/dnsutils/dns.go diff --git a/Makefile b/Makefile index 909a418..4bc8a53 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ BUILD_DIR = bin MAIN_PROGRAM_NAME = k8spider -default: build build-static +default: build build-static check-size # build build: @@ -11,6 +11,10 @@ build: build-static: GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go - upx $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + upx -9 $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + +check-size: + ls -alh $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)* + clean: rm -rf $(BUILD_DIR) \ No newline at end of file diff --git a/cmd/all/all.go b/cmd/all/all.go index 4ff6097..ab3335e 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -48,6 +48,7 @@ var AllCmd = &cobra.Command{ log.Warnf("ParseStringToIPNet failed: %v", err) return } + var finalRecord define.Records if command.Opts.MultiThreadingMode { finalRecord = RunMultiThread(ipNets, command.Opts.ThreadingNum) @@ -55,6 +56,7 @@ var AllCmd = &cobra.Command{ finalRecord = Run(ipNets) } printer.PrintResult(finalRecord, command.Opts.OutputFile) + PostRun(finalRecord) }, } @@ -78,6 +80,9 @@ func RunMultiThread(net *net.IPNet, count int) (finalRecord define.Records) { } func PostRun(finalRecord define.Records) { + if finalRecord == nil || len(finalRecord) == 0 { + return + } log.Info("Extract Namespaces: ") list := post.RecordsDumpNameSpace(finalRecord, command.Opts.Zone) for _, ns := range list { diff --git a/cmd/dnsutils/dns.go b/cmd/dnsutils/dns.go new file mode 100644 index 0000000..c95b46a --- /dev/null +++ b/cmd/dnsutils/dns.go @@ -0,0 +1,46 @@ +package dnsutils + +import ( + "strings" + + command "github.com/esonhugh/k8spider/cmd" + "github.com/esonhugh/k8spider/pkg" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var queryType string + +func init() { + DNSCmd.PersistentFlags().StringVarP(&queryType, "type", "t", "A", "query type") + command.RootCmd.AddCommand(DNSCmd) +} + +var DNSCmd = &cobra.Command{ + Use: "dns", + Aliases: []string{"dig"}, + Short: "dns is a command to query dns server", + Run: func(cmd *cobra.Command, args []string) { + var querier pkg.DnsQuery + switch strings.ToLower(queryType) { + case "a", "aaaa": + querier = pkg.QueryA + case "ptr": + querier = pkg.QueryPTR + case "srv": + querier = pkg.QuerySRV + case "txt": + querier = pkg.QueryTXT + default: + querier = pkg.QueryA + } + for _, query := range args { + res, err := querier(query) + if err != nil { + log.Warnf("Query %s failed: %v", query, err) + continue + } + log.Infof("Query [%d] %s: %v", queryType, query, res) + } + }, +} diff --git a/main.go b/main.go index 2ee8f92..ca4739d 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "github.com/esonhugh/k8spider/cmd" _ "github.com/esonhugh/k8spider/cmd/all" _ "github.com/esonhugh/k8spider/cmd/axfr" + _ "github.com/esonhugh/k8spider/cmd/dnsutils" _ "github.com/esonhugh/k8spider/cmd/neighbor" _ "github.com/esonhugh/k8spider/cmd/service" _ "github.com/esonhugh/k8spider/cmd/subnet" diff --git a/pkg/query_utils.go b/pkg/query_utils.go index 2cb09ea..361080f 100644 --- a/pkg/query_utils.go +++ b/pkg/query_utils.go @@ -2,6 +2,7 @@ package pkg import ( "context" + "fmt" "net" "regexp" "strings" @@ -89,7 +90,9 @@ func WarpDnsServer(dnsServer string) *SpiderResolver { return d.DialContext(ctx, network, dnsServer) }, }, - ctx: ctx, + ctx: ctx, + filter: []*regexp.Regexp{}, + contains: []string{}, } } @@ -146,3 +149,28 @@ func (s *SpiderResolver) TXTRecord(domain string) ([]string, error) { func TXTRecord(domain string) (txts []string, err error) { return NetResolver.TXTRecord(domain) } + +type DnsQuery func(domain string) ([]string, error) + +var ( + QueryPTR DnsQuery = func(domain string) ([]string, error) { + return PTRRecord(net.ParseIP(domain)), nil + } + QueryA DnsQuery = func(domain string) ([]string, error) { + res, err := ARecord(domain) + var ret []string + for _, r := range res { + ret = append(ret, r.String()) + } + return ret, err + } + QueryTXT DnsQuery = TXTRecord + QuerySRV DnsQuery = func(domain string) ([]string, error) { + _, res, err := SRVRecord(domain) + var ret []string + for _, r := range res { + ret = append(ret, fmt.Sprintf("%s:%d", r.Target, r.Port)) + } + return ret, err + } +) From 8f5540301221b0ae8f97451c27b39b689228b97e Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Fri, 22 Nov 2024 00:32:59 +0800 Subject: [PATCH 04/12] update: dns utils --- .github/workflows/build.yaml | 5 +++++ .goreleaser.yaml | 5 ++++- Makefile | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e9d0c0..3165870 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,6 +22,11 @@ jobs: with: go-version: ${{ env.GO_VERSION }} + - name: Install UPX + uses: crazy-max/ghaction-upx@v3 + with: + install-only: true + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 with: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b738726..5706652 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,10 +3,11 @@ project_name: k8spider before: hooks: - go mod tidy - - go generate ./... builds: - env: - CGO_ENABLED=0 + ldflags: + - -s -w goos: - darwin - windows @@ -22,6 +23,8 @@ archives: format_overrides: - goos: windows format: zip +upx: + - enabled: true checksum: name_template: "checksums.txt" diff --git a/Makefile b/Makefile index 4bc8a53..fae0c30 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ build: build-static: GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go - upx -9 $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + upx --ultra-brute $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static check-size: ls -alh $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)* From d030203e8d771f8fb127496bb45e9cceba29b448 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Fri, 22 Nov 2024 23:14:00 +0800 Subject: [PATCH 05/12] update: best compress upx --- .goreleaser.yaml | 5 +++++ Makefile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 5706652..a510f18 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -23,8 +23,13 @@ archives: format_overrides: - goos: windows format: zip + upx: - enabled: true + compress: best + brute: true + lzma: true + checksum: name_template: "checksums.txt" diff --git a/Makefile b/Makefile index fae0c30..5c551c7 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ build: build-static: GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go - upx --ultra-brute $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + upx --lzma --brute $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static check-size: ls -alh $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)* From d07dacb72fd6e56fe826ef68ba26b78b23d4e543 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Sat, 23 Nov 2024 16:06:58 +0800 Subject: [PATCH 06/12] update: metrics tracer --- pkg/metrics/matcher.go | 60 +++++++++++++++++++++++++++++++++---- pkg/metrics/metrics_test.go | 31 +++++++------------ pkg/metrics/parser.go | 14 ++++----- 3 files changed, 72 insertions(+), 33 deletions(-) diff --git a/pkg/metrics/matcher.go b/pkg/metrics/matcher.go index de1b9c7..ff5a9ba 100644 --- a/pkg/metrics/matcher.go +++ b/pkg/metrics/matcher.go @@ -1,12 +1,60 @@ package metrics -type MatchRules = []MetricMatcher +import "errors" + +type MatchRules []*MetricMatcher func GenerateMatchRules() MatchRules { - return MatchRules{ - { - Header: "", - Labels: []Label{}, - }, + return make(MatchRules, 0) +} + +func DefaultMatchRules() MatchRules { + return []*MetricMatcher{ + NewMetricMatcher("configmap").AddLabel("namespace").AddLabel("configmap"), + NewMetricMatcher("secret").AddLabel("namespace").AddLabel("secret"), + + NewMetricMatcher("node").AddLabel("node").AddLabel("kernel_version"). + AddLabel("os_image").AddLabel("container_runtime_version"). + AddLabel("provider_id").AddLabel("internal_ip"), + + NewMetricMatcher("pod").AddLabel("namespace").AddLabel("pod").AddLabel("node"). + AddLabel("host_ip").AddLabel("pod_ip"), + NewMetricMatcher("container").SetHeader("kube_pod_container_info").AddLabel("namespace"). + AddLabel("pod").AddLabel("container").AddLabel("image"), + NewMetricMatcher("cronjob").AddLabel("namespace").AddLabel("cronjob"). + AddLabel("schedule"), + + NewMetricMatcher("service_account").SetHeader("kube_pod_service_account"). + AddLabel("namespace").AddLabel("pod").AddLabel("service_account"), + + NewMetricMatcher("service").AddLabel("namespace").AddLabel("service"). + AddLabel("cluster_ip"), + NewMetricMatcher("endpoint_address").SetHeader("kube_endpoint_address"). + AddLabel("namespace").AddLabel("endpoint").AddLabel("ip"), + NewMetricMatcher("endpoint_port").SetHeader("kube_endpoint_ports"). + AddLabel("namespace").AddLabel("endpoint").AddLabel("port_number"), + } +} + +func (m MatchRules) Compile() error { + var err error = nil + for i := range m { + e := m[i].Compile() + if e != nil { + err = errors.Join(err, e) + } + } + return err +} + +func (m MatchRules) Match(target string) (*MetricMatcher, error) { + for _, r := range m { + _, e := r.Match(target) + if e != nil { + continue + } else { + return r, nil + } } + return nil, errors.New("no match found") } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 5214aa6..004c624 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -6,37 +6,28 @@ import ( "testing" ) -var rule = []*MetricMatcher{ - NewMetricMatcher("configmap").AddLabel("namespace").AddLabel("configmap"), - NewMetricMatcher("pod").AddLabel("namespace").AddLabel("pod").AddLabel("host_ip").AddLabel("pod_ip"), -} - func TestMetrics(t *testing.T) { - f, err := os.Open("./metrics_output.txt") + f, err := os.Open("./metrics") if err != nil { t.Fatalf("open file failed: %v", err) t.Fail() } defer f.Close() scanner := bufio.NewScanner(f) - for i := range rule { - err := rule[i].Compile() - if err != nil { - t.Fatalf("compile rule failed: %v", err) - t.Fail() - } + + rule := DefaultMatchRules() + if err := rule.Compile(); err != nil { + t.Fatalf("compile rule failed: %v", err) + t.Fail() } for scanner.Scan() { line := scanner.Text() - for _, r := range rule { - res, e := r.Match(line) - _ = res - if e != nil { - continue - } else { - t.Log(r.DumpString()) - } + res, err := rule.Match(line) + if err != nil { + continue + } else { + t.Logf("matched: %s", res.DumpString()) } } } diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index af9f4f5..232f5ca 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -28,11 +28,12 @@ type Label struct { } type MetricMatcher struct { - Name string `json:"name"` + Name string `json:"type"` Header string `json:"-"` Labels []Label `json:"labels"` grok *grok.Grok `json:"-"` finalPattern string + ptr any `json:"-"` } func NewMetricMatcher(label_name string) *MetricMatcher { @@ -74,7 +75,7 @@ func (mt *MetricMatcher) Compile() error { } func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) { - if !strings.HasPrefix(target, mt.Header) { + if !strings.HasPrefix(target, mt.Header+"{") { log.Debugf("not match: %s", target) if COMMON_MATCH_GROK.MatchString(target) { res, err = COMMON_MATCH_GROK.ParseString(target) @@ -92,15 +93,14 @@ func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) return nil, errors.New("can't match") } else { res, err = mt.grok.ParseString(target) - if res != nil && len(res) != 0 { - mt.setResult(res) - } + mt.setResult(res) return res, err } } -func (mt *MetricMatcher) SetHeader(header string) { - mt.Header = `^` + header + `{` +func (mt *MetricMatcher) SetHeader(header string) *MetricMatcher { + mt.Header = header + return mt } func (mt *MetricMatcher) AddLabel(label string) *MetricMatcher { From 4551e3c8f735f5269444fd88e5ec9ab84e51426c Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Sun, 1 Dec 2024 23:55:05 +0800 Subject: [PATCH 07/12] update: Resource parse --- pkg/metrics/metrics_test.go | 11 ++++++- pkg/metrics/parser.go | 9 +++-- pkg/metrics/resource.go | 65 +++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 pkg/metrics/resource.go diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 004c624..e3cf498 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -7,7 +7,7 @@ import ( ) func TestMetrics(t *testing.T) { - f, err := os.Open("./metrics") + f, err := os.Open("./metrics_output.txt") if err != nil { t.Fatalf("open file failed: %v", err) t.Fail() @@ -21,6 +21,11 @@ func TestMetrics(t *testing.T) { t.Fail() } + output, err := os.OpenFile("./output.txt", os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + t.Fatalf("open output file failed: %v", err) + } + defer output.Close() for scanner.Scan() { line := scanner.Text() res, err := rule.Match(line) @@ -28,6 +33,10 @@ func TestMetrics(t *testing.T) { continue } else { t.Logf("matched: %s", res.DumpString()) + // _, _ = output.WriteString(res.DumpString() + "\n") } } + var res ResourceList = ConvertToResource(rule) + _, _ = output.WriteString(res.JSON() + "\n") + t.Logf(res.JSON()) } diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 232f5ca..153482b 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -36,9 +36,9 @@ type MetricMatcher struct { ptr any `json:"-"` } -func NewMetricMatcher(label_name string) *MetricMatcher { +func NewMetricMatcher(t string) *MetricMatcher { return &MetricMatcher{ - Name: label_name, + Name: t, grok: grok.New(), Labels: make([]Label, 0), } @@ -75,7 +75,7 @@ func (mt *MetricMatcher) Compile() error { } func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) { - if !strings.HasPrefix(target, mt.Header+"{") { + if !strings.HasPrefix(target, mt.Header) { log.Debugf("not match: %s", target) if COMMON_MATCH_GROK.MatchString(target) { res, err = COMMON_MATCH_GROK.ParseString(target) @@ -93,6 +93,9 @@ func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) return nil, errors.New("can't match") } else { res, err = mt.grok.ParseString(target) + if err != nil || res == nil || len(res) == 0 { + return nil, errors.New("match failed, no result found") + } mt.setResult(res) return res, err } diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go new file mode 100644 index 0000000..46474d4 --- /dev/null +++ b/pkg/metrics/resource.go @@ -0,0 +1,65 @@ +package metrics + +import "encoding/json" + +type Resource struct { + Namespace string `json:"namespace"` + Type string `json:"type"` + Name string `json:"name"` + Spec map[string]string `json:"spec"` +} + +func NewResource(t string) *Resource { + return &Resource{ + Type: t, + Spec: make(map[string]string), + } +} + +func (r *Resource) AddLabelSpec(l Label) { + r.Spec[l.Key] = l.Value +} + +func (r *Resource) AddSpec(key string, value string) { + r.Spec[key] = value +} + +type ResourceList []*Resource + +func (rl *ResourceList) JSON() string { + b, _ := json.Marshal(rl) + return string(b) +} + +func ConvertToResource(r []*MetricMatcher) []*Resource { + var res []*Resource + for _, m := range r { + var resource *Resource + if m.Name == "endpoint_address" || m.Name == "endpoint_port" { + for i, c := range res { + if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { + resource = res[i] + } else { + resource = NewResource("endpoint") + } + } + } else { + resource = NewResource(m.Name) + } + + resource.Namespace = m.FindLabel("namespace") + if m.Name == "endpoint_address" || m.Name == "endpoint_port" { + resource.Name = m.FindLabel("endpoint") + } else { + resource.Name = m.FindLabel(m.Name) + } + // merge endpoint_address and endpoint_port + for _, l := range m.Labels { + if l.Key != "namespace" && l.Key != m.Name { + resource.AddLabelSpec(l) + } + } + res = append(res, resource) + } + return res +} From f591f06b8cf4784a233ce2eae165664be170fbfc Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Mon, 2 Dec 2024 00:45:17 +0800 Subject: [PATCH 08/12] fix: parsing problem --- pkg/metrics/metrics_test.go | 27 +++++++++++++++++++++++++-- pkg/metrics/parser.go | 8 ++++++++ pkg/metrics/resource.go | 24 +++++++++++++++++++----- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index e3cf498..e201c68 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -2,6 +2,7 @@ package metrics import ( "bufio" + "encoding/json" "os" "testing" ) @@ -26,6 +27,7 @@ func TestMetrics(t *testing.T) { t.Fatalf("open output file failed: %v", err) } defer output.Close() + var rx []*MetricMatcher for scanner.Scan() { line := scanner.Text() res, err := rule.Match(line) @@ -33,10 +35,31 @@ func TestMetrics(t *testing.T) { continue } else { t.Logf("matched: %s", res.DumpString()) - // _, _ = output.WriteString(res.DumpString() + "\n") + _, _ = output.WriteString(res.DumpString() + "\n") + rx = append(rx, res.CopyData()) } } - var res ResourceList = ConvertToResource(rule) +} + +func TestConvertToResource(t *testing.T) { + output, err := os.OpenFile("./output.txt", os.O_CREATE|os.O_RDONLY, 0644) + if err != nil { + t.Fatalf("open output file failed: %v", err) + } + defer output.Close() + var rules []*MetricMatcher + scanner := bufio.NewScanner(output) + for scanner.Scan() { + line := scanner.Text() + var r *MetricMatcher + e := json.Unmarshal([]byte(line), &r) + if e != nil { + t.Logf("unmarshal failed: %v", e) + continue + } + rules = append(rules, r) + } + var res ResourceList = ConvertToResource(rules) _, _ = output.WriteString(res.JSON() + "\n") t.Logf(res.JSON()) } diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 153482b..35ab005 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -148,3 +148,11 @@ func init() { log.Fatalf("compile pattern failed: %v", err) } } + +func (mt *MetricMatcher) CopyData() *MetricMatcher { + return &MetricMatcher{ + Name: mt.Name, + Header: mt.Header, + Labels: mt.Labels, + } +} diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index 46474d4..f9aed08 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -12,7 +12,7 @@ type Resource struct { func NewResource(t string) *Resource { return &Resource{ Type: t, - Spec: make(map[string]string), + Spec: make(map[string]string, 4), } } @@ -26,19 +26,29 @@ func (r *Resource) AddSpec(key string, value string) { type ResourceList []*Resource -func (rl *ResourceList) JSON() string { - b, _ := json.Marshal(rl) +func (r *Resource) JSON() string { + b, _ := json.Marshal(r) return string(b) } +func (rl *ResourceList) JSON() string { + var res = "" + for _, r := range *rl { + res += r.JSON() + "\n" + } + return res +} + func ConvertToResource(r []*MetricMatcher) []*Resource { var res []*Resource for _, m := range r { var resource *Resource + var addFlag = true if m.Name == "endpoint_address" || m.Name == "endpoint_port" { for i, c := range res { if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { resource = res[i] + addFlag = false } else { resource = NewResource("endpoint") } @@ -53,13 +63,17 @@ func ConvertToResource(r []*MetricMatcher) []*Resource { } else { resource.Name = m.FindLabel(m.Name) } + resource.Namespace = m.FindLabel("namespace") + // merge endpoint_address and endpoint_port for _, l := range m.Labels { - if l.Key != "namespace" && l.Key != m.Name { + if l.Key != "namespace" && l.Key != resource.Type { resource.AddLabelSpec(l) } } - res = append(res, resource) + if addFlag { + res = append(res, resource) + } } return res } From 359c65789385496661c537df94c3f600ce2af8e2 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Mon, 2 Dec 2024 23:28:46 +0800 Subject: [PATCH 09/12] update: label of Resource has same key but diff value --- pkg/metrics/matcher.go | 2 +- pkg/metrics/metrics_test.go | 2 +- pkg/metrics/resource.go | 38 ++++++++++++++++++++----------------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/pkg/metrics/matcher.go b/pkg/metrics/matcher.go index ff5a9ba..edc55a5 100644 --- a/pkg/metrics/matcher.go +++ b/pkg/metrics/matcher.go @@ -28,7 +28,7 @@ func DefaultMatchRules() MatchRules { AddLabel("namespace").AddLabel("pod").AddLabel("service_account"), NewMetricMatcher("service").AddLabel("namespace").AddLabel("service"). - AddLabel("cluster_ip"), + AddLabel("cluster_ip").AddLabel("external_name").AddLabel("load_balancer_ip"), NewMetricMatcher("endpoint_address").SetHeader("kube_endpoint_address"). AddLabel("namespace").AddLabel("endpoint").AddLabel("ip"), NewMetricMatcher("endpoint_port").SetHeader("kube_endpoint_ports"). diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index e201c68..f248c46 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -8,7 +8,7 @@ import ( ) func TestMetrics(t *testing.T) { - f, err := os.Open("./metrics_output.txt") + f, err := os.Open("./metrics") if err != nil { t.Fatalf("open file failed: %v", err) t.Fail() diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index f9aed08..528ff61 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -3,25 +3,31 @@ package metrics import "encoding/json" type Resource struct { - Namespace string `json:"namespace"` - Type string `json:"type"` - Name string `json:"name"` - Spec map[string]string `json:"spec"` + Namespace string `json:"namespace"` + Type string `json:"type"` + Name string `json:"name"` + Spec map[string][]string `json:"spec"` } func NewResource(t string) *Resource { return &Resource{ Type: t, - Spec: make(map[string]string, 4), + Spec: make(map[string][]string, 4), } } func (r *Resource) AddLabelSpec(l Label) { - r.Spec[l.Key] = l.Value + if l.Value == "" { + return + } + if _, ok := r.Spec[l.Key]; !ok { + r.Spec[l.Key] = make([]string, 0) + } + r.Spec[l.Key] = append(r.Spec[l.Key], l.Value) } func (r *Resource) AddSpec(key string, value string) { - r.Spec[key] = value + r.AddLabelSpec(Label{Key: key, Value: value}) } type ResourceList []*Resource @@ -44,30 +50,28 @@ func ConvertToResource(r []*MetricMatcher) []*Resource { for _, m := range r { var resource *Resource var addFlag = true + + resourceType := m.Name if m.Name == "endpoint_address" || m.Name == "endpoint_port" { for i, c := range res { if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { resource = res[i] addFlag = false - } else { - resource = NewResource("endpoint") } } - } else { - resource = NewResource(m.Name) + resourceType = "endpoint" } - resource.Namespace = m.FindLabel("namespace") - if m.Name == "endpoint_address" || m.Name == "endpoint_port" { - resource.Name = m.FindLabel("endpoint") - } else { - resource.Name = m.FindLabel(m.Name) + if addFlag { + resource = NewResource(resourceType) } + resource.Namespace = m.FindLabel("namespace") + resource.Name = m.FindLabel(resourceType) // merge endpoint_address and endpoint_port for _, l := range m.Labels { - if l.Key != "namespace" && l.Key != resource.Type { + if l.Key != "namespace" && l.Key != resourceType { resource.AddLabelSpec(l) } } From 2886634ad4b7011e34bd04a46f6318d1f5b610b5 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Mon, 2 Dec 2024 23:46:38 +0800 Subject: [PATCH 10/12] update: metrics to resource merge hooks --- pkg/metrics/resource.go | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index 528ff61..79ebeae 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -45,24 +45,43 @@ func (rl *ResourceList) JSON() string { return res } -func ConvertToResource(r []*MetricMatcher) []*Resource { +type ResourceMergeHook func(m *MetricMatcher, resource ResourceList) (res *Resource, addFlag bool) + +var EndpointMergeHook ResourceMergeHook = func(m *MetricMatcher, res ResourceList) (r *Resource, addFlag bool) { + if m.Name == "endpoint_address" || m.Name == "endpoint_port" { + for i, c := range res { + if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { + r = res[i] + return r, false + } + } + return NewResource("endpoint"), true + } + return nil, true +} + +func ConvertToResource(r []*MetricMatcher, hooks ...ResourceMergeHook) []*Resource { var res []*Resource + if len(hooks) == 0 { + hooks = append(hooks, EndpointMergeHook) + } + for _, m := range r { var resource *Resource var addFlag = true - resourceType := m.Name - if m.Name == "endpoint_address" || m.Name == "endpoint_port" { - for i, c := range res { - if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { - resource = res[i] - addFlag = false - } + for _, hook := range hooks { + resource, addFlag = hook(m, res) + if resource != nil { + break } - resourceType = "endpoint" } - if addFlag { + resourceType := m.Name + if resource != nil { + resourceType = resource.Type + } + if resource == nil && addFlag { resource = NewResource(resourceType) } From 48017ddf7b4e19187511e009c2e3a18dfa79e5dc Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Sat, 7 Dec 2024 03:59:02 +0800 Subject: [PATCH 11/12] feat: complete dev of kube metrics parse --- cmd/metrics/parser.go | 93 +++++++++++++++++++++++++++++++++++++ main.go | 1 + pkg/metrics/metrics_test.go | 3 +- pkg/metrics/resource.go | 27 +++++++++-- 4 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 cmd/metrics/parser.go diff --git a/cmd/metrics/parser.go b/cmd/metrics/parser.go new file mode 100644 index 0000000..019cfcf --- /dev/null +++ b/cmd/metrics/parser.go @@ -0,0 +1,93 @@ +package metrics + +import ( + "bufio" + "io" + "net/http" + "os" + "strings" + + cmdx "github.com/esonhugh/k8spider/cmd" + "github.com/esonhugh/k8spider/pkg/metrics" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var MetricOpt struct { + From string +} + +func init() { + cmdx.RootCmd.AddCommand(MetricCmd) + MetricCmd.PersistentFlags().StringVarP(&MetricOpt.From, "metric", "m", "", "metrics from (file / remote url)") + +} + +var MetricCmd = &cobra.Command{ + Use: "metric", + Short: "parse kube stat metrics to readable resource", + Run: func(cmd *cobra.Command, args []string) { + if MetricOpt.From == "" { + return + } + log.Debugf("parse metrics from %v", MetricOpt.From) + rule := metrics.DefaultMatchRules() + if err := rule.Compile(); err != nil { + log.Fatalf("compile rule failed: %v", err) + } + log.Debugf("compiled rules completed, start to get resource \n") + + ot := output() + + var r io.Reader + if strings.HasPrefix("http://", MetricOpt.From) || strings.HasPrefix("https://", MetricOpt.From) { + resp, err := http.Get(MetricOpt.From) + if err != nil { + log.Fatalf("get metrics from %v failed: %v", MetricOpt.From, err) + } + defer resp.Body.Close() + r = resp.Body + } else { + f, err := os.OpenFile(MetricOpt.From, os.O_RDONLY, 0666) + if err != nil { + log.Fatalf("open file %v failed: %v", MetricOpt.From, err) + } + defer f.Close() + r = f + } + log.Debugf("start to parse metrics line by line\n") + + var rx []*metrics.MetricMatcher + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + res, err := rule.Match(line) + if err != nil { + continue + } else { + log.Debugf("matched: %s", res.DumpString()) + rx = append(rx, res.CopyData()) + } + } + if err := scanner.Err(); err != nil { + log.Warnf("scan metrics failed and break out, reason: %v", err) + } + var res metrics.ResourceList = metrics.ConvertToResource(rx) + log.Debugf("parse metrics completed, start to print result\n") + + res.Print(ot) + }, +} + +func output() io.WriteCloser { + if cmdx.Opts.OutputFile != "" { + f, err := os.OpenFile(cmdx.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Warnf("create output file failed: %v", err) + return nil + } + return f + } else { + return os.Stdout + } +} diff --git a/main.go b/main.go index ca4739d..57547d5 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( _ "github.com/esonhugh/k8spider/cmd/all" _ "github.com/esonhugh/k8spider/cmd/axfr" _ "github.com/esonhugh/k8spider/cmd/dnsutils" + _ "github.com/esonhugh/k8spider/cmd/metrics" _ "github.com/esonhugh/k8spider/cmd/neighbor" _ "github.com/esonhugh/k8spider/cmd/service" _ "github.com/esonhugh/k8spider/cmd/subnet" diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index f248c46..245b6fb 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -60,6 +60,5 @@ func TestConvertToResource(t *testing.T) { rules = append(rules, r) } var res ResourceList = ConvertToResource(rules) - _, _ = output.WriteString(res.JSON() + "\n") - t.Logf(res.JSON()) + res.Print(os.Stderr) } diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index 79ebeae..8ad0982 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -1,6 +1,13 @@ package metrics -import "encoding/json" +import ( + "encoding/json" + "fmt" + "io" + "os" + + log "github.com/sirupsen/logrus" +) type Resource struct { Namespace string `json:"namespace"` @@ -37,12 +44,22 @@ func (r *Resource) JSON() string { return string(b) } -func (rl *ResourceList) JSON() string { - var res = "" +func (rl *ResourceList) Print(writer ...io.Writer) { + var W io.Writer + if len(writer) == 0 { + W = os.Stdout + } else { + w := io.MultiWriter(writer...) + W = io.MultiWriter(os.Stdout, w) + } for _, r := range *rl { - res += r.JSON() + "\n" + data, err := json.Marshal(r.JSON()) + if err != nil { + log.Error(err) + return + } + _, _ = fmt.Fprintf(W, "%v\n", string(data)) } - return res } type ResourceMergeHook func(m *MetricMatcher, resource ResourceList) (res *Resource, addFlag bool) From dae0d7c21ad143673c236cf664d3db95b924dde2 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Sat, 7 Dec 2024 04:21:40 +0800 Subject: [PATCH 12/12] fix: bug of print error data --- cmd/metrics/parser.go | 1 + pkg/metrics/parser.go | 10 +++++++--- pkg/metrics/resource.go | 18 ++---------------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/cmd/metrics/parser.go b/cmd/metrics/parser.go index 019cfcf..42566a4 100644 --- a/cmd/metrics/parser.go +++ b/cmd/metrics/parser.go @@ -58,6 +58,7 @@ var MetricCmd = &cobra.Command{ log.Debugf("start to parse metrics line by line\n") var rx []*metrics.MetricMatcher + scanner := bufio.NewScanner(r) for scanner.Scan() { line := scanner.Text() diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go index 35ab005..11a1340 100644 --- a/pkg/metrics/parser.go +++ b/pkg/metrics/parser.go @@ -150,9 +150,13 @@ func init() { } func (mt *MetricMatcher) CopyData() *MetricMatcher { + var newLabel []Label + for _, label := range mt.Labels { + newLabel = append(newLabel, label) + } return &MetricMatcher{ - Name: mt.Name, - Header: mt.Header, - Labels: mt.Labels, + Name: strings.Clone(mt.Name), + Header: strings.Clone(mt.Header), + Labels: newLabel, } } diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go index 8ad0982..2c112b8 100644 --- a/pkg/metrics/resource.go +++ b/pkg/metrics/resource.go @@ -4,9 +4,6 @@ import ( "encoding/json" "fmt" "io" - "os" - - log "github.com/sirupsen/logrus" ) type Resource struct { @@ -45,20 +42,9 @@ func (r *Resource) JSON() string { } func (rl *ResourceList) Print(writer ...io.Writer) { - var W io.Writer - if len(writer) == 0 { - W = os.Stdout - } else { - w := io.MultiWriter(writer...) - W = io.MultiWriter(os.Stdout, w) - } + var w io.Writer = io.MultiWriter(writer...) for _, r := range *rl { - data, err := json.Marshal(r.JSON()) - if err != nil { - log.Error(err) - return - } - _, _ = fmt.Fprintf(W, "%v\n", string(data)) + _, _ = fmt.Fprintf(w, "%v\n", r.JSON()) } }