diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e9d0c0..3165870 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,6 +22,11 @@ jobs: with: go-version: ${{ env.GO_VERSION }} + - name: Install UPX + uses: crazy-max/ghaction-upx@v3 + with: + install-only: true + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v3 with: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b738726..a510f18 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,10 +3,11 @@ project_name: k8spider before: hooks: - go mod tidy - - go generate ./... builds: - env: - CGO_ENABLED=0 + ldflags: + - -s -w goos: - darwin - windows @@ -23,6 +24,13 @@ archives: - goos: windows format: zip +upx: + - enabled: true + compress: best + brute: true + lzma: true + + checksum: name_template: "checksums.txt" snapshot: diff --git a/Makefile b/Makefile index 909a418..5c551c7 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ BUILD_DIR = bin MAIN_PROGRAM_NAME = k8spider -default: build build-static +default: build build-static check-size # build build: @@ -11,6 +11,10 @@ build: build-static: GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go - upx $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + upx --lzma --brute $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static + +check-size: + ls -alh $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)* + clean: rm -rf $(BUILD_DIR) \ No newline at end of file diff --git a/cmd/all/all.go b/cmd/all/all.go index 4ff6097..ab3335e 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -48,6 +48,7 @@ var AllCmd = &cobra.Command{ log.Warnf("ParseStringToIPNet failed: %v", err) return } + var finalRecord define.Records if command.Opts.MultiThreadingMode { finalRecord = RunMultiThread(ipNets, command.Opts.ThreadingNum) @@ -55,6 +56,7 @@ var AllCmd = &cobra.Command{ finalRecord = Run(ipNets) } printer.PrintResult(finalRecord, command.Opts.OutputFile) + PostRun(finalRecord) }, } @@ -78,6 +80,9 @@ func RunMultiThread(net *net.IPNet, count int) (finalRecord define.Records) { } func PostRun(finalRecord define.Records) { + if finalRecord == nil || len(finalRecord) == 0 { + return + } log.Info("Extract Namespaces: ") list := post.RecordsDumpNameSpace(finalRecord, command.Opts.Zone) for _, ns := range list { diff --git a/cmd/dnsutils/dns.go b/cmd/dnsutils/dns.go new file mode 100644 index 0000000..c95b46a --- /dev/null +++ b/cmd/dnsutils/dns.go @@ -0,0 +1,46 @@ +package dnsutils + +import ( + "strings" + + command "github.com/esonhugh/k8spider/cmd" + "github.com/esonhugh/k8spider/pkg" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var queryType string + +func init() { + DNSCmd.PersistentFlags().StringVarP(&queryType, "type", "t", "A", "query type") + command.RootCmd.AddCommand(DNSCmd) +} + +var DNSCmd = &cobra.Command{ + Use: "dns", + Aliases: []string{"dig"}, + Short: "dns is a command to query dns server", + Run: func(cmd *cobra.Command, args []string) { + var querier pkg.DnsQuery + switch strings.ToLower(queryType) { + case "a", "aaaa": + querier = pkg.QueryA + case "ptr": + querier = pkg.QueryPTR + case "srv": + querier = pkg.QuerySRV + case "txt": + querier = pkg.QueryTXT + default: + querier = pkg.QueryA + } + for _, query := range args { + res, err := querier(query) + if err != nil { + log.Warnf("Query %s failed: %v", query, err) + continue + } + log.Infof("Query [%d] %s: %v", queryType, query, res) + } + }, +} diff --git a/cmd/metrics/parser.go b/cmd/metrics/parser.go new file mode 100644 index 0000000..42566a4 --- /dev/null +++ b/cmd/metrics/parser.go @@ -0,0 +1,94 @@ +package metrics + +import ( + "bufio" + "io" + "net/http" + "os" + "strings" + + cmdx "github.com/esonhugh/k8spider/cmd" + "github.com/esonhugh/k8spider/pkg/metrics" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var MetricOpt struct { + From string +} + +func init() { + cmdx.RootCmd.AddCommand(MetricCmd) + MetricCmd.PersistentFlags().StringVarP(&MetricOpt.From, "metric", "m", "", "metrics from (file / remote url)") + +} + +var MetricCmd = &cobra.Command{ + Use: "metric", + Short: "parse kube stat metrics to readable resource", + Run: func(cmd *cobra.Command, args []string) { + if MetricOpt.From == "" { + return + } + log.Debugf("parse metrics from %v", MetricOpt.From) + rule := metrics.DefaultMatchRules() + if err := rule.Compile(); err != nil { + log.Fatalf("compile rule failed: %v", err) + } + log.Debugf("compiled rules completed, start to get resource \n") + + ot := output() + + var r io.Reader + if strings.HasPrefix("http://", MetricOpt.From) || strings.HasPrefix("https://", MetricOpt.From) { + resp, err := http.Get(MetricOpt.From) + if err != nil { + log.Fatalf("get metrics from %v failed: %v", MetricOpt.From, err) + } + defer resp.Body.Close() + r = resp.Body + } else { + f, err := os.OpenFile(MetricOpt.From, os.O_RDONLY, 0666) + if err != nil { + log.Fatalf("open file %v failed: %v", MetricOpt.From, err) + } + defer f.Close() + r = f + } + log.Debugf("start to parse metrics line by line\n") + + var rx []*metrics.MetricMatcher + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + res, err := rule.Match(line) + if err != nil { + continue + } else { + log.Debugf("matched: %s", res.DumpString()) + rx = append(rx, res.CopyData()) + } + } + if err := scanner.Err(); err != nil { + log.Warnf("scan metrics failed and break out, reason: %v", err) + } + var res metrics.ResourceList = metrics.ConvertToResource(rx) + log.Debugf("parse metrics completed, start to print result\n") + + res.Print(ot) + }, +} + +func output() io.WriteCloser { + if cmdx.Opts.OutputFile != "" { + f, err := os.OpenFile(cmdx.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Warnf("create output file failed: %v", err) + return nil + } + return f + } else { + return os.Stdout + } +} diff --git a/go.mod b/go.mod index 7a361e2..d393d69 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module github.com/esonhugh/k8spider -go 1.19 +go 1.21.0 + +toolchain go1.23.2 require ( + github.com/elastic/go-grok v0.3.1 github.com/miekg/dns v1.1.58 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 @@ -10,9 +13,9 @@ require ( require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/tools v0.17.0 // indirect diff --git a/go.sum b/go.sum index d3ed1fe..1d5334b 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= +github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -16,16 +20,15 @@ github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/main.go b/main.go index 2ee8f92..57547d5 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,8 @@ import ( "github.com/esonhugh/k8spider/cmd" _ "github.com/esonhugh/k8spider/cmd/all" _ "github.com/esonhugh/k8spider/cmd/axfr" + _ "github.com/esonhugh/k8spider/cmd/dnsutils" + _ "github.com/esonhugh/k8spider/cmd/metrics" _ "github.com/esonhugh/k8spider/cmd/neighbor" _ "github.com/esonhugh/k8spider/cmd/service" _ "github.com/esonhugh/k8spider/cmd/subnet" diff --git a/pkg/metrics/matcher.go b/pkg/metrics/matcher.go new file mode 100644 index 0000000..edc55a5 --- /dev/null +++ b/pkg/metrics/matcher.go @@ -0,0 +1,60 @@ +package metrics + +import "errors" + +type MatchRules []*MetricMatcher + +func GenerateMatchRules() MatchRules { + return make(MatchRules, 0) +} + +func DefaultMatchRules() MatchRules { + return []*MetricMatcher{ + NewMetricMatcher("configmap").AddLabel("namespace").AddLabel("configmap"), + NewMetricMatcher("secret").AddLabel("namespace").AddLabel("secret"), + + NewMetricMatcher("node").AddLabel("node").AddLabel("kernel_version"). + AddLabel("os_image").AddLabel("container_runtime_version"). + AddLabel("provider_id").AddLabel("internal_ip"), + + NewMetricMatcher("pod").AddLabel("namespace").AddLabel("pod").AddLabel("node"). + AddLabel("host_ip").AddLabel("pod_ip"), + NewMetricMatcher("container").SetHeader("kube_pod_container_info").AddLabel("namespace"). + AddLabel("pod").AddLabel("container").AddLabel("image"), + NewMetricMatcher("cronjob").AddLabel("namespace").AddLabel("cronjob"). + AddLabel("schedule"), + + NewMetricMatcher("service_account").SetHeader("kube_pod_service_account"). + AddLabel("namespace").AddLabel("pod").AddLabel("service_account"), + + NewMetricMatcher("service").AddLabel("namespace").AddLabel("service"). + AddLabel("cluster_ip").AddLabel("external_name").AddLabel("load_balancer_ip"), + NewMetricMatcher("endpoint_address").SetHeader("kube_endpoint_address"). + AddLabel("namespace").AddLabel("endpoint").AddLabel("ip"), + NewMetricMatcher("endpoint_port").SetHeader("kube_endpoint_ports"). + AddLabel("namespace").AddLabel("endpoint").AddLabel("port_number"), + } +} + +func (m MatchRules) Compile() error { + var err error = nil + for i := range m { + e := m[i].Compile() + if e != nil { + err = errors.Join(err, e) + } + } + return err +} + +func (m MatchRules) Match(target string) (*MetricMatcher, error) { + for _, r := range m { + _, e := r.Match(target) + if e != nil { + continue + } else { + return r, nil + } + } + return nil, errors.New("no match found") +} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 0000000..245b6fb --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "bufio" + "encoding/json" + "os" + "testing" +) + +func TestMetrics(t *testing.T) { + f, err := os.Open("./metrics") + if err != nil { + t.Fatalf("open file failed: %v", err) + t.Fail() + } + defer f.Close() + scanner := bufio.NewScanner(f) + + rule := DefaultMatchRules() + if err := rule.Compile(); err != nil { + t.Fatalf("compile rule failed: %v", err) + t.Fail() + } + + output, err := os.OpenFile("./output.txt", os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + t.Fatalf("open output file failed: %v", err) + } + defer output.Close() + var rx []*MetricMatcher + for scanner.Scan() { + line := scanner.Text() + res, err := rule.Match(line) + if err != nil { + continue + } else { + t.Logf("matched: %s", res.DumpString()) + _, _ = output.WriteString(res.DumpString() + "\n") + rx = append(rx, res.CopyData()) + } + } +} + +func TestConvertToResource(t *testing.T) { + output, err := os.OpenFile("./output.txt", os.O_CREATE|os.O_RDONLY, 0644) + if err != nil { + t.Fatalf("open output file failed: %v", err) + } + defer output.Close() + var rules []*MetricMatcher + scanner := bufio.NewScanner(output) + for scanner.Scan() { + line := scanner.Text() + var r *MetricMatcher + e := json.Unmarshal([]byte(line), &r) + if e != nil { + t.Logf("unmarshal failed: %v", e) + continue + } + rules = append(rules, r) + } + var res ResourceList = ConvertToResource(rules) + res.Print(os.Stderr) +} diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go new file mode 100644 index 0000000..11a1340 --- /dev/null +++ b/pkg/metrics/parser.go @@ -0,0 +1,162 @@ +package metrics + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/elastic/go-grok" + log "github.com/sirupsen/logrus" +) + +var ( + MetricsPatterns = map[string]string{ + "METRIC_HEAD": "^%{WORD:metric_name}{", + "LAST_LABEL": `(%{DATA:last_label_name}="%{DATA:last_label_value}",?)`, + "NUMBER_SCI": "%{NUMBER}(e%{NUMBER})?", + "METRIC_TAIL": `%{SPACE}%{NUMBER_SCI:metric_value}$`, + "NORMAL_EXPR": `^%{WORD:metric_name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}=\"%{DATA:last_label_value}\")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$`, + } + COMMON_MATCH_GROK = grok.New() +) + +const COMMON_MATCH = `^%{WORD:name}{((namespace="%{DATA:namespace}")|(%{DATA:last_label_name}="%{DATA:last_label_value}")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$` + +type Label struct { + Key string `json:"key"` + Value string `json:"value""` +} + +type MetricMatcher struct { + Name string `json:"type"` + Header string `json:"-"` + Labels []Label `json:"labels"` + grok *grok.Grok `json:"-"` + finalPattern string + ptr any `json:"-"` +} + +func NewMetricMatcher(t string) *MetricMatcher { + return &MetricMatcher{ + Name: t, + grok: grok.New(), + Labels: make([]Label, 0), + } +} + +// example: ^%{WORD:name}{((namespace="%{DATA:ns}")|(%{DATA:last_label_name}="%{DATA:last_label_value}")(,|))*}%{SPACE}%{NUMBER}(e%{NUMBER})?$ +// ^%{WORD:name}{ +// ( +// (namespace="%{DATA:ns}") +// | +// (%{DATA:last_label_name}="%{DATA:last_label_value}") +// (,|) +// ) +// * +// }%{SPACE}%{NUMBER}(e%{NUMBER})?$ + +func (mt *MetricMatcher) Compile() error { + if err := mt.grok.AddPatterns(MetricsPatterns); err != nil { + return err + } + if mt.Header == "" { + mt.Header = `kube_` + mt.Name + `_info` // custom head + } + header := mt.Header + var body []string + for _, label := range mt.Labels { + body = append(body, "("+label.Key+`=`+`"%{DATA:`+label.Key+`}",?)`) + } + tail := `%{METRIC_TAIL}` + pattern := header + "{" + "(" + strings.Join(body, "|") + "|%{LAST_LABEL})*" + "}" + tail + mt.finalPattern = pattern + log.Debugln("generated pattern: ", pattern) + return mt.grok.Compile(pattern, true) +} + +func (mt *MetricMatcher) Match(target string) (res map[string]string, err error) { + if !strings.HasPrefix(target, mt.Header) { + log.Debugf("not match: %s", target) + if COMMON_MATCH_GROK.MatchString(target) { + res, err = COMMON_MATCH_GROK.ParseString(target) + if err != nil { + return nil, errors.Join( + errors.New("match failed, in common expr"), + err, + ) + } + return res, errors.Join( + errors.New("match failed, in custom expr, but success in common expr, check ret result to get more detail"), + err, + ) + } + return nil, errors.New("can't match") + } else { + res, err = mt.grok.ParseString(target) + if err != nil || res == nil || len(res) == 0 { + return nil, errors.New("match failed, no result found") + } + mt.setResult(res) + return res, err + } +} + +func (mt *MetricMatcher) SetHeader(header string) *MetricMatcher { + mt.Header = header + return mt +} + +func (mt *MetricMatcher) AddLabel(label string) *MetricMatcher { + mt.Labels = append(mt.Labels, Label{ + Key: label, + Value: "", + }) + return mt +} + +func (mt *MetricMatcher) setResult(res map[string]string) { + for i, label := range mt.Labels { + if v, ok := res[label.Key]; ok { + mt.Labels[i].Value = v + } else { + log.Debugf("label %s not found in result", label.Key) // keep it empty + } + } +} + +func (mt *MetricMatcher) FindLabel(Key string) string { + for _, label := range mt.Labels { + if label.Key == Key { + return label.Value + } + } + return "" +} + +func (mt *MetricMatcher) DumpString() string { + b, _ := json.Marshal(mt) + return string(b) +} + +func init() { + err := COMMON_MATCH_GROK.AddPatterns(MetricsPatterns) + if err != nil { + log.Fatalf("add patterns failed: %v", err) + } + err = COMMON_MATCH_GROK.Compile(COMMON_MATCH, true) + if err != nil { + log.Fatalf("compile pattern failed: %v", err) + } +} + +func (mt *MetricMatcher) CopyData() *MetricMatcher { + var newLabel []Label + for _, label := range mt.Labels { + newLabel = append(newLabel, label) + } + return &MetricMatcher{ + Name: strings.Clone(mt.Name), + Header: strings.Clone(mt.Header), + Labels: newLabel, + } +} diff --git a/pkg/metrics/resource.go b/pkg/metrics/resource.go new file mode 100644 index 0000000..2c112b8 --- /dev/null +++ b/pkg/metrics/resource.go @@ -0,0 +1,105 @@ +package metrics + +import ( + "encoding/json" + "fmt" + "io" +) + +type Resource struct { + Namespace string `json:"namespace"` + Type string `json:"type"` + Name string `json:"name"` + Spec map[string][]string `json:"spec"` +} + +func NewResource(t string) *Resource { + return &Resource{ + Type: t, + Spec: make(map[string][]string, 4), + } +} + +func (r *Resource) AddLabelSpec(l Label) { + if l.Value == "" { + return + } + if _, ok := r.Spec[l.Key]; !ok { + r.Spec[l.Key] = make([]string, 0) + } + r.Spec[l.Key] = append(r.Spec[l.Key], l.Value) +} + +func (r *Resource) AddSpec(key string, value string) { + r.AddLabelSpec(Label{Key: key, Value: value}) +} + +type ResourceList []*Resource + +func (r *Resource) JSON() string { + b, _ := json.Marshal(r) + return string(b) +} + +func (rl *ResourceList) Print(writer ...io.Writer) { + var w io.Writer = io.MultiWriter(writer...) + for _, r := range *rl { + _, _ = fmt.Fprintf(w, "%v\n", r.JSON()) + } +} + +type ResourceMergeHook func(m *MetricMatcher, resource ResourceList) (res *Resource, addFlag bool) + +var EndpointMergeHook ResourceMergeHook = func(m *MetricMatcher, res ResourceList) (r *Resource, addFlag bool) { + if m.Name == "endpoint_address" || m.Name == "endpoint_port" { + for i, c := range res { + if m.FindLabel("namespace") == c.Namespace && m.FindLabel("endpoint") == c.Name { + r = res[i] + return r, false + } + } + return NewResource("endpoint"), true + } + return nil, true +} + +func ConvertToResource(r []*MetricMatcher, hooks ...ResourceMergeHook) []*Resource { + var res []*Resource + if len(hooks) == 0 { + hooks = append(hooks, EndpointMergeHook) + } + + for _, m := range r { + var resource *Resource + var addFlag = true + + for _, hook := range hooks { + resource, addFlag = hook(m, res) + if resource != nil { + break + } + } + + resourceType := m.Name + if resource != nil { + resourceType = resource.Type + } + if resource == nil && addFlag { + resource = NewResource(resourceType) + } + + resource.Namespace = m.FindLabel("namespace") + resource.Name = m.FindLabel(resourceType) + + // merge endpoint_address and endpoint_port + for _, l := range m.Labels { + if l.Key != "namespace" && l.Key != resourceType { + resource.AddLabelSpec(l) + } + } + if addFlag { + res = append(res, resource) + } + } + return res +} diff --git a/pkg/query_utils.go b/pkg/query_utils.go index 2cb09ea..361080f 100644 --- a/pkg/query_utils.go +++ b/pkg/query_utils.go @@ -2,6 +2,7 @@ package pkg import ( "context" + "fmt" "net" "regexp" "strings" @@ -89,7 +90,9 @@ func WarpDnsServer(dnsServer string) *SpiderResolver { return d.DialContext(ctx, network, dnsServer) }, }, - ctx: ctx, + ctx: ctx, + filter: []*regexp.Regexp{}, + contains: []string{}, } } @@ -146,3 +149,28 @@ func (s *SpiderResolver) TXTRecord(domain string) ([]string, error) { func TXTRecord(domain string) (txts []string, err error) { return NetResolver.TXTRecord(domain) } + +type DnsQuery func(domain string) ([]string, error) + +var ( + QueryPTR DnsQuery = func(domain string) ([]string, error) { + return PTRRecord(net.ParseIP(domain)), nil + } + QueryA DnsQuery = func(domain string) ([]string, error) { + res, err := ARecord(domain) + var ret []string + for _, r := range res { + ret = append(ret, r.String()) + } + return ret, err + } + QueryTXT DnsQuery = TXTRecord + QuerySRV DnsQuery = func(domain string) ([]string, error) { + _, res, err := SRVRecord(domain) + var ret []string + for _, r := range res { + ret = append(ret, fmt.Sprintf("%s:%d", r.Target, r.Port)) + } + return ret, err + } +)