From 96c5362ca3569b8863cf9d291154c573f17c36c0 Mon Sep 17 00:00:00 2001 From: Esonhugh Date: Thu, 21 Mar 2024 13:01:47 +0800 Subject: [PATCH] update: spider spider-mutlithreading with super cool statusbar metrics --- cmd/all/all.go | 2 +- cmd/subnet/subnet.go | 2 +- go.mod | 7 +++++++ go.sum | 17 +++++++++++++++++ pkg/mutli/subnet.go | 22 ++++++++++++++++++---- pkg/scanner/scanner.go | 19 +++++++++++++++---- 6 files changed, 59 insertions(+), 10 deletions(-) diff --git a/cmd/all/all.go b/cmd/all/all.go index af23219..9fc0995 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -46,7 +46,7 @@ var AllCmd = &cobra.Command{ } func Run(net *net.IPNet) { - var records define.Records = scanner.ScanSubnet(net) + var records define.Records = scanner.ScanSubnet(net, nil) if records == nil || len(records) == 0 { log.Warnf("ScanSubnet Found Nothing") return diff --git a/cmd/subnet/subnet.go b/cmd/subnet/subnet.go index 4e9064d..b45445d 100644 --- a/cmd/subnet/subnet.go +++ b/cmd/subnet/subnet.go @@ -39,7 +39,7 @@ var SubNetCmd = &cobra.Command{ } func Run(net *net.IPNet) { - var records define.Records = scanner.ScanSubnet(net) + var records define.Records = scanner.ScanSubnet(net, nil) if records == nil || len(records) == 0 { log.Warnf("ScanSubnet Found Nothing") return diff --git a/go.mod b/go.mod index 7a361e2..7936946 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,20 @@ module github.com/esonhugh/k8spider go 1.19 require ( + github.com/cheggaaa/pb/v3 v3.1.5 github.com/miekg/dns v1.1.58 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 ) require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.0 // indirect golang.org/x/mod v0.14.0 // indirect diff --git a/go.sum b/go.sum index d3ed1fe..bcc59fd 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,28 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk= +github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -27,6 +42,8 @@ golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= diff --git a/pkg/mutli/subnet.go b/pkg/mutli/subnet.go index 63c0802..688854b 100644 --- a/pkg/mutli/subnet.go +++ b/pkg/mutli/subnet.go @@ -4,6 +4,7 @@ import ( "net" "sync" + "github.com/cheggaaa/pb/v3" "github.com/esonhugh/k8spider/define" "github.com/esonhugh/k8spider/pkg" "github.com/esonhugh/k8spider/pkg/scanner" @@ -27,21 +28,34 @@ func (s *SubnetScanner) ScanSubnet(subnet *net.IPNet) <-chan []define.Record { } out := make(chan []define.Record, 100) go func() { + var pblist []*pb.ProgressBar + var pool *pb.Pool if subnets, err := pkg.SubnetShift(subnet, 4); err != nil { - go s.scan(subnet, out) + go s.scan(subnet, out, nil) } else { + // start pool for _, sn := range subnets { - go s.scan(sn, out) + pblist = append(pblist, pb.New(len(pkg.ParseIPNetToIPs(sn)))) + } + pool, err = pb.StartPool(pblist...) + if err != nil { + panic(err) + } + for i, sn := range subnets { + go s.scan(sn, out, pblist[i]) } } s.wg.Wait() close(out) + if len(pblist) > 0 { + _ = pool.Stop() + } }() return out } -func (s *SubnetScanner) scan(subnet *net.IPNet, to chan []define.Record) { +func (s *SubnetScanner) scan(subnet *net.IPNet, to chan []define.Record, pb *pb.ProgressBar) { s.wg.Add(1) - to <- scanner.ScanSubnet(subnet) + to <- scanner.ScanSubnet(subnet, pb) s.wg.Done() } diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go index c5dfabf..8ad0790 100644 --- a/pkg/scanner/scanner.go +++ b/pkg/scanner/scanner.go @@ -4,14 +4,25 @@ import ( "net" "strings" + "github.com/cheggaaa/pb/v3" "github.com/esonhugh/k8spider/define" "github.com/esonhugh/k8spider/pkg" "github.com/miekg/dns" log "github.com/sirupsen/logrus" ) -func ScanSubnet(subnet *net.IPNet) (records []define.Record) { - for _, ip := range pkg.ParseIPNetToIPs(subnet) { +var BarTemplate = `{{ string . "subnet" }} {{ bar . "[" "-" (cycle . ">" ) "." "]"}} {{speed . }} {{percent .}}` + +func ScanSubnet(subnet *net.IPNet, optPb *pb.ProgressBar) (records []define.Record) { + subnetIPlist := pkg.ParseIPNetToIPs(subnet) + var bar *pb.ProgressBar + if optPb != nil { + bar = optPb.SetTemplateString(BarTemplate).Start() + } else { + bar = pb.ProgressBarTemplate(BarTemplate).Start(len(subnetIPlist)) + } + bar.Set("subnet", subnet.String()) + for _, ip := range subnetIPlist { ptr := pkg.PTRRecord(ip) if len(ptr) > 0 { for _, domain := range ptr { @@ -19,10 +30,10 @@ func ScanSubnet(subnet *net.IPNet) (records []define.Record) { r := define.Record{Ip: ip, SvcDomain: domain} records = append(records, r) } - } else { - continue } + bar.Increment() } + bar.Finish() return }