From 543ab3cc0057dda50c8aa346015a3c0c16843408 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 5 Sep 2023 10:55:08 +0200 Subject: [PATCH] Add feature to specify the allowed dhcp cidrs (#51) --- go.mod | 1 + go.sum | 2 + internal/leases/leases.go | 16 +--- internal/leases/leases_test.go | 2 +- internal/leases/parser.go | 23 ++++-- internal/leases/parser_test.go | 12 +-- internal/leases/types.go | 16 ---- internal/reporter/reporter.go | 136 +++++++++++++++++++++---------- main.go | 8 +- {domain => pkg/config}/config.go | 17 +++- 10 files changed, 143 insertions(+), 90 deletions(-) rename {domain => pkg/config}/config.go (87%) diff --git a/go.mod b/go.mod index 4ec7263..f511b9f 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 + golang.org/x/sync v0.3.0 ) require ( diff --git a/go.sum b/go.sum index a4ffa81..496bdf6 100644 --- a/go.sum +++ b/go.sum @@ -294,6 +294,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/leases/leases.go b/internal/leases/leases.go index 4e547a8..033e763 100644 --- a/internal/leases/leases.go +++ b/internal/leases/leases.go @@ -1,7 +1,6 @@ package leases import ( - "fmt" "os" "time" ) @@ -31,18 +30,9 @@ func (l Leases) LatestByMac() map[string]Lease { } func ReadLeases(leaseFile string) (Leases, error) { - leasesContent := mustRead(leaseFile) - leases, err := Parse(leasesContent) + leasesContent, err := os.ReadFile(leaseFile) if err != nil { - return nil, fmt.Errorf("could not parse leases file:%w", err) + return nil, err } - return leases, nil -} - -func mustRead(name string) string { - c, err := os.ReadFile(name) - if err != nil { - panic(err) - } - return string(c) + return parse(string(leasesContent)) } diff --git a/internal/leases/leases_test.go b/internal/leases/leases_test.go index f33d0c5..d11a73b 100644 --- a/internal/leases/leases_test.go +++ b/internal/leases/leases_test.go @@ -9,7 +9,7 @@ import ( func TestFilterActive(t *testing.T) { assert := assert.New(t) - l, err := Parse(LEASES_CONTENT) + l, err := parse(sampleLeaseContent) assert.NoError(err) assert.Equal(Leases{}, l.FilterActive()) } diff --git a/internal/leases/parser.go b/internal/leases/parser.go index e96c3ac..5e68ac9 100644 --- a/internal/leases/parser.go +++ b/internal/leases/parser.go @@ -1,17 +1,21 @@ package leases import ( + "errors" "regexp" "time" ) -const DATE_FORMAT = "2006/01/02 15:04:05" -const LEASE_REGEX = `(?ms)lease\s+(?P[^\s]+)\s+{.*?starts\s\d+\s(?P[\d\/]+\s[\d\:]+);.*?ends\s\d+\s(?P[\d\/]+\s[\d\:]+);.*?hardware\sethernet\s(?P[\w\:]+);.*?}` +const ( + leaseDateFormat = "2006/01/02 15:04:05" + leaseRegex = `(?ms)lease\s+(?P[^\s]+)\s+{.*?starts\s\d+\s(?P[\d\/]+\s[\d\:]+);.*?ends\s\d+\s(?P[\d\/]+\s[\d\:]+);.*?hardware\sethernet\s(?P[\w\:]+);.*?}` +) -func Parse(contents string) (Leases, error) { +func parse(contents string) (Leases, error) { leases := Leases{} - var re = regexp.MustCompile(LEASE_REGEX) + var re = regexp.MustCompile(leaseRegex) matches := re.FindAllStringSubmatch(contents, -1) + var errs []error for _, m := range matches { rm := make(map[string]string) for i, name := range re.SubexpNames() { @@ -19,13 +23,13 @@ func Parse(contents string) (Leases, error) { rm[name] = m[i] } } - begin, err := time.Parse(DATE_FORMAT, rm["begin"]) + begin, err := time.Parse(leaseDateFormat, rm["begin"]) if err != nil { - panic(err) + errs = append(errs, err) } - end, err := time.Parse(DATE_FORMAT, rm["end"]) + end, err := time.Parse(leaseDateFormat, rm["end"]) if err != nil { - panic(err) + errs = append(errs, err) } l := Lease{ @@ -36,5 +40,8 @@ func Parse(contents string) (Leases, error) { } leases = append(leases, l) } + if len(errs) > 0 { + return leases, errors.Join(errs...) + } return leases, nil } diff --git a/internal/leases/parser_test.go b/internal/leases/parser_test.go index 1480fe4..87848a4 100644 --- a/internal/leases/parser_test.go +++ b/internal/leases/parser_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -var LEASES_CONTENT = ` +var sampleLeaseContent = ` lease 192.168.2.27 { starts 4 2019/06/27 13:30:21; ends 4 2019/06/27 13:40:21; @@ -34,11 +34,11 @@ lease 192.168.2.30 { func TestParse(t *testing.T) { assert := assert.New(t) - l, err := Parse(LEASES_CONTENT) + l, err := parse(sampleLeaseContent) assert.NoError(err) - b, _ := time.Parse(DATE_FORMAT, "2019/06/27 13:30:21") - e, _ := time.Parse(DATE_FORMAT, "2019/06/27 13:40:21") + b, _ := time.Parse(leaseDateFormat, "2019/06/27 13:30:21") + e, _ := time.Parse(leaseDateFormat, "2019/06/27 13:40:21") lease1 := Lease{ Mac: "ac:1f:6b:35:ac:62", Ip: "192.168.2.27", @@ -46,8 +46,8 @@ func TestParse(t *testing.T) { End: e, } - b, _ = time.Parse(DATE_FORMAT, "2019/06/27 06:40:06") - e, _ = time.Parse(DATE_FORMAT, "2019/06/27 06:50:06") + b, _ = time.Parse(leaseDateFormat, "2019/06/27 06:40:06") + e, _ = time.Parse(leaseDateFormat, "2019/06/27 06:50:06") lease2 := Lease{ Mac: "ac:1f:6b:35:ab:2d", Ip: "192.168.2.30", diff --git a/internal/leases/types.go b/internal/leases/types.go index 06177d5..162ca99 100644 --- a/internal/leases/types.go +++ b/internal/leases/types.go @@ -27,19 +27,3 @@ type ReportItem struct { IndicatorLED *string PowerMetric *models.V1PowerMetric } - -func NewReportItem(l Lease, log *zap.SugaredLogger) *ReportItem { - return &ReportItem{ - Lease: l, - Log: log, - } -} - -func (i *ReportItem) MacContainedIn(macs []string) bool { - for _, m := range macs { - if m == i.Mac { - return true - } - } - return false -} diff --git a/internal/reporter/reporter.go b/internal/reporter/reporter.go index 073e176..ee18dea 100644 --- a/internal/reporter/reporter.go +++ b/internal/reporter/reporter.go @@ -1,33 +1,39 @@ package reporter import ( + "fmt" + "net/netip" "os" "os/signal" - "sync" + "slices" "syscall" "time" - "github.com/metal-stack/metal-bmc/domain" "github.com/metal-stack/metal-bmc/internal/leases" + "github.com/metal-stack/metal-bmc/pkg/config" metalgo "github.com/metal-stack/metal-go" "github.com/metal-stack/metal-go/api/client/machine" "github.com/metal-stack/metal-go/api/models" "go.uber.org/zap" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" ) // reporter reports information about bmc, bios and dhcp ip of bmc to metal-api type reporter struct { - cfg *domain.Config + cfg *config.Config log *zap.SugaredLogger client metalgo.Client + sem *semaphore.Weighted } // New will create a reporter for MachineIpmiReports -func New(log *zap.SugaredLogger, cfg *domain.Config, client metalgo.Client) (*reporter, error) { +func New(log *zap.SugaredLogger, cfg *config.Config, client metalgo.Client) (*reporter, error) { return &reporter{ cfg: cfg, log: log, client: client, + sem: semaphore.NewWeighted(1), }, nil } @@ -35,40 +41,12 @@ func (r reporter) Run() { periodic := time.NewTicker(r.cfg.ReportInterval) signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) - for { select { case <-periodic.C: - ls, err := leases.ReadLeases(r.cfg.LeaseFile) - if err != nil { - r.log.Fatalw("could not parse leases file", "error", err) - } - active := ls.FilterActive() - byMac := active.LatestByMac() - r.log.Infow("reporting leases to metal-api", "all", len(ls), "active", len(active), "uniqueActive", len(byMac)) - - mtx := new(sync.Mutex) - var items []*leases.ReportItem - - wg := new(sync.WaitGroup) - wg.Add(len(byMac)) - - for _, l := range byMac { - item := leases.NewReportItem(l, r.log) - go func() { - item.EnrichWithBMCDetails(r.cfg.IpmiPort, r.cfg.IpmiUser, r.cfg.IpmiPassword) - mtx.Lock() - items = append(items, item) - wg.Done() - mtx.Unlock() - }() - } - - wg.Wait() - - err = r.report(items) + err := r.collectAndReport() if err != nil { - r.log.Warnw("could not report ipmi addresses", "error", err) + r.log.Errorw("collect and report", "error", err) } case <-signals: return @@ -76,6 +54,87 @@ func (r reporter) Run() { } } +func (r reporter) collectAndReport() error { + if !r.sem.TryAcquire(1) { + r.log.Warn("lease reporting is still running") + return nil + } + defer r.sem.Release(1) + + start := time.Now() + ls, err := leases.ReadLeases(r.cfg.LeaseFile) + if err != nil { + r.log.Errorw("could not parse leases file, partial results will considered", "error", err) + } + if len(ls) == 0 { + r.log.Warn("empty leases returned, nothing to report") + return nil + } + active := ls.FilterActive() + byMac := active.LatestByMac() + r.log.Infow("consider reporting leases to metal-api", "all", len(ls), "active", len(active), "uniqueActive", len(byMac)) + + var items []*leases.ReportItem + for _, l := range byMac { + l := l + if !r.isInAllowedCidr(l.Ip) { + continue + } + + if slices.Contains(r.cfg.IgnoreMacs, l.Mac) { + continue + } + + item := &leases.ReportItem{ + Lease: l, + Log: r.log, + } + items = append(items, item) + } + r.log.Infow("reporting leases to metal-api", "count", len(items)) + + g := new(errgroup.Group) + // Allow 20 goroutines run in parallel at max + g.SetLimit(20) + for _, item := range items { + item := item + g.Go(func() error { + item.EnrichWithBMCDetails(r.cfg.IpmiPort, r.cfg.IpmiUser, r.cfg.IpmiPassword) + return nil + }) + } + err = g.Wait() + if err != nil { + r.log.Errorw("could not enrich all ipmi details", "error", err) + } + + err = r.report(items) + if err != nil { + return fmt.Errorf("could not report ipmi addresses %w", err) + } + r.log.Infow("reporting leases to metal-api", "took", time.Since(start)) + return nil +} + +func (r reporter) isInAllowedCidr(ip string) bool { + parsedIP, err := netip.ParseAddr(ip) + if err != nil { + r.log.Errorw("given ip is not parsable", "ip", ip, "error", err) + return false + } + for _, cidr := range r.cfg.AllowedCidrs { + cidr := cidr + pfx, err := netip.ParsePrefix(cidr) + if err != nil { + return false + } + if pfx.Contains(parsedIP) { + return true + } + } + return false +} + // report will send all gathered information about machines to the metal-api func (r reporter) report(items []*leases.ReportItem) error { partitionID := r.cfg.PartitionID @@ -83,15 +142,8 @@ func (r reporter) report(items []*leases.ReportItem) error { for _, item := range items { item := item - mac := item.Mac - - if item.MacContainedIn(r.cfg.IgnoreMacs) { - continue - } - - ip := item.Ip if item.UUID == nil { - r.log.Errorw("could not determine uuid of device", "mac", mac, "ip", ip) + r.log.Errorw("could not determine uuid of device", "mac", item.Mac, "ip", item.Ip) continue } diff --git a/main.go b/main.go index 1a364bf..02d4046 100644 --- a/main.go +++ b/main.go @@ -3,8 +3,8 @@ package main import ( "fmt" - "github.com/metal-stack/metal-bmc/domain" "github.com/metal-stack/metal-bmc/internal/bmc" + "github.com/metal-stack/metal-bmc/pkg/config" metalgo "github.com/metal-stack/metal-go" "github.com/metal-stack/metal-bmc/internal/reporter" @@ -16,11 +16,15 @@ import ( ) func main() { - var cfg domain.Config + var cfg config.Config if err := envconfig.Process("METAL_BMC", &cfg); err != nil { panic(fmt.Errorf("bad configuration: %w", err)) } + if err := cfg.Validate(); err != nil { + panic(fmt.Errorf("bad configuration: %w", err)) + } + level, err := zap.ParseAtomicLevel(cfg.LogLevel) if err != nil { panic(fmt.Errorf("can't initialize zap logger: %w", err)) diff --git a/domain/config.go b/pkg/config/config.go similarity index 87% rename from domain/config.go rename to pkg/config/config.go index dec66fe..2ca3970 100644 --- a/domain/config.go +++ b/pkg/config/config.go @@ -1,12 +1,13 @@ -package domain +package config import ( + "net/netip" "net/url" "time" ) type Config struct { - // Valid log levels are: DEBUG, INFO, WARN, ERROR, FATAL and PANIC + // Valid log levels are: DEBUG, INFO, WARN, ERROR, FATAL LogLevel string `required:"false" default:"debug" desc:"set log level" split_words:"true"` PartitionID string `required:"true" desc:"set the partition ID" envconfig:"partition_id"` @@ -19,6 +20,7 @@ type Config struct { IpmiUser string `required:"false" default:"ADMIN" desc:"the ipmi user" split_words:"true"` IpmiPassword string `required:"false" default:"ADMIN" desc:"the ipmi password" split_words:"true"` IgnoreMacs []string `required:"false" desc:"mac addresses to ignore" split_words:"true"` + AllowedCidrs []string `required:"false" default:"0.0.0.0/0" desc:"filters dhcp leases" split_words:"true"` // NSQ connection parameters MQAddress string `required:"false" default:"localhost:4161" desc:"set the MQ server address" envconfig:"mq_address"` @@ -34,3 +36,14 @@ type Config struct { ConsoleCertFile string `required:"false" default:"cert.pem" desc:"cert file" envconfig:"console_cert_file"` ConsoleKeyFile string `required:"false" default:"key.pem" desc:"key file" envconfig:"console_key_file"` } + +func (c *Config) Validate() error { + for _, cidr := range c.AllowedCidrs { + cidr := cidr + _, err := netip.ParsePrefix(cidr) + if err != nil { + return err + } + } + return nil +}