From 7f15f777f8a1b71ae14120baeeab2f05fa0a0df8 Mon Sep 17 00:00:00 2001 From: Atul-source Date: Mon, 24 Mar 2025 12:53:52 +0530 Subject: [PATCH 1/5] added ipv4 type monitor key Signed-off-by: Atul-source --- bpfprogs/bpf.go | 28 +++++++++----- bpfprogs/bpfmap.go | 82 ++++++++++++++++++++++++++++++++++++++--- bpfprogs/bpfmap_test.go | 2 +- models/l3afd.go | 5 ++- restart/restart.go | 2 +- stats/metrics.go | 12 ++++-- 6 files changed, 107 insertions(+), 24 deletions(-) diff --git a/bpfprogs/bpf.go b/bpfprogs/bpf.go index 2d3f802d..958f7791 100644 --- a/bpfprogs/bpf.go +++ b/bpfprogs/bpf.go @@ -21,7 +21,6 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "strings" "sync" "time" @@ -713,7 +712,7 @@ func (b *BPF) GetBPFMap(mapName string) (*BPFMap, error) { } // Add eBPF map into BPFMaps list -func (b *BPF) AddMetricsBPFMap(mapName, aggregator string, key, samplesLength int) error { +func (b *BPF) AddMetricsBPFMap(mapName, key, aggregator string, samplesLength int) error { var tmpMetricsBPFMap MetricsBPFMap bpfMap, err := b.GetBPFMap(mapName) if err != nil { @@ -725,8 +724,8 @@ func (b *BPF) AddMetricsBPFMap(mapName, aggregator string, key, samplesLength in tmpMetricsBPFMap.Aggregator = aggregator tmpMetricsBPFMap.Values = ring.New(samplesLength) - log.Info().Msgf("added Metrics map ID %d Name %s Type %s Key %d Aggregator %s", tmpMetricsBPFMap.MapID, tmpMetricsBPFMap.Name, tmpMetricsBPFMap.Type, tmpMetricsBPFMap.Key, tmpMetricsBPFMap.Aggregator) - map_key := mapName + strconv.Itoa(key) + aggregator + log.Info().Msgf("added Metrics map ID %d Name %s Type %s Key %s Aggregator %s", tmpMetricsBPFMap.MapID, tmpMetricsBPFMap.Name, tmpMetricsBPFMap.Type, tmpMetricsBPFMap.Key, tmpMetricsBPFMap.Aggregator) + map_key := mapName + key + aggregator b.MetricsBpfMaps[map_key] = &tmpMetricsBPFMap return nil @@ -735,17 +734,26 @@ func (b *BPF) AddMetricsBPFMap(mapName, aggregator string, key, samplesLength in // This method to fetch values from bpf maps and publish to metrics func (b *BPF) MonitorMaps(ifaceName string, intervals int) error { for _, element := range b.Program.MonitorMaps { - log.Debug().Msgf("monitor maps element %s key %d aggregator %s", element.Name, element.Key, element.Aggregator) - mapKey := element.Name + strconv.Itoa(element.Key) + element.Aggregator + log.Debug().Msgf("monitor maps element %s key %s aggregator %s", element.Name, element.Key, element.Aggregator) + mapKey := element.Name + element.Key + element.Aggregator _, ok := b.MetricsBpfMaps[mapKey] if !ok { - if err := b.AddMetricsBPFMap(element.Name, element.Aggregator, element.Key, intervals); err != nil { - return fmt.Errorf("unable to fetch map %s key %d aggregator %s : %w", element.Name, element.Key, element.Aggregator, err) + if err := b.AddMetricsBPFMap(element.Name, element.Key, element.Aggregator, intervals); err != nil { + return fmt.Errorf("unable to fetch map %s key %s aggregator %s : %w", element.Name, element.Key, element.Aggregator, err) } } bpfMap := b.MetricsBpfMaps[mapKey] - MetricName := element.Name + "_" + strconv.Itoa(element.Key) + "_" + element.Aggregator - stats.SetValue(bpfMap.GetValue(), stats.BPFMonitorMap, b.Program.Name, MetricName, ifaceName) + if element.Key != "*" { + stats.SetValue(bpfMap.GetValue(element.KeyType), stats.BPFMonitorMap, b.Program.Name, element.Name, ifaceName, bpfMap.Aggregator, element.Key) + } else { + allKV, err := bpfMap.GetAllKeysAndValues(element.KeyType) + if err != nil { + return fmt.Errorf("GetAllKeysAndValues failed with error %w", err) + } + for _, kv := range allKV { + stats.SetValue(kv.Value, stats.BPFMonitorMap, b.Program.Name, element.Name, ifaceName, bpfMap.Aggregator, kv.Key) + } + } } return nil } diff --git a/bpfprogs/bpfmap.go b/bpfprogs/bpfmap.go index b2bc56e4..6faef8d3 100644 --- a/bpfprogs/bpfmap.go +++ b/bpfprogs/bpfmap.go @@ -8,6 +8,8 @@ import ( "errors" "fmt" "math" + "net/netip" + "strconv" "unsafe" "github.com/cilium/ebpf" @@ -24,10 +26,15 @@ type BPFMap struct { BPFProg *BPF `json:"-"` } +type MetricKeyValue struct { + Key string + Value float64 +} + // This stores Metrics map details. type MetricsBPFMap struct { BPFMap - Key int + Key string Values *ring.Ring Aggregator string LastValue float64 @@ -87,7 +94,7 @@ func (b *BPFMap) Update(key, value int) error { // max-rate - this calculates delta requests / sec and stores absolute value. // avg - stores the values in the circular queue // We can implement more aggregate function as needed. -func (b *MetricsBPFMap) GetValue() float64 { +func (b *MetricsBPFMap) GetValue(keyType string) float64 { ebpfMap, err := ebpf.NewMapFromID(b.MapID) if err != nil { // We have observed in smaller configuration VM's, if we restart BPF's @@ -107,11 +114,33 @@ func (b *MetricsBPFMap) GetValue() float64 { } } defer ebpfMap.Close() - var value int64 - if err = ebpfMap.Lookup(unsafe.Pointer(&b.Key), unsafe.Pointer(&value)); err != nil { - log.Warn().Err(err).Msgf("GetValue Lookup failed : Name %s ID %d", b.Name, b.MapID) - return 0 + switch keyType { + case "int": + var k int + k, err = strconv.Atoi(b.Key) + if err != nil { + log.Warn().Err(err).Msgf("conversion of key to int failed : Name %s ID %d", b.Name, b.MapID) + return 0 + } + if err = ebpfMap.Lookup(unsafe.Pointer(&k), unsafe.Pointer(&value)); err != nil { + log.Warn().Err(err).Msgf("GetValue Lookup failed : Name %s ID %d", b.Name, b.MapID) + return 0 + } + break + case "ipv4": + ip, err := netip.ParseAddr(b.Key) + if err != nil { + log.Warn().Err(err).Msgf("Parsing of IP address failed %v : Name %s ID %d", b.Key, b.Name, b.MapID) + return 0 + } + if err = ebpfMap.Lookup(ip, unsafe.Pointer(&value)); err != nil { + log.Warn().Err(err).Msgf("GetValue Lookup failed : Name %s ID %d", b.Name, b.MapID) + return 0 + } + break + default: + fmt.Errorf("unsupported keytype") } var retVal float64 @@ -133,6 +162,47 @@ func (b *MetricsBPFMap) GetValue() float64 { return retVal } +func (b *MetricsBPFMap) GetAllKeysAndValues(KeyType string) ([]MetricKeyValue, error) { + var result []MetricKeyValue + ebpfMap, err := ebpf.NewMapFromID(b.MapID) + if err != nil { + return nil, fmt.Errorf("access new map from ID failed %w", err) + } + defer ebpfMap.Close() + if KeyType == "ipv4" { + var key netip.Addr + var val int64 + iter := ebpfMap.Iterate() + for iter.Next(&key, &val) { + result = append(result, MetricKeyValue{ + Key: key.String(), + Value: float64(val), + }) + } + if iter.Err() != nil { + return nil, iter.Err() + } + } else { + if KeyType == "int" { + var key int + var val int64 + iter := ebpfMap.Iterate() + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&val)) { + result = append(result, MetricKeyValue{ + Key: strconv.Itoa(key), + Value: float64(val), + }) + } + if iter.Err() != nil { + return nil, iter.Err() + } + } else { + return nil, fmt.Errorf("unsupported keytype") + } + } + return result, nil +} + // This method finds the max value in the circular list func (b *MetricsBPFMap) MaxValue() float64 { tmp := b.Values diff --git a/bpfprogs/bpfmap_test.go b/bpfprogs/bpfmap_test.go index a5fac304..ddf2a116 100644 --- a/bpfprogs/bpfmap_test.go +++ b/bpfprogs/bpfmap_test.go @@ -79,7 +79,7 @@ func TestMetricsBPFMapAvgValue(t *testing.T) { t.Run(tt.name, func(t *testing.T) { metricsMap := &MetricsBPFMap{ Values: TestValues, - Key: 0, + Key: "0", Aggregator: tt.args.aggregator, LastValue: 0, } diff --git a/models/l3afd.go b/models/l3afd.go index 00741d62..90be67c2 100644 --- a/models/l3afd.go +++ b/models/l3afd.go @@ -69,8 +69,9 @@ type BPFProgram struct { // L3afDNFMetricsMap defines BPF map type L3afDNFMetricsMap struct { Name string `json:"name"` // BPF map name - Key int `json:"key"` // Index of the bpf map + Key string `json:"key"` // Index of the bpf map Aggregator string `json:"aggregator"` // Aggregation function names + KeyType string `json:"key_type"` // Type of key } // KeyValue defines struct for key and value @@ -122,7 +123,7 @@ type MetaColl struct { type MetaMetricsBPFMap struct { MapName string - Key int + Key string Values []float64 Aggregator string LastValue float64 diff --git a/restart/restart.go b/restart/restart.go index fac22371..962f14af 100644 --- a/restart/restart.go +++ b/restart/restart.go @@ -274,7 +274,7 @@ func SetMetrics(t models.L3AFALLHOSTDATA) { getValueofLabel("version", f.Labels), getValueofLabel("direction", f.Labels), getValueofLabel("interface_name", f.Labels)) } else if len(getValueofLabel("map_name", f.Labels)) > 0 { stats.SetValue(f.Value, getGaugeVecByMetricName(f.MetricName), getValueofLabel("ebpf_program", f.Labels), - getValueofLabel("map_name", f.Labels), getValueofLabel("interface_name", f.Labels)) + getValueofLabel("map_name", f.Labels), getValueofLabel("interface_name", f.Labels), getValueofLabel("aggregator", f.Labels), getValueofLabel("key", f.Labels)) } else { stats.Set(f.Value, getGaugeVecByMetricName(f.MetricName), getValueofLabel("ebpf_program", f.Labels), getValueofLabel("direction", f.Labels), getValueofLabel("interface_name", f.Labels)) diff --git a/stats/metrics.go b/stats/metrics.go index 87d2c702..e3025f6d 100644 --- a/stats/metrics.go +++ b/stats/metrics.go @@ -5,6 +5,7 @@ package stats import ( "errors" + "fmt" "net" "net/http" @@ -108,7 +109,7 @@ func SetupMetrics(hostname, daemonName, metricsAddr string) { Name: "BPFMonitorMap", Help: "This value indicates BPF program monitor counters", }, - []string{"host", "ebpf_program", "map_name", "interface_name"}, + []string{"host", "ebpf_program", "map_name", "interface_name", "key", "aggregator"}, ) if err := prometheus.Register(bpfMonitorMapVec); err != nil { @@ -197,7 +198,7 @@ func Set(value float64, gaugeVec *prometheus.GaugeVec, ebpfProgram, direction, i } // Set gaugevec metrics value with given mapName and other fields -func SetValue(value float64, gaugeVec *prometheus.GaugeVec, ebpfProgram, mapName, ifaceName string) { +func SetValue(value float64, gaugeVec *prometheus.GaugeVec, ebpfProgram, mapName, ifaceName, aggregator, key string) { if gaugeVec == nil { log.Warn().Msg("Metrics: gauge vector is nil and needs to be initialized before SetValue") @@ -208,11 +209,14 @@ func SetValue(value float64, gaugeVec *prometheus.GaugeVec, ebpfProgram, mapName "ebpf_program": ebpfProgram, "map_name": mapName, "interface_name": ifaceName, + "key": key, + "aggregator": aggregator, }), ) if err != nil { - log.Warn().Msgf("Metrics: unable to fetch gauge with fields: ebpf_program: %s, map_name: %s, interface_name: %s", - ebpfProgram, mapName, ifaceName) + log.Warn().Msgf("Metrics: unable to fetch gauge with fields: ebpf_program: %s, map_name: %s, interface_name: %s , key %s , aggregator %s", + ebpfProgram, mapName, ifaceName, key, aggregator) + fmt.Println(err) return } bpfGauge.Set(value) From 5465d1415cf80e89d506f853f6283704e7674fa3 Mon Sep 17 00:00:00 2001 From: Atul-source Date: Mon, 24 Mar 2025 14:29:28 +0530 Subject: [PATCH 2/5] added temporory changes to run tests Signed-off-by: Atul-source --- .github/workflows/ci-e2e.yaml | 2 +- bpfprogs/bpfmap.go | 5 ++--- bpfprogs/bpfmap_test.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-e2e.yaml b/.github/workflows/ci-e2e.yaml index f5ee4703..1c26f90d 100644 --- a/.github/workflows/ci-e2e.yaml +++ b/.github/workflows/ci-e2e.yaml @@ -33,7 +33,7 @@ jobs: - name: Prep run: | sudo cp -r /home/runner/work/l3afd/l3afd /root - sudo git clone https://github.com/l3af-project/l3af-arch.git /root/l3af-arch + sudo git clone -b output-all-metrics https://github.com/Atul-source/l3af-arch.git /root/l3af-arch sudo bash /root/l3af-arch/dev_environment/e2e_test/prep_env.sh sudo bash /root/l3af-arch/dev_environment/setup_linux_dev_env.sh --ci-build hm=$(hostname) diff --git a/bpfprogs/bpfmap.go b/bpfprogs/bpfmap.go index 6faef8d3..6d55ab2c 100644 --- a/bpfprogs/bpfmap.go +++ b/bpfprogs/bpfmap.go @@ -127,7 +127,6 @@ func (b *MetricsBPFMap) GetValue(keyType string) float64 { log.Warn().Err(err).Msgf("GetValue Lookup failed : Name %s ID %d", b.Name, b.MapID) return 0 } - break case "ipv4": ip, err := netip.ParseAddr(b.Key) if err != nil { @@ -138,9 +137,9 @@ func (b *MetricsBPFMap) GetValue(keyType string) float64 { log.Warn().Err(err).Msgf("GetValue Lookup failed : Name %s ID %d", b.Name, b.MapID) return 0 } - break default: - fmt.Errorf("unsupported keytype") + log.Warn().Msgf("unsupported keytype %s", keyType) + return 0 } var retVal float64 diff --git a/bpfprogs/bpfmap_test.go b/bpfprogs/bpfmap_test.go index ddf2a116..bf854e8b 100644 --- a/bpfprogs/bpfmap_test.go +++ b/bpfprogs/bpfmap_test.go @@ -43,7 +43,7 @@ func TestMetricsBPFMapMaxValue(t *testing.T) { t.Run(tt.name, func(t *testing.T) { metricsMap := &MetricsBPFMap{ Values: TestValues, - Key: 0, + Key: "0", Aggregator: tt.args.aggregator, LastValue: 0, } From 683d2139aa3f6567236b8172ec5c739a5a2603a5 Mon Sep 17 00:00:00 2001 From: Atul-source Date: Wed, 23 Apr 2025 10:45:56 +0530 Subject: [PATCH 3/5] Your commit message From 29e5dbaaec8ceac8e5403723fd4280e3c3966c61 Mon Sep 17 00:00:00 2001 From: Atul-source Date: Wed, 23 Apr 2025 10:54:40 +0530 Subject: [PATCH 4/5] testing From aee1bbe5394dc327d151c38ad7758717d010a0a5 Mon Sep 17 00:00:00 2001 From: Atul-source Date: Wed, 23 Apr 2025 11:09:22 +0530 Subject: [PATCH 5/5] testing