Merge pull request #625 from kubescape/prometheus

Prometheus

Author: matthyx

pkg/containerwatcher/v2/containercallback.go

@@ -62,6 +62,7 @@ func (cw *ContainerWatcher) containerCallbackAsync(notif containercollection.Pub
 			helpers.String("k8s workload", k8sContainerID),
 			helpers.String("ContainerImageDigest", notif.Container.Runtime.ContainerImageDigest),
 			helpers.String("ContainerImageName", notif.Container.Runtime.ContainerImageName))
+		cw.metrics.ReportContainerStart()
 		// Check if Pod has a label of max sniffing time
 		sniffingTime := utils.AddJitter(cw.cfg.MaxSniffingTime, cw.cfg.MaxJitterPercentage)
 		if podLabelMaxSniffingTime, ok := notif.Container.K8s.PodLabels[MaxSniffingTimeLabel]; ok {
@@ -89,6 +90,7 @@ func (cw *ContainerWatcher) containerCallbackAsync(notif containercollection.Pub
 			helpers.String("k8s workload", k8sContainerID),
 			helpers.String("ContainerImageDigest", notif.Container.Runtime.ContainerImageDigest),
 			helpers.String("ContainerImageName", notif.Container.Runtime.ContainerImageName))
+		cw.metrics.ReportContainerStop()
 		cw.objectCache.K8sObjectCache().DeleteSharedContainerData(notif.Container.Runtime.ContainerID)
 	}
 }

pkg/containerwatcher/v2/event_handler_factory.go

@@ -245,10 +245,10 @@ func (ehf *EventHandlerFactory) registerHandlers(
 	ehf.handlers[utils.HTTPEventType] = []Manager{containerProfileManager, ruleManager, metrics}
 
 	// Ptrace events
-	ehf.handlers[utils.PtraceEventType] = []Manager{ruleManager}
+	ehf.handlers[utils.PtraceEventType] = []Manager{ruleManager, metrics}
 
 	// IoUring events
-	ehf.handlers[utils.IoUringEventType] = []Manager{ruleManager, rulePolicy}
+	ehf.handlers[utils.IoUringEventType] = []Manager{ruleManager, metrics, rulePolicy}
 
 	// Note: SyscallEventType is not registered here because the syscall tracer
 	// doesn't generate events - it only provides a peek function for other components

pkg/metricsmanager/metrics_manager_interface.go

@@ -15,4 +15,6 @@ type MetricsManager interface {
 	ReportRuleProcessed(ruleID string)
 	ReportRuleAlert(ruleID string)
 	ReportEbpfStats(stats *top.Event[toptypes.Stats])
+	ReportContainerStart()
+	ReportContainerStop()
 }

pkg/metricsmanager/metrics_manager_mock.go

@@ -53,3 +53,7 @@ func (m *MetricsMock) ReportRuleAlert(ruleID string) {
 
 func (m *MetricsMock) ReportEbpfStats(stats *top.Event[toptypes.Stats]) {
 }
+
+func (m *MetricsMock) ReportContainerStart() {}
+
+func (m *MetricsMock) ReportContainerStop() {}

pkg/metricsmanager/prometheus/prometheus.go

@@ -32,6 +32,12 @@ type PrometheusMetric struct {
 	ebpfCapabilityCounter prometheus.Counter
 	ebpfRandomXCounter    prometheus.Counter
 	ebpfFailedCounter     prometheus.Counter
+	ebpfSymlinkCounter    prometheus.Counter
+	ebpfHardlinkCounter   prometheus.Counter
+	ebpfSSHCounter        prometheus.Counter
+	ebpfHTTPCounter       prometheus.Counter
+	ebpfPtraceCounter     prometheus.Counter
+	ebpfIoUringCounter    prometheus.Counter
 	ruleCounter           *prometheus.CounterVec
 	alertCounter          *prometheus.CounterVec
 
@@ -45,6 +51,10 @@ type PrometheusMetric struct {
 	programCpuUsageGauge      *prometheus.GaugeVec
 	programPerCpuUsageGauge   *prometheus.GaugeVec
 
+	// Container metrics
+	containerStartCounter prometheus.Counter
+	containerStopCounter  prometheus.Counter
+
 	// Cache to avoid allocating Labels maps on every call
 	ruleCounterCache  map[string]prometheus.Counter
 	alertCounterCache map[string]prometheus.Counter
@@ -85,6 +95,30 @@ func NewPrometheusMetric() *PrometheusMetric {
 			Name: "node_agent_ebpf_event_failure_counter",
 			Help: "The total number of failed events received from the eBPF probe",
 		}),
+		ebpfSymlinkCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_symlink_counter",
+			Help: "The total number of symlink events received from the eBPF probe",
+		}),
+		ebpfHardlinkCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_hardlink_counter",
+			Help: "The total number of hardlink events received from the eBPF probe",
+		}),
+		ebpfSSHCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_ssh_counter",
+			Help: "The total number of SSH events received from the eBPF probe",
+		}),
+		ebpfHTTPCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_http_counter",
+			Help: "The total number of HTTP events received from the eBPF probe",
+		}),
+		ebpfPtraceCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_ptrace_counter",
+			Help: "The total number of ptrace events received from the eBPF probe",
+		}),
+		ebpfIoUringCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_iouring_counter",
+			Help: "The total number of io_uring events received from the eBPF probe",
+		}),
 		ruleCounter: promauto.NewCounterVec(prometheus.CounterOpts{
 			Name: "node_agent_rule_counter",
 			Help: "The total number of rules processed by the engine",
@@ -135,6 +169,16 @@ func NewPrometheusMetric() *PrometheusMetric {
 			Help: "Per-CPU usage of programs by program ID",
 		}, []string{programTypeLabel, programNameLabel}),
 
+		// Container metrics
+		containerStartCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_container_start_counter",
+			Help: "The total number of container start events",
+		}),
+		containerStopCounter: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "node_agent_container_stop_counter",
+			Help: "The total number of container stop events",
+		}),
+
 		// Initialize counter caches
 		ruleCounterCache:  make(map[string]prometheus.Counter),
 		alertCounterCache: make(map[string]prometheus.Counter),
@@ -161,7 +205,14 @@ func (p *PrometheusMetric) Destroy() {
 	prometheus.Unregister(p.ebpfFailedCounter)
 	prometheus.Unregister(p.ruleCounter)
 	prometheus.Unregister(p.alertCounter)
-
+	prometheus.Unregister(p.ebpfSymlinkCounter)
+	prometheus.Unregister(p.ebpfHardlinkCounter)
+	prometheus.Unregister(p.ebpfSSHCounter)
+	prometheus.Unregister(p.ebpfHTTPCounter)
+	prometheus.Unregister(p.ebpfPtraceCounter)
+	prometheus.Unregister(p.ebpfIoUringCounter)
+	prometheus.Unregister(p.containerStartCounter)
+	prometheus.Unregister(p.containerStopCounter)
 	// Unregister program ID metrics
 	prometheus.Unregister(p.programRuntimeGauge)
 	prometheus.Unregister(p.programRunCountGauge)
@@ -175,6 +226,8 @@ func (p *PrometheusMetric) Destroy() {
 
 func (p *PrometheusMetric) ReportEvent(eventType utils.EventType) {
 	switch eventType {
+	case utils.CapabilitiesEventType:
+		p.ebpfCapabilityCounter.Inc()
 	case utils.ExecveEventType:
 		p.ebpfExecCounter.Inc()
 	case utils.OpenEventType:
@@ -183,12 +236,22 @@ func (p *PrometheusMetric) ReportEvent(eventType utils.EventType) {
 		p.ebpfNetworkCounter.Inc()
 	case utils.DnsEventType:
 		p.ebpfDNSCounter.Inc()
-	case utils.SyscallEventType:
-		p.ebpfSyscallCounter.Inc()
-	case utils.CapabilitiesEventType:
-		p.ebpfCapabilityCounter.Inc()
 	case utils.RandomXEventType:
 		p.ebpfRandomXCounter.Inc()
+	case utils.SymlinkEventType:
+		p.ebpfSymlinkCounter.Inc()
+	case utils.HardlinkEventType:
+		p.ebpfHardlinkCounter.Inc()
+	case utils.SSHEventType:
+		p.ebpfSSHCounter.Inc()
+	case utils.HTTPEventType:
+		p.ebpfHTTPCounter.Inc()
+	case utils.PtraceEventType:
+		p.ebpfPtraceCounter.Inc()
+	case utils.IoUringEventType:
+		p.ebpfIoUringCounter.Inc()
+	case utils.SyscallEventType:
+		p.ebpfSyscallCounter.Inc()
 	}
 }
 
@@ -271,3 +334,11 @@ func (p *PrometheusMetric) ReportEbpfStats(stats *top.Event[toptypes.Stats]) {
 		p.programPerCpuUsageGauge.With(labels).Set(stat.PerCpuUsage)
 	}
 }
+
+func (p *PrometheusMetric) ReportContainerStart() {
+	p.containerStartCounter.Inc()
+}
+
+func (p *PrometheusMetric) ReportContainerStop() {
+	p.containerStopCounter.Inc()
+}