diff --git a/.mk/bc.mk b/.mk/bc.mk index c0b5f2db8..f7e7640c1 100644 --- a/.mk/bc.mk +++ b/.mk/bc.mk @@ -22,7 +22,8 @@ define PROGRAMS "xfrm_input_kprobe": "kprobe", "xfrm_input_kretprobe": "kretprobe", "xfrm_output_kprobe": "kprobe", - "xfrm_output_kretprobe": "kretprobe" + "xfrm_output_kretprobe": "kretprobe", + "probe_entry_SSL_write":"uprobe", } endef @@ -38,7 +39,8 @@ define MAPS "filter_map":"lpm_trie", "peer_filter_map":"lpm_trie", "ipsec_ingress_map":"hash", - "ipsec_egress_map":"hash" + "ipsec_egress_map":"hash", + "ssl_data_event_map":"ringbuf" } endef diff --git a/Dockerfile b/Dockerfile index 2662e64f7..98b696314 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ ARG TARGETARCH # Build the manager binary FROM docker.io/library/golang:1.24 as builder -ARG TARGETARCH ARG LDFLAGS WORKDIR /opt/app-root diff --git a/Makefile b/Makefile index aa3ebd979..84ff8e8d1 100644 --- a/Makefile +++ b/Makefile @@ -205,7 +205,7 @@ create-and-deploy-kind-cluster: prereqs ## Create a kind cluster and deploy the .PHONY: destroy-kind-cluster destroy-kind-cluster: ## Destroy the kind cluster. - oc delete -f scripts/agent.yml + kubectl delete -f scripts/agent.yml kind delete cluster ##@ Images diff --git a/bpf/configs.h b/bpf/configs.h index 0a61ac1de..8bf24bb22 100644 --- a/bpf/configs.h +++ b/bpf/configs.h @@ -15,4 +15,5 @@ volatile const u8 enable_network_events_monitoring = 0; volatile const u8 network_events_monitoring_groupid = 0; volatile const u8 enable_pkt_translation_tracking = 0; volatile const u8 enable_ipsec = 0; +volatile const u8 enable_ssl = 0; #endif //__CONFIGS_H__ diff --git a/bpf/flows.c b/bpf/flows.c index 6602c1904..ecbf46300 100644 --- a/bpf/flows.c +++ b/bpf/flows.c @@ -57,6 +57,11 @@ */ #include "ipsec.h" +/* + * Defines ssl tracker + */ +#include "openssl_tracker.h" + // return 0 on success, 1 if capacity reached static __always_inline int add_observed_intf(flow_metrics *value, pkt_info *pkt, u32 if_index, u8 direction) { diff --git a/bpf/maps_definition.h b/bpf/maps_definition.h index 72cec2000..08a7662ba 100644 --- a/bpf/maps_definition.h +++ b/bpf/maps_definition.h @@ -97,4 +97,11 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } ipsec_egress_map SEC(".maps"); +// Ringbuf for SSL data events +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 27); // 16KB * 1000 events/sec * 5sec "eviction time" = ~128MB + __uint(pinning, LIBBPF_PIN_BY_NAME); +} ssl_data_event_map SEC(".maps"); + #endif //__MAPS_DEFINITION_H__ diff --git a/bpf/openssl_tracker.h b/bpf/openssl_tracker.h new file mode 100644 index 000000000..ef9d16cb4 --- /dev/null +++ b/bpf/openssl_tracker.h @@ -0,0 +1,65 @@ +/* + * OpenSSL monitoring uprobe/uretprobe eBPF hook. + */ + +#ifndef __OPENSSL_TRACKER_H__ +#define __OPENSSL_TRACKER_H__ + +#include "utils.h" + +static inline void generate_SSL_data_event(struct pt_regs *ctx, u64 pid_tgid, u8 ssl_type, + const char *buf, int len) { + if (len <= 0) { + return; + } + + struct ssl_data_event_t *event; + event = bpf_ringbuf_reserve(&ssl_data_event_map, sizeof(*event), 0); + if (!event) { + return; + } + event->timestamp_ns = bpf_ktime_get_ns(); + event->pid_tgid = pid_tgid; + event->ssl_type = ssl_type; + event->data_len = len < MAX_DATA_SIZE_OPENSSL ? len : MAX_DATA_SIZE_OPENSSL; + bpf_probe_read_user(&event->data, event->data_len, buf); + bpf_ringbuf_submit(event, 0); +} + +// int SSL_write(SSL *ssl, const void *buf, int num); +// https://github.com/openssl/openssl/blob/master/ssl/ssl_lib.c#L2666 +SEC("uprobe/SSL_write") +int probe_entry_SSL_write(struct pt_regs *ctx) { + if (enable_ssl == 0) { + return 0; + } + + u64 pid_tgid = bpf_get_current_pid_tgid(); + + BPF_PRINTK("openssl uprobe/SSL_write pid: %d\n", pid_tgid); + // https://github.com/openssl/openssl/blob/master/ssl/ssl_local.h#L1233 + void *ssl = (void *)PT_REGS_PARM1(ctx); + + u32 ssl_type; + int ret; + + ret = bpf_probe_read_user(&ssl_type, sizeof(ssl_type), (u32 *)ssl); + if (ret) { + BPF_PRINTK("(OPENSSL) bpf_probe_read ssl_type_ptr failed, ret: %d\n", ret); + return 0; + } + const char *buf = (const char *)PT_REGS_PARM2(ctx); + int num = (int)PT_REGS_PARM3(ctx); // Third parameter: number of bytes to write + + BPF_PRINTK("openssl uprobe/SSL_write type: %d, buf: %p, num: %d\n", ssl_type, buf, num); + + // Read the data immediately in the uprobe (before SSL_write processes it) + // This captures the plaintext before encryption + if (num > 0) { + generate_SSL_data_event(ctx, pid_tgid, ssl_type, buf, num); + } + + return 0; +} + +#endif /* __OPENSSL_TRACKER_H__ */ \ No newline at end of file diff --git a/bpf/types.h b/bpf/types.h index 500620de4..412583def 100644 --- a/bpf/types.h +++ b/bpf/types.h @@ -277,4 +277,17 @@ struct filter_value_t { // Force emitting enums/structs into the ELF const static struct filter_value_t *unused12 __attribute__((unused)); +#define MAX_DATA_SIZE_OPENSSL 1024 * 16 +// SSL data event +struct ssl_data_event_t { + u64 timestamp_ns; + u64 pid_tgid; + s32 data_len; + u8 ssl_type; + char data[MAX_DATA_SIZE_OPENSSL]; +} ssl_data_event; + +// Force emitting enums/structs into the ELF +const static struct ssl_data_event_t *unused13 __attribute__((unused)); + #endif /* __TYPES_H__ */ diff --git a/examples/test-ssl-host.sh b/examples/test-ssl-host.sh new file mode 100755 index 000000000..d5e4c356a --- /dev/null +++ b/examples/test-ssl-host.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Test SSL tracking with HOST processes (not containers) +# +# This script tests SSL/TLS tracking functionality by executing HTTPS requests +# directly on cluster nodes (host processes) and verifying that the NetObserv +# agent captures SSL events via eBPF uprobes. +# +# Prerequisites: +# - Kubernetes cluster (kind/minikube/etc) +# - NetObserv agent deployed with EnableSSL: true +# - Agent configured with correct OpenSSL library path +# +# Note: Some tests (TLS 1.3, HTTP/2) are optional and won't cause failure +# if not supported on the node. + +# Don't exit on error - we want to run all tests and report results +# Steps to test on Kind cluster: +# make create-and-deploy-kind-cluster +# export KUBECONFIG=$(pwd)/scripts/kubeconfig +# ./examples/test-ssl-host.sh + +set +e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}=== Testing SSL with Host Process ===${NC}" +echo "" +echo "This will run various SSL/TLS tests on each cluster node directly on the host" +echo "This should trigger the SSL uprobes since the host process uses" +echo "the same libssl.so that the agent attached to." +echo "" + +# Get all node names +NODES=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') + +# Counter for tests +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 + +run_test() { + local node=$1 + local test_name=$2 + local curl_cmd=$3 + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + echo -e "${YELLOW}[TEST $TOTAL_TESTS] $test_name${NC}" + + if docker exec $node bash -c "$curl_cmd" > /dev/null 2>&1; then + echo -e "${GREEN}✓ Request completed successfully${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + return 0 + else + echo -e "${RED}✗ Request failed${NC}" + FAILED_TESTS=$((FAILED_TESTS + 1)) + return 1 + fi +} + +check_ssl_events() { + local agent_pod=$1 + local test_desc=$2 + + echo -e "${BLUE}Checking logs for SSL events after $test_desc:${NC}" + local ssl_events=$(kubectl logs -n netobserv-privileged $agent_pod --tail=100 | grep 'SSL EVENT' | tail -5) + + if [ -z "$ssl_events" ]; then + echo -e "${YELLOW}No SSL events found in logs${NC}" + else + echo -e "${GREEN}SSL events found:${NC}" + echo "$ssl_events" + fi + echo "" +} + +for NODE in $NODES; do + echo "=========================================" + echo -e "${BLUE}Testing node: $NODE${NC}" + echo "=========================================" + + # Get the agent pod running on this node + AGENT_POD=$(kubectl get pods -n netobserv-privileged -l k8s-app=netobserv-ebpf-agent \ + --field-selector spec.nodeName=$NODE \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | head -n1) + + if [ -z "$AGENT_POD" ]; then + echo -e "${RED}Warning: No agent pod found on node $NODE, skipping...${NC}" + continue + fi + + echo -e "${GREEN}Agent pod: $AGENT_POD${NC}" + echo "" + + # Show diagnostic information + echo -e "${BLUE}Node diagnostics:${NC}" + echo -n " curl version: " + docker exec $NODE curl --version 2>/dev/null | head -1 || echo "unknown" + echo -n " OpenSSL version: " + docker exec $NODE openssl version 2>/dev/null || echo "unknown" + echo -n " libssl location: " + docker exec $NODE bash -c "ls -la /usr/lib*/libssl.so* 2>/dev/null | head -1 || echo 'not found in standard location'" + echo "" + + # Check if agent has SSL tracking enabled + echo -e "${BLUE}Agent SSL tracking status:${NC}" + if kubectl logs -n netobserv-privileged $AGENT_POD --tail=100 | grep -q "SSL tracking enabled"; then + echo -e " ${GREEN}✓ SSL tracking is enabled${NC}" + kubectl logs -n netobserv-privileged $AGENT_POD --tail=100 | grep "SSL tracking enabled" | tail -1 + else + echo -e " ${YELLOW}⚠ SSL tracking status unclear (check agent configuration)${NC}" + fi + echo "" + + # Test 1: Basic HTTPS GET with HTTP/1.1 + run_test "$NODE" "Basic HTTPS GET with HTTP/1.1" \ + "curl -s --http1.1 --max-time 10 https://httpbin.org/get" + + # Test 2: HTTPS POST with data + run_test "$NODE" "HTTPS POST with JSON data" \ + "curl -s --http1.1 --max-time 10 -X POST https://httpbin.org/post -H 'Content-Type: application/json' -d '{\"test\":\"data\"}'" + + # Test 3: HTTPS with TLS 1.2 + run_test "$NODE" "HTTPS with TLS 1.2 explicitly" \ + "curl -s --tlsv1.2 --tls-max 1.2 --max-time 10 https://www.howsmyssl.com/a/check" + + # Test 4: HTTPS with TLS 1.3 (optional - may not be supported) + echo -e "${YELLOW}[TEST $((TOTAL_TESTS + 1))] HTTPS with TLS 1.3 explicitly (optional)${NC}" + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + + # First check if TLS 1.3 is supported + if docker exec $NODE bash -c "curl --help all 2>/dev/null | grep -q tlsv1.3" 2>/dev/null; then + if docker exec $NODE bash -c "curl -s --tlsv1.3 --max-time 10 https://www.howsmyssl.com/a/check" > /dev/null 2>&1; then + echo -e "${GREEN}✓ Request completed successfully (TLS 1.3 supported)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + else + # Try alternative endpoint + if docker exec $NODE bash -c "curl -s --tlsv1.3 --max-time 10 https://www.cloudflare.com" > /dev/null 2>&1; then + echo -e "${GREEN}✓ Request completed successfully with alternative endpoint${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + else + echo -e "${YELLOW}⚠ TLS 1.3 option exists but connection failed (this is OK)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + fi + fi + else + echo -e "${YELLOW}⚠ TLS 1.3 not supported by curl on this node (skipped)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + fi + + # Test 5: HTTPS with headers + run_test "$NODE" "HTTPS with custom headers" \ + "curl -s --http1.1 --max-time 10 -H 'User-Agent: NetObserv-Test/1.0' -H 'X-Test-Header: SSL-Tracking' https://httpbin.org/headers" + + # Test 6: Different endpoint - github API + run_test "$NODE" "HTTPS to GitHub API" \ + "curl -s --http1.1 --max-time 10 https://api.github.com" + + # Test 7: Different endpoint - Google + run_test "$NODE" "HTTPS to Google" \ + "curl -s --http1.1 --max-time 10 -L https://www.google.com" + + # Test 8: HTTPS with large response + run_test "$NODE" "HTTPS with large response (1KB)" \ + "curl -s --http1.1 --max-time 10 https://httpbin.org/bytes/1024" + + # Test 9: HTTPS with HTTP/2 (optional - may not be supported) + echo -e "${YELLOW}[TEST $((TOTAL_TESTS + 1))] HTTPS with HTTP/2 (optional)${NC}" + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + + if docker exec $NODE bash -c "curl --help all 2>/dev/null | grep -q http2" 2>/dev/null; then + if docker exec $NODE bash -c "curl -s --http2 --max-time 10 https://www.google.com" > /dev/null 2>&1; then + echo -e "${GREEN}✓ Request completed successfully (HTTP/2 supported)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + else + echo -e "${YELLOW}⚠ HTTP/2 option exists but connection failed (this is OK)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + fi + else + echo -e "${YELLOW}⚠ HTTP/2 not supported by curl on this node (skipped)${NC}" + PASSED_TESTS=$((PASSED_TESTS + 1)) + fi + + echo "" + check_ssl_events "$AGENT_POD" "all tests" + + echo -e "${BLUE}Detailed SSL event analysis:${NC}" + kubectl logs -n netobserv-privileged $AGENT_POD --tail=200 | grep -A 2 "SSL EVENT" | tail -20 || echo "No detailed SSL events found" + + echo "" + echo -e "${BLUE}Node $NODE test summary:${NC}" + echo " Total tests: $TOTAL_TESTS" + echo -e " ${GREEN}Passed: $PASSED_TESTS${NC}" + echo -e " ${RED}Failed: $FAILED_TESTS${NC}" + echo "" +done + +echo "=========================================" +echo -e "${BLUE}Test completed for all nodes${NC}" +echo "=========================================" +echo "" +echo -e "${BLUE}Overall Summary:${NC}" +echo " Total tests executed: $TOTAL_TESTS" +echo -e " ${GREEN}Passed: $PASSED_TESTS${NC}" +echo -e " ${RED}Failed: $FAILED_TESTS${NC}" +echo "" + +# Calculate pass rate +if [ $TOTAL_TESTS -gt 0 ]; then + PASS_RATE=$((PASSED_TESTS * 100 / TOTAL_TESTS)) + echo " Pass rate: ${PASS_RATE}%" +fi diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index ce14b3d53..ea03bb4a2 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -86,7 +86,8 @@ type Flows struct { promoServer *http.Server sampleDecoder *ovnobserv.SampleDecoder - metrics *metrics.Metrics + metrics *metrics.Metrics + rbSSLTracer *flow.RingBufTracer } // ebpfFlowFetcher abstracts the interface of ebpf.FlowFetcher to allow dependency injection in tests @@ -97,6 +98,7 @@ type ebpfFlowFetcher interface { LookupAndDeleteMap(*metrics.Metrics) map[ebpf.BpfFlowId]model.BpfFlowContent DeleteMapsStaleEntries(timeOut time.Duration) ReadRingBuf() (ringbuf.Record, error) + ReadSSLRingBuf() (ringbuf.Record, error) } // FlowsAgent instantiates a new agent, given a configuration. @@ -175,6 +177,8 @@ func FlowsAgent(cfg *config.Agent) (*Flows, error) { BpfManBpfFSPath: cfg.BpfManBpfFSPath, EnableIPsecTracker: cfg.EnableIPsecTracking, FilterConfig: filterRules, + EnableSSL: cfg.EnableSSL, + OpenSSLPath: cfg.OpenSSLPath, } fetcher, err := tracer.NewFlowFetcher(ebpfConfig, m) @@ -206,6 +210,10 @@ func flowsAgent( mapTracer := flow.NewMapTracer(fetcher, cfg.CacheActiveTimeout, cfg.StaleEntriesEvictTimeout, m, s, cfg.EnableUDNMapping) rbTracer := flow.NewRingBufTracer(fetcher, mapTracer, cfg.CacheActiveTimeout, m) + var rbSSLTracer *flow.RingBufTracer + if cfg.EnableSSL { + rbSSLTracer = flow.NewSSLRingBufTracer(fetcher, mapTracer, cfg.CacheActiveTimeout, m) + } accounter := flow.NewAccounter(cfg.CacheMaxFlows, cfg.CacheActiveTimeout, time.Now, monotime.Now, m, s, cfg.EnableUDNMapping) limiter := flow.NewCapacityLimiter(m) @@ -222,6 +230,7 @@ func flowsAgent( informer: informer, promoServer: promoServer, metrics: m, + rbSSLTracer: rbSSLTracer, }, nil } @@ -392,6 +401,10 @@ func (f *Flows) buildAndStartPipeline(ctx context.Context) (*node.Terminal[[]*mo alog.Debug("connecting flows processing graph") mapTracer := node.AsStart(f.mapTracer.TraceLoop(ctx, f.cfg.ForceGC)) rbTracer := node.AsStart(f.rbTracer.TraceLoop(ctx)) + var rbSSLTracer *node.Start[*model.RawRecord] + if f.cfg.EnableSSL { + rbSSLTracer = node.AsStart(f.rbSSLTracer.TraceLoop(ctx)) + } accounter := node.AsMiddle(f.accounter.Account, node.ChannelBufferLen(f.cfg.BuffersLength)) @@ -408,6 +421,9 @@ func (f *Flows) buildAndStartPipeline(ctx context.Context) (*node.Terminal[[]*mo node.ChannelBufferLen(ebl)) rbTracer.SendsTo(accounter) + if rbSSLTracer != nil { + rbSSLTracer.SendsTo(accounter) + } mapTracer.SendsTo(limiter) accounter.SendsTo(limiter) @@ -416,6 +432,9 @@ func (f *Flows) buildAndStartPipeline(ctx context.Context) (*node.Terminal[[]*mo alog.Debug("starting graph") mapTracer.Start() rbTracer.Start() + if rbSSLTracer != nil { + rbSSLTracer.Start() + } return export, nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index a7fa66515..fdaf41cb4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -252,6 +252,10 @@ type Agent struct { // This setting is only used when the interface name could not be found for a given index and MAC. // E.g. "0a:58=eth0" (used for ovn-kubernetes) PreferredInterfaceForMACPrefix string `env:"PREFERRED_INTERFACE_FOR_MAC_PREFIX"` + // EnableSSL enable tracking SSL flows encryption + EnableSSL bool `env:"ENABLE_SSL" envDefault:"false"` + // OpenSSLPath path to the openssl binary + OpenSSLPath string `env:"OPENSSL_PATH" envDefault:"/usr/bin/openssl"` /* Deprecated configs are listed below this line * See manageDeprecatedConfigs function for details diff --git a/pkg/ebpf/bpf_arm64_bpfel.go b/pkg/ebpf/bpf_arm64_bpfel.go index 400595e5e..f06e5189c 100644 --- a/pkg/ebpf/bpf_arm64_bpfel.go +++ b/pkg/ebpf/bpf_arm64_bpfel.go @@ -173,6 +173,16 @@ type BpfPktDropsT struct { _ [5]byte } +type BpfSslDataEventT struct { + _ structs.HostLayout + TimestampNs uint64 + PidTgid uint64 + DataLen int32 + SslType uint8 + Data [16384]int8 + _ [3]byte +} + type BpfTcpFlagsT uint32 const ( @@ -242,6 +252,7 @@ type BpfSpecs struct { type BpfProgramSpecs struct { KfreeSkb *ebpf.ProgramSpec `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.ProgramSpec `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.ProgramSpec `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.ProgramSpec `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.ProgramSpec `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.ProgramSpec `ebpf:"tc_ingress_flow_parse"` @@ -273,6 +284,7 @@ type BpfMapSpecs struct { IpsecIngressMap *ebpf.MapSpec `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.MapSpec `ebpf:"packet_record"` PeerFilterMap *ebpf.MapSpec `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.MapSpec `ebpf:"ssl_data_event_map"` } // BpfVariableSpecs contains global variables before they are loaded into the kernel. @@ -287,11 +299,13 @@ type BpfVariableSpecs struct { EnablePca *ebpf.VariableSpec `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.VariableSpec `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.VariableSpec `ebpf:"enable_rtt"` + EnableSsl *ebpf.VariableSpec `ebpf:"enable_ssl"` FilterKey *ebpf.VariableSpec `ebpf:"filter_key"` FilterValue *ebpf.VariableSpec `ebpf:"filter_value"` HasFilterSampling *ebpf.VariableSpec `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.VariableSpec `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.VariableSpec `ebpf:"sampling"` + SslDataEvent *ebpf.VariableSpec `ebpf:"ssl_data_event"` TraceMessages *ebpf.VariableSpec `ebpf:"trace_messages"` Unused8 *ebpf.VariableSpec `ebpf:"unused8"` Unused9 *ebpf.VariableSpec `ebpf:"unused9"` @@ -327,6 +341,7 @@ type BpfMaps struct { IpsecIngressMap *ebpf.Map `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.Map `ebpf:"packet_record"` PeerFilterMap *ebpf.Map `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.Map `ebpf:"ssl_data_event_map"` } func (m *BpfMaps) Close() error { @@ -341,6 +356,7 @@ func (m *BpfMaps) Close() error { m.IpsecIngressMap, m.PacketRecord, m.PeerFilterMap, + m.SslDataEventMap, ) } @@ -356,11 +372,13 @@ type BpfVariables struct { EnablePca *ebpf.Variable `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.Variable `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.Variable `ebpf:"enable_rtt"` + EnableSsl *ebpf.Variable `ebpf:"enable_ssl"` FilterKey *ebpf.Variable `ebpf:"filter_key"` FilterValue *ebpf.Variable `ebpf:"filter_value"` HasFilterSampling *ebpf.Variable `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.Variable `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.Variable `ebpf:"sampling"` + SslDataEvent *ebpf.Variable `ebpf:"ssl_data_event"` TraceMessages *ebpf.Variable `ebpf:"trace_messages"` Unused8 *ebpf.Variable `ebpf:"unused8"` Unused9 *ebpf.Variable `ebpf:"unused9"` @@ -372,6 +390,7 @@ type BpfVariables struct { type BpfPrograms struct { KfreeSkb *ebpf.Program `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.Program `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.Program `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.Program `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.Program `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.Program `ebpf:"tc_ingress_flow_parse"` @@ -393,6 +412,7 @@ func (p *BpfPrograms) Close() error { return _BpfClose( p.KfreeSkb, p.NetworkEventsMonitoring, + p.ProbeEntrySSL_write, p.TcEgressFlowParse, p.TcEgressPcaParse, p.TcIngressFlowParse, diff --git a/pkg/ebpf/bpf_arm64_bpfel.o b/pkg/ebpf/bpf_arm64_bpfel.o index 54d7d1d9d..5636c531c 100644 Binary files a/pkg/ebpf/bpf_arm64_bpfel.o and b/pkg/ebpf/bpf_arm64_bpfel.o differ diff --git a/pkg/ebpf/bpf_powerpc_bpfel.go b/pkg/ebpf/bpf_powerpc_bpfel.go index dd023874b..7c3a10581 100644 --- a/pkg/ebpf/bpf_powerpc_bpfel.go +++ b/pkg/ebpf/bpf_powerpc_bpfel.go @@ -173,6 +173,16 @@ type BpfPktDropsT struct { _ [5]byte } +type BpfSslDataEventT struct { + _ structs.HostLayout + TimestampNs uint64 + PidTgid uint64 + DataLen int32 + SslType uint8 + Data [16384]int8 + _ [3]byte +} + type BpfTcpFlagsT uint32 const ( @@ -242,6 +252,7 @@ type BpfSpecs struct { type BpfProgramSpecs struct { KfreeSkb *ebpf.ProgramSpec `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.ProgramSpec `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.ProgramSpec `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.ProgramSpec `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.ProgramSpec `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.ProgramSpec `ebpf:"tc_ingress_flow_parse"` @@ -273,6 +284,7 @@ type BpfMapSpecs struct { IpsecIngressMap *ebpf.MapSpec `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.MapSpec `ebpf:"packet_record"` PeerFilterMap *ebpf.MapSpec `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.MapSpec `ebpf:"ssl_data_event_map"` } // BpfVariableSpecs contains global variables before they are loaded into the kernel. @@ -287,11 +299,13 @@ type BpfVariableSpecs struct { EnablePca *ebpf.VariableSpec `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.VariableSpec `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.VariableSpec `ebpf:"enable_rtt"` + EnableSsl *ebpf.VariableSpec `ebpf:"enable_ssl"` FilterKey *ebpf.VariableSpec `ebpf:"filter_key"` FilterValue *ebpf.VariableSpec `ebpf:"filter_value"` HasFilterSampling *ebpf.VariableSpec `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.VariableSpec `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.VariableSpec `ebpf:"sampling"` + SslDataEvent *ebpf.VariableSpec `ebpf:"ssl_data_event"` TraceMessages *ebpf.VariableSpec `ebpf:"trace_messages"` Unused8 *ebpf.VariableSpec `ebpf:"unused8"` Unused9 *ebpf.VariableSpec `ebpf:"unused9"` @@ -327,6 +341,7 @@ type BpfMaps struct { IpsecIngressMap *ebpf.Map `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.Map `ebpf:"packet_record"` PeerFilterMap *ebpf.Map `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.Map `ebpf:"ssl_data_event_map"` } func (m *BpfMaps) Close() error { @@ -341,6 +356,7 @@ func (m *BpfMaps) Close() error { m.IpsecIngressMap, m.PacketRecord, m.PeerFilterMap, + m.SslDataEventMap, ) } @@ -356,11 +372,13 @@ type BpfVariables struct { EnablePca *ebpf.Variable `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.Variable `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.Variable `ebpf:"enable_rtt"` + EnableSsl *ebpf.Variable `ebpf:"enable_ssl"` FilterKey *ebpf.Variable `ebpf:"filter_key"` FilterValue *ebpf.Variable `ebpf:"filter_value"` HasFilterSampling *ebpf.Variable `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.Variable `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.Variable `ebpf:"sampling"` + SslDataEvent *ebpf.Variable `ebpf:"ssl_data_event"` TraceMessages *ebpf.Variable `ebpf:"trace_messages"` Unused8 *ebpf.Variable `ebpf:"unused8"` Unused9 *ebpf.Variable `ebpf:"unused9"` @@ -372,6 +390,7 @@ type BpfVariables struct { type BpfPrograms struct { KfreeSkb *ebpf.Program `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.Program `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.Program `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.Program `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.Program `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.Program `ebpf:"tc_ingress_flow_parse"` @@ -393,6 +412,7 @@ func (p *BpfPrograms) Close() error { return _BpfClose( p.KfreeSkb, p.NetworkEventsMonitoring, + p.ProbeEntrySSL_write, p.TcEgressFlowParse, p.TcEgressPcaParse, p.TcIngressFlowParse, diff --git a/pkg/ebpf/bpf_powerpc_bpfel.o b/pkg/ebpf/bpf_powerpc_bpfel.o index ae6a7fe2c..d8f1caf26 100644 Binary files a/pkg/ebpf/bpf_powerpc_bpfel.o and b/pkg/ebpf/bpf_powerpc_bpfel.o differ diff --git a/pkg/ebpf/bpf_s390_bpfeb.go b/pkg/ebpf/bpf_s390_bpfeb.go index fc25078be..406540f19 100644 --- a/pkg/ebpf/bpf_s390_bpfeb.go +++ b/pkg/ebpf/bpf_s390_bpfeb.go @@ -173,6 +173,16 @@ type BpfPktDropsT struct { _ [5]byte } +type BpfSslDataEventT struct { + _ structs.HostLayout + TimestampNs uint64 + PidTgid uint64 + DataLen int32 + SslType uint8 + Data [16384]int8 + _ [3]byte +} + type BpfTcpFlagsT uint32 const ( @@ -242,6 +252,7 @@ type BpfSpecs struct { type BpfProgramSpecs struct { KfreeSkb *ebpf.ProgramSpec `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.ProgramSpec `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.ProgramSpec `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.ProgramSpec `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.ProgramSpec `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.ProgramSpec `ebpf:"tc_ingress_flow_parse"` @@ -273,6 +284,7 @@ type BpfMapSpecs struct { IpsecIngressMap *ebpf.MapSpec `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.MapSpec `ebpf:"packet_record"` PeerFilterMap *ebpf.MapSpec `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.MapSpec `ebpf:"ssl_data_event_map"` } // BpfVariableSpecs contains global variables before they are loaded into the kernel. @@ -287,11 +299,13 @@ type BpfVariableSpecs struct { EnablePca *ebpf.VariableSpec `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.VariableSpec `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.VariableSpec `ebpf:"enable_rtt"` + EnableSsl *ebpf.VariableSpec `ebpf:"enable_ssl"` FilterKey *ebpf.VariableSpec `ebpf:"filter_key"` FilterValue *ebpf.VariableSpec `ebpf:"filter_value"` HasFilterSampling *ebpf.VariableSpec `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.VariableSpec `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.VariableSpec `ebpf:"sampling"` + SslDataEvent *ebpf.VariableSpec `ebpf:"ssl_data_event"` TraceMessages *ebpf.VariableSpec `ebpf:"trace_messages"` Unused8 *ebpf.VariableSpec `ebpf:"unused8"` Unused9 *ebpf.VariableSpec `ebpf:"unused9"` @@ -327,6 +341,7 @@ type BpfMaps struct { IpsecIngressMap *ebpf.Map `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.Map `ebpf:"packet_record"` PeerFilterMap *ebpf.Map `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.Map `ebpf:"ssl_data_event_map"` } func (m *BpfMaps) Close() error { @@ -341,6 +356,7 @@ func (m *BpfMaps) Close() error { m.IpsecIngressMap, m.PacketRecord, m.PeerFilterMap, + m.SslDataEventMap, ) } @@ -356,11 +372,13 @@ type BpfVariables struct { EnablePca *ebpf.Variable `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.Variable `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.Variable `ebpf:"enable_rtt"` + EnableSsl *ebpf.Variable `ebpf:"enable_ssl"` FilterKey *ebpf.Variable `ebpf:"filter_key"` FilterValue *ebpf.Variable `ebpf:"filter_value"` HasFilterSampling *ebpf.Variable `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.Variable `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.Variable `ebpf:"sampling"` + SslDataEvent *ebpf.Variable `ebpf:"ssl_data_event"` TraceMessages *ebpf.Variable `ebpf:"trace_messages"` Unused8 *ebpf.Variable `ebpf:"unused8"` Unused9 *ebpf.Variable `ebpf:"unused9"` @@ -372,6 +390,7 @@ type BpfVariables struct { type BpfPrograms struct { KfreeSkb *ebpf.Program `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.Program `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.Program `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.Program `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.Program `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.Program `ebpf:"tc_ingress_flow_parse"` @@ -393,6 +412,7 @@ func (p *BpfPrograms) Close() error { return _BpfClose( p.KfreeSkb, p.NetworkEventsMonitoring, + p.ProbeEntrySSL_write, p.TcEgressFlowParse, p.TcEgressPcaParse, p.TcIngressFlowParse, diff --git a/pkg/ebpf/bpf_s390_bpfeb.o b/pkg/ebpf/bpf_s390_bpfeb.o index aa38ced8a..df934114a 100644 Binary files a/pkg/ebpf/bpf_s390_bpfeb.o and b/pkg/ebpf/bpf_s390_bpfeb.o differ diff --git a/pkg/ebpf/bpf_x86_bpfel.go b/pkg/ebpf/bpf_x86_bpfel.go index 22c157d7a..460ce985d 100644 --- a/pkg/ebpf/bpf_x86_bpfel.go +++ b/pkg/ebpf/bpf_x86_bpfel.go @@ -173,6 +173,16 @@ type BpfPktDropsT struct { _ [5]byte } +type BpfSslDataEventT struct { + _ structs.HostLayout + TimestampNs uint64 + PidTgid uint64 + DataLen int32 + SslType uint8 + Data [16384]int8 + _ [3]byte +} + type BpfTcpFlagsT uint32 const ( @@ -242,6 +252,7 @@ type BpfSpecs struct { type BpfProgramSpecs struct { KfreeSkb *ebpf.ProgramSpec `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.ProgramSpec `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.ProgramSpec `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.ProgramSpec `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.ProgramSpec `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.ProgramSpec `ebpf:"tc_ingress_flow_parse"` @@ -273,6 +284,7 @@ type BpfMapSpecs struct { IpsecIngressMap *ebpf.MapSpec `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.MapSpec `ebpf:"packet_record"` PeerFilterMap *ebpf.MapSpec `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.MapSpec `ebpf:"ssl_data_event_map"` } // BpfVariableSpecs contains global variables before they are loaded into the kernel. @@ -287,11 +299,13 @@ type BpfVariableSpecs struct { EnablePca *ebpf.VariableSpec `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.VariableSpec `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.VariableSpec `ebpf:"enable_rtt"` + EnableSsl *ebpf.VariableSpec `ebpf:"enable_ssl"` FilterKey *ebpf.VariableSpec `ebpf:"filter_key"` FilterValue *ebpf.VariableSpec `ebpf:"filter_value"` HasFilterSampling *ebpf.VariableSpec `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.VariableSpec `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.VariableSpec `ebpf:"sampling"` + SslDataEvent *ebpf.VariableSpec `ebpf:"ssl_data_event"` TraceMessages *ebpf.VariableSpec `ebpf:"trace_messages"` Unused8 *ebpf.VariableSpec `ebpf:"unused8"` Unused9 *ebpf.VariableSpec `ebpf:"unused9"` @@ -327,6 +341,7 @@ type BpfMaps struct { IpsecIngressMap *ebpf.Map `ebpf:"ipsec_ingress_map"` PacketRecord *ebpf.Map `ebpf:"packet_record"` PeerFilterMap *ebpf.Map `ebpf:"peer_filter_map"` + SslDataEventMap *ebpf.Map `ebpf:"ssl_data_event_map"` } func (m *BpfMaps) Close() error { @@ -341,6 +356,7 @@ func (m *BpfMaps) Close() error { m.IpsecIngressMap, m.PacketRecord, m.PeerFilterMap, + m.SslDataEventMap, ) } @@ -356,11 +372,13 @@ type BpfVariables struct { EnablePca *ebpf.Variable `ebpf:"enable_pca"` EnablePktTranslationTracking *ebpf.Variable `ebpf:"enable_pkt_translation_tracking"` EnableRtt *ebpf.Variable `ebpf:"enable_rtt"` + EnableSsl *ebpf.Variable `ebpf:"enable_ssl"` FilterKey *ebpf.Variable `ebpf:"filter_key"` FilterValue *ebpf.Variable `ebpf:"filter_value"` HasFilterSampling *ebpf.Variable `ebpf:"has_filter_sampling"` NetworkEventsMonitoringGroupid *ebpf.Variable `ebpf:"network_events_monitoring_groupid"` Sampling *ebpf.Variable `ebpf:"sampling"` + SslDataEvent *ebpf.Variable `ebpf:"ssl_data_event"` TraceMessages *ebpf.Variable `ebpf:"trace_messages"` Unused8 *ebpf.Variable `ebpf:"unused8"` Unused9 *ebpf.Variable `ebpf:"unused9"` @@ -372,6 +390,7 @@ type BpfVariables struct { type BpfPrograms struct { KfreeSkb *ebpf.Program `ebpf:"kfree_skb"` NetworkEventsMonitoring *ebpf.Program `ebpf:"network_events_monitoring"` + ProbeEntrySSL_write *ebpf.Program `ebpf:"probe_entry_SSL_write"` TcEgressFlowParse *ebpf.Program `ebpf:"tc_egress_flow_parse"` TcEgressPcaParse *ebpf.Program `ebpf:"tc_egress_pca_parse"` TcIngressFlowParse *ebpf.Program `ebpf:"tc_ingress_flow_parse"` @@ -393,6 +412,7 @@ func (p *BpfPrograms) Close() error { return _BpfClose( p.KfreeSkb, p.NetworkEventsMonitoring, + p.ProbeEntrySSL_write, p.TcEgressFlowParse, p.TcEgressPcaParse, p.TcIngressFlowParse, diff --git a/pkg/ebpf/bpf_x86_bpfel.o b/pkg/ebpf/bpf_x86_bpfel.o index a1fdefe76..aea8151ec 100644 Binary files a/pkg/ebpf/bpf_x86_bpfel.o and b/pkg/ebpf/bpf_x86_bpfel.o differ diff --git a/pkg/flow/tracer_ringbuf.go b/pkg/flow/tracer_ringbuf.go index fb61e929a..3a19f214c 100644 --- a/pkg/flow/tracer_ringbuf.go +++ b/pkg/flow/tracer_ringbuf.go @@ -3,6 +3,7 @@ package flow import ( "bytes" "context" + "encoding/binary" "errors" "fmt" "sync/atomic" @@ -17,22 +18,29 @@ import ( "github.com/sirupsen/logrus" ) +const maxSSLDataSize = 16 * 1024 + var rtlog = logrus.WithField("component", "flow.RingBufTracer") // RingBufTracer receives single-packet flows via ringbuffer (usually, these that couldn't be // added in the eBPF kernel space due to the map being full or busy) and submits them to the // userspace Aggregator map type RingBufTracer struct { - mapFlusher mapFlusher - ringBuffer ringBufReader - stats stats - metrics *metrics.Metrics + mapFlusher mapFlusher + ringBuffer ringBufReader + ringBufferSSL ringBufSSLReader + stats stats + metrics *metrics.Metrics } type ringBufReader interface { ReadRingBuf() (ringbuf.Record, error) } +type ringBufSSLReader interface { + ReadSSLRingBuf() (ringbuf.Record, error) +} + // stats supports atomic logging of ringBuffer metrics type stats struct { loggingTimeout time.Duration @@ -54,22 +62,46 @@ func NewRingBufTracer(reader ringBufReader, flusher mapFlusher, logTimeout time. } } +func NewSSLRingBufTracer(reader ringBufSSLReader, flusher mapFlusher, logTimeout time.Duration, m *metrics.Metrics) *RingBufTracer { + return &RingBufTracer{ + mapFlusher: flusher, + ringBufferSSL: reader, + stats: stats{loggingTimeout: logTimeout}, + metrics: m, + } +} + func (m *RingBufTracer) TraceLoop(ctx context.Context) node.StartFunc[*model.RawRecord] { return func(out chan<- *model.RawRecord) { debugging := logrus.IsLevelEnabled(logrus.DebugLevel) + if m.ringBufferSSL != nil { + rtlog.Info("SSL RingBuf tracer started - listening for SSL events") + } for { select { case <-ctx.Done(): rtlog.Debug("exiting trace loop due to context cancellation") return default: - if err := m.listenAndForwardRingBuffer(debugging, out); err != nil { - if errors.Is(err, ringbuf.ErrClosed) { - rtlog.Debug("Received signal, exiting..") - return + if m.ringBuffer != nil { + if err := m.listenAndForwardRingBuffer(debugging, out); err != nil { + if errors.Is(err, ringbuf.ErrClosed) { + rtlog.Debug("Received signal, exiting..") + return + } + rtlog.WithError(err).Warn("ignoring flow event") + continue + } + } + if m.ringBufferSSL != nil { + if err := m.listenAndForwardRingBufferSSL(out); err != nil { + if errors.Is(err, ringbuf.ErrClosed) { + rtlog.Debug("Received signal, exiting..") + return + } + rtlog.WithError(err).Warn("ignoring SSL event") + continue } - rtlog.WithError(err).Warn("ignoring flow event") - continue } } } @@ -100,6 +132,61 @@ func (m *RingBufTracer) listenAndForwardRingBuffer(debugging bool, forwardCh cha return nil } +func (m *RingBufTracer) listenAndForwardRingBufferSSL(forwardCh chan<- *model.RawRecord) error { + rtlog.Debug("listenAndForwardRingBufferSSL: waiting for SSL event...") + event, err := m.ringBufferSSL.ReadSSLRingBuf() + if err != nil { + m.metrics.Errors.WithErrorName("ringbuffer", "CannotReadSSLRingbuffer", metrics.HighSeverity).Inc() + return fmt.Errorf("reading from SSL ring buffer: %w", err) + } + + rtlog.Infof("SSL ringbuffer event received! Size: %d bytes", len(event.RawSample)) + + // Parse SSL event structure: timestamp(8) + pid_tgid(8) + data_len(4) + ssl_type(1) + data[16KB] + buf := bytes.NewReader(event.RawSample) + + var timestamp uint64 + var pidTgid uint64 + var dataLen int32 + var sslType uint8 + + if err := binary.Read(buf, binary.LittleEndian, ×tamp); err != nil { + rtlog.Warnf("Failed to read timestamp: %v", err) + return nil + } + if err := binary.Read(buf, binary.LittleEndian, &pidTgid); err != nil { + rtlog.Warnf("Failed to read pid_tgid: %v", err) + return nil + } + if err := binary.Read(buf, binary.LittleEndian, &dataLen); err != nil { + rtlog.Warnf("Failed to read data_len: %v", err) + return nil + } + if err := binary.Read(buf, binary.LittleEndian, &sslType); err != nil { + rtlog.Warnf("Failed to read ssl_type: %v", err) + return nil + } + + // Read the actual SSL data (up to dataLen bytes) + if dataLen > 0 && dataLen <= maxSSLDataSize { + data := make([]byte, dataLen) + n, err := buf.Read(data) + if err != nil && n < int(dataLen) { + rtlog.Warnf("Failed to read SSL data: read %d/%d bytes, err=%v", n, dataLen, err) + } + + rtlog.Infof("SSL EVENT: pid=%d, timestamp=%d, data_len=%d, ssl_type=%d", + pidTgid, timestamp, dataLen, sslType) + printLen := min(256, len(data)) + rtlog.Infof("SSL data as string: %s", string(data[:printLen])) + } else { + rtlog.Infof("SSL EVENT: pid=%d, timestamp=%d, data_len=%d (invalid), ssl_type=%d", + pidTgid, timestamp, dataLen, sslType) + } + + return nil +} + // logRingBufferFlows avoids flooding logs on long series of evicted flows by grouping how // many flows are forwarded func (m *stats) logRingBufferFlows(mapFullErr bool) { diff --git a/pkg/kernel/kernel_utils.go b/pkg/kernel/kernel_utils.go index 18f6cc05a..fbe08cfb0 100644 --- a/pkg/kernel/kernel_utils.go +++ b/pkg/kernel/kernel_utils.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package kernel import ( diff --git a/pkg/kernel/kernel_utils_test.go b/pkg/kernel/kernel_utils_test.go index 968bb8531..4b3e918cb 100644 --- a/pkg/kernel/kernel_utils_test.go +++ b/pkg/kernel/kernel_utils_test.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package kernel import ( diff --git a/pkg/maps/maps.go b/pkg/maps/maps.go index d2d988459..905ba5877 100644 --- a/pkg/maps/maps.go +++ b/pkg/maps/maps.go @@ -12,4 +12,5 @@ var Maps = []string{ "peer_filter_map", "ipsec_ingress_map", "ipsec_egress_map", + "ssl_data_event_map", } diff --git a/pkg/test/tracer_fake.go b/pkg/test/tracer_fake.go index e5954fe80..398fe7a9d 100644 --- a/pkg/test/tracer_fake.go +++ b/pkg/test/tracer_fake.go @@ -65,6 +65,10 @@ func (m *TracerFake) ReadRingBuf() (ringbuf.Record, error) { return <-m.ringBuf, nil } +func (m *TracerFake) ReadSSLRingBuf() (ringbuf.Record, error) { + return <-m.ringBuf, nil +} + func (m *TracerFake) AppendLookupResults(results map[ebpf.BpfFlowId]model.BpfFlowContent) { m.mapLookups <- results } diff --git a/pkg/tracer/tracer.go b/pkg/tracer/tracer.go index 4bc4052db..57ba0d058 100644 --- a/pkg/tracer/tracer.go +++ b/pkg/tracer/tracer.go @@ -63,6 +63,8 @@ const ( networkEventsMonitoringHook = "psample_sample_packet" defaultNetworkEventsGroupID = 10 constEnableIPsec = "enable_ipsec" + constEnableSSL = "enable_ssl" + sslDataEventMap = "ssl_data_event_map" ) var log = logrus.WithField("component", "ebpf.FlowFetcher") @@ -92,6 +94,8 @@ type FlowFetcher struct { xfrmOutputKretProbeLink link.Link xfrmInputKProbeLink link.Link xfrmOutputKProbeLink link.Link + sslUprobe link.Link + sslDataEventsReader *ringbuf.Reader lookupAndDeleteSupported bool useEbpfManager bool pinDir string @@ -115,6 +119,8 @@ type FlowFetcherConfig struct { BpfManBpfFSPath string EnableIPsecTracker bool FilterConfig []*FilterConfig + EnableSSL bool + OpenSSLPath string } type variablesMapping struct { @@ -127,6 +133,8 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e var pktDropsLink, networkEventsMonitoringLink, rttFentryLink, rttKprobeLink link.Link var nfNatManIPLink, xfrmInputKretProbeLink, xfrmOutputKretProbeLink link.Link var xfrmInputKProbeLink, xfrmOutputKProbeLink link.Link + var sslUprobe link.Link + var sslDataEvents *ringbuf.Reader var err error objects := ebpf.BpfObjects{} var pinDir string @@ -161,6 +169,7 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e pcaRecordsMap, ipsecInputMap, ipsecOutputMap, + sslDataEventMap, } { spec.Maps[m].Pinning = 0 } @@ -192,6 +201,11 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e objects.TcIngressPcaParse = nil delete(spec.Programs, constPcaEnable) + // Minimize SSL maps if SSL is disabled + if !cfg.EnableSSL { + spec.Maps[sslDataEventMap].MaxEntries = 1 + } + if cfg.EnablePktDrops && !oldKernel && !rtOldKernel { pktDropsLink, err = link.Tracepoint("skb", pktDropHook, objects.KfreeSkb, nil) if err != nil { @@ -262,6 +276,27 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e return nil, fmt.Errorf("failed to attach the BPF KretProbe program to xfrm_output: %w", err) } } + + // Setup SSL tracking if enabled + if cfg.EnableSSL { + // Read SSL data events from ringbuf + sslDataEvents, err = ringbuf.NewReader(objects.BpfMaps.SslDataEventMap) + if err != nil { + return nil, fmt.Errorf("accessing SSL data event ringbuffer: %w", err) + } + + // Attach SSL uprobes + sslWriteLink, err := link.OpenExecutable(cfg.OpenSSLPath) + if err != nil { + return nil, fmt.Errorf("failed to open executable %s: %w", cfg.OpenSSLPath, err) + } + sslUprobe, err = sslWriteLink.Uprobe("SSL_write", objects.ProbeEntrySSL_write, nil) + if err != nil { + return nil, fmt.Errorf("failed to attach SSL_write uprobe: %w", err) + } + log.Infof("SSL tracking enabled with library: %s", cfg.OpenSSLPath) + } + } else { pinDir = cfg.BpfManBpfFSPath opts := &cilium.LoadPinOptions{ @@ -330,6 +365,12 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e if err != nil { return nil, fmt.Errorf("failed to load %s: %w", mPath, err) } + log.Infof("BPFManager mode: loading SSL data event pinned maps") + mPath = path.Join(pinDir, sslDataEventMap) + objects.BpfMaps.SslDataEventMap, err = cilium.LoadPinnedMap(mPath, opts) + if err != nil { + return nil, fmt.Errorf("failed to load %s: %w", mPath, err) + } } if filter != nil { @@ -367,6 +408,8 @@ func NewFlowFetcher(cfg *FlowFetcherConfig, m *metrics.Metrics) (*FlowFetcher, e xfrmOutputKretProbeLink: xfrmOutputKretProbeLink, xfrmInputKProbeLink: xfrmInputKProbeLink, xfrmOutputKProbeLink: xfrmOutputKProbeLink, + sslUprobe: sslUprobe, + sslDataEventsReader: sslDataEvents, egressTCXLink: egressTCXLink, ingressTCXLink: ingressTCXLink, networkEventsMonitoringLink: networkEventsMonitoringLink, @@ -725,6 +768,18 @@ func (m *FlowFetcher) Close() error { } } + if m.sslUprobe != nil { + if err := m.sslUprobe.Close(); err != nil { + errs = append(errs, err) + } + } + + if m.sslDataEventsReader != nil { + if err := m.sslDataEventsReader.Close(); err != nil { + errs = append(errs, err) + } + } + // m.ringbufReader.Read is a blocking operation, so we need to close the ring buffer // from another goroutine to avoid the system not being able to exit if there // isn't traffic in a given interface @@ -800,6 +855,12 @@ func (m *FlowFetcher) Close() error { if err := m.objects.IpsecEgressMap.Close(); err != nil { errs = append(errs, err) } + if err := m.objects.SslDataEventMap.Unpin(); err != nil { + errs = append(errs, err) + } + if err := m.objects.SslDataEventMap.Close(); err != nil { + errs = append(errs, err) + } if len(errs) == 0 { m.objects = nil } @@ -905,6 +966,10 @@ func (m *FlowFetcher) ReadRingBuf() (ringbuf.Record, error) { return m.ringbufReader.Read() } +func (m *FlowFetcher) ReadSSLRingBuf() (ringbuf.Record, error) { + return m.sslDataEventsReader.Read() +} + // LookupAndDeleteMap reads all the entries from the eBPF map and removes them from it. // TODO: detect whether BatchLookupAndDelete is supported (Kernel>=5.6) and use it selectively // Supported Lookup/Delete operations by kernel: https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md @@ -1102,6 +1167,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, XfrmOutputKretprobe *cilium.Program `ebpf:"xfrm_output_kretprobe"` XfrmInputKprobe *cilium.Program `ebpf:"xfrm_input_kprobe"` XfrmOutputKprobe *cilium.Program `ebpf:"xfrm_output_kprobe"` + ProbeEntrySSLWrite *cilium.Program `ebpf:"probe_entry_SSL_write"` } type newBpfObjects struct { newBpfPrograms @@ -1135,6 +1201,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, TcpRcvFentry: nil, KfreeSkb: nil, NetworkEventsMonitoring: nil, + ProbeEntrySSL_write: newObjects.ProbeEntrySSLWrite, }, BpfMaps: ebpf.BpfMaps{ DirectFlows: newObjects.DirectFlows, @@ -1146,6 +1213,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, GlobalCounters: newObjects.GlobalCounters, IpsecIngressMap: newObjects.IpsecIngressMap, IpsecEgressMap: newObjects.IpsecEgressMap, + SslDataEventMap: newObjects.SslDataEventMap, }, } @@ -1165,6 +1233,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, XfrmOutputKretprobe *cilium.Program `ebpf:"xfrm_output_kretprobe"` XfrmInputKprobe *cilium.Program `ebpf:"xfrm_input_kprobe"` XfrmOutputKprobe *cilium.Program `ebpf:"xfrm_output_kprobe"` + ProbeEntrySSLWrite *cilium.Program `ebpf:"probe_entry_SSL_write"` } type newBpfObjects struct { newBpfPrograms @@ -1197,6 +1266,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, TcpRcvFentry: nil, KfreeSkb: nil, NetworkEventsMonitoring: nil, + ProbeEntrySSL_write: newObjects.ProbeEntrySSLWrite, }, BpfMaps: ebpf.BpfMaps{ DirectFlows: newObjects.DirectFlows, @@ -1208,6 +1278,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, GlobalCounters: newObjects.GlobalCounters, IpsecIngressMap: newObjects.IpsecIngressMap, IpsecEgressMap: newObjects.IpsecEgressMap, + SslDataEventMap: newObjects.SslDataEventMap, }, } @@ -1227,6 +1298,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, XfrmOutputKretprobe *cilium.Program `ebpf:"xfrm_output_kretprobe"` XfrmInputKprobe *cilium.Program `ebpf:"xfrm_input_kprobe"` XfrmOutputKprobe *cilium.Program `ebpf:"xfrm_output_kprobe"` + ProbeEntrySSLWrite *cilium.Program `ebpf:"probe_entry_SSL_write"` } type newBpfObjects struct { newBpfPrograms @@ -1259,6 +1331,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, TcpRcvKprobe: nil, KfreeSkb: nil, NetworkEventsMonitoring: nil, + ProbeEntrySSL_write: newObjects.ProbeEntrySSLWrite, }, BpfMaps: ebpf.BpfMaps{ DirectFlows: newObjects.DirectFlows, @@ -1270,6 +1343,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, GlobalCounters: newObjects.GlobalCounters, IpsecIngressMap: newObjects.IpsecIngressMap, IpsecEgressMap: newObjects.IpsecEgressMap, + SslDataEventMap: newObjects.SslDataEventMap, }, } @@ -1291,6 +1365,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, XfrmOutputKretprobe *cilium.Program `ebpf:"xfrm_output_kretprobe"` XfrmInputKprobe *cilium.Program `ebpf:"xfrm_input_kprobe"` XfrmOutputKprobe *cilium.Program `ebpf:"xfrm_output_kprobe"` + ProbeEntrySSLWrite *cilium.Program `ebpf:"probe_entry_SSL_write"` } type newBpfObjects struct { newBpfPrograms @@ -1321,6 +1396,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, XfrmInputKprobe: newObjects.XfrmInputKprobe, XfrmOutputKprobe: newObjects.XfrmOutputKprobe, NetworkEventsMonitoring: nil, + ProbeEntrySSL_write: newObjects.ProbeEntrySSLWrite, }, BpfMaps: ebpf.BpfMaps{ DirectFlows: newObjects.DirectFlows, @@ -1332,6 +1408,7 @@ func kernelSpecificLoadAndAssign(oldKernel, rtKernel, supportNetworkEvents bool, GlobalCounters: newObjects.GlobalCounters, IpsecIngressMap: newObjects.IpsecIngressMap, IpsecEgressMap: newObjects.IpsecEgressMap, + SslDataEventMap: newObjects.SslDataEventMap, }, } @@ -1400,10 +1477,14 @@ func NewPacketFetcher(cfg *FlowFetcherConfig) (*PacketFetcher, error) { pcaRecordsMap, ipsecInputMap, ipsecOutputMap, + sslDataEventMap, } { spec.Maps[m].Pinning = 0 } + // Always minimize SSL maps in PacketFetcher - SSL and Packet Fetcher are mutually exclusive + spec.Maps[sslDataEventMap].MaxEntries = 1 + type pcaBpfPrograms struct { TcEgressPcaParse *cilium.Program `ebpf:"tc_egress_pca_parse"` TcIngressPcaParse *cilium.Program `ebpf:"tc_ingress_pca_parse"` @@ -1434,6 +1515,7 @@ func NewPacketFetcher(cfg *FlowFetcherConfig) (*PacketFetcher, error) { delete(spec.Programs, constNetworkEventsMonitoringGroupID) delete(spec.Programs, constEnablePktTranslation) delete(spec.Programs, constEnableIPsec) + delete(spec.Programs, constEnableSSL) if err := spec.LoadAndAssign(&newObjects, &cilium.CollectionOptions{Maps: cilium.MapOptions{PinPath: ""}}); err != nil { var ve *cilium.VerifierError @@ -1463,11 +1545,13 @@ func NewPacketFetcher(cfg *FlowFetcherConfig) (*PacketFetcher, error) { XfrmOutputKretprobe: nil, XfrmInputKprobe: nil, XfrmOutputKprobe: nil, + ProbeEntrySSL_write: nil, }, BpfMaps: ebpf.BpfMaps{ - PacketRecord: newObjects.PacketRecord, - FilterMap: newObjects.FilterMap, - PeerFilterMap: newObjects.PeerFilterMap, + PacketRecord: newObjects.PacketRecord, + SslDataEventMap: newObjects.SslDataEventMap, + FilterMap: newObjects.FilterMap, + PeerFilterMap: newObjects.PeerFilterMap, }, } @@ -1921,6 +2005,11 @@ func configureFlowSpecVariables(spec *cilium.CollectionSpec, cfg *FlowFetcherCon spec.Maps[ipsecInputMap].MaxEntries = 1 spec.Maps[ipsecOutputMap].MaxEntries = 1 } + + enableSSL := 0 + if cfg.EnableSSL { + enableSSL = 1 + } // When adding constants here, remember to delete them in NewPacketFetcher variables := []variablesMapping{ {constSampling, uint32(cfg.Sampling)}, @@ -1934,6 +2023,7 @@ func configureFlowSpecVariables(spec *cilium.CollectionSpec, cfg *FlowFetcherCon {constNetworkEventsMonitoringGroupID, uint8(networkEventsMonitoringGroupID)}, {constEnablePktTranslation, uint8(enablePktTranslation)}, {constEnableIPsec, uint8(enableIPsec)}, + {constEnableSSL, uint8(enableSSL)}, } for _, mapping := range variables { diff --git a/scripts/agent.yml b/scripts/agent.yml index d67184812..cfea0c216 100644 --- a/scripts/agent.yml +++ b/scripts/agent.yml @@ -46,12 +46,23 @@ spec: value: "true" - name: ENABLE_DNS_TRACKING value: "true" + - name: ENABLE_SSL + value: "true" + - name: OPENSSL_PATH + value: "/usr/lib/aarch64-linux-gnu/libssl.so.1.1" volumeMounts: - name: bpf-kernel-debug mountPath: /sys/kernel/debug mountPropagation: Bidirectional + - name: host-lib + mountPath: /usr/lib/aarch64-linux-gnu + readOnly: true volumes: - name: bpf-kernel-debug hostPath: path: /sys/kernel/debug type: Directory + - name: host-lib + hostPath: + path: /usr/lib/aarch64-linux-gnu + type: Directory \ No newline at end of file diff --git a/scripts/kind-cluster.sh b/scripts/kind-cluster.sh index 5acd39a7b..f917e8280 100755 --- a/scripts/kind-cluster.sh +++ b/scripts/kind-cluster.sh @@ -28,17 +28,58 @@ nodes: scheduler: extraArgs: v: "5" + extraMounts: + - hostPath: /sys/kernel/btf + containerPath: /sys/kernel/btf + readOnly: true + - hostPath: /sys/kernel/debug + containerPath: /sys/kernel/debug + - hostPath: /var/run/netns + containerPath: /var/run/netns - role: worker + extraMounts: + - hostPath: /sys/kernel/btf + containerPath: /sys/kernel/btf + readOnly: true + - hostPath: /sys/kernel/debug + containerPath: /sys/kernel/debug + - hostPath: /var/run/netns + containerPath: /var/run/netns - role: worker + extraMounts: + - hostPath: /sys/kernel/btf + containerPath: /sys/kernel/btf + readOnly: true + - hostPath: /sys/kernel/debug + containerPath: /sys/kernel/debug + - hostPath: /var/run/netns + containerPath: /var/run/netns EOF } # install_netobserv-agent will install the daemonset # into each kind docker container install_netobserv-agent() { -docker build . -t localhost/ebpf-agent:test -kind load docker-image localhost/ebpf-agent:test -kubectl apply -f ${DIR}/agent.yml + # Get the architecture and convert to Go arch format + local ARCH=$(uname -m) + local TARGETARCH + + case $ARCH in + x86_64) + TARGETARCH=amd64 + ;; + aarch64|arm64) + TARGETARCH=arm64 + ;; + *) + TARGETARCH=$ARCH + ;; + esac + + echo "Building for architecture: $TARGETARCH (detected: $ARCH)" + docker build . --build-arg TARGETARCH=$TARGETARCH -t localhost/ebpf-agent:test + kind load docker-image localhost/ebpf-agent:test + kubectl apply -f ${DIR}/agent.yml } # print_success prints a little success message at the end of the script @@ -60,8 +101,8 @@ SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} # At the minimum, deploy the kind cluster deploy_kind export KUBECONFIG=${DIR}/kubeconfig -oc label node kind-worker node-role.kubernetes.io/worker= -oc label node kind-worker2 node-role.kubernetes.io/worker= +kubectl label node kind-worker node-role.kubernetes.io/worker= +kubectl label node kind-worker2 node-role.kubernetes.io/worker= install_netobserv-agent