Backport net_fd connection parsing for more reliable black-box propag…

…ation in Go (#718)
grafana · Apr 4, 2024 · 4f9d076 · 4f9d076
1 parent f205789
commit 4f9d076
Show file tree

Hide file tree

Showing 21 changed files with 72 additions and 37 deletions.
diff --git a/bpf/go_nethttp.c b/bpf/go_nethttp.c
@@ -626,6 +626,29 @@ static __always_inline void read_ip_and_port(u8 *dst_ip, u16 *dst_port, void *sr
     }
 }
 
+static __always_inline void get_conn_info_from_fd(void *fd_ptr, connection_info_t *info) {
+    if (fd_ptr) {
+        void *laddr_ptr = 0;
+        void *raddr_ptr = 0;
+
+        bpf_probe_read(&laddr_ptr, sizeof(laddr_ptr), (void *)(fd_ptr + fd_laddr_pos + 8)); // find laddr
+        bpf_probe_read(&raddr_ptr, sizeof(raddr_ptr), (void *)(fd_ptr + fd_raddr_pos + 8)); // find raddr
+
+        if (laddr_ptr && raddr_ptr) {
+            bpf_dbg_printk("laddr %llx, raddr %llx", laddr_ptr, raddr_ptr);
+
+            // read local
+            read_ip_and_port(info->s_addr, &info->s_port, laddr_ptr);
+
+            // read remote
+            read_ip_and_port(info->d_addr, &info->d_port, raddr_ptr);
+
+            sort_connection_info(info);
+            //dbg_print_http_connection_info(info);
+        }
+    }
+}
+
 // HTTP black-box context propagation
 static __always_inline void get_conn_info(void *conn_ptr, connection_info_t *info) {
     if (conn_ptr) {
@@ -634,47 +657,34 @@ static __always_inline void get_conn_info(void *conn_ptr, connection_info_t *inf
 
         bpf_dbg_printk("Found fd ptr %llx", fd_ptr);
 
-        if (fd_ptr) {
-            void *laddr_ptr = 0;
-            void *raddr_ptr = 0;
-
-            bpf_probe_read(&laddr_ptr, sizeof(laddr_ptr), (void *)(fd_ptr + fd_laddr_pos + 8)); // find laddr
-            bpf_probe_read(&raddr_ptr, sizeof(raddr_ptr), (void *)(fd_ptr + fd_raddr_pos + 8)); // find raddr
-
-            if (laddr_ptr && raddr_ptr) {
-                bpf_dbg_printk("laddr %llx, raddr %llx", laddr_ptr, raddr_ptr);
-
-                // read local
-                read_ip_and_port(info->s_addr, &info->s_port, laddr_ptr);
-
-                // read remote
-                read_ip_and_port(info->d_addr, &info->d_port, raddr_ptr);
-
-                sort_connection_info(info);
-                //dbg_print_http_connection_info(info);
-            }
-        }
+        get_conn_info_from_fd(fd_ptr, info);
     }
 }
 
 SEC("uprobe/connServe")
 int uprobe_connServe(struct pt_regs *ctx) {
-    bpf_dbg_printk("=== uprobe/proc http conn serve === ");
+    void *goroutine_addr = GOROUTINE_PTR(ctx);
+    bpf_dbg_printk("=== uprobe/proc http conn serve goroutine %lx === ", goroutine_addr);
 
-    void *c_ptr = GO_PARAM1(ctx);
-    if (c_ptr) {
-        void *rwc_ptr = c_ptr + 8 + c_rwc_pos; // embedded struct
-        if (rwc_ptr) {
-            void *conn_ptr = 0;
-            bpf_probe_read(&conn_ptr, sizeof(conn_ptr), (void *)(rwc_ptr + rwc_conn_pos)); // find conn
-            if (conn_ptr) {
-                void *goroutine_addr = GOROUTINE_PTR(ctx);
-                connection_info_t conn = {0};
-                get_conn_info(conn_ptr, &conn);
+    connection_info_t conn = {0};
+    bpf_map_update_elem(&ongoing_http_server_connections, &goroutine_addr, &conn, BPF_ANY);
 
-                bpf_map_update_elem(&ongoing_http_server_connections, &goroutine_addr, &conn, BPF_ANY);
-            }
-        }
+    return 0;
+}
+
+SEC("uprobe/netFdRead")
+int uprobe_netFdRead(struct pt_regs *ctx) {
+    void *goroutine_addr = GOROUTINE_PTR(ctx);
+    bpf_dbg_printk("=== uprobe/proc netFD read goroutine %lx === ", goroutine_addr);
+
+    connection_info_t *conn = bpf_map_lookup_elem(&ongoing_http_server_connections, &goroutine_addr);
+
+    if (conn) {
+        bpf_dbg_printk("Found existing server connection, parsing FD information for socket tuples");
+
+        void *fd_ptr = GO_PARAM1(ctx);
+        get_conn_info_from_fd(fd_ptr, conn);
+        //dbg_print_http_connection_info(conn);
     }
 
     return 0;

diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_bpfel_arm64.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_bpfel_x86.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_arm64.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_debug_bpfel_x86.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_arm64.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_tp_bpfel_x86.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_arm64.o
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.go
diff --git a/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o b/pkg/internal/ebpf/nethttp/bpf_tp_debug_bpfel_x86.o
diff --git a/pkg/internal/ebpf/nethttp/nethttp.go b/pkg/internal/ebpf/nethttp/nethttp.go
@@ -165,6 +165,9 @@ func (p *Tracer) GoProbes() map[string]ebpfcommon.FunctionPrograms {
 			Start: p.bpfObjects.UprobeConnServe,
 			End:   p.bpfObjects.UprobeConnServeRet,
 		},
+		"net.(*netFD).Read": { // for http server too
+			Start: p.bpfObjects.UprobeNetFdRead,
+		},
 		"net/http.(*persistConn).roundTrip": { // http client
 			Start: p.bpfObjects.UprobePersistConnRoundTrip,
 		},

diff --git a/pkg/internal/netolly/ebpf/net_bpfel_arm64.go b/pkg/internal/netolly/ebpf/net_bpfel_arm64.go
diff --git a/pkg/internal/netolly/ebpf/net_bpfel_x86.go b/pkg/internal/netolly/ebpf/net_bpfel_x86.go
diff --git a/pkg/internal/netolly/ebpf/tracer.go b/pkg/internal/netolly/ebpf/tracer.go
@@ -37,7 +37,7 @@ import (
 )
 
 // $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
-//go:generate bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -type flow_metrics_t -type flow_id_t  -type flow_record_t -target amd64,arm64 Net ../../../../bpf/flows.c -- -I../../../../bpf/headers
+//go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -type flow_metrics_t -type flow_id_t  -type flow_record_t -target amd64,arm64 Net ../../../../bpf/flows.c -- -I../../../../bpf/headers
 
 const (
 	qdiscType = "clsact"