Merge pull request #205 from grcevski/wrapped_resp

Read the statusCode on WriteHeader

bpf/go_nethttp.c

@@ -48,9 +48,33 @@ int uprobe_ServeHTTP(struct pt_regs *ctx) {
     return 0;
 }
 
-SEC("uprobe/ServeHTTP_return")
-int uprobe_ServeHttp_return(struct pt_regs *ctx) {
-    bpf_dbg_printk("=== uprobe/ServeHTTP_return === ");
+SEC("uprobe/startBackgroundRead")
+int uprobe_startBackgroundRead(struct pt_regs *ctx) {
+    bpf_dbg_printk("=== uprobe/proc startBackgroundRead === ");
+
+    void *goroutine_addr = GOROUTINE_PTR(ctx);
+    bpf_dbg_printk("goroutine_addr %lx", goroutine_addr);
+
+    // This code is here for keepalive support on HTTP requests. Since the connection is not
+    // established everytime, we set the initial goroutine start on the new read initiation.
+    goroutine_metadata *g_metadata = bpf_map_lookup_elem(&ongoing_goroutines, &goroutine_addr);
+    if (!g_metadata) {
+        goroutine_metadata metadata = {
+            .timestamp = bpf_ktime_get_ns(),
+            .parent = (u64)goroutine_addr,
+        };
+
+        if (bpf_map_update_elem(&ongoing_goroutines, &goroutine_addr, &metadata, BPF_ANY)) {
+            bpf_dbg_printk("can't update active goroutine");
+        }
+    }
+
+    return 0;
+}
+
+SEC("uprobe/WriteHeader")
+int uprobe_WriteHeader(struct pt_regs *ctx) {
+    bpf_dbg_printk("=== uprobe/WriteHeader === ");
     void *goroutine_addr = GOROUTINE_PTR(ctx);
     bpf_dbg_printk("goroutine_addr %lx", goroutine_addr);
 
@@ -80,10 +104,18 @@ int uprobe_ServeHttp_return(struct pt_regs *ctx) {
         trace->go_start_monotime_ns = invocation->start_monotime_ns;
     }
 
-    // Read arguments from the original set of registers
+    // Read the response argument
+    void *resp_ptr = GO_PARAM1(ctx);
 
     // Get request struct
-    void *req_ptr = GO_PARAM4(&(invocation->regs));
+    void *req_ptr = 0;
+    bpf_probe_read(&req_ptr, sizeof(req_ptr), (void *)(resp_ptr + resp_req_pos));
+
+    if (!req_ptr) {
+        bpf_printk("can't find req inside the response value");
+        bpf_ringbuf_discard(trace, 0);
+        return 0;
+    }
 
     // Get method from Request.Method
     if (!read_go_str("method", req_ptr, method_ptr_pos, &trace->method, sizeof(trace->method))) {
@@ -117,43 +149,14 @@ int uprobe_ServeHttp_return(struct pt_regs *ctx) {
     }
     bpf_probe_read(&trace->content_length, sizeof(trace->content_length), (void *)(req_ptr + content_length_ptr_pos));
 
-    // get return code from http.ResponseWriter (interface)
-    // assuming implementation of http.ResponseWriter is http.response
-    // TODO: this is really a nonportable assumption
-    void *resp_ptr = GO_PARAM3(&(invocation->regs));
-
-    bpf_probe_read(&trace->status, sizeof(trace->status), (void *)(resp_ptr + status_ptr_pos));
+    trace->status = (u16)(((u64)GO_PARAM2(ctx)) & 0x0ffff);
 
     // submit the completed trace via ringbuffer
     bpf_ringbuf_submit(trace, get_flags());
 
     return 0;
 }
 
-SEC("uprobe/startBackgroundRead")
-int uprobe_startBackgroundRead(struct pt_regs *ctx) {
-    bpf_dbg_printk("=== uprobe/proc startBackgroundRead === ");
-
-    void *goroutine_addr = GOROUTINE_PTR(ctx);
-    bpf_dbg_printk("goroutine_addr %lx", goroutine_addr);
-
-    // This code is here for keepalive support on HTTP requests. Since the connection is not
-    // established everytime, we set the initial goroutine start on the new read initiation.
-    goroutine_metadata *g_metadata = bpf_map_lookup_elem(&ongoing_goroutines, &goroutine_addr);
-    if (!g_metadata) {
-        goroutine_metadata metadata = {
-            .timestamp = bpf_ktime_get_ns(),
-            .parent = (u64)goroutine_addr,
-        };
-
-        if (bpf_map_update_elem(&ongoing_goroutines, &goroutine_addr, &metadata, BPF_ANY)) {
-            bpf_dbg_printk("can't update active goroutine");
-        }
-    }
-
-    return 0;
-}
-
 /* HTTP Client. We expect to see HTTP client in both HTTP server and gRPC server calls.*/
 
 SEC("uprobe/clientSend")

bpf/go_nethttp.h

@@ -25,5 +25,6 @@ volatile const u64 status_code_ptr_pos;
 volatile const u64 remoteaddr_ptr_pos;
 volatile const u64 host_ptr_pos;
 volatile const u64 content_length_ptr_pos;
+volatile const u64 resp_req_pos;
 
 #endif

bpf/go_runtime.c

@@ -75,6 +75,9 @@ int uprobe_proc_goexit1(struct pt_regs *ctx) {
     bpf_dbg_printk("goroutine_addr %lx", goroutine_addr);
 
     bpf_map_delete_elem(&ongoing_goroutines, &goroutine_addr);
+    // We also clean-up ongoing_server_requests so that we can handle hijacked requests, where the ServeHTTP
+    // finishes, but we never call WriteHeader to clean-up the ongoing requests
+    bpf_map_delete_elem(&ongoing_server_requests, &goroutine_addr);
 
     return 0;
 }

bpf/http_ssl.h

@@ -112,7 +112,7 @@ static __always_inline void handle_ssl_buf(u64 id, ssl_args_t *args, int bytes_l
     if (args && bytes_len > 0) {
         void *ssl = ((void *)args->ssl);
         u64 ssl_ptr = (u64)ssl;
-        bpf_printk("SSL_buf id=%d ssl=%llx", id, ssl);
+        bpf_dbg_printk("SSL_buf id=%d ssl=%llx", id, ssl);
         connection_info_t *conn = bpf_map_lookup_elem(&ssl_to_conn, &ssl);
 
         if (!conn) {

configs/offsets/tracker_input.json

@@ -14,7 +14,8 @@
         "Path"
       ],
       "net/http.response": [
-        "status"
+        "status",
+        "req"
       ],
       "net/http.Response": [
         "StatusCode"

pkg/ebpf/nethttp/nethttp.go

@@ -61,6 +61,7 @@ func (p *Tracer) Constants(_ *exec.FileInfo, offsets *goexec.Offsets) map[string
 		"remoteaddr_ptr_pos",
 		"host_ptr_pos",
 		"content_length_ptr_pos",
+		"resp_req_pos",
 	} {
 		constants[s] = offsets.Field[s]
 	}
@@ -80,11 +81,13 @@ func (p *Tracer) GoProbes() map[string]ebpfcommon.FunctionPrograms {
 		"net/http.HandlerFunc.ServeHTTP": {
 			Required: true,
 			Start:    p.bpfObjects.UprobeServeHTTP,
-			End:      p.bpfObjects.UprobeServeHttpReturn,
 		},
 		"net/http.(*connReader).startBackgroundRead": {
 			Start: p.bpfObjects.UprobeStartBackgroundRead,
 		},
+		"net/http.(*response).WriteHeader": {
+			Start: p.bpfObjects.UprobeWriteHeader,
+		},
 		"net/http.(*Client).send": {
 			Start: p.bpfObjects.UprobeClientSend,
 			End:   p.bpfObjects.UprobeClientSendReturn,
@@ -123,7 +126,9 @@ func (p *GinTracer) GoProbes() map[string]ebpfcommon.FunctionPrograms {
 		"github.com/gin-gonic/gin.(*Engine).ServeHTTP": {
 			Required: true,
 			Start:    p.bpfObjects.UprobeServeHTTP,
-			End:      p.bpfObjects.UprobeServeHttpReturn,
+		},
+		"net/http.(*response).WriteHeader": {
+			Start: p.bpfObjects.UprobeWriteHeader,
 		},
 	}
 }

pkg/goexec/offsets.json

@@ -197,6 +197,18 @@
       }
     },
     "net/http.response": {
+      "req": {
+        "versions": {
+          "oldest": "1.17.0",
+          "newest": "1.20.6"
+        },
+        "offsets": [
+          {
+            "offset": 8,
+            "since": "1.17.0"
+          }
+        ]
+      },
       "status": {
         "versions": {
           "oldest": "1.17.0",

pkg/goexec/structmembers.go

@@ -52,6 +52,7 @@ var structMembers = map[string]structInfo{
 		lib: "go",
 		fields: map[string]string{
 			"status": "status_ptr_pos",
+			"req":    "resp_req_pos",
 		},
 	},
 	"net/http.Response": {

pkg/goexec/structmembers_test.go

@@ -69,6 +69,7 @@ func TestGoOffsetsFromDwarf(t *testing.T) {
 		"status_ptr_pos":        uint64(120),
 		"tcp_addr_ip_ptr_pos":   uint64(0),
 		"tcp_addr_port_ptr_pos": uint64(24),
+		"resp_req_pos":          uint64(8),
 	}, offsets)
 }
 

test/integration/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - "8080:8080"
       - "8081:8081"
       - "8082:8082"
+      - "8083:8083"
       - "50051:50051"
     environment:
       LOG_LEVEL: DEBUG

test/integration/red_test.go

@@ -19,11 +19,12 @@ import (
 )
 
 const (
-	instrumentedServiceStdURL     = "http://localhost:8080"
-	instrumentedServiceGinURL     = "http://localhost:8081"
-	instrumentedServiceGorillaURL = "http://localhost:8082"
-	prometheusHostPort            = "localhost:9090"
-	jaegerQueryURL                = "http://localhost:16686/api/traces"
+	instrumentedServiceStdURL        = "http://localhost:8080"
+	instrumentedServiceGinURL        = "http://localhost:8081"
+	instrumentedServiceGorillaURL    = "http://localhost:8082"
+	instrumentedServiceGorillaMidURL = "http://localhost:8083"
+	prometheusHostPort               = "localhost:9090"
+	jaegerQueryURL                   = "http://localhost:16686/api/traces"
 
 	testTimeout = 20 * time.Second
 )
@@ -41,6 +42,7 @@ func testREDMetricsHTTP(t *testing.T) {
 		instrumentedServiceStdURL,
 		instrumentedServiceGorillaURL,
 		instrumentedServiceGinURL,
+		instrumentedServiceGorillaMidURL,
 	} {
 		t.Run(testCaseURL, func(t *testing.T) {
 			waitForTestComponents(t, testCaseURL)

test/integration/red_test_dotnet.go

@@ -17,6 +17,8 @@ func testREDMetricsDotNetHTTP(t *testing.T) {
 	}
 }
 
+/*
+// Disabled because of: https://github.com/grafana/ebpf-autoinstrument/issues/208
 func testREDMetricsDotNetHTTPS(t *testing.T) {
 	for _, testCaseURL := range []string{
 		"https://localhost:7034",
@@ -27,3 +29,4 @@ func testREDMetricsDotNetHTTPS(t *testing.T) {
 		})
 	}
 }
+*/

test/integration/suites_test.go

@@ -233,6 +233,9 @@ func TestSuite_DotNet(t *testing.T) {
 	t.Run("BPF pinning folder unmounted", testBPFPinningUnmounted)
 }
 
+/*
+// Disabled for now as we randomly fail to register 3 events, but only get 2
+// Issue: https://github.com/grafana/ebpf-autoinstrument/issues/208
 func TestSuite_DotNetTLS(t *testing.T) {
 	compose, err := docker.ComposeSuite("docker-compose-dotnet.yml", path.Join(pathOutput, "test-suite-dotnet-tls.log"))
 	compose.Env = append(compose.Env, `OPEN_PORT=7033`, `EXECUTABLE_NAME=`, `TEST_SERVICE_PORTS=7034:7033`, `TESTSERVER_DOCKERFILE_SUFFIX=_tls`)
@@ -243,6 +246,7 @@ func TestSuite_DotNetTLS(t *testing.T) {
 	require.NoError(t, compose.Close())
 	t.Run("BPF pinning folder unmounted", testBPFPinningUnmounted)
 }
+*/
 
 func TestSuite_Python(t *testing.T) {
 	compose, err := docker.ComposeSuite("docker-compose-python.yml", path.Join(pathOutput, "test-suite-python.log"))