Dump metrics and traces on file after integration tests (grafana#1521)

* Dump metrics and traces on file after integration tests

* Fix linting

test/integration/components/kube/kind.go

@@ -3,11 +3,14 @@ package kube
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -26,6 +29,10 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
 	"sigs.k8s.io/e2e-framework/support/kind"
+
+	"github.com/grafana/beyla/test/integration/components/jaeger"
+	"github.com/grafana/beyla/test/integration/components/prom"
+	"github.com/grafana/beyla/test/integration/k8s/common/testpath"
 )
 
 const (
@@ -47,6 +54,8 @@ type Kind struct {
 	deployManifests []string
 	localImages     []string
 	logsDir         string
+	promEndpoint    string
+	jaegerEndpoint  string
 }
 
 // Option that can be passed to the NewKind function in order to change the configuration
@@ -68,12 +77,29 @@ func KindConfig(filePath string) Option {
 }
 
 // ExportLogs can be passed to NewKind to specify the folder where the kubernetes logs will be exported after the tests.
+// Default: k8s.KindLogs
 func ExportLogs(folder string) Option {
 	return func(k *Kind) {
 		k.logsDir = folder
 	}
 }
 
+// ExportPrometheus overrides the prometheus host:port, where all the stored metrics will be collected from
+// before the Kind cluster is shut down. Default: localhost:39090
+func ExportPrometheus(hostPort string) Option {
+	return func(k *Kind) {
+		k.promEndpoint = hostPort
+	}
+}
+
+// ExportJaeger overrides a jaeger host:port, where all the stored traces will be collected from
+// before the Kind cluster is shut down. Default: localhost:36686
+func ExportJaeger(hostPort string) Option {
+	return func(k *Kind) {
+		k.jaegerEndpoint = hostPort
+	}
+}
+
 // Timeout for long-running operations (e.g. deployments, readiness probes...)
 func Timeout(t time.Duration) Option {
 	return func(k *Kind) {
@@ -91,9 +117,12 @@ func LocalImage(nameTag string) Option {
 // NewKind creates a kind cluster given a name and set of Option instances.
 func NewKind(kindClusterName string, options ...Option) *Kind {
 	k := &Kind{
-		testEnv:     env.New(),
-		clusterName: kindClusterName,
-		timeout:     2 * time.Minute,
+		testEnv:        env.New(),
+		clusterName:    kindClusterName,
+		timeout:        2 * time.Minute,
+		promEndpoint:   "localhost:39090",
+		jaegerEndpoint: "localhost:36686",
+		logsDir:        testpath.KindLogs,
 	}
 	for _, option := range options {
 		option(k)
@@ -128,6 +157,8 @@ func (k *Kind) Run(m *testing.M) {
 	code := k.testEnv.Setup(funcs...).
 		Finish(
 			k.exportLogs(),
+			k.exportAllMetrics(),
+			k.exportAllTraces(),
 			k.deleteLabeled(),
 			envfuncs.DestroyCluster(k.clusterName),
 		).Run(m)
@@ -149,6 +180,46 @@ func (k *Kind) exportLogs() env.Func {
 	}
 }
 
+func (k *Kind) exportAllMetrics() env.Func {
+	return func(ctx context.Context, _ *envconf.Config) (context.Context, error) {
+		if k.promEndpoint == "" {
+			return ctx, nil
+		}
+		_ = os.MkdirAll(path.Join(k.logsDir, k.clusterName), 0755)
+		out, err := os.Create(path.Join(k.logsDir, k.clusterName, "prometheus_metrics.txt"))
+		if err != nil {
+			log().Error("creating prometheus export file", "error", err)
+			return ctx, nil
+		}
+		defer out.Close()
+		if err := DumpMetrics(out, k.promEndpoint); err != nil {
+			log().Error("dumping prometheus metrics", "error", err)
+			return ctx, nil
+		}
+		return ctx, nil
+	}
+}
+
+func (k *Kind) exportAllTraces() env.Func {
+	return func(ctx context.Context, _ *envconf.Config) (context.Context, error) {
+		if k.promEndpoint == "" {
+			return ctx, nil
+		}
+		_ = os.MkdirAll(path.Join(k.logsDir, k.clusterName), 0755)
+		out, err := os.Create(path.Join(k.logsDir, k.clusterName, "jaeger_traces.txt"))
+		if err != nil {
+			log().Error("creating jaeger export file", "error", err)
+			return ctx, nil
+		}
+		defer out.Close()
+		if err := DumpTraces(out, k.jaegerEndpoint); err != nil {
+			log().Error("dumping jaeger traces", "error", err)
+			return ctx, nil
+		}
+		return ctx, nil
+	}
+}
+
 // deleteLabeled sends a kill signal to all the Beyla instances before tearing down the
 // kind cluster, in order to force them to write the coverage information
 // This method assumes that all the beyla pod instances are labeled as "teardown=delete"
@@ -343,3 +414,67 @@ func (k *Kind) loadLocalImage(tag string) env.Func {
 		return ctx, fmt.Errorf("couldn't load image %q from local registry: %w", tag, err)
 	}
 }
+
+func DumpMetrics(out io.Writer, promHostPort string) error {
+	if _, err := fmt.Fprintf(out, "===== Dumping metrics from %s ====\n", promHostPort); err != nil {
+		return err
+	}
+	pq := prom.Client{HostPort: promHostPort}
+	results, err := pq.Query(`{__name__!=""}`)
+	if err != nil {
+		return err
+	}
+	for _, res := range results {
+		fmt.Fprint(out, res.Metric["__name__"])
+		fmt.Fprint(out, "{")
+		for k, v := range res.Metric {
+			if k == "__name__" {
+				continue
+			}
+			fmt.Fprintf(out, `%s="%s",`, k, v)
+		}
+		fmt.Fprintf(out, "} ")
+		for _, v := range res.Value {
+			fmt.Fprintf(out, "%v ", v)
+		}
+		fmt.Fprintln(out)
+	}
+	return nil
+}
+
+func DumpTraces(out io.Writer, jaegerHostPort string) error {
+	if !strings.HasPrefix(jaegerHostPort, "http") {
+		jaegerHostPort = "http://" + jaegerHostPort
+	}
+	if _, err := fmt.Fprintf(out, "===== Dumping traces from %s ====\n", jaegerHostPort); err != nil {
+		return fmt.Errorf("writing output: %w", err)
+	}
+	// get services
+	res, err := http.Get(jaegerHostPort + "/api/services")
+	if err != nil {
+		return fmt.Errorf("getting services: %w", err)
+	}
+	svcs := jaeger.Services{}
+	if err := json.NewDecoder(res.Body).Decode(&svcs); err != nil {
+		return fmt.Errorf("decoding services: %w", err)
+	}
+	for _, svcName := range svcs.Data {
+		fmt.Fprintf(out, "---- Service: %s ----\n", svcName)
+		res, err := http.Get(jaegerHostPort + "/api/traces?service=" + svcName)
+		if err != nil {
+			fmt.Fprintln(out, "! ERROR getting trace:", err)
+			continue
+		}
+		tq := jaeger.TracesQuery{}
+		if err := json.NewDecoder(res.Body).Decode(&tq); err != nil {
+			fmt.Fprintln(out, "! ERROR decoding trace:", err)
+			continue
+		}
+		for _, trace := range tq.Data {
+			if err := json.NewEncoder(out).Encode(trace); err != nil {
+				fmt.Fprintln(out, "! ERROR encoding trace:", err)
+			}
+		}
+	}
+	return nil
+}

test/integration/k8s/common/k8s_common.go

@@ -1,28 +1,25 @@
 package k8s
 
-import "path"
+import (
+	"path"
 
-var (
-	PathRoot            = path.Join("..", "..", "..", "..")
-	PathOutput          = path.Join(PathRoot, "testoutput")
-	PathKindLogs        = path.Join(PathOutput, "kind")
-	PathIntegrationTest = path.Join(PathRoot, "test", "integration")
-	PathComponents      = path.Join(PathIntegrationTest, "components")
-	PathManifests       = path.Join(PathIntegrationTest, "k8s", "manifests")
+	"github.com/grafana/beyla/test/integration/k8s/common/testpath"
+)
 
-	DockerfileTestServer       = path.Join(PathComponents, "testserver", "Dockerfile")
-	DockerfileBeyla            = path.Join(PathComponents, "beyla", "Dockerfile")
-	DockerfileBeylaK8sCache    = path.Join(PathComponents, "beyla-k8s-cache", "Dockerfile")
-	DockerfilePinger           = path.Join(PathComponents, "grpcpinger", "Dockerfile")
-	DockerfilePythonTestServer = path.Join(PathComponents, "pythonserver", "Dockerfile_8083")
-	DockerfileHTTPPinger       = path.Join(PathComponents, "httppinger", "Dockerfile")
+var (
+	DockerfileTestServer       = path.Join(testpath.Components, "testserver", "Dockerfile")
+	DockerfileBeyla            = path.Join(testpath.Components, "beyla", "Dockerfile")
+	DockerfileBeylaK8sCache    = path.Join(testpath.Components, "beyla-k8s-cache", "Dockerfile")
+	DockerfilePinger           = path.Join(testpath.Components, "grpcpinger", "Dockerfile")
+	DockerfilePythonTestServer = path.Join(testpath.Components, "pythonserver", "Dockerfile_8083")
+	DockerfileHTTPPinger       = path.Join(testpath.Components, "httppinger", "Dockerfile")
 
-	PingerManifest               = path.Join(PathManifests, "/06-instrumented-client.template.yml")
-	GrpcPingerManifest           = path.Join(PathManifests, "/06-instrumented-grpc-client.template.yml")
-	UninstrumentedPingerManifest = path.Join(PathManifests, "/06-uninstrumented-client.template.yml")
-	UninstrumentedAppManifest    = path.Join(PathManifests, "/05-uninstrumented-service.yml")
-	PingerManifestProm           = path.Join(PathManifests, "/06-instrumented-client-prom.template.yml")
-	GrpcPingerManifestProm       = path.Join(PathManifests, "/06-instrumented-grpc-client-prom.template.yml")
+	PingerManifest               = path.Join(testpath.Manifests, "/06-instrumented-client.template.yml")
+	GrpcPingerManifest           = path.Join(testpath.Manifests, "/06-instrumented-grpc-client.template.yml")
+	UninstrumentedPingerManifest = path.Join(testpath.Manifests, "/06-uninstrumented-client.template.yml")
+	UninstrumentedAppManifest    = path.Join(testpath.Manifests, "/05-uninstrumented-service.yml")
+	PingerManifestProm           = path.Join(testpath.Manifests, "/06-instrumented-client-prom.template.yml")
+	GrpcPingerManifestProm       = path.Join(testpath.Manifests, "/06-instrumented-grpc-client-prom.template.yml")
 )
 
 // Pinger stores the configuration data of a local pod that will be used to

test/integration/k8s/common/k8s_metrics_testfuncs.go

@@ -4,10 +4,7 @@ package k8s
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"net/http"
-	"os"
 	"slices"
 	"testing"
 	"time"
@@ -18,7 +15,6 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
-	"github.com/grafana/beyla/test/integration/components/jaeger"
 	"github.com/grafana/beyla/test/integration/components/kube"
 	"github.com/grafana/beyla/test/integration/components/prom"
 )
@@ -291,66 +287,3 @@ func testMetricsDecoration(
 		return ctx
 	}
 }
-
-func DumpMetricsAfterFail(t *testing.T, queryURL string) {
-	if !t.Failed() {
-		return
-	}
-	fmt.Printf("===== Dumping metrics from %s ====\n", queryURL)
-	pq := prom.Client{HostPort: queryURL}
-	results, err := pq.Query(`{__name__!=""}`)
-	if err != nil {
-		fmt.Printf("ERROR: %s\n", err)
-		return
-	}
-	for _, res := range results {
-		fmt.Printf(res.Metric["__name__"])
-		fmt.Printf("{")
-		for k, v := range res.Metric {
-			if k == "__name__" {
-				continue
-			}
-			fmt.Printf(`%s="%s",`, k, v)
-		}
-		fmt.Print("} ")
-		for _, v := range res.Value {
-			fmt.Printf("%s ", v)
-		}
-		fmt.Println()
-	}
-}
-
-func DumpTracesAfterFail(t *testing.T, hostURL string) {
-	if !t.Failed() {
-		return
-	}
-	fmt.Printf("===== Dumping traces from %s ====\n", hostURL)
-	// get services
-	res, err := http.Get(hostURL + "/api/services")
-	if err != nil {
-		fmt.Println("ERROR getting services:", err)
-		return
-	}
-	svcs := jaeger.Services{}
-	if err := json.NewDecoder(res.Body).Decode(&svcs); err != nil {
-		fmt.Println("ERROR decoding services:", err)
-		return
-	}
-	for _, svcName := range svcs.Data {
-		fmt.Printf("---- Service: %s ----\n", svcName)
-		res, err := http.Get(hostURL + "/api/traces?service=" + svcName)
-		if err != nil {
-			fmt.Println("ERROR getting service:", err)
-			return
-		}
-		tq := jaeger.TracesQuery{}
-		if err := json.NewDecoder(res.Body).Decode(&tq); err != nil {
-			fmt.Println("ERROR decoding service:", err)
-			continue
-		}
-		for _, trace := range tq.Data {
-			json.NewEncoder(os.Stdout).Encode(trace)
-			fmt.Println()
-		}
-	}
-}

test/integration/k8s/common/testpath/testpath.go

@@ -0,0 +1,12 @@
+package testpath
+
+import "path"
+
+var (
+	Root            = path.Join("..", "..", "..", "..")
+	Output          = path.Join(Root, "testoutput")
+	KindLogs        = path.Join(Output, "kind")
+	IntegrationTest = path.Join(Root, "test", "integration")
+	Components      = path.Join(IntegrationTest, "components")
+	Manifests       = path.Join(IntegrationTest, "k8s", "manifests")
+)

test/integration/k8s/daemonset/k8s_daemonset_main_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/grafana/beyla/test/integration/components/docker"
 	"github.com/grafana/beyla/test/integration/components/kube"
 	k8s "github.com/grafana/beyla/test/integration/k8s/common"
+	"github.com/grafana/beyla/test/integration/k8s/common/testpath"
 	"github.com/grafana/beyla/test/tools"
 )
 
@@ -37,21 +38,20 @@ func TestMain(m *testing.M) {
 	}
 
 	cluster = kube.NewKind("test-kind-cluster-daemonset",
-		kube.ExportLogs(k8s.PathKindLogs),
-		kube.KindConfig(k8s.PathManifests+"/00-kind.yml"),
+		kube.KindConfig(testpath.Manifests+"/00-kind.yml"),
 		kube.LocalImage("testserver:dev"),
 		kube.LocalImage("beyla:dev"),
 		kube.LocalImage("grpcpinger:dev"),
 		kube.LocalImage("quay.io/prometheus/prometheus:v2.53.0"),
 		kube.LocalImage("otel/opentelemetry-collector-contrib:0.103.0"),
 		kube.LocalImage("jaegertracing/all-in-one:1.57"),
-		kube.Deploy(k8s.PathManifests+"/01-volumes.yml"),
-		kube.Deploy(k8s.PathManifests+"/01-serviceaccount.yml"),
-		kube.Deploy(k8s.PathManifests+"/02-prometheus-otelscrape.yml"),
-		kube.Deploy(k8s.PathManifests+"/03-otelcol.yml"),
-		kube.Deploy(k8s.PathManifests+"/04-jaeger.yml"),
-		kube.Deploy(k8s.PathManifests+"/05-uninstrumented-service.yml"),
-		kube.Deploy(k8s.PathManifests+"/06-beyla-daemonset.yml"),
+		kube.Deploy(testpath.Manifests+"/01-volumes.yml"),
+		kube.Deploy(testpath.Manifests+"/01-serviceaccount.yml"),
+		kube.Deploy(testpath.Manifests+"/02-prometheus-otelscrape.yml"),
+		kube.Deploy(testpath.Manifests+"/03-otelcol.yml"),
+		kube.Deploy(testpath.Manifests+"/04-jaeger.yml"),
+		kube.Deploy(testpath.Manifests+"/05-uninstrumented-service.yml"),
+		kube.Deploy(testpath.Manifests+"/06-beyla-daemonset.yml"),
 	)
 
 	cluster.Run(m)