Enhance profiling and allow traces to profiles (#1006)

Co-authored-by: Will Browne <[email protected]>

backend/setup.go

@@ -5,8 +5,10 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"runtime"
 	"strconv"
 
+	pyroscopepprof "github.com/grafana/pyroscope-go/godeltaprof/http/pprof"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
@@ -23,11 +25,14 @@ const (
 	// PluginProfilingEnabledEnv is a constant for the GF_PLUGIN_PROFILING_ENABLED environment variable used to enable pprof.
 	PluginProfilingEnabledEnv = "GF_PLUGIN_PROFILING_ENABLED"
 
-	// PluginProfilerPortEnvDeprecated is a constant for the GF_PLUGINS_PROFILER_PORT environment variable use to specify a pprof port (default 6060).
+	// PluginProfilerPortEnvDeprecated is a constant for the GF_PLUGINS_PROFILER_PORT environment variable used to specify a pprof port (default 6060).
 	PluginProfilerPortEnvDeprecated = "GF_PLUGINS_PROFILER_PORT" // nolint:gosec
-	// PluginProfilingPortEnv is a constant for the GF_PLUGIN_PROFILING_PORT environment variable use to specify a pprof port (default 6060).
+	// PluginProfilingPortEnv is a constant for the GF_PLUGIN_PROFILING_PORT environment variable used to specify a pprof port (default 6060).
 	PluginProfilingPortEnv = "GF_PLUGIN_PROFILING_PORT" // nolint:gosec
 
+	// PluginProfilingContentionEnv is a constant for the GF_PLUGIN_PROFILING_CONTENTION environment variable used to enable contention profiling (report of all goroutine blocking and mutex contention events) (default false).
+	PluginProfilingContentionEnv = "GF_PLUGIN_PROFILING_CONTENTION" // nolint:gosec
+
 	// PluginTracingOpenTelemetryOTLPAddressEnv is a constant for the GF_INSTANCE_OTLP_ADDRESS
 	// environment variable used to specify the OTLP address.
 	PluginTracingOpenTelemetryOTLPAddressEnv = "GF_INSTANCE_OTLP_ADDRESS" // nolint:gosec
@@ -75,32 +80,56 @@ func setupProfiler(pluginID string) {
 		// compare value to plugin name
 		if value == pluginID {
 			profilerEnabled = true
+			Logger.Warn("Use of GF_PLUGINS_PROFILER environment variable is deprecated and won't be supported in the future. Please use GF_PLUGIN_PROFILING_ENABLED instead.")
 		}
 	} else if value, ok = os.LookupEnv(PluginProfilingEnabledEnv); ok {
 		if value == "true" {
 			profilerEnabled = true
 		}
 	}
 
-	Logger.Debug("Profiler", "enabled", profilerEnabled)
 	if profilerEnabled {
 		profilerPort := "6060"
 		for _, env := range []string{PluginProfilerPortEnvDeprecated, PluginProfilingPortEnv} {
 			if value, ok := os.LookupEnv(env); ok {
 				profilerPort = value
+
+				if env == PluginProfilerPortEnvDeprecated {
+					Logger.Warn("Use of GF_PLUGINS_PROFILER_PORT environment variable is deprecated and won't be supported in the future. Please use GF_PLUGIN_PROFILING_PORT instead.")
+				}
+
 				break
 			}
 		}
-		Logger.Info("Profiler", "port", profilerPort)
+
+		contentionProfiling := false
+		if value, ok := os.LookupEnv(PluginProfilingContentionEnv); ok {
+			if value == "true" {
+				contentionProfiling = true
+			}
+		}
+
+		Logger.Info("Profiling enabled", "port", profilerPort, "contentionProfilingEnabled", contentionProfiling)
 		portConfig := fmt.Sprintf(":%s", profilerPort)
 
+		if contentionProfiling {
+			runtime.SetBlockProfileRate(1)
+			runtime.SetMutexProfileFraction(1)
+		}
+
 		r := http.NewServeMux()
 		r.HandleFunc("/debug/pprof/", pprof.Index)
 		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
 		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
 
+		// Register godeltaprof endpoints which are more suitable for continuous profiling,
+		// see https://github.com/grafana/pyroscope-go/tree/main/godeltaprof.
+		r.HandleFunc("/debug/pprof/delta_heap", pyroscopepprof.Heap)
+		r.HandleFunc("/debug/pprof/delta_block", pyroscopepprof.Block)
+		r.HandleFunc("/debug/pprof/delta_mutex", pyroscopepprof.Mutex)
+
 		go func() {
 			//nolint:gosec
 			if err := http.ListenAndServe(portConfig, r); err != nil {
@@ -147,6 +176,7 @@ func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
 		if err != nil {
 			return fmt.Errorf("new trace provider: %w", err)
 		}
+
 		pf, err := tracerprovider.NewTextMapPropagator(tracingCfg.propagation)
 		if err != nil {
 			return fmt.Errorf("new propagator format: %w", err)
@@ -158,8 +188,8 @@ func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
 	}
 
 	enabled := tracingCfg.isEnabled()
-	Logger.Debug("Tracing", "enabled", enabled)
 	if enabled {
+		Logger.Info("Tracing enabled")
 		Logger.Debug(
 			"Tracing configuration",
 			"propagation", tracingCfg.propagation,

go.mod

@@ -10,11 +10,14 @@ require (
 	github.com/getkin/kin-openapi v0.124.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
+	github.com/grafana/otel-profiling-go v0.5.1
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.7
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-plugin v1.6.1
 	github.com/invopop/jsonschema v0.12.0 // for schema codgen+extraction
 	github.com/json-iterator/go v1.1.12
+	github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6
 	github.com/magefile/mage v1.15.0
 	github.com/mattetti/filebuffer v1.0.1
 	github.com/mitchellh/reflectwalk v1.0.2
@@ -74,8 +77,7 @@ require (
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -46,6 +46,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
 github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -81,6 +82,10 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8=
+github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.7 h1:C11j63y7gymiW8VugJ9ZW0pWfxTZugdSJyC48olk5KY=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.7/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE=
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
@@ -109,8 +114,8 @@ github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpR
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
+github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -200,8 +205,6 @@ github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3/go.mod h1:1xEUf2abjfP9
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM=
 github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0=
-github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae h1:oyiy3uBj1F4O3AaFh7hUGBrJjAssJhKyAbwxtkslxqo=
-github.com/vectordotdev/go-datemath v0.1.1-0.20220323213446-f3954d0b18ae/go.mod h1:PnwzbSst7KD3vpBzzlntZU5gjVa455Uqa5QPiKSYJzQ=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -220,16 +223,20 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.26.0 h1:RH76Cl2pfOLLoCtxAPax9c
 go.opentelemetry.io/contrib/propagators/jaeger v1.26.0/go.mod h1:W/cylm0ZtJK1uxsuTqoYGYPnqpZ8CeVGgW7TwfXPsGw=
 go.opentelemetry.io/contrib/samplers/jaegerremote v0.20.0 h1:ja+d7Aea/9PgGxB63+E0jtRFpma717wubS0KFkZpmYw=
 go.opentelemetry.io/contrib/samplers/jaegerremote v0.20.0/go.mod h1:Yc1eg51SJy7xZdOTyg1xyFcwE+ghcWh3/0hKeLo6Wlo=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
 go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 h1:Waw9Wfpo/IXzOI8bCB7DIk+0JZcqqsyn1JFnAc+iam8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0/go.mod h1:wnJIG4fOqyynOnnQF/eQb4/16VlX2EJAHhHgqIqWfAo=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
 go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
+go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
 go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=
 go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
 go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
 go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
@@ -270,6 +277,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/tracerprovider/profiling_tracerprovider.go

@@ -0,0 +1,27 @@
+package tracerprovider
+
+import (
+	"context"
+
+	otelpyroscope "github.com/grafana/otel-profiling-go"
+	trace "go.opentelemetry.io/otel/trace"
+)
+
+type profilingTracerProvider struct {
+	trace.TracerProvider
+	wrappedTp TracerProvider
+}
+
+// newProfilingTracerProvider creates a new tracer provider that annotates pprof
+// samples with span_id label. This allows to establish a relationship
+// between pprof profiles and reported tracing spans.
+func newProfilingTracerProvider(tp TracerProvider) *profilingTracerProvider {
+	return &profilingTracerProvider{
+		TracerProvider: otelpyroscope.NewTracerProvider(tp),
+		wrappedTp:      tp,
+	}
+}
+
+func (tp *profilingTracerProvider) Shutdown(ctx context.Context) error {
+	return tp.wrappedTp.Shutdown(ctx)
+}

internal/tracerprovider/tracerprovider.go

@@ -84,7 +84,12 @@ func NewTracerProvider(address string, samplerOpts SamplerOptions, opts tracing.
 	if err != nil {
 		return nil, fmt.Errorf("new otel SamplerType: %w", err)
 	}
-	return newOtelTracerProvider(exp, sampler, opts.CustomAttributes...)
+	tp, err := newOtelTracerProvider(exp, sampler, opts.CustomAttributes...)
+	if err != nil {
+		return nil, err
+	}
+
+	return newProfilingTracerProvider(tp), nil
 }
 
 // NewTextMapPropagator takes a string-like value and returns the corresponding propagation.TextMapPropagator.