From 8fc7bdfaf72898c714b368b309829b6298837c12 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 16:16:27 +0100 Subject: [PATCH 01/12] Export internal metrics using OTEL metrics --- charts/beyla/Chart.yaml | 2 +- charts/beyla/README.md | 2 +- charts/beyla/templates/daemon-set.yaml | 2 +- docs/sources/configure/options.md | 10 +- pkg/beyla/config.go | 8 ++ pkg/beyla/config_test.go | 1 + pkg/components/beyla.go | 29 ++-- pkg/export/otel/metrics.go | 1 - pkg/export/otel/metrics_internal.go | 176 +++++++++++++++++++++++++ pkg/internal/imetrics/imetrics.go | 3 +- 10 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 pkg/export/otel/metrics_internal.go diff --git a/charts/beyla/Chart.yaml b/charts/beyla/Chart.yaml index 6ea8bb172..cc14e3ea0 100644 --- a/charts/beyla/Chart.yaml +++ b/charts/beyla/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: beyla -version: 1.6.1 +version: 1.6.2 appVersion: 1.9.7 description: eBPF-based autoinstrumentation HTTP, HTTP2 and gRPC services, as well as network metrics. home: https://grafana.com/oss/beyla-ebpf/ diff --git a/charts/beyla/README.md b/charts/beyla/README.md index ea7daa0d6..d5413bd37 100644 --- a/charts/beyla/README.md +++ b/charts/beyla/README.md @@ -1,6 +1,6 @@ # beyla -![Version: 1.6.1](https://img.shields.io/badge/Version-1.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.9.7](https://img.shields.io/badge/AppVersion-1.9.7-informational?style=flat-square) +![Version: 1.6.2](https://img.shields.io/badge/Version-1.6.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.9.7](https://img.shields.io/badge/AppVersion-1.9.7-informational?style=flat-square) eBPF-based autoinstrumentation HTTP, HTTP2 and gRPC services, as well as network metrics. diff --git a/charts/beyla/templates/daemon-set.yaml b/charts/beyla/templates/daemon-set.yaml index 9efafd303..d5a65aa88 100644 --- a/charts/beyla/templates/daemon-set.yaml +++ b/charts/beyla/templates/daemon-set.yaml @@ -74,7 +74,7 @@ spec: containerPort: {{ .Values.service.targetPort | default .Values.config.data.prometheus_export.port }} protocol: TCP {{- end }} - {{- if (and (or (.Values.service.internalMetrics.targetPort) (.Values.config.data.internal_metrics)) (not (eq .Values.config.data.prometheus_export.port .Values.config.data.internal_metrics.prometheus.port))) }} + {{- if (and (or (.Values.service.internalMetrics.targetPort) (.Values.config.data.internal_metrics.prometheus)) (not (eq .Values.config.data.prometheus_export.port .Values.config.data.internal_metrics.prometheus.port))) }} - name: {{ .Values.service.internalMetrics.portName }} containerPort: {{ .Values.service.internalMetrics.targetPort | default .Values.config.data.internal_metrics.prometheus.port }} protocol: TCP diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index 03ef1ae7b..bd7254fd3 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1410,9 +1410,7 @@ gRPC application metrics, while the rest of the **instrumentations** are be disa YAML section `internal_metrics`. This component reports certain internal metrics about the behavior -of the auto-instrumentation tool. Currently, only [Prometheus](https://prometheus.io/) export -is supported. It is enabled if the `internal_metrics` section -contains a `prometheus` subsection with the `port` property set. +of the auto-instrumentation tool. Currently, both [Prometheus](https://prometheus.io/) and [OTEL](https://opentelemetry.io/) metrics export are supported. Prometheus export is enabled if the `internal_metrics` section contains a `prometheus` subsection with the `port` property set. OTEL metrics export is enabled if the `internal_metrics` section contains an `otel_metrics` property set to `true`. Example: @@ -1444,6 +1442,12 @@ same values, this `internal_metrics.prometheus.path` value can be different from `prometheus_export.path`, to keep both metric families separated, or the same (both metric families are listed in the same scrape endpoint). +| YAML | Environment variable | Type | Default | +| ----------- | ---------------------------------------- | ---- | ------- | +| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | bool | `false` | + +Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics will be exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. + ## YAML file example ```yaml diff --git a/pkg/beyla/config.go b/pkg/beyla/config.go index 2230643fa..a987c71d1 100644 --- a/pkg/beyla/config.go +++ b/pkg/beyla/config.go @@ -101,6 +101,7 @@ var DefaultConfig = Config{ Printer: false, // Deprecated: use TracePrinter instead TracePrinter: debug.TracePrinterDisabled, InternalMetrics: imetrics.Config{ + OTELMetrics: false, // disabled by default Prometheus: imetrics.PrometheusConfig{ Port: 0, // disabled by default Path: "/internal/metrics", @@ -289,6 +290,13 @@ func (c *Config) Validate() error { return ConfigError("wildcard_char can only be a single character, multiple characters are not allowed") } + if c.InternalMetrics.OTELMetrics && c.InternalMetrics.Prometheus.Port != 0 { + return ConfigError("you can't enable both OTEL and Prometheus internal metrics") + } + if c.InternalMetrics.OTELMetrics && !c.Metrics.Enabled() && !c.Grafana.OTLP.MetricsEnabled() { + return ConfigError("you can't enable OTEL internal metrics without enabling OTEL metrics") + } + return nil } diff --git a/pkg/beyla/config_test.go b/pkg/beyla/config_test.go index b091c4695..2636df2ee 100644 --- a/pkg/beyla/config_test.go +++ b/pkg/beyla/config_test.go @@ -243,6 +243,7 @@ func TestConfigValidate(t *testing.T) { {"BEYLA_TRACE_PRINTER": "json_indent", "BEYLA_EXECUTABLE_NAME": "foo"}, {"BEYLA_TRACE_PRINTER": "counter", "BEYLA_EXECUTABLE_NAME": "foo"}, {"BEYLA_PROMETHEUS_PORT": "8080", "BEYLA_EXECUTABLE_NAME": "foo", "INSTRUMENT_FUNC_NAME": "bar"}, + {"BEYLA_INTERNAL_METRICS_OTEL_METRICS": "true", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "localhost:1234", "BEYLA_EXECUTABLE_NAME": "foo"}, } for n, tc := range testCases { t.Run(fmt.Sprint("case", n), func(t *testing.T) { diff --git a/pkg/components/beyla.go b/pkg/components/beyla.go index e9ec04323..45f66ea73 100644 --- a/pkg/components/beyla.go +++ b/pkg/components/beyla.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/beyla/pkg/beyla" "github.com/grafana/beyla/pkg/export/attributes" + "github.com/grafana/beyla/pkg/export/otel" "github.com/grafana/beyla/pkg/internal/appolly" "github.com/grafana/beyla/pkg/internal/connector" "github.com/grafana/beyla/pkg/internal/imetrics" @@ -20,7 +21,10 @@ import ( // RunBeyla in the foreground process. This is a blocking function and won't exit // until both the AppO11y and NetO11y components end func RunBeyla(ctx context.Context, cfg *beyla.Config) error { - ctxInfo := buildCommonContextInfo(ctx, cfg) + ctxInfo, err := buildCommonContextInfo(ctx, cfg) + if err != nil { + return fmt.Errorf("can't build common context info: %w", err) + } wg := sync.WaitGroup{} app := cfg.Enabled(beyla.FeatureAppO11y) @@ -111,7 +115,7 @@ func mustSkip(cfg *beyla.Config) string { // from the user-provided configuration func buildCommonContextInfo( ctx context.Context, config *beyla.Config, -) *global.ContextInfo { +) (*global.ContextInfo, error) { promMgr := &connector.PrometheusManager{} ctxInfo := &global.ContextInfo{ Prometheus: promMgr, @@ -126,7 +130,20 @@ func buildCommonContextInfo( RestrictLocalNode: config.Attributes.Kubernetes.MetaRestrictLocalNode, }), } + if config.Attributes.HostID.Override == "" { + ctxInfo.FetchHostID(ctx, config.Attributes.HostID.FetchTimeout) + } else { + ctxInfo.HostID = config.Attributes.HostID.Override + } switch { + case config.InternalMetrics.OTELMetrics: + var err error + config.Metrics.Grafana = &config.Grafana.OTLP + slog.Debug("reporting internal metrics as OpenTelemetry") + ctxInfo.Metrics, err = otel.NewInternalMetricsReporter(ctx, ctxInfo, &config.Metrics) + if err != nil { + return nil, fmt.Errorf("can't start OpenTelemetry metrics: %w", err) + } case config.InternalMetrics.Prometheus.Port != 0: slog.Debug("reporting internal metrics as Prometheus") ctxInfo.Metrics = imetrics.NewPrometheusReporter(&config.InternalMetrics.Prometheus, promMgr, nil) @@ -143,13 +160,7 @@ func buildCommonContextInfo( attributeGroups(config, ctxInfo) - if config.Attributes.HostID.Override == "" { - ctxInfo.FetchHostID(ctx, config.Attributes.HostID.FetchTimeout) - } else { - ctxInfo.HostID = config.Attributes.HostID.Override - } - - return ctxInfo + return ctxInfo, nil } // attributeGroups specifies, based in the provided configuration, which groups of attributes diff --git a/pkg/export/otel/metrics.go b/pkg/export/otel/metrics.go index e8b745cbd..54d4ed1f8 100644 --- a/pkg/export/otel/metrics.go +++ b/pkg/export/otel/metrics.go @@ -651,7 +651,6 @@ func isExponentialAggregation(mc *MetricsConfig, mlog *slog.Logger) bool { return false } -// TODO: restore as private func InstantiateMetricsExporter(ctx context.Context, cfg *MetricsConfig, log *slog.Logger) (sdkmetric.Exporter, error) { var err error var exporter sdkmetric.Exporter diff --git a/pkg/export/otel/metrics_internal.go b/pkg/export/otel/metrics_internal.go new file mode 100644 index 000000000..c8d8ff635 --- /dev/null +++ b/pkg/export/otel/metrics_internal.go @@ -0,0 +1,176 @@ +package otel + +import ( + "context" + "log/slog" + "runtime" + "time" + + "go.opentelemetry.io/otel/attribute" + instrument "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.19.0" + + "github.com/google/uuid" + "github.com/grafana/beyla/pkg/buildinfo" + "github.com/grafana/beyla/pkg/internal/pipe/global" +) + +// InternalMetricsReporter is an internal metrics Reporter that exports to OTEL +type InternalMetricsReporter struct { + ctx context.Context + tracerFlushes instrument.Float64Histogram + otelMetricExports instrument.Float64Counter + otelMetricExportErrs instrument.Float64Counter + otelTraceExports instrument.Float64Counter + otelTraceExportErrs instrument.Float64Counter + prometheusRequests instrument.Float64Counter + instrumentedProcesses instrument.Int64UpDownCounter + beylaInfo instrument.Int64Gauge +} + +func imlog() *slog.Logger { + return slog.With("component", "otel.InternalMetricsReporter") +} + +func NewInternalMetricsReporter(ctx context.Context, ctxInfo *global.ContextInfo, metrics *MetricsConfig) (*InternalMetricsReporter, error) { + log := imlog() + log.Debug("instantiating internal metrics exporter provider") + exporter, err := InstantiateMetricsExporter(context.Background(), metrics, log) + if err != nil { + return nil, err + } + + res := newResourceInternal(ctxInfo.HostID) + provider, err := newInternalMeterProvider(res, &exporter, metrics.Interval) + meter := provider.Meter("beyla_internal") + + if err != nil { + log.Error("", "error", err) + return nil, err + } + tracerFlushes, err := meter.Float64Histogram( + "beyla.ebpf.tracer.flushes", + instrument.WithDescription("Length of the groups of traces flushed from the eBPF tracer to the next pipeline stage"), + instrument.WithUnit("1"), + ) + if err != nil { + return nil, err + } + + otelMetricExports, err := meter.Float64Counter( + "beyla.otel.metric.exports", + instrument.WithDescription("Length of the metric batches submitted to the remote OTEL collector"), + ) + if err != nil { + return nil, err + } + + otelMetricExportErrs, err := meter.Float64Counter( + "beyla.otel.metric.export.errors", + instrument.WithDescription("Error count on each failed OTEL metric export"), + ) + if err != nil { + return nil, err + } + + otelTraceExports, err := meter.Float64Counter( + "beyla.otel.trace.exports", + instrument.WithDescription("Length of the trace batches submitted to the remote OTEL collector"), + ) + if err != nil { + return nil, err + } + + otelTraceExportErrs, err := meter.Float64Counter( + "beyla.otel.trace.export.errors", + instrument.WithDescription("Error count on each failed OTEL trace export"), + ) + if err != nil { + return nil, err + } + + instrumentedProcesses, err := meter.Int64UpDownCounter( + "beyla.instrumented.processes", + instrument.WithDescription("Instrumented processes by Beyla"), + ) + if err != nil { + return nil, err + } + + beylaInfo, err := meter.Int64Gauge( + "beyla.internal.build.info", + instrument.WithDescription("A metric with a constant '1' value labeled by version, revision, branch, goversion from which Beyla was built, the goos and goarch for the build."), + ) + if err != nil { + return nil, err + } + + return &InternalMetricsReporter{ + ctx: ctx, + tracerFlushes: tracerFlushes, + otelMetricExports: otelMetricExports, + otelMetricExportErrs: otelMetricExportErrs, + otelTraceExports: otelTraceExports, + otelTraceExportErrs: otelTraceExportErrs, + instrumentedProcesses: instrumentedProcesses, + beylaInfo: beylaInfo, + }, nil +} + +func newResourceInternal(hostID string) *resource.Resource { + attrs := []attribute.KeyValue{ + semconv.ServiceName("beyla-internal"), + semconv.ServiceInstanceID(uuid.New().String()), + semconv.TelemetrySDKLanguageKey.String(semconv.TelemetrySDKLanguageGo.Value.AsString()), + // We set the SDK name as Beyla, so we can distinguish beyla generated metrics from other SDKs + semconv.TelemetrySDKNameKey.String("beyla"), + semconv.HostID(hostID), + } + + return resource.NewWithAttributes(semconv.SchemaURL, attrs...) +} + +func newInternalMeterProvider(res *resource.Resource, exporter *metric.Exporter, interval time.Duration) (*metric.MeterProvider, error) { + meterProvider := metric.NewMeterProvider( + metric.WithResource(res), + metric.WithReader(metric.NewPeriodicReader(*exporter, metric.WithInterval(interval))), + ) + return meterProvider, nil +} + +func (p *InternalMetricsReporter) Start(ctx context.Context) { + p.beylaInfo.Record(ctx, 1, instrument.WithAttributes(attribute.String("goarch", runtime.GOARCH), attribute.String("goos", runtime.GOOS), attribute.String("goversion", runtime.Version()), attribute.String("version", buildinfo.Version), attribute.String("revision", buildinfo.Revision))) +} + +func (p *InternalMetricsReporter) TracerFlush(len int) { + p.tracerFlushes.Record(p.ctx, float64(len)) +} + +func (p *InternalMetricsReporter) OTELMetricExport(len int) { + p.otelMetricExports.Add(p.ctx, float64(len)) +} + +func (p *InternalMetricsReporter) OTELMetricExportError(err error) { + p.otelMetricExportErrs.Add(p.ctx, 1) +} + +func (p *InternalMetricsReporter) OTELTraceExport(len int) { + p.otelTraceExports.Add(p.ctx, float64(len)) +} + +func (p *InternalMetricsReporter) OTELTraceExportError(err error) { + p.otelTraceExportErrs.Add(p.ctx, 1) +} + +func (p *InternalMetricsReporter) PrometheusRequest(port, path string) { +} + +func (p *InternalMetricsReporter) InstrumentProcess(processName string) { + p.instrumentedProcesses.Add(p.ctx, 1, instrument.WithAttributes(attribute.String("process_name", processName))) +} + +func (p *InternalMetricsReporter) UninstrumentProcess(processName string) { + p.instrumentedProcesses.Add(p.ctx, -1, instrument.WithAttributes(attribute.String("process_name", processName))) +} diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index 2917683ca..ab1eee95a 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -7,7 +7,8 @@ import ( // Config options for the different metrics exporters type Config struct { - Prometheus PrometheusConfig `yaml:"prometheus,omitempty"` + Prometheus PrometheusConfig `yaml:"prometheus,omitempty"` + OTELMetrics bool `yaml:"otel_metrics,omitempty" env:"BEYLA_INTERNAL_METRICS_OTEL_METRICS"` } // Reporter of internal metrics From 66eb32c9ece64c326576757527ec0073aecfbcfc Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 16:23:17 +0100 Subject: [PATCH 02/12] fmt --- pkg/export/otel/metrics_internal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/export/otel/metrics_internal.go b/pkg/export/otel/metrics_internal.go index c8d8ff635..e3c1765a4 100644 --- a/pkg/export/otel/metrics_internal.go +++ b/pkg/export/otel/metrics_internal.go @@ -6,13 +6,13 @@ import ( "runtime" "time" + "github.com/google/uuid" "go.opentelemetry.io/otel/attribute" instrument "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.19.0" - "github.com/google/uuid" "github.com/grafana/beyla/pkg/buildinfo" "github.com/grafana/beyla/pkg/internal/pipe/global" ) From 64fca59bdd1863e2e98ac8aaa3910eea76dfc1ba Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 16:23:27 +0100 Subject: [PATCH 03/12] vale --- docs/sources/configure/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index bd7254fd3..a47d2f264 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1444,9 +1444,9 @@ or the same (both metric families are listed in the same scrape endpoint). | YAML | Environment variable | Type | Default | | ----------- | ---------------------------------------- | ---- | ------- | -| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | bool | `false` | +| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | boolean | `false` | -Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics will be exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. +Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics are exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. ## YAML file example From 429cb240b774a7d7f642be0fba81547dde161654 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 16:27:10 +0100 Subject: [PATCH 04/12] chart lint --- charts/beyla/templates/daemon-set.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/beyla/templates/daemon-set.yaml b/charts/beyla/templates/daemon-set.yaml index d5a65aa88..d7843caad 100644 --- a/charts/beyla/templates/daemon-set.yaml +++ b/charts/beyla/templates/daemon-set.yaml @@ -74,7 +74,7 @@ spec: containerPort: {{ .Values.service.targetPort | default .Values.config.data.prometheus_export.port }} protocol: TCP {{- end }} - {{- if (and (or (.Values.service.internalMetrics.targetPort) (.Values.config.data.internal_metrics.prometheus)) (not (eq .Values.config.data.prometheus_export.port .Values.config.data.internal_metrics.prometheus.port))) }} + {{- if (and (or (.Values.service.internalMetrics.targetPort) ((and .Values.config.data.internal_metrics .Values.config.data.internal_metrics.prometheus))) (not (eq .Values.config.data.prometheus_export.port .Values.config.data.internal_metrics.prometheus.port))) }} - name: {{ .Values.service.internalMetrics.portName }} containerPort: {{ .Values.service.internalMetrics.targetPort | default .Values.config.data.internal_metrics.prometheus.port }} protocol: TCP From 9fe2ec744c241284000e1ef7ea2c73d08ec94ac4 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 16:34:18 +0100 Subject: [PATCH 05/12] lint --- pkg/export/otel/metrics_internal.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/export/otel/metrics_internal.go b/pkg/export/otel/metrics_internal.go index e3c1765a4..f76d5e33e 100644 --- a/pkg/export/otel/metrics_internal.go +++ b/pkg/export/otel/metrics_internal.go @@ -25,7 +25,6 @@ type InternalMetricsReporter struct { otelMetricExportErrs instrument.Float64Counter otelTraceExports instrument.Float64Counter otelTraceExportErrs instrument.Float64Counter - prometheusRequests instrument.Float64Counter instrumentedProcesses instrument.Int64UpDownCounter beylaInfo instrument.Int64Gauge } @@ -153,7 +152,7 @@ func (p *InternalMetricsReporter) OTELMetricExport(len int) { } func (p *InternalMetricsReporter) OTELMetricExportError(err error) { - p.otelMetricExportErrs.Add(p.ctx, 1) + p.otelMetricExportErrs.Add(p.ctx, 1, instrument.WithAttributes(attribute.String("error", err.Error()))) } func (p *InternalMetricsReporter) OTELTraceExport(len int) { @@ -161,10 +160,10 @@ func (p *InternalMetricsReporter) OTELTraceExport(len int) { } func (p *InternalMetricsReporter) OTELTraceExportError(err error) { - p.otelTraceExportErrs.Add(p.ctx, 1) + p.otelTraceExportErrs.Add(p.ctx, 1, instrument.WithAttributes(attribute.String("error", err.Error()))) } -func (p *InternalMetricsReporter) PrometheusRequest(port, path string) { +func (p *InternalMetricsReporter) PrometheusRequest(_, _ string) { } func (p *InternalMetricsReporter) InstrumentProcess(processName string) { From 8b9bba32889fcd38cdd3be24496f5d4c8deb2a5c Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 2 Dec 2024 17:04:28 +0100 Subject: [PATCH 06/12] Fix values --- docs/sources/configure/options.md | 2 +- pkg/beyla/config_test.go | 2 +- pkg/internal/imetrics/imetrics.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index a47d2f264..8854f8304 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1444,7 +1444,7 @@ or the same (both metric families are listed in the same scrape endpoint). | YAML | Environment variable | Type | Default | | ----------- | ---------------------------------------- | ---- | ------- | -| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | boolean | `false` | +| `otel_metrics` | `BEYLA_INTERNAL_OTEL_METRICS` | boolean | `false` | Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics are exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. diff --git a/pkg/beyla/config_test.go b/pkg/beyla/config_test.go index 2636df2ee..a69fad37e 100644 --- a/pkg/beyla/config_test.go +++ b/pkg/beyla/config_test.go @@ -243,7 +243,7 @@ func TestConfigValidate(t *testing.T) { {"BEYLA_TRACE_PRINTER": "json_indent", "BEYLA_EXECUTABLE_NAME": "foo"}, {"BEYLA_TRACE_PRINTER": "counter", "BEYLA_EXECUTABLE_NAME": "foo"}, {"BEYLA_PROMETHEUS_PORT": "8080", "BEYLA_EXECUTABLE_NAME": "foo", "INSTRUMENT_FUNC_NAME": "bar"}, - {"BEYLA_INTERNAL_METRICS_OTEL_METRICS": "true", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "localhost:1234", "BEYLA_EXECUTABLE_NAME": "foo"}, + {"BEYLA_INTERNAL_OTEL_METRICS": "true", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "localhost:1234", "BEYLA_EXECUTABLE_NAME": "foo"}, } for n, tc := range testCases { t.Run(fmt.Sprint("case", n), func(t *testing.T) { diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index ab1eee95a..de3cd89a2 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -8,7 +8,7 @@ import ( // Config options for the different metrics exporters type Config struct { Prometheus PrometheusConfig `yaml:"prometheus,omitempty"` - OTELMetrics bool `yaml:"otel_metrics,omitempty" env:"BEYLA_INTERNAL_METRICS_OTEL_METRICS"` + OTELMetrics bool `yaml:"otel_metrics,omitempty" env:"BEYLA_INTERNAL_OTEL_METRICS"` } // Reporter of internal metrics From 0a2288029d020e142477aa1ba8c348186f910a55 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 3 Dec 2024 12:45:48 +0100 Subject: [PATCH 07/12] Wip refactor --- docs/sources/configure/options.md | 2 +- pkg/beyla/config.go | 6 +++--- pkg/beyla/config_test.go | 1 + pkg/components/beyla.go | 4 ++-- pkg/internal/imetrics/imetrics.go | 26 ++++++++++++++++++++++++-- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index 8854f8304..a47d2f264 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1444,7 +1444,7 @@ or the same (both metric families are listed in the same scrape endpoint). | YAML | Environment variable | Type | Default | | ----------- | ---------------------------------------- | ---- | ------- | -| `otel_metrics` | `BEYLA_INTERNAL_OTEL_METRICS` | boolean | `false` | +| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | boolean | `false` | Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics are exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. diff --git a/pkg/beyla/config.go b/pkg/beyla/config.go index a987c71d1..15bb3ba28 100644 --- a/pkg/beyla/config.go +++ b/pkg/beyla/config.go @@ -101,7 +101,7 @@ var DefaultConfig = Config{ Printer: false, // Deprecated: use TracePrinter instead TracePrinter: debug.TracePrinterDisabled, InternalMetrics: imetrics.Config{ - OTELMetrics: false, // disabled by default + Exporter: imetrics.InternalMetricsExporterDisabled, Prometheus: imetrics.PrometheusConfig{ Port: 0, // disabled by default Path: "/internal/metrics", @@ -290,10 +290,10 @@ func (c *Config) Validate() error { return ConfigError("wildcard_char can only be a single character, multiple characters are not allowed") } - if c.InternalMetrics.OTELMetrics && c.InternalMetrics.Prometheus.Port != 0 { + if c.InternalMetrics.Exporter == imetrics.InternalMetricsExporterOTEL && c.InternalMetrics.Prometheus.Port != 0 { return ConfigError("you can't enable both OTEL and Prometheus internal metrics") } - if c.InternalMetrics.OTELMetrics && !c.Metrics.Enabled() && !c.Grafana.OTLP.MetricsEnabled() { + if c.InternalMetrics.Exporter == imetrics.InternalMetricsExporterOTEL && !c.Metrics.Enabled() && !c.Grafana.OTLP.MetricsEnabled() { return ConfigError("you can't enable OTEL internal metrics without enabling OTEL metrics") } diff --git a/pkg/beyla/config_test.go b/pkg/beyla/config_test.go index a69fad37e..a2f3ea4ef 100644 --- a/pkg/beyla/config_test.go +++ b/pkg/beyla/config_test.go @@ -174,6 +174,7 @@ network: RequestSizeHistogram: []float64{0, 10, 20, 22}, }}, InternalMetrics: imetrics.Config{ + Exporter: imetrics.InternalMetricsExporterPrometheus, Prometheus: imetrics.PrometheusConfig{ Port: 3210, Path: "/internal/metrics", diff --git a/pkg/components/beyla.go b/pkg/components/beyla.go index 45f66ea73..c7dd2d806 100644 --- a/pkg/components/beyla.go +++ b/pkg/components/beyla.go @@ -136,7 +136,7 @@ func buildCommonContextInfo( ctxInfo.HostID = config.Attributes.HostID.Override } switch { - case config.InternalMetrics.OTELMetrics: + case config.InternalMetrics.Exporter == imetrics.InternalMetricsExporterOTEL: var err error config.Metrics.Grafana = &config.Grafana.OTLP slog.Debug("reporting internal metrics as OpenTelemetry") @@ -144,7 +144,7 @@ func buildCommonContextInfo( if err != nil { return nil, fmt.Errorf("can't start OpenTelemetry metrics: %w", err) } - case config.InternalMetrics.Prometheus.Port != 0: + case config.InternalMetrics.Exporter == imetrics.InternalMetricsExporterPrometheus || config.InternalMetrics.Prometheus.Port != 0: slog.Debug("reporting internal metrics as Prometheus") ctxInfo.Metrics = imetrics.NewPrometheusReporter(&config.InternalMetrics.Prometheus, promMgr, nil) // Prometheus manager also has its own internal metrics, so we need to pass the imetrics reporter diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index de3cd89a2..880973f72 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -3,12 +3,34 @@ package imetrics import ( "context" + "log/slog" ) +type InternalMetricsExporter string + +const ( + InternalMetricsExporterDisabled = InternalMetricsExporter("disabled") + InternalMetricsExporterPrometheus = InternalMetricsExporter("prometheus") + InternalMetricsExporterOTEL = InternalMetricsExporter("otel") +) + +func mlog() *slog.Logger { + return slog.With("component", "debug.TracePrinter") +} + +func (t InternalMetricsExporter) Valid() bool { + switch t { + case InternalMetricsExporterDisabled, InternalMetricsExporterPrometheus, InternalMetricsExporterOTEL: + return true + } + + return false +} + // Config options for the different metrics exporters type Config struct { - Prometheus PrometheusConfig `yaml:"prometheus,omitempty"` - OTELMetrics bool `yaml:"otel_metrics,omitempty" env:"BEYLA_INTERNAL_OTEL_METRICS"` + Prometheus PrometheusConfig `yaml:"prometheus,omitempty"` + Exporter InternalMetricsExporter `yaml:"exporter,omitempty" env:"BEYLA_INTERNAL_METRICS_EXPORTER"` } // Reporter of internal metrics From 6389a3f79be33effc3b8222f030c8af9ac79fc3b Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 7 Jan 2025 12:46:17 +0100 Subject: [PATCH 08/12] Fix test --- pkg/beyla/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/beyla/config_test.go b/pkg/beyla/config_test.go index a2f3ea4ef..a95eefd85 100644 --- a/pkg/beyla/config_test.go +++ b/pkg/beyla/config_test.go @@ -174,7 +174,7 @@ network: RequestSizeHistogram: []float64{0, 10, 20, 22}, }}, InternalMetrics: imetrics.Config{ - Exporter: imetrics.InternalMetricsExporterPrometheus, + Exporter: imetrics.InternalMetricsExporterDisabled, Prometheus: imetrics.PrometheusConfig{ Port: 3210, Path: "/internal/metrics", From 9e615d13f5b5f97e70fe853876be0f77522d3bcd Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 7 Jan 2025 12:55:36 +0100 Subject: [PATCH 09/12] Remove unused code --- pkg/internal/imetrics/imetrics.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index 880973f72..2122e6b32 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -3,7 +3,6 @@ package imetrics import ( "context" - "log/slog" ) type InternalMetricsExporter string @@ -14,10 +13,6 @@ const ( InternalMetricsExporterOTEL = InternalMetricsExporter("otel") ) -func mlog() *slog.Logger { - return slog.With("component", "debug.TracePrinter") -} - func (t InternalMetricsExporter) Valid() bool { switch t { case InternalMetricsExporterDisabled, InternalMetricsExporterPrometheus, InternalMetricsExporterOTEL: From 85acf94e1405dfe7619a9db66eab7a4aa626f731 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 7 Jan 2025 13:00:58 +0100 Subject: [PATCH 10/12] Update docs --- docs/sources/configure/options.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index a47d2f264..d507e8b04 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1410,17 +1410,24 @@ gRPC application metrics, while the rest of the **instrumentations** are be disa YAML section `internal_metrics`. This component reports certain internal metrics about the behavior -of the auto-instrumentation tool. Currently, both [Prometheus](https://prometheus.io/) and [OTEL](https://opentelemetry.io/) metrics export are supported. Prometheus export is enabled if the `internal_metrics` section contains a `prometheus` subsection with the `port` property set. OTEL metrics export is enabled if the `internal_metrics` section contains an `otel_metrics` property set to `true`. +of the auto-instrumentation tool. Currently, both [Prometheus](https://prometheus.io/) and [OTEL](https://opentelemetry.io/) metrics export are supported. Prometheus export is enabled if the `internal_metrics` section has the `exporter` set to `prometheus` and contains a `prometheus` subsection with the `port` property set. OTEL metrics export is enabled if the `internal_metrics` section has the `exporter` set to `otel` using the endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. Example: ```yaml internal_metrics: + exporter: prometheus prometheus: port: 6060 path: /internal/metrics ``` +| YAML | Environment variable | Type | Default | +| ----------- | ---------------------------------------- | ---- | ------- | +| `exporter` | `BEYLA_INTERNAL_METRICS_EXPORTER` | string | `disabled` | + +Specifies the internal metrics exporter. Accepted values are `disabled`, `prometheus` and `otel`. + | YAML | Environment variable | Type | Default | | ------ | ---------------------------------------- | ---- | ------- | | `port` | `BEYLA_INTERNAL_METRICS_PROMETHEUS_PORT` | int | (unset) | @@ -1442,11 +1449,6 @@ same values, this `internal_metrics.prometheus.path` value can be different from `prometheus_export.path`, to keep both metric families separated, or the same (both metric families are listed in the same scrape endpoint). -| YAML | Environment variable | Type | Default | -| ----------- | ---------------------------------------- | ---- | ------- | -| `otel_metrics` | `BEYLA_INTERNAL_METRICS_OTEL` | boolean | `false` | - -Specifies whether to enable the internal metrics exporter for OpenTelemetry metrics. If set to `true`, the internal metrics are exported to the OpenTelemetry endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. ## YAML file example From db272dee0f29a8afa06baa4531ebe53d1e490fb4 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 7 Jan 2025 17:57:06 +0100 Subject: [PATCH 11/12] feedback --- docs/sources/configure/options.md | 2 +- pkg/export/otel/metrics_internal.go | 3 ++- pkg/export/otel/metrics_net.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index d507e8b04..d63510b69 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -1410,7 +1410,7 @@ gRPC application metrics, while the rest of the **instrumentations** are be disa YAML section `internal_metrics`. This component reports certain internal metrics about the behavior -of the auto-instrumentation tool. Currently, both [Prometheus](https://prometheus.io/) and [OTEL](https://opentelemetry.io/) metrics export are supported. Prometheus export is enabled if the `internal_metrics` section has the `exporter` set to `prometheus` and contains a `prometheus` subsection with the `port` property set. OTEL metrics export is enabled if the `internal_metrics` section has the `exporter` set to `otel` using the endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. +of the auto-instrumentation tool. [Prometheus](https://prometheus.io/) and [OTEL](https://opentelemetry.io/) metrics export are supported. Prometheus export is enabled if the `internal_metrics` section has the `exporter` set to `prometheus` and contains a `prometheus` subsection with the `port` property set. OTEL metrics export is enabled if the `internal_metrics` section has the `exporter` set to `otel` using the endpoint specified in the `otel_metrics_export` section or `grafana.otlp` section. Example: diff --git a/pkg/export/otel/metrics_internal.go b/pkg/export/otel/metrics_internal.go index f76d5e33e..f87112b9a 100644 --- a/pkg/export/otel/metrics_internal.go +++ b/pkg/export/otel/metrics_internal.go @@ -38,6 +38,7 @@ func NewInternalMetricsReporter(ctx context.Context, ctxInfo *global.ContextInfo log.Debug("instantiating internal metrics exporter provider") exporter, err := InstantiateMetricsExporter(context.Background(), metrics, log) if err != nil { + log.Error("can't instantiate metrics exporter", "error", err) return nil, err } @@ -46,7 +47,7 @@ func NewInternalMetricsReporter(ctx context.Context, ctxInfo *global.ContextInfo meter := provider.Meter("beyla_internal") if err != nil { - log.Error("", "error", err) + log.Error("can't instantiate meter provider", "error", err) return nil, err } tracerFlushes, err := meter.Float64Histogram( diff --git a/pkg/export/otel/metrics_net.go b/pkg/export/otel/metrics_net.go index 0a4fcecd3..7f7353f53 100644 --- a/pkg/export/otel/metrics_net.go +++ b/pkg/export/otel/metrics_net.go @@ -85,14 +85,14 @@ func newMetricsExporter(ctx context.Context, ctxInfo *global.ContextInfo, cfg *N log.Debug("instantiating network metrics exporter provider") exporter, err := InstantiateMetricsExporter(context.Background(), cfg.Metrics, log) if err != nil { - log.Error("", "error", err) + log.Error("can't instantiate metrics exporter", "error", err) return nil, err } provider, err := newMeterProvider(newResource(ctxInfo.HostID), &exporter, cfg.Metrics.Interval) if err != nil { - log.Error("", "error", err) + log.Error("can't instantiate meter provider", "error", err) return nil, err } From 5f1d101a8acd324d625459fdd90b5c934d6dcf62 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Wed, 15 Jan 2025 12:04:14 +0100 Subject: [PATCH 12/12] restore