From 418724ddcc69215fc73116449ed491acf7683b20 Mon Sep 17 00:00:00 2001 From: hainenber Date: Sat, 18 Nov 2023 13:35:50 +0700 Subject: [PATCH] feat(otelcol/exp/prom): add support for resource_to_telemetry_conversion Signed-off-by: hainenber --- CHANGELOG.md | 2 + .../prometheus/internal/convert/convert.go | 50 +++- .../internal/convert/convert_test.go | 249 +++++++++++++++++- .../otelcol/exporter/prometheus/prometheus.go | 31 ++- .../exporter/prometheus/prometheus_test.go | 27 +- .../components/otelcol.exporter.prometheus.md | 1 + 6 files changed, 312 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4b5dc5b418..7d1f5670f8d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,8 @@ v0.38.0-rc.0 (2023-11-16) - Updated windows exporter to use prometheus-community/windows_exporter commit 1836cd1. (@mattdurham) +- Added support to convert resource attributes to Prometheus-compatible datapoint attributes in `otelcol.exporter.prometheus`. (@hainenber) + ### Bugfixes - Set exit code 1 on grafana-agentctl non-runnable command. (@fgouteroux) diff --git a/component/otelcol/exporter/prometheus/internal/convert/convert.go b/component/otelcol/exporter/prometheus/internal/convert/convert.go index 3e2a2578c5a4..0a7039e7195e 100644 --- a/component/otelcol/exporter/prometheus/internal/convert/convert.go +++ b/component/otelcol/exporter/prometheus/internal/convert/convert.go @@ -65,6 +65,8 @@ type Options struct { IncludeScopeLabels bool // AddMetricSuffixes controls whether suffixes are added to metric names. Defaults to true. AddMetricSuffixes bool + // ResourceToTelemetryConversion controls whether to convert resource attributes to Prometheus-compatible datapoint attributes + ResourceToTelemetryConversion bool } var _ consumer.Metrics = (*Converter)(nil) @@ -131,6 +133,7 @@ func (conv *Converter) consumeResourceMetrics(app storage.Appender, rm pmetric.R Type: textparse.MetricTypeGauge, Help: "Target metadata", }) + resAttrs := rm.Resource().Attributes() memResource := conv.getOrCreateResource(rm.Resource()) if conv.getOpts().IncludeTargetInfo { @@ -144,7 +147,7 @@ func (conv *Converter) consumeResourceMetrics(app storage.Appender, rm pmetric.R for smcount := 0; smcount < rm.ScopeMetrics().Len(); smcount++ { sm := rm.ScopeMetrics().At(smcount) - conv.consumeScopeMetrics(app, memResource, sm) + conv.consumeScopeMetrics(app, memResource, sm, resAttrs) } } @@ -219,7 +222,7 @@ func (conv *Converter) getOrCreateResource(res pcommon.Resource) *memorySeries { return entry } -func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *memorySeries, sm pmetric.ScopeMetrics) { +func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *memorySeries, sm pmetric.ScopeMetrics, resAttrs pcommon.Map) { scopeMD := conv.createOrUpdateMetadata("otel_scope_info", metadata.Metadata{ Type: textparse.MetricTypeGauge, }) @@ -236,7 +239,7 @@ func (conv *Converter) consumeScopeMetrics(app storage.Appender, memResource *me for mcount := 0; mcount < sm.Metrics().Len(); mcount++ { m := sm.Metrics().At(mcount) - conv.consumeMetric(app, memResource, memScope, m) + conv.consumeMetric(app, memResource, memScope, m, resAttrs) } } @@ -274,20 +277,27 @@ func (conv *Converter) getOrCreateScope(res *memorySeries, scope pcommon.Instrum return entry } -func (conv *Converter) consumeMetric(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeMetric(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { switch m.Type() { case pmetric.MetricTypeGauge: - conv.consumeGauge(app, memResource, memScope, m) + conv.consumeGauge(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeSum: - conv.consumeSum(app, memResource, memScope, m) + conv.consumeSum(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeHistogram: - conv.consumeHistogram(app, memResource, memScope, m) + conv.consumeHistogram(app, memResource, memScope, m, resAttrs) case pmetric.MetricTypeSummary: - conv.consumeSummary(app, memResource, memScope, m) + conv.consumeSummary(app, memResource, memScope, m, resAttrs) } } -func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func joinAttributeMaps(from, to pcommon.Map) { + from.Range(func(k string, v pcommon.Value) bool { + v.CopyTo(to.PutEmpty(k)) + return true + }) +} + +func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) metricMD := conv.createOrUpdateMetadata(metricName, metadata.Metadata{ @@ -302,6 +312,10 @@ func (conv *Converter) consumeGauge(app storage.Appender, memResource *memorySer for dpcount := 0; dpcount < m.Gauge().DataPoints().Len(); dpcount++ { dp := m.Gauge().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + memSeries := conv.getOrCreateSeries(memResource, memScope, metricName, dp.Attributes()) if err := writeSeries(app, memSeries, dp, getNumberDataPointValue(dp)); err != nil { level.Error(conv.log).Log("msg", "failed to write metric sample", metricName, "err", err) @@ -389,7 +403,7 @@ func getNumberDataPointValue(dp pmetric.NumberDataPoint) float64 { return 0 } -func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) // Excerpt from the spec: @@ -430,6 +444,10 @@ func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySerie for dpcount := 0; dpcount < m.Sum().DataPoints().Len(); dpcount++ { dp := m.Sum().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + memSeries := conv.getOrCreateSeries(memResource, memScope, metricName, dp.Attributes()) val := getNumberDataPointValue(dp) @@ -447,7 +465,7 @@ func (conv *Converter) consumeSum(app storage.Appender, memResource *memorySerie } } -func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) if m.Histogram().AggregationTemporality() != pmetric.AggregationTemporalityCumulative { @@ -469,6 +487,10 @@ func (conv *Converter) consumeHistogram(app storage.Appender, memResource *memor for dpcount := 0; dpcount < m.Histogram().DataPoints().Len(); dpcount++ { dp := m.Histogram().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + // Sum metric if dp.HasSum() { sumMetric := conv.getOrCreateSeries(memResource, memScope, metricName+"_sum", dp.Attributes()) @@ -606,7 +628,7 @@ func (conv *Converter) convertExemplar(otelExemplar pmetric.Exemplar, ts time.Ti } } -func (conv *Converter) consumeSummary(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric) { +func (conv *Converter) consumeSummary(app storage.Appender, memResource *memorySeries, memScope *memorySeries, m pmetric.Metric, resAttrs pcommon.Map) { metricName := prometheus.BuildCompliantName(m, "", conv.opts.AddMetricSuffixes) metricMD := conv.createOrUpdateMetadata(metricName, metadata.Metadata{ @@ -621,6 +643,10 @@ func (conv *Converter) consumeSummary(app storage.Appender, memResource *memoryS for dpcount := 0; dpcount < m.Summary().DataPoints().Len(); dpcount++ { dp := m.Summary().DataPoints().At(dpcount) + if conv.getOpts().ResourceToTelemetryConversion { + joinAttributeMaps(resAttrs, dp.Attributes()) + } + // Sum metric { sumMetric := conv.getOrCreateSeries(memResource, memScope, metricName+"_sum", dp.Attributes()) diff --git a/component/otelcol/exporter/prometheus/internal/convert/convert_test.go b/component/otelcol/exporter/prometheus/internal/convert/convert_test.go index 80a6bce1a55b..746902c32fc0 100644 --- a/component/otelcol/exporter/prometheus/internal/convert/convert_test.go +++ b/component/otelcol/exporter/prometheus/internal/convert/convert_test.go @@ -18,12 +18,13 @@ func TestConverter(t *testing.T) { input string expect string - showTimestamps bool - includeTargetInfo bool - includeScopeInfo bool - includeScopeLabels bool - addMetricSuffixes bool - enableOpenMetrics bool + showTimestamps bool + includeTargetInfo bool + includeScopeInfo bool + includeScopeLabels bool + addMetricSuffixes bool + enableOpenMetrics bool + resourceToTelemetryConversion bool }{ { name: "Gauge", @@ -838,6 +839,233 @@ func TestConverter(t *testing.T) { addMetricSuffixes: true, enableOpenMetrics: true, }, + { + name: "Gauge: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [{ + "name": "test_metric_gauge", + "gauge": { + "data_points": [{ + "as_double": 1234.56 + }] + } + }] + }] + }] + }`, + expect: ` + # TYPE test_metric_gauge gauge + test_metric_gauge{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 1234.56 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Summary: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [{ + "name": "test_metric_summary", + "unit": "seconds", + "summary": { + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "count": 333, + "sum": 100, + "quantile_values": [ + { "quantile": 0, "value": 100 }, + { "quantile": 0.5, "value": 400 }, + { "quantile": 1, "value": 500 } + ] + }] + } + }] + }] + }] + }`, + expect: ` + # TYPE test_metric_summary summary + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="0.0"} 100.0 + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="0.5"} 400.0 + test_metric_summary{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",quantile="1.0"} 500.0 + test_metric_summary_sum{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 100.0 + test_metric_summary_count{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 333 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Histogram: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [ + { + "name": "test_metric_histogram", + "unit": "seconds", + "histogram": { + "aggregation_temporality": 2, + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "count": 333, + "sum": 100, + "bucket_counts": [0, 111, 0, 222], + "explicit_bounds": [0.25, 0.5, 0.75, 1.0], + "exemplars":[ + { + "time_unix_nano": 1000000001, + "as_double": 0.3, + "span_id": "aaaaaaaaaaaaaaaa", + "trace_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + { + "time_unix_nano": 1000000003, + "as_double": 1.5, + "span_id": "cccccccccccccccc", + "trace_id": "cccccccccccccccccccccccccccccccc" + }, + { + "time_unix_nano": 1000000002, + "as_double": 0.5, + "span_id": "bbbbbbbbbbbbbbbb", + "trace_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + ] + }] + } + } + ] + }] + }] + }`, + expect: ` + # TYPE test_metric_histogram histogram + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.25"} 0 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.5"} 111 # {span_id="aaaaaaaaaaaaaaaa",trace_id="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} 0.3 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="0.75"} 111 # {span_id="bbbbbbbbbbbbbbbb",trace_id="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} 0.5 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="1.0"} 333 + test_metric_histogram_bucket{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test",le="+Inf"} 333 # {span_id="cccccccccccccccc",trace_id="cccccccccccccccccccccccccccccccc"} 1.5 + test_metric_histogram_sum{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 100.0 + test_metric_histogram_count{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 333 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, + { + name: "Monotonic sum: convert resource attributes to metric label", + input: `{ + "resource_metrics": [{ + "resource": { + "attributes": [{ + "key": "service.name", + "value": { "stringValue": "myservice" } + }, { + "key": "service.instance.id", + "value": { "stringValue": "instance" } + }, { + "key": "raw", + "value": { "stringValue": "test" } + },{ + "key": "foo.one", + "value": { "stringValue": "foo" } + }, { + "key": "bar.one", + "value": { "stringValue": "bar" } + }] + }, + "scope_metrics": [{ + "metrics": [ + { + "name": "test_metric_mono_sum_total", + "unit": "seconds", + "sum": { + "aggregation_temporality": 2, + "is_monotonic": true, + "data_points": [{ + "start_time_unix_nano": 1000000000, + "time_unix_nano": 1000000000, + "as_double": 15, + "exemplars":[ + { + "time_unix_nano": 1000000001, + "as_double": 0.3, + "span_id": "aaaaaaaaaaaaaaaa", + "trace_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }] + } + } + ] + }] + }] + }`, + expect: ` + # TYPE test_metric_mono_sum counter + test_metric_mono_sum_total{bar_one="bar",foo_one="foo",instance="instance",service_instance_id="instance",job="myservice",service_name="myservice",raw="test"} 15.0 # {span_id="aaaaaaaaaaaaaaaa",trace_id="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} 0.3 + `, + enableOpenMetrics: true, + resourceToTelemetryConversion: true, + }, } decoder := &pmetric.JSONUnmarshaler{} @@ -851,10 +1079,11 @@ func TestConverter(t *testing.T) { l := util.TestLogger(t) conv := convert.New(l, appenderAppendable{Inner: &app}, convert.Options{ - IncludeTargetInfo: tc.includeTargetInfo, - IncludeScopeInfo: tc.includeScopeInfo, - IncludeScopeLabels: tc.includeScopeLabels, - AddMetricSuffixes: tc.addMetricSuffixes, + IncludeTargetInfo: tc.includeTargetInfo, + IncludeScopeInfo: tc.includeScopeInfo, + IncludeScopeLabels: tc.includeScopeLabels, + AddMetricSuffixes: tc.addMetricSuffixes, + ResourceToTelemetryConversion: tc.resourceToTelemetryConversion, }) require.NoError(t, conv.ConsumeMetrics(context.Background(), payload)) diff --git a/component/otelcol/exporter/prometheus/prometheus.go b/component/otelcol/exporter/prometheus/prometheus.go index 0dd9ef49e69f..272b944af00a 100644 --- a/component/otelcol/exporter/prometheus/prometheus.go +++ b/component/otelcol/exporter/prometheus/prometheus.go @@ -31,21 +31,23 @@ func init() { // Arguments configures the otelcol.exporter.prometheus component. type Arguments struct { - IncludeTargetInfo bool `river:"include_target_info,attr,optional"` - IncludeScopeInfo bool `river:"include_scope_info,attr,optional"` - IncludeScopeLabels bool `river:"include_scope_labels,attr,optional"` - GCFrequency time.Duration `river:"gc_frequency,attr,optional"` - ForwardTo []storage.Appendable `river:"forward_to,attr"` - AddMetricSuffixes bool `river:"add_metric_suffixes,attr,optional"` + IncludeTargetInfo bool `river:"include_target_info,attr,optional"` + IncludeScopeInfo bool `river:"include_scope_info,attr,optional"` + IncludeScopeLabels bool `river:"include_scope_labels,attr,optional"` + GCFrequency time.Duration `river:"gc_frequency,attr,optional"` + ForwardTo []storage.Appendable `river:"forward_to,attr"` + AddMetricSuffixes bool `river:"add_metric_suffixes,attr,optional"` + ResourceToTelemetryConversion bool `river:"resource_to_telemetry_conversion,attr,optional"` } // DefaultArguments holds defaults values. var DefaultArguments = Arguments{ - IncludeTargetInfo: true, - IncludeScopeInfo: false, - IncludeScopeLabels: true, - GCFrequency: 5 * time.Minute, - AddMetricSuffixes: true, + IncludeTargetInfo: true, + IncludeScopeInfo: false, + IncludeScopeLabels: true, + GCFrequency: 5 * time.Minute, + AddMetricSuffixes: true, + ResourceToTelemetryConversion: false, } // SetToDefault implements river.Defaulter. @@ -151,8 +153,9 @@ func (c *Component) Update(newConfig component.Arguments) error { func convertArgumentsToConvertOptions(args Arguments) convert.Options { return convert.Options{ - IncludeTargetInfo: args.IncludeTargetInfo, - IncludeScopeInfo: args.IncludeScopeInfo, - AddMetricSuffixes: args.AddMetricSuffixes, + IncludeTargetInfo: args.IncludeTargetInfo, + IncludeScopeInfo: args.IncludeScopeInfo, + AddMetricSuffixes: args.AddMetricSuffixes, + ResourceToTelemetryConversion: args.ResourceToTelemetryConversion, } } diff --git a/component/otelcol/exporter/prometheus/prometheus_test.go b/component/otelcol/exporter/prometheus/prometheus_test.go index 2939c8962346..7e642ff9b585 100644 --- a/component/otelcol/exporter/prometheus/prometheus_test.go +++ b/component/otelcol/exporter/prometheus/prometheus_test.go @@ -23,12 +23,13 @@ func TestArguments_UnmarshalRiver(t *testing.T) { forward_to = [] `, expected: prometheus.Arguments{ - IncludeTargetInfo: true, - IncludeScopeInfo: false, - IncludeScopeLabels: true, - GCFrequency: 5 * time.Minute, - AddMetricSuffixes: true, - ForwardTo: []storage.Appendable{}, + IncludeTargetInfo: true, + IncludeScopeInfo: false, + IncludeScopeLabels: true, + GCFrequency: 5 * time.Minute, + AddMetricSuffixes: true, + ForwardTo: []storage.Appendable{}, + ResourceToTelemetryConversion: false, }, }, { @@ -39,15 +40,17 @@ func TestArguments_UnmarshalRiver(t *testing.T) { include_scope_labels = false gc_frequency = "1s" add_metric_suffixes = false + resource_to_telemetry_conversion = true forward_to = [] `, expected: prometheus.Arguments{ - IncludeTargetInfo: false, - IncludeScopeInfo: true, - IncludeScopeLabels: false, - GCFrequency: 1 * time.Second, - AddMetricSuffixes: false, - ForwardTo: []storage.Appendable{}, + IncludeTargetInfo: false, + IncludeScopeInfo: true, + IncludeScopeLabels: false, + GCFrequency: 1 * time.Second, + AddMetricSuffixes: false, + ForwardTo: []storage.Appendable{}, + ResourceToTelemetryConversion: true, }, }, { diff --git a/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md b/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md index 8c8f81d133d1..932d0742d9d9 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.prometheus.md @@ -46,6 +46,7 @@ Name | Type | Description | Defaul `add_metric_suffixes` | `boolean` | Whether to add type and unit suffixes to metrics names. | `true` | no `gc_frequency` | `duration` | How often to clean up stale metrics from memory. | `"5m"` | no `forward_to` | `list(receiver)` | Where to forward converted Prometheus metrics. | | yes +`resource_to_telemetry_conversion` | `boolean` | Whether to convert OTEL resource attribute to Prometheus-compatible datapoint attribute | `false` | no By default, OpenTelemetry resources are converted into `target_info` metrics. OpenTelemetry instrumentation scopes are converted into `otel_scope_info`