From d61087172e6b65340986de380b066be519581cda Mon Sep 17 00:00:00 2001 From: Steven Berler Date: Wed, 13 Sep 2023 14:57:36 -0700 Subject: [PATCH] add nil_to_zero YACE config --- CHANGELOG.md | 2 + .../prometheus/exporter/cloudwatch/config.go | 27 ++- .../exporter/cloudwatch/config_test.go | 203 +++++++++++++++++- .../internal/build/cloudwatch_exporter.go | 2 + .../staticconvert/testdata/integrations.river | 8 +- .../staticconvert/testdata/integrations.yaml | 4 +- .../prometheus.exporter.cloudwatch.md | 15 +- .../cloudwatch-exporter-config.md | 16 ++ .../cloudwatch_exporter/config.go | 27 ++- .../cloudwatch_exporter/config_test.go | 197 ++++++++++++++++- 10 files changed, 466 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e89ac0f11174..ea101c4ce73b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,8 @@ Main (unreleased) - Flow: In `prometheus.exporter.blackbox`, allow setting labels for individual targets. (@spartan0x117) +- Add optional `nil_to_zero` config flag for `YACE` which can be set in the `static`, `discovery`, or `metric` config blocks. (@berler) + ### Enhancements - Clustering: allow advertise interfaces to be configurable, with the possibility to select all available interfaces. (@wildum) diff --git a/component/prometheus/exporter/cloudwatch/config.go b/component/prometheus/exporter/cloudwatch/config.go index bec644c30f13..879e9838ee0b 100644 --- a/component/prometheus/exporter/cloudwatch/config.go +++ b/component/prometheus/exporter/cloudwatch/config.go @@ -13,7 +13,7 @@ import ( var addCloudwatchTimestamp = false // Avoid producing absence of values in metrics -var nilToZero = true +var defaultNilToZero = true var defaults = Arguments{ Debug: false, @@ -53,6 +53,7 @@ type DiscoveryJob struct { Type string `river:"type,attr"` DimensionNameRequirements []string `river:"dimension_name_requirements,attr,optional"` Metrics []Metric `river:"metric,block"` + NilToZero *bool `river:"nil_to_zero,attr,optional"` } // Tags represents a series of tags configured on an AWS resource. Each tag is a @@ -67,6 +68,7 @@ type StaticJob struct { Namespace string `river:"namespace,attr"` Dimensions Dimensions `river:"dimensions,attr"` Metrics []Metric `river:"metric,block"` + NilToZero *bool `river:"nil_to_zero,attr,optional"` } // RegionAndRoles exposes for each supported job, the AWS regions and IAM roles in which the agent should perform the @@ -90,6 +92,7 @@ type Metric struct { Statistics []string `river:"statistics,attr"` Period time.Duration `river:"period,attr"` Length time.Duration `river:"length,attr,optional"` + NilToZero *bool `river:"nil_to_zero,attr,optional"` } // SetToDefault implements river.Defaulter. @@ -150,7 +153,7 @@ func toYACERoles(rs []Role) []yaceConf.Role { return yaceRoles } -func toYACEMetrics(ms []Metric) []*yaceConf.Metric { +func toYACEMetrics(ms []Metric, jobNilToZero *bool) []*yaceConf.Metric { yaceMetrics := []*yaceConf.Metric{} for _, m := range ms { periodSeconds := int64(m.Period.Seconds()) @@ -159,6 +162,10 @@ func toYACEMetrics(ms []Metric) []*yaceConf.Metric { if m.Length != 0 { lengthSeconds = int64(m.Length.Seconds()) } + nilToZero := m.NilToZero + if nilToZero == nil { + nilToZero = jobNilToZero + } yaceMetrics = append(yaceMetrics, &yaceConf.Metric{ Name: m.Name, Statistics: m.Statistics, @@ -175,7 +182,7 @@ func toYACEMetrics(ms []Metric) []*yaceConf.Metric { // this with RoundingPeriod (see toYACEDiscoveryJob), we should omit this setting. Delay: 0, - NilToZero: &nilToZero, + NilToZero: nilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }) } @@ -190,6 +197,10 @@ func toYACEStaticJob(sj StaticJob) *yaceConf.Static { Value: value, }) } + nilToZero := sj.NilToZero + if nilToZero == nil { + nilToZero = &defaultNilToZero + } return &yaceConf.Static{ Name: sj.Name, Regions: sj.Auth.Regions, @@ -197,11 +208,15 @@ func toYACEStaticJob(sj StaticJob) *yaceConf.Static { Namespace: sj.Namespace, CustomTags: sj.CustomTags.toYACE(), Dimensions: dims, - Metrics: toYACEMetrics(sj.Metrics), + Metrics: toYACEMetrics(sj.Metrics, nilToZero), } } func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job { + nilToZero := rj.NilToZero + if nilToZero == nil { + nilToZero = &defaultNilToZero + } job := &yaceConf.Job{ Regions: rj.Auth.Regions, Roles: toYACERoles(rj.Auth.Roles), @@ -218,10 +233,10 @@ func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job { Period: 0, Length: 0, Delay: 0, - NilToZero: &nilToZero, + NilToZero: nilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, - Metrics: toYACEMetrics(rj.Metrics), + Metrics: toYACEMetrics(rj.Metrics, nilToZero), } return job } diff --git a/component/prometheus/exporter/cloudwatch/config_test.go b/component/prometheus/exporter/cloudwatch/config_test.go index fbdfbd47ac0f..37ad2ce01ec5 100644 --- a/component/prometheus/exporter/cloudwatch/config_test.go +++ b/component/prometheus/exporter/cloudwatch/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" ) +var truePtr = true var falsePtr = false const invalidDiscoveryJobType = ` @@ -102,6 +103,71 @@ discovery { } ` +const staticJobNilToZeroConfig = ` +sts_region = "us-east-2" +debug = true +static "super_ec2_instance_id" { + regions = ["us-east-2"] + namespace = "AWS/EC2" + dimensions = { + "InstanceId" = "i01u29u12ue1u2c", + } + metric { + name = "CPUUsage" + statistics = ["Sum", "Average"] + period = "1m" + } + // setting nil_to_zero on the job level + nil_to_zero = false +} +` + +const staticJobNilToZeroMetricConfig = ` +sts_region = "us-east-2" +debug = true +static "super_ec2_instance_id" { + regions = ["us-east-2"] + namespace = "AWS/EC2" + dimensions = { + "InstanceId" = "i01u29u12ue1u2c", + } + metric { + name = "CPUUsage" + statistics = ["Sum", "Average"] + period = "1m" + // setting nil_to_zero on the metric level + nil_to_zero = false + } +} +` + +const discoveryJobNilToZeroConfig = ` +sts_region = "us-east-2" +debug = true +discovery_exported_tags = { "ec2" = ["name"] } +discovery { + type = "sqs" + regions = ["us-east-2"] + search_tags = { + "scrape" = "true", + } + // setting nil_to_zero on the job level + nil_to_zero = false + metric { + name = "NumberOfMessagesSent" + statistics = ["Sum", "Average"] + period = "1m" + } + metric { + name = "NumberOfMessagesReceived" + statistics = ["Sum", "Average"] + period = "1m" + // setting nil_to_zero on the metric level + nil_to_zero = true + } +} +` + func TestCloudwatchComponentConfig(t *testing.T) { type testcase struct { raw string @@ -150,7 +216,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Period: 60, Length: 60, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }}, }, @@ -184,7 +250,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Period: 60, Length: 60, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, { @@ -193,7 +259,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Period: 60, Length: 60, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -203,7 +269,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Length: 0, Delay: 0, AddCloudwatchTimestamp: &falsePtr, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, }, }, { @@ -221,7 +287,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Period: 60, Length: 60, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -231,7 +297,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Length: 0, Delay: 0, AddCloudwatchTimestamp: &falsePtr, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, }, }, { @@ -250,7 +316,128 @@ func TestCloudwatchComponentConfig(t *testing.T) { Period: 60, Length: 3600, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }, + }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceConf.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &defaultNilToZero, + }, + }, + }, + }, + }, + }, + "static job nil to zero": { + raw: staticJobNilToZeroConfig, + expected: yaceConf.ScrapeConf{ + APIVersion: "v1alpha1", + StsRegion: "us-east-2", + Discovery: yaceConf.Discovery{}, + Static: []*yaceConf.Static{ + { + Name: "super_ec2_instance_id", + // assert an empty role is used as default. IMPORTANT since this + // is what YACE looks for delegating to the environment role + Roles: []yaceConf.Role{{}}, + Regions: []string{"us-east-2"}, + Namespace: "AWS/EC2", + CustomTags: []yaceModel.Tag{}, + Dimensions: []yaceConf.Dimension{ + { + Name: "InstanceId", + Value: "i01u29u12ue1u2c", + }, + }, + Metrics: []*yaceConf.Metric{{ + Name: "CPUUsage", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }}, + }, + }, + }, + }, + "static job nil to zero metric": { + raw: staticJobNilToZeroMetricConfig, + expected: yaceConf.ScrapeConf{ + APIVersion: "v1alpha1", + StsRegion: "us-east-2", + Discovery: yaceConf.Discovery{}, + Static: []*yaceConf.Static{ + { + Name: "super_ec2_instance_id", + // assert an empty role is used as default. IMPORTANT since this + // is what YACE looks for delegating to the environment role + Roles: []yaceConf.Role{{}}, + Regions: []string{"us-east-2"}, + Namespace: "AWS/EC2", + CustomTags: []yaceModel.Tag{}, + Dimensions: []yaceConf.Dimension{ + { + Name: "InstanceId", + Value: "i01u29u12ue1u2c", + }, + }, + Metrics: []*yaceConf.Metric{{ + Name: "CPUUsage", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }}, + }, + }, + }, + }, + "discovery job nil to zero config": { + raw: discoveryJobNilToZeroConfig, + expected: yaceConf.ScrapeConf{ + APIVersion: "v1alpha1", + StsRegion: "us-east-2", + Discovery: yaceConf.Discovery{ + ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics{ + "ec2": []string{"name"}, + }, + Jobs: []*yaceConf.Job{ + { + Regions: []string{"us-east-2"}, + // assert an empty role is used as default. IMPORTANT since this + // is what YACE looks for delegating to the environment role + Roles: []yaceConf.Role{{}}, + Type: "sqs", + SearchTags: []yaceModel.Tag{{ + Key: "scrape", Value: "true", + }}, + CustomTags: []yaceModel.Tag{}, + Metrics: []*yaceConf.Metric{ + { + Name: "NumberOfMessagesSent", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }, + { + Name: "NumberOfMessagesReceived", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: &truePtr, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -260,7 +447,7 @@ func TestCloudwatchComponentConfig(t *testing.T) { Length: 0, Delay: 0, AddCloudwatchTimestamp: &falsePtr, - NilToZero: &nilToZero, + NilToZero: &falsePtr, }, }, }, diff --git a/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go b/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go index 120b6ed546c2..610980866e7c 100644 --- a/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go +++ b/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go @@ -52,6 +52,7 @@ func toDiscoveryJob(job *cloudwatch_exporter.DiscoveryJob) cloudwatch.DiscoveryJ Type: job.Type, DimensionNameRequirements: job.DimensionNameRequirements, Metrics: toMetrics(job.Metrics), + NilToZero: job.NilToZero, } } @@ -92,5 +93,6 @@ func toMetric(metric cloudwatch_exporter.Metric) cloudwatch.Metric { Statistics: metric.Statistics, Period: metric.Period, Length: metric.Length, + NilToZero: metric.NilToZero, } } diff --git a/converter/internal/staticconvert/testdata/integrations.river b/converter/internal/staticconvert/testdata/integrations.river index e411f4c556d5..8cd4df3921ca 100644 --- a/converter/internal/staticconvert/testdata/integrations.river +++ b/converter/internal/staticconvert/testdata/integrations.river @@ -83,9 +83,10 @@ prometheus.exporter.cloudwatch "integrations_cloudwatch_exporter" { type = "AWS/EC2" metric { - name = "CPUUtilization" - statistics = ["Average"] - period = "5m0s" + name = "CPUUtilization" + statistics = ["Average"] + period = "5m0s" + nil_to_zero = false } metric { @@ -93,6 +94,7 @@ prometheus.exporter.cloudwatch "integrations_cloudwatch_exporter" { statistics = ["Average"] period = "5m0s" } + nil_to_zero = true } decoupled_scraping { } diff --git a/converter/internal/staticconvert/testdata/integrations.yaml b/converter/internal/staticconvert/testdata/integrations.yaml index 27884f65f6ac..300ec2db9980 100644 --- a/converter/internal/staticconvert/testdata/integrations.yaml +++ b/converter/internal/staticconvert/testdata/integrations.yaml @@ -37,11 +37,13 @@ integrations: - type: AWS/EC2 regions: - us-east-2 + nil_to_zero: true metrics: - name: CPUUtilization period: 5m statistics: - Average + nil_to_zero: false - name: NetworkPacketsIn period: 5m statistics: @@ -169,4 +171,4 @@ integrations: scrape_timeout: 1m scrape_integration: true statsd_exporter: - enabled: true \ No newline at end of file + enabled: true diff --git a/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md b/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md index 9fc5c1aa7fec..9c61670126aa 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md @@ -198,6 +198,7 @@ different `search_tags`. | `custom_tags` | `map(string)` | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`. | `{}` | no | | `search_tags` | `map(string)` | List of key / value pairs to use for tag filtering (all must match). Value can be a regex. | `{}` | no | | `dimension_name_requirements` | `list(string)` | List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included. | `{}` | no | +| `nil_to_zero` | `bool` | When `true`, `NaN` metric values are converted to 0. Individual metrics can override this value in the [metric][] block. | `true` | no | [supported-services]: #supported-services-in-discovery-jobs @@ -252,6 +253,7 @@ You can configure the `static` block one or multiple times to scrape metrics wit | `namespace` | `string` | CloudWatch metric namespace. | | yes | | `dimensions` | `map(string)` | CloudWatch metric dimensions as a list of name / value pairs. Must uniquely define all metrics in this job. | | yes | | `custom_tags` | `map(string)` | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`. | `{}` | no | +| `nil_to_zero` | `bool` | When `true`, `NaN` metric values are converted to 0. Individual metrics can override this value in the [metric][] block. | `true` | no | All dimensions must be specified when scraping single metrics like the example above. For example, `AWS/Logs` metrics require `Resource`, `Service`, `Class`, and `Type` dimensions to be specified. The same applies to CloudWatch custom @@ -265,12 +267,13 @@ metrics. Follow [this guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/viewing_metrics_with_cloudwatch.html) on how to explore metrics, to easily pick the ones you need. -| Name | Type | Description | Default | Required | -| ------------ | -------------- | ---------------------------------------------------------------- | --------------------------------------------------------- | -------- | -| `name` | `string` | Metric name. | | yes | -| `statistics` | `list(string)` | List of statistics to scrape. Ex: `"Minimum"`, `"Maximum"`, etc. | | yes | -| `period` | `duration` | See [period][] section below. | | yes | -| `length` | `duration` | See [period][] section below. | Calculated based on `period`. See [period][] for details. | no | +| Name | Type | Description | Default | Required | +| ------------- | -------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | -------- | +| `name` | `string` | Metric name. | | yes | +| `statistics` | `list(string)` | List of statistics to scrape. Ex: `"Minimum"`, `"Maximum"`, etc. | | yes | +| `period` | `duration` | See [period][] section below. | | yes | +| `length` | `duration` | See [period][] section below. | Calculated based on `period`. See [period][] for details. | no | +| `nil_to_zero` | `bool` | When `true`, `NaN` metric values are converted to 0. | The value of `nil_to_zero` in the parent [static][] or [discovery][] block (`true` if not set in the parent block). | no | [period]: #period-and-length diff --git a/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md b/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md index c054957652ff..9f297c2815ed 100644 --- a/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md +++ b/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md @@ -177,15 +177,18 @@ discovery: - type: AWS/EC2 regions: - us-east-2 + nil_to_zero: true metrics: - name: CPUUtilization period: 5m statistics: - Average + nil_to_zero: true - name: NetworkPacketsIn period: 5m statistics: - Average + nil_to_zero: true ``` Configuration reference: @@ -211,6 +214,9 @@ Configuration reference: # Optional: List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included. dimension_name_requirements: [ ] + # Optional: Flag that controls if `NaN` metric values are converted to 0. Default `true`. This can be overridden in the config of each metric. + nil_to_zero: + # Required: List of metric definitions to scrape. metrics: [ ] ``` @@ -234,15 +240,18 @@ static: dimensions: - name: InstanceId value: i-0e43cee369aa44b52 + nil_to_zero: true metrics: - name: CPUUtilization period: 5m statistics: - Average + nil_to_zero: true - name: NetworkPacketsIn period: 5m statistics: - Average + nil_to_zero: true ``` All dimensions need to be specified when scraping single metrics like the example above. For example `AWS/Logs` metrics @@ -271,6 +280,9 @@ Configuration reference: # `custom_tag_{Key}`. custom_tags: [ ] + # Optional: Flag that controls if `NaN` metric values are converted to 0. Default `true`. This can be overridden in the config of each metric. + nil_to_zero: + # Required: List of metric definitions to scrape. metrics: [ ] ``` @@ -328,6 +340,10 @@ pick the ones you need. # Optional: See the `period and length` section below. length: [ | default = calculated based on `period` ] + + # Optional: Flag that controls if `NaN` metric values are converted to 0. + # When not set, the value defaults to the setting in the parent static or discovery block (`true` if not set in the parent block). + nil_to_zero: ``` ### Period and length diff --git a/pkg/integrations/cloudwatch_exporter/config.go b/pkg/integrations/cloudwatch_exporter/config.go index 4b5ad8aa1921..b3a326c65cd3 100644 --- a/pkg/integrations/cloudwatch_exporter/config.go +++ b/pkg/integrations/cloudwatch_exporter/config.go @@ -29,7 +29,7 @@ const ( var addCloudwatchTimestamp = false // Avoid producing absence of values in metrics -var nilToZero = true +var defaultNilToZero = true func init() { integrations.RegisterIntegration(&Config{}) @@ -70,6 +70,7 @@ type DiscoveryJob struct { Type string `yaml:"type"` DimensionNameRequirements []string `yaml:"dimension_name_requirements"` Metrics []Metric `yaml:"metrics"` + NilToZero *bool `yaml:"nil_to_zero,omitempty"` } // StaticJob will scrape metrics that match all defined dimensions. @@ -80,6 +81,7 @@ type StaticJob struct { Namespace string `yaml:"namespace"` Dimensions []Dimension `yaml:"dimensions"` Metrics []Metric `yaml:"metrics"` + NilToZero *bool `yaml:"nil_to_zero,omitempty"` } // InlineRegionAndRoles exposes for each supported job, the AWS regions and IAM roles in which the agent should perform the @@ -113,6 +115,7 @@ type Metric struct { Statistics []string `yaml:"statistics"` Period time.Duration `yaml:"period"` Length time.Duration `yaml:"length"` + NilToZero *bool `yaml:"nil_to_zero,omitempty"` } // Name returns the name of the integration this config is for. @@ -204,6 +207,10 @@ func PatchYACEDefaults(yc *yaceConf.ScrapeConf) { } func toYACEStaticJob(job StaticJob) *yaceConf.Static { + nilToZero := job.NilToZero + if nilToZero == nil { + nilToZero = &defaultNilToZero + } return &yaceConf.Static{ Name: job.Name, Regions: job.Regions, @@ -211,7 +218,7 @@ func toYACEStaticJob(job StaticJob) *yaceConf.Static { Namespace: job.Namespace, CustomTags: toYACETags(job.CustomTags), Dimensions: toYACEDimensions(job.Dimensions), - Metrics: toYACEMetrics(job.Metrics), + Metrics: toYACEMetrics(job.Metrics, nilToZero), } } @@ -228,12 +235,16 @@ func toYACEDimensions(dim []Dimension) []yaceConf.Dimension { func toYACEDiscoveryJob(job *DiscoveryJob) *yaceConf.Job { roles := toYACERoles(job.Roles) + nilToZero := job.NilToZero + if nilToZero == nil { + nilToZero = &defaultNilToZero + } yaceJob := yaceConf.Job{ Regions: job.Regions, Roles: roles, CustomTags: toYACETags(job.CustomTags), Type: job.Type, - Metrics: toYACEMetrics(job.Metrics), + Metrics: toYACEMetrics(job.Metrics, nilToZero), SearchTags: toYACETags(job.SearchTags), DimensionNameRequirements: job.DimensionNameRequirements, @@ -247,14 +258,14 @@ func toYACEDiscoveryJob(job *DiscoveryJob) *yaceConf.Job { Period: 0, Length: 0, Delay: 0, - NilToZero: &nilToZero, + NilToZero: nilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, } return &yaceJob } -func toYACEMetrics(metrics []Metric) []*yaceConf.Metric { +func toYACEMetrics(metrics []Metric, jobNilToZero *bool) []*yaceConf.Metric { yaceMetrics := []*yaceConf.Metric{} for _, metric := range metrics { periodSeconds := int64(metric.Period.Seconds()) @@ -263,6 +274,10 @@ func toYACEMetrics(metrics []Metric) []*yaceConf.Metric { if metric.Length != 0 { lengthSeconds = int64(metric.Length.Seconds()) } + nilToZero := metric.NilToZero + if nilToZero == nil { + nilToZero = jobNilToZero + } yaceMetrics = append(yaceMetrics, &yaceConf.Metric{ Name: metric.Name, @@ -280,7 +295,7 @@ func toYACEMetrics(metrics []Metric) []*yaceConf.Metric { // this with RoundingPeriod (see toYACEDiscoveryJob), we should omit this setting. Delay: 0, - NilToZero: &nilToZero, + NilToZero: nilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }) } diff --git a/pkg/integrations/cloudwatch_exporter/config_test.go b/pkg/integrations/cloudwatch_exporter/config_test.go index fe97841c2e22..8548f37c7aec 100644 --- a/pkg/integrations/cloudwatch_exporter/config_test.go +++ b/pkg/integrations/cloudwatch_exporter/config_test.go @@ -122,6 +122,66 @@ static: - Average ` +// for testing nilToZero at the DiscoveryJob, StaticJob, and Metric level +const configString3 = ` +sts_region: us-east-2 +discovery: + exported_tags: + AWS/EC2: + - name + - type + jobs: + - type: AWS/EC2 + search_tags: + - key: instance_type + value: spot + regions: + - us-east-2 + roles: + - role_arn: arn:aws:iam::878167871295:role/yace_testing + custom_tags: + - key: alias + value: tesis + nil_to_zero: false + metrics: + - name: CPUUtilization + period: 5m + statistics: + - Maximum + - Average + - type: s3 + regions: + - us-east-2 + roles: + - role_arn: arn:aws:iam::878167871295:role/yace_testing + dimension_name_requirements: + - BucketName + nil_to_zero: true + metrics: + - name: BucketSizeBytes + period: 5m + length: 1h + nil_to_zero: false + statistics: + - Sum +static: + - regions: + - us-east-2 + name: custom_tesis_metrics + namespace: CoolApp + dimensions: + - name: PURCHASES_SERVICE + value: CoolService + - name: APP_VERSION + value: 1.0 + nil_to_zero: false + metrics: + - name: KPIs + period: 5m + statistics: + - Average +` + var falsePtr = false var truePtr = true @@ -161,7 +221,7 @@ var expectedConfig = yaceConf.ScrapeConf{ Period: 300, Length: 300, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -171,7 +231,7 @@ var expectedConfig = yaceConf.ScrapeConf{ Length: 0, Delay: 0, AddCloudwatchTimestamp: &falsePtr, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, }, }, { @@ -193,7 +253,7 @@ var expectedConfig = yaceConf.ScrapeConf{ Period: 300, Length: 3600, // 1 hour Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -203,7 +263,7 @@ var expectedConfig = yaceConf.ScrapeConf{ Length: 0, Delay: 0, AddCloudwatchTimestamp: &falsePtr, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, }, }, }, @@ -232,7 +292,122 @@ var expectedConfig = yaceConf.ScrapeConf{ Length: 300, Statistics: []string{"Average"}, Delay: 0, - NilToZero: &nilToZero, + NilToZero: &defaultNilToZero, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }, + }, + }, + }, +} + +var expectedConfig3 = yaceConf.ScrapeConf{ + APIVersion: "v1alpha1", + StsRegion: "us-east-2", + Discovery: yaceConf.Discovery{ + ExportedTagsOnMetrics: map[string][]string{ + "AWS/EC2": {"name", "type"}, + }, + Jobs: []*yaceConf.Job{ + { + Type: "AWS/EC2", + Regions: []string{"us-east-2"}, + Roles: []yaceConf.Role{ + { + RoleArn: "arn:aws:iam::878167871295:role/yace_testing", + }, + }, + CustomTags: []yaceModel.Tag{ + { + Key: "alias", + Value: "tesis", + }, + }, + SearchTags: []yaceModel.Tag{ + { + Key: "instance_type", + Value: "spot", + }, + }, + Metrics: []*yaceConf.Metric{ + { + Name: "CPUUtilization", + Statistics: []string{"Maximum", "Average"}, + // Defaults get configured from general settings + Period: 300, + Length: 300, + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }, + }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceConf.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &falsePtr, + }, + }, + { + Type: "s3", + Regions: []string{"us-east-2"}, + Roles: []yaceConf.Role{ + { + RoleArn: "arn:aws:iam::878167871295:role/yace_testing", + }, + }, + SearchTags: []yaceModel.Tag{}, + CustomTags: []yaceModel.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, + Metrics: []*yaceConf.Metric{ + { + Name: "BucketSizeBytes", + Statistics: []string{"Sum"}, + // Defaults get configured from general settings + Period: 300, + Length: 3600, // 1 hour + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &addCloudwatchTimestamp, + }, + }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceConf.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &truePtr, + }, + }, + }, + }, + Static: []*yaceConf.Static{ + { + Name: "custom_tesis_metrics", + Regions: []string{"us-east-2"}, + Roles: []yaceConf.Role{{}}, + Namespace: "CoolApp", + CustomTags: []yaceModel.Tag{}, + Dimensions: []yaceConf.Dimension{ + { + Name: "PURCHASES_SERVICE", + Value: "CoolService", + }, + { + Name: "APP_VERSION", + Value: "1.0", + }, + }, + Metrics: []*yaceConf.Metric{ + { + Name: "KPIs", + Period: 300, + Length: 300, + Statistics: []string{"Average"}, + Delay: 0, + NilToZero: &falsePtr, AddCloudwatchTimestamp: &addCloudwatchTimestamp, }, }, @@ -261,6 +436,18 @@ func TestTranslateConfigToYACEConfig(t *testing.T) { require.EqualValues(t, falsePtr, fipsEnabled2) } +func TestTranslateNilToZeroConfigToYACEConfig(t *testing.T) { + c := Config{} + err := yaml.Unmarshal([]byte(configString3), &c) + require.NoError(t, err, "failed to unmarshal config") + + yaceConf, fipsEnabled, err := ToYACEConfig(&c) + require.NoError(t, err, "failed to translate to YACE configuration") + + require.EqualValues(t, expectedConfig3, yaceConf) + require.EqualValues(t, truePtr, fipsEnabled) +} + func TestCloudwatchExporterConfigInstanceKey(t *testing.T) { cfg1 := &Config{ STSRegion: "us-east-2",