diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4b5e46e611..47f70f87f51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ Main (unreleased) - Flow: Add a new stage `non_indexed_labels` to attach non-indexed labels from extracted data to log line entry. (@vlad-diachenko) +- Allow specification of `dimension_name_requirements` for Cloudwatch discovery exports. (@cvdv-au) + - `loki.write` now exposes basic WAL support. (@thepalbi) - `loki.write` WAL now exposes a last segment reclaimed metric. (@thepalbi) diff --git a/component/prometheus/exporter/cloudwatch/config.go b/component/prometheus/exporter/cloudwatch/config.go index 8ac9e6fdb7ea..cbcb68764786 100644 --- a/component/prometheus/exporter/cloudwatch/config.go +++ b/component/prometheus/exporter/cloudwatch/config.go @@ -35,11 +35,12 @@ type TagsPerNamespace = cloudwatch_exporter.TagsPerNamespace // DiscoveryJob configures a discovery job for a given service. type DiscoveryJob struct { - Auth RegionAndRoles `river:",squash"` - CustomTags Tags `river:"custom_tags,attr,optional"` - SearchTags Tags `river:"search_tags,attr,optional"` - Type string `river:"type,attr"` - Metrics []Metric `river:"metric,block"` + Auth RegionAndRoles `river:",squash"` + CustomTags Tags `river:"custom_tags,attr,optional"` + SearchTags Tags `river:"search_tags,attr,optional"` + Type string `river:"type,attr"` + DimensionNameRequirements []string `river:"dimension_name_requirements,attr,optional"` + Metrics []Metric `river:"metric,block"` } // Tags represents a series of tags configured on an AWS resource. Each tag is a @@ -190,11 +191,12 @@ func toYACEStaticJob(sj StaticJob) *yaceConf.Static { func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job { job := &yaceConf.Job{ - Regions: rj.Auth.Regions, - Roles: toYACERoles(rj.Auth.Roles), - Type: rj.Type, - CustomTags: rj.CustomTags.toYACE(), - SearchTags: rj.SearchTags.toYACE(), + Regions: rj.Auth.Regions, + Roles: toYACERoles(rj.Auth.Roles), + Type: rj.Type, + CustomTags: rj.CustomTags.toYACE(), + SearchTags: rj.SearchTags.toYACE(), + DimensionNameRequirements: rj.DimensionNameRequirements, // By setting RoundingPeriod to nil, the exporter will align the start and end times for retrieving CloudWatch // metrics, with the smallest period in the retrieved batch. RoundingPeriod: nil, diff --git a/component/prometheus/exporter/cloudwatch/config_test.go b/component/prometheus/exporter/cloudwatch/config_test.go index 8e54bdd1f49a..8a74e0c72780 100644 --- a/component/prometheus/exporter/cloudwatch/config_test.go +++ b/component/prometheus/exporter/cloudwatch/config_test.go @@ -92,6 +92,7 @@ discovery { role { role_arn = "arn:aws:iam::878167871295:role/yace_testing" } + dimension_name_requirements = ["BucketName"] metric { name = "BucketSizeBytes" statistics = ["Sum"] @@ -238,9 +239,10 @@ func TestCloudwatchComponentConfig(t *testing.T) { Roles: []yaceConf.Role{{ RoleArn: "arn:aws:iam::878167871295:role/yace_testing", }}, - Type: "s3", - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, + Type: "s3", + SearchTags: []yaceModel.Tag{}, + CustomTags: []yaceModel.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, Metrics: []*yaceConf.Metric{ { Name: "BucketSizeBytes", diff --git a/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md b/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md index 8f9e877abc36..1a0805627e91 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md @@ -189,12 +189,13 @@ prometheus.exporter.cloudwatch "discover_instances" { You can configure the `discovery` block one or multiple times to scrape metrics from different services or with different `search_tags`. -| Name | Type | Description | Default | Required | -|---------------|----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------| -| `regions` | `list(string)` | List of AWS regions. | | yes | -| `type` | `string` | Cloudwatch service alias (`"alb"`, `"ec2"`, etc) or namespace name (`"AWS/EC2"`, `"AWS/S3"`, etc). See [supported-services][] for a complete list. | | yes | -| `custom_tags` | `map(string)` | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`. | `{}` | no | -| `search_tags` | `map(string)` | List of key / value pairs to use for tag filtering (all must match). Value can be a regex. | `{}` | no | +| Name | Type | Description | Default | Required | +|--------------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------| +| `regions` | `list(string)` | List of AWS regions. | | yes | +| `type` | `string` | Cloudwatch service alias (`"alb"`, `"ec2"`, etc) or namespace name (`"AWS/EC2"`, `"AWS/S3"`, etc). See [supported-services][] for a complete list. | | yes | +| `custom_tags` | `map(string)` | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`. | `{}` | no | +| `search_tags` | `map(string)` | List of key / value pairs to use for tag filtering (all must match). Value can be a regex. | `{}` | no | +| `dimension_name_requirements` | `list(string)` | List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included. | `{}` | no | [supported-services]: #supported-services-in-discovery-jobs diff --git a/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md b/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md index 9776240c56cc..6d1af354e100 100644 --- a/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md +++ b/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md @@ -198,6 +198,9 @@ Configuration reference: # the following format: `custom_tag_{Key}`. custom_tags: [ ] + # Optional: List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included. + dimension_name_requirements: [ ] + # Required: List of metric definitions to scrape. metrics: [ ] ``` diff --git a/pkg/integrations/cloudwatch_exporter/config.go b/pkg/integrations/cloudwatch_exporter/config.go index d9c770573bf0..c97fe2be39de 100644 --- a/pkg/integrations/cloudwatch_exporter/config.go +++ b/pkg/integrations/cloudwatch_exporter/config.go @@ -55,11 +55,12 @@ type TagsPerNamespace map[string][]string // DiscoveryJob configures a discovery job for a given service. type DiscoveryJob struct { - InlineRegionAndRoles `yaml:",inline"` - InlineCustomTags `yaml:",inline"` - SearchTags []Tag `yaml:"search_tags"` - Type string `yaml:"type"` - Metrics []Metric `yaml:"metrics"` + InlineRegionAndRoles `yaml:",inline"` + InlineCustomTags `yaml:",inline"` + SearchTags []Tag `yaml:"search_tags"` + Type string `yaml:"type"` + DimensionNameRequirements []string `yaml:"dimension_name_requirements"` + Metrics []Metric `yaml:"metrics"` } // StaticJob will scrape metrics that match all defined dimensions. @@ -206,12 +207,13 @@ func toYACEDimensions(dim []Dimension) []yaceConf.Dimension { func toYACEDiscoveryJob(job *DiscoveryJob) *yaceConf.Job { roles := toYACERoles(job.Roles) yaceJob := yaceConf.Job{ - Regions: job.Regions, - Roles: roles, - CustomTags: toYACETags(job.CustomTags), - Type: job.Type, - Metrics: toYACEMetrics(job.Metrics), - SearchTags: toYACETags(job.SearchTags), + Regions: job.Regions, + Roles: roles, + CustomTags: toYACETags(job.CustomTags), + Type: job.Type, + Metrics: toYACEMetrics(job.Metrics), + SearchTags: toYACETags(job.SearchTags), + DimensionNameRequirements: job.DimensionNameRequirements, // By setting RoundingPeriod to nil, the exporter will align the start and end times for retrieving CloudWatch // metrics, with the smallest period in the retrieved batch. diff --git a/pkg/integrations/cloudwatch_exporter/config_test.go b/pkg/integrations/cloudwatch_exporter/config_test.go index cb4afd9a452c..fe97841c2e22 100644 --- a/pkg/integrations/cloudwatch_exporter/config_test.go +++ b/pkg/integrations/cloudwatch_exporter/config_test.go @@ -40,6 +40,8 @@ discovery: - us-east-2 roles: - role_arn: arn:aws:iam::878167871295:role/yace_testing + dimension_name_requirements: + - BucketName metrics: - name: BucketSizeBytes period: 5m @@ -95,6 +97,8 @@ discovery: - us-east-2 roles: - role_arn: arn:aws:iam::878167871295:role/yace_testing + dimension_name_requirements: + - BucketName metrics: - name: BucketSizeBytes period: 5m @@ -178,8 +182,9 @@ var expectedConfig = yaceConf.ScrapeConf{ RoleArn: "arn:aws:iam::878167871295:role/yace_testing", }, }, - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, + SearchTags: []yaceModel.Tag{}, + CustomTags: []yaceModel.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, Metrics: []*yaceConf.Metric{ { Name: "BucketSizeBytes",