YACE dimension_name_requirements support (#4705)

grafana · Aug 10, 2023 · 4507bfa · 4507bfa
1 parent 2aa87ea
commit 4507bfa
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 32 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,8 @@ Main (unreleased)
 
 - Flow: Add a new stage `non_indexed_labels` to attach non-indexed labels from extracted data to log line entry. (@vlad-diachenko)
 
+- Allow specification of `dimension_name_requirements` for Cloudwatch discovery exports. (@cvdv-au)
+
 - `loki.write` now exposes basic WAL support. (@thepalbi)
 
 - `loki.write` WAL now exposes a last segment reclaimed metric. (@thepalbi)

diff --git a/component/prometheus/exporter/cloudwatch/config.go b/component/prometheus/exporter/cloudwatch/config.go
@@ -35,11 +35,12 @@ type TagsPerNamespace = cloudwatch_exporter.TagsPerNamespace
 
 // DiscoveryJob configures a discovery job for a given service.
 type DiscoveryJob struct {
-	Auth       RegionAndRoles `river:",squash"`
-	CustomTags Tags           `river:"custom_tags,attr,optional"`
-	SearchTags Tags           `river:"search_tags,attr,optional"`
-	Type       string         `river:"type,attr"`
-	Metrics    []Metric       `river:"metric,block"`
+	Auth                      RegionAndRoles `river:",squash"`
+	CustomTags                Tags           `river:"custom_tags,attr,optional"`
+	SearchTags                Tags           `river:"search_tags,attr,optional"`
+	Type                      string         `river:"type,attr"`
+	DimensionNameRequirements []string       `river:"dimension_name_requirements,attr,optional"`
+	Metrics                   []Metric       `river:"metric,block"`
 }
 
 // Tags represents a series of tags configured on an AWS resource. Each tag is a
@@ -190,11 +191,12 @@ func toYACEStaticJob(sj StaticJob) *yaceConf.Static {
 
 func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job {
 	job := &yaceConf.Job{
-		Regions:    rj.Auth.Regions,
-		Roles:      toYACERoles(rj.Auth.Roles),
-		Type:       rj.Type,
-		CustomTags: rj.CustomTags.toYACE(),
-		SearchTags: rj.SearchTags.toYACE(),
+		Regions:                   rj.Auth.Regions,
+		Roles:                     toYACERoles(rj.Auth.Roles),
+		Type:                      rj.Type,
+		CustomTags:                rj.CustomTags.toYACE(),
+		SearchTags:                rj.SearchTags.toYACE(),
+		DimensionNameRequirements: rj.DimensionNameRequirements,
 		// By setting RoundingPeriod to nil, the exporter will align the start and end times for retrieving CloudWatch
 		// metrics, with the smallest period in the retrieved batch.
 		RoundingPeriod: nil,

diff --git a/component/prometheus/exporter/cloudwatch/config_test.go b/component/prometheus/exporter/cloudwatch/config_test.go
@@ -92,6 +92,7 @@ discovery {
 	role {
 		role_arn = "arn:aws:iam::878167871295:role/yace_testing"
 	}
+	dimension_name_requirements = ["BucketName"]
 	metric {
 		name = "BucketSizeBytes"
 		statistics = ["Sum"]
@@ -238,9 +239,10 @@ func TestCloudwatchComponentConfig(t *testing.T) {
 							Roles: []yaceConf.Role{{
 								RoleArn: "arn:aws:iam::878167871295:role/yace_testing",
 							}},
-							Type:       "s3",
-							SearchTags: []yaceModel.Tag{},
-							CustomTags: []yaceModel.Tag{},
+							Type:                      "s3",
+							SearchTags:                []yaceModel.Tag{},
+							CustomTags:                []yaceModel.Tag{},
+							DimensionNameRequirements: []string{"BucketName"},
 							Metrics: []*yaceConf.Metric{
 								{
 									Name:                   "BucketSizeBytes",

diff --git a/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md b/docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
@@ -189,12 +189,13 @@ prometheus.exporter.cloudwatch "discover_instances" {
 You can configure the `discovery` block one or multiple times to scrape metrics from different services or with
 different `search_tags`.
 
-| Name          | Type           | Description                                                                                                                                                  | Default | Required |
-|---------------|----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------|
-| `regions`     | `list(string)` | List of AWS regions.                                                                                                                                         |         | yes      |
-| `type`        | `string`       | Cloudwatch service alias (`"alb"`, `"ec2"`, etc) or namespace name (`"AWS/EC2"`, `"AWS/S3"`, etc). See [supported-services][] for a complete list.           |         | yes      |
-| `custom_tags` | `map(string)`  | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`. | `{}`    | no       |
-| `search_tags` | `map(string)`  | List of key / value pairs to use for tag filtering (all must match). Value can be a regex.                                                                   | `{}`    | no       |
+| Name                           | Type           | Description                                                                                                                                                                                                                                                | Default | Required |
+|--------------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------|
+| `regions`                      | `list(string)` | List of AWS regions.                                                                                                                                                                                                                                       |         | yes      |
+| `type`                         | `string`       | Cloudwatch service alias (`"alb"`, `"ec2"`, etc) or namespace name (`"AWS/EC2"`, `"AWS/S3"`, etc). See [supported-services][] for a complete list.                                                                                                         |         | yes      |
+| `custom_tags`                  | `map(string)`  | Custom tags to be added as a list of key / value pairs. When exported to Prometheus format, the label name follows the following format: `custom_tag_{key}`.                                                                                               | `{}`    | no       |
+| `search_tags`                  | `map(string)`  | List of key / value pairs to use for tag filtering (all must match). Value can be a regex.                                                                                                                                                                 | `{}`    | no       |
+| `dimension_name_requirements`  | `list(string)` | List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included. | `{}`    | no       |
 
 [supported-services]: #supported-services-in-discovery-jobs
 

diff --git a/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md b/docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
@@ -198,6 +198,9 @@ Configuration reference:
   # the following format: `custom_tag_{Key}`.
   custom_tags: [ <aws_tag> ]
 
+  # Optional: List of metric dimensions to query. Before querying metric values, the total list of metrics will be filtered to only those that contain exactly this list of dimensions. An empty or undefined list results in all dimension combinations being included.
+  dimension_name_requirements: [ <string> ]
+
   # Required: List of metric definitions to scrape.
   metrics: [ <metric> ]
 ```

diff --git a/pkg/integrations/cloudwatch_exporter/config.go b/pkg/integrations/cloudwatch_exporter/config.go
@@ -55,11 +55,12 @@ type TagsPerNamespace map[string][]string
 
 // DiscoveryJob configures a discovery job for a given service.
 type DiscoveryJob struct {
-	InlineRegionAndRoles `yaml:",inline"`
-	InlineCustomTags     `yaml:",inline"`
-	SearchTags           []Tag    `yaml:"search_tags"`
-	Type                 string   `yaml:"type"`
-	Metrics              []Metric `yaml:"metrics"`
+	InlineRegionAndRoles      `yaml:",inline"`
+	InlineCustomTags          `yaml:",inline"`
+	SearchTags                []Tag    `yaml:"search_tags"`
+	Type                      string   `yaml:"type"`
+	DimensionNameRequirements []string `yaml:"dimension_name_requirements"`
+	Metrics                   []Metric `yaml:"metrics"`
 }
 
 // StaticJob will scrape metrics that match all defined dimensions.
@@ -206,12 +207,13 @@ func toYACEDimensions(dim []Dimension) []yaceConf.Dimension {
 func toYACEDiscoveryJob(job *DiscoveryJob) *yaceConf.Job {
 	roles := toYACERoles(job.Roles)
 	yaceJob := yaceConf.Job{
-		Regions:    job.Regions,
-		Roles:      roles,
-		CustomTags: toYACETags(job.CustomTags),
-		Type:       job.Type,
-		Metrics:    toYACEMetrics(job.Metrics),
-		SearchTags: toYACETags(job.SearchTags),
+		Regions:                   job.Regions,
+		Roles:                     roles,
+		CustomTags:                toYACETags(job.CustomTags),
+		Type:                      job.Type,
+		Metrics:                   toYACEMetrics(job.Metrics),
+		SearchTags:                toYACETags(job.SearchTags),
+		DimensionNameRequirements: job.DimensionNameRequirements,
 
 		// By setting RoundingPeriod to nil, the exporter will align the start and end times for retrieving CloudWatch
 		// metrics, with the smallest period in the retrieved batch.

diff --git a/pkg/integrations/cloudwatch_exporter/config_test.go b/pkg/integrations/cloudwatch_exporter/config_test.go
@@ -40,6 +40,8 @@ discovery:
         - us-east-2
       roles:
         - role_arn: arn:aws:iam::878167871295:role/yace_testing
+      dimension_name_requirements:
+        - BucketName
       metrics:
         - name: BucketSizeBytes
           period: 5m
@@ -95,6 +97,8 @@ discovery:
         - us-east-2
       roles:
         - role_arn: arn:aws:iam::878167871295:role/yace_testing
+      dimension_name_requirements:
+        - BucketName
       metrics:
         - name: BucketSizeBytes
           period: 5m
@@ -178,8 +182,9 @@ var expectedConfig = yaceConf.ScrapeConf{
 						RoleArn: "arn:aws:iam::878167871295:role/yace_testing",
 					},
 				},
-				SearchTags: []yaceModel.Tag{},
-				CustomTags: []yaceModel.Tag{},
+				SearchTags:                []yaceModel.Tag{},
+				CustomTags:                []yaceModel.Tag{},
+				DimensionNameRequirements: []string{"BucketName"},
 				Metrics: []*yaceConf.Metric{
 					{
 						Name:       "BucketSizeBytes",