Skip to content

Commit

Permalink
add nil_to_zero YACE config
Browse files Browse the repository at this point in the history
  • Loading branch information
@berler
berler committed Sep 14, 2023
1 parent 7ebf224 commit d610871
Show file tree
Hide file tree
Showing 10 changed files with 466 additions and 35 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ Main (unreleased)

- Flow: In `prometheus.exporter.blackbox`, allow setting labels for individual targets. (@spartan0x117)

- Add optional `nil_to_zero` config flag for `YACE` which can be set in the `static`, `discovery`, or `metric` config blocks. (@berler)

### Enhancements

- Clustering: allow advertise interfaces to be configurable, with the possibility to select all available interfaces. (@wildum)
Expand Down
27 changes: 21 additions & 6 deletions component/prometheus/exporter/cloudwatch/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
var addCloudwatchTimestamp = false

// Avoid producing absence of values in metrics
var nilToZero = true
var defaultNilToZero = true

var defaults = Arguments{
Debug: false,
Expand Down Expand Up @@ -53,6 +53,7 @@ type DiscoveryJob struct {
Type string `river:"type,attr"`
DimensionNameRequirements []string `river:"dimension_name_requirements,attr,optional"`
Metrics []Metric `river:"metric,block"`
NilToZero *bool `river:"nil_to_zero,attr,optional"`
}

// Tags represents a series of tags configured on an AWS resource. Each tag is a
Expand All @@ -67,6 +68,7 @@ type StaticJob struct {
Namespace string `river:"namespace,attr"`
Dimensions Dimensions `river:"dimensions,attr"`
Metrics []Metric `river:"metric,block"`
NilToZero *bool `river:"nil_to_zero,attr,optional"`
}

// RegionAndRoles exposes for each supported job, the AWS regions and IAM roles in which the agent should perform the
Expand All @@ -90,6 +92,7 @@ type Metric struct {
Statistics []string `river:"statistics,attr"`
Period time.Duration `river:"period,attr"`
Length time.Duration `river:"length,attr,optional"`
NilToZero *bool `river:"nil_to_zero,attr,optional"`
}

// SetToDefault implements river.Defaulter.
Expand Down Expand Up @@ -150,7 +153,7 @@ func toYACERoles(rs []Role) []yaceConf.Role {
return yaceRoles
}

func toYACEMetrics(ms []Metric) []*yaceConf.Metric {
func toYACEMetrics(ms []Metric, jobNilToZero *bool) []*yaceConf.Metric {
yaceMetrics := []*yaceConf.Metric{}
for _, m := range ms {
periodSeconds := int64(m.Period.Seconds())
Expand All @@ -159,6 +162,10 @@ func toYACEMetrics(ms []Metric) []*yaceConf.Metric {
if m.Length != 0 {
lengthSeconds = int64(m.Length.Seconds())
}
nilToZero := m.NilToZero
if nilToZero == nil {
nilToZero = jobNilToZero
}
yaceMetrics = append(yaceMetrics, &yaceConf.Metric{
Name: m.Name,
Statistics: m.Statistics,
Expand All @@ -175,7 +182,7 @@ func toYACEMetrics(ms []Metric) []*yaceConf.Metric {
// this with RoundingPeriod (see toYACEDiscoveryJob), we should omit this setting.
Delay: 0,

NilToZero: &nilToZero,
NilToZero: nilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
})
}
Expand All @@ -190,18 +197,26 @@ func toYACEStaticJob(sj StaticJob) *yaceConf.Static {
Value: value,
})
}
nilToZero := sj.NilToZero
if nilToZero == nil {
nilToZero = &defaultNilToZero
}
return &yaceConf.Static{
Name: sj.Name,
Regions: sj.Auth.Regions,
Roles: toYACERoles(sj.Auth.Roles),
Namespace: sj.Namespace,
CustomTags: sj.CustomTags.toYACE(),
Dimensions: dims,
Metrics: toYACEMetrics(sj.Metrics),
Metrics: toYACEMetrics(sj.Metrics, nilToZero),
}
}

func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job {
nilToZero := rj.NilToZero
if nilToZero == nil {
nilToZero = &defaultNilToZero
}
job := &yaceConf.Job{
Regions: rj.Auth.Regions,
Roles: toYACERoles(rj.Auth.Roles),
Expand All @@ -218,10 +233,10 @@ func toYACEDiscoveryJob(rj DiscoveryJob) *yaceConf.Job {
Period: 0,
Length: 0,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: nilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
Metrics: toYACEMetrics(rj.Metrics),
Metrics: toYACEMetrics(rj.Metrics, nilToZero),
}
return job
}
203 changes: 195 additions & 8 deletions component/prometheus/exporter/cloudwatch/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
)

var truePtr = true
var falsePtr = false

const invalidDiscoveryJobType = `
Expand Down Expand Up @@ -102,6 +103,71 @@ discovery {
}
`

const staticJobNilToZeroConfig = `
sts_region = "us-east-2"
debug = true
static "super_ec2_instance_id" {
regions = ["us-east-2"]
namespace = "AWS/EC2"
dimensions = {
"InstanceId" = "i01u29u12ue1u2c",
}
metric {
name = "CPUUsage"
statistics = ["Sum", "Average"]
period = "1m"
}
// setting nil_to_zero on the job level
nil_to_zero = false
}
`

const staticJobNilToZeroMetricConfig = `
sts_region = "us-east-2"
debug = true
static "super_ec2_instance_id" {
regions = ["us-east-2"]
namespace = "AWS/EC2"
dimensions = {
"InstanceId" = "i01u29u12ue1u2c",
}
metric {
name = "CPUUsage"
statistics = ["Sum", "Average"]
period = "1m"
// setting nil_to_zero on the metric level
nil_to_zero = false
}
}
`

const discoveryJobNilToZeroConfig = `
sts_region = "us-east-2"
debug = true
discovery_exported_tags = { "ec2" = ["name"] }
discovery {
type = "sqs"
regions = ["us-east-2"]
search_tags = {
"scrape" = "true",
}
// setting nil_to_zero on the job level
nil_to_zero = false
metric {
name = "NumberOfMessagesSent"
statistics = ["Sum", "Average"]
period = "1m"
}
metric {
name = "NumberOfMessagesReceived"
statistics = ["Sum", "Average"]
period = "1m"
// setting nil_to_zero on the metric level
nil_to_zero = true
}
}
`

func TestCloudwatchComponentConfig(t *testing.T) {
type testcase struct {
raw string
Expand Down Expand Up @@ -150,7 +216,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
}},
},
Expand Down Expand Up @@ -184,7 +250,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
{
Expand All @@ -193,7 +259,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
},
Expand All @@ -203,7 +269,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Length: 0,
Delay: 0,
AddCloudwatchTimestamp: &falsePtr,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
},
},
{
Expand All @@ -221,7 +287,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
},
Expand All @@ -231,7 +297,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Length: 0,
Delay: 0,
AddCloudwatchTimestamp: &falsePtr,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
},
},
{
Expand All @@ -250,7 +316,128 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Period: 60,
Length: 3600,
Delay: 0,
NilToZero: &nilToZero,
NilToZero: &defaultNilToZero,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
},
RoundingPeriod: nil,
JobLevelMetricFields: yaceConf.JobLevelMetricFields{
Period: 0,
Length: 0,
Delay: 0,
AddCloudwatchTimestamp: &falsePtr,
NilToZero: &defaultNilToZero,
},
},
},
},
},
},
"static job nil to zero": {
raw: staticJobNilToZeroConfig,
expected: yaceConf.ScrapeConf{
APIVersion: "v1alpha1",
StsRegion: "us-east-2",
Discovery: yaceConf.Discovery{},
Static: []*yaceConf.Static{
{
Name: "super_ec2_instance_id",
// assert an empty role is used as default. IMPORTANT since this
// is what YACE looks for delegating to the environment role
Roles: []yaceConf.Role{{}},
Regions: []string{"us-east-2"},
Namespace: "AWS/EC2",
CustomTags: []yaceModel.Tag{},
Dimensions: []yaceConf.Dimension{
{
Name: "InstanceId",
Value: "i01u29u12ue1u2c",
},
},
Metrics: []*yaceConf.Metric{{
Name: "CPUUsage",
Statistics: []string{"Sum", "Average"},
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &falsePtr,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
}},
},
},
},
},
"static job nil to zero metric": {
raw: staticJobNilToZeroMetricConfig,
expected: yaceConf.ScrapeConf{
APIVersion: "v1alpha1",
StsRegion: "us-east-2",
Discovery: yaceConf.Discovery{},
Static: []*yaceConf.Static{
{
Name: "super_ec2_instance_id",
// assert an empty role is used as default. IMPORTANT since this
// is what YACE looks for delegating to the environment role
Roles: []yaceConf.Role{{}},
Regions: []string{"us-east-2"},
Namespace: "AWS/EC2",
CustomTags: []yaceModel.Tag{},
Dimensions: []yaceConf.Dimension{
{
Name: "InstanceId",
Value: "i01u29u12ue1u2c",
},
},
Metrics: []*yaceConf.Metric{{
Name: "CPUUsage",
Statistics: []string{"Sum", "Average"},
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &falsePtr,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
}},
},
},
},
},
"discovery job nil to zero config": {
raw: discoveryJobNilToZeroConfig,
expected: yaceConf.ScrapeConf{
APIVersion: "v1alpha1",
StsRegion: "us-east-2",
Discovery: yaceConf.Discovery{
ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics{
"ec2": []string{"name"},
},
Jobs: []*yaceConf.Job{
{
Regions: []string{"us-east-2"},
// assert an empty role is used as default. IMPORTANT since this
// is what YACE looks for delegating to the environment role
Roles: []yaceConf.Role{{}},
Type: "sqs",
SearchTags: []yaceModel.Tag{{
Key: "scrape", Value: "true",
}},
CustomTags: []yaceModel.Tag{},
Metrics: []*yaceConf.Metric{
{
Name: "NumberOfMessagesSent",
Statistics: []string{"Sum", "Average"},
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &falsePtr,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
{
Name: "NumberOfMessagesReceived",
Statistics: []string{"Sum", "Average"},
Period: 60,
Length: 60,
Delay: 0,
NilToZero: &truePtr,
AddCloudwatchTimestamp: &addCloudwatchTimestamp,
},
},
Expand All @@ -260,7 +447,7 @@ func TestCloudwatchComponentConfig(t *testing.T) {
Length: 0,
Delay: 0,
AddCloudwatchTimestamp: &falsePtr,
NilToZero: &nilToZero,
NilToZero: &falsePtr,
},
},
},
Expand Down
2 changes: 2 additions & 0 deletions converter/internal/staticconvert/internal/build/cloudwatch_exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ func toDiscoveryJob(job *cloudwatch_exporter.DiscoveryJob) cloudwatch.DiscoveryJ
Type: job.Type,
DimensionNameRequirements: job.DimensionNameRequirements,
Metrics: toMetrics(job.Metrics),
NilToZero: job.NilToZero,
}
}

Expand Down Expand Up @@ -92,5 +93,6 @@ func toMetric(metric cloudwatch_exporter.Metric) cloudwatch.Metric {
Statistics: metric.Statistics,
Period: metric.Period,
Length: metric.Length,
NilToZero: metric.NilToZero,
}
}
Loading

0 comments on commit d610871

Please sign in to comment.