Skip to content

Commit

Permalink
Add active_series_additional_custom_trackers config option (#10428)
Browse files Browse the repository at this point in the history 
* Add active_series_additional_custom_trackers config option

Signed-off-by: Marco Pracucci <[email protected]>

* Add changelog entry

Signed-off-by: Marco Pracucci <[email protected]>

* Address copy review

Signed-off-by: Marco Pracucci <[email protected]>

* Make MergeCustomTrackersConfig() a no-op if there's nothing to merge

Signed-off-by: Marco Pracucci <[email protected]>

* Fixed race in ActiveSeriesCustomTrackersConfig()

Signed-off-by: Marco Pracucci <[email protected]>

* Fix issue with atomic

Signed-off-by: Marco Pracucci <[email protected]>

* Fixed TestRuntimeConfigLoader_ShouldLoadAnchoredYAML

Signed-off-by: Marco Pracucci <[email protected]>

* Added TestRuntimeConfigLoader_ActiveSeriesCustomTrackersMergingShouldNotInterfereBetweenTenants

Signed-off-by: Marco Pracucci <[email protected]>

* Removed addressed TODO

Signed-off-by: Marco Pracucci <[email protected]>

* Remove if condition

Signed-off-by: Marco Pracucci <[email protected]>

---------

Signed-off-by: Marco Pracucci <[email protected]>
  • Loading branch information
@pracucci
pracucci authored Jan 16, 2025
1 parent b812677 commit 12c09fd
Show file tree
Hide file tree
Showing 17 changed files with 1,200 additions and 40 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
* [ENHANCEMENT] Ruler: When rule concurrency is enabled for a rule group, its rules will now be reordered and run in batches based on their dependencies. This increases the number of rules that can potentially run concurrently. Note that the global and tenant-specific limits still apply #10400
* [ENHANCEMENT] Query-frontend: include more information about read consistency in trace spans produced when using experimental ingest storage. #10412
* [ENHANCEMENT] Ingester: Hide tokens in ingester ring status page when ingest storage is enabled #10399
* [ENHANCEMENT] Ingester: add `active_series_additional_custom_trackers` configuration, in addition to the already existing `active_series_custom_trackers`. The `active_series_additional_custom_trackers` configuration allows you to configure additional custom trackers that get merged with `active_series_custom_trackers` at runtime. #10428
* [BUGFIX] Distributor: Use a boolean to track changes while merging the ReplicaDesc components, rather than comparing the objects directly. #10185
* [BUGFIX] Querier: fix timeout responding to query-frontend when response size is very close to `-querier.frontend-client.grpc-max-send-msg-size`. #10154
* [BUGFIX] Query-frontend and querier: show warning/info annotations in some cases where they were missing (if a lazy querier was used). #10277
Expand Down
12 changes: 11 additions & 1 deletion cmd/mimir/config-descriptor.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4020,13 +4020,23 @@
"kind": "field",
"name": "active_series_custom_trackers",
"required": false,
"desc": "Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero).",
"desc": "Custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count is exposed in the custom trackers metric labeled using the tracker name (map key). Zero-valued counts are not exposed and are removed when they go back to zero.",
"fieldValue": null,
"fieldDefaultValue": {},
"fieldFlag": "ingester.active-series-custom-trackers",
"fieldType": "map of tracker name (string) to matcher (string)",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "active_series_additional_custom_trackers",
"required": false,
"desc": "Additional custom trackers for active metrics merged on top of the base custom trackers. You can use this configuration option to define the base custom trackers globally for all tenants, and then use the additional trackers to add extra trackers on a per-tenant basis.",
"fieldValue": null,
"fieldDefaultValue": {},
"fieldType": "map of tracker name (string) to matcher (string)",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "out_of_order_time_window",
Expand Down
5 changes: 3 additions & 2 deletions development/mimir-ingest-storage/config/mimir.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@ ingest_storage:
address: kafka_1:9092
topic: mimir-ingest
last_produced_offset_poll_interval: 500ms
startup_fetch_concurrency: 15
ongoing_fetch_concurrency: 2
fetch_concurrency_max: 15

ingester:
track_ingester_owned_series: true
active_series_metrics_update_period: 10s
active_series_metrics_idle_timeout: 1m

partition_ring:
min_partition_owners_count: 1
Expand Down
7 changes: 7 additions & 0 deletions development/mimir-ingest-storage/config/runtime.yaml
View file
Original file line number Diff line number Diff line change
@@ -1 +1,8 @@
# This file can be used to set overrides or other runtime config.
overrides:
anonymous:
active_series_custom_trackers:
base_mimir_write: '{job="mimir-read-write-mode/mimir-write"}'
base_mimir_read: '{job="mimir-read-write-mode/mimir-read"}'
active_series_additional_custom_trackers:
additional_mimir_backend: '{job="mimir-read-write-mode/mimir-backend"}'
25 changes: 19 additions & 6 deletions docs/sources/mimir/configure/configuration-parameters/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3359,20 +3359,33 @@ The `limits` block configures default and per-tenant limits imposed by component
# CLI flag: -ingester.ooo-native-histograms-ingestion-enabled
[ooo_native_histograms_ingestion_enabled: <boolean> | default = false]

# (advanced) Additional custom trackers for active metrics. If there are active
# series matching a provided matcher (map value), the count will be exposed in
# the custom trackers metric labeled using the tracker name (map key). Zero
# valued counts are not exposed (and removed when they go back to zero).
# (advanced) Custom trackers for active metrics. If there are active series
# matching a provided matcher (map value), the count is exposed in the custom
# trackers metric labeled using the tracker name (map key). Zero-valued counts
# are not exposed and are removed when they go back to zero.
# Example:
# The following configuration will count the active series coming from dev and
# prod namespaces for each tenant and label them as {name="dev"} and
# The following configuration counts the active series coming from dev and
# prod namespaces for each tenant and labels them as {name="dev"} and
# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.
# active_series_custom_trackers:
# dev: '{namespace=~"dev-.*"}'
# prod: '{namespace=~"prod-.*"}'
# CLI flag: -ingester.active-series-custom-trackers
[active_series_custom_trackers: <map of tracker name (string) to matcher (string)> | default = ]

# (advanced) Additional custom trackers for active metrics merged on top of the
# base custom trackers. You can use this configuration option to define the base
# custom trackers globally for all tenants, and then use the additional trackers
# to add extra trackers on a per-tenant basis.
# Example:
# The following configuration counts the active series coming from dev and
# prod namespaces for each tenant and labels them as {name="dev"} and
# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.
# active_series_additional_custom_trackers:
# dev: '{namespace=~"dev-.*"}'
# prod: '{namespace=~"prod-.*"}'
[active_series_additional_custom_trackers: <map of tracker name (string) to matcher (string)> | default = ]

# (experimental) Non-zero value enables out-of-order support for most recent
# samples that are within the time window in relation to the TSDB's maximum
# time, i.e., within [db.maxTime-timeWindow, db.maxTime]). The ingester will
Expand Down
46 changes: 44 additions & 2 deletions pkg/ingester/activeseries/model/custom_trackers_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ type CustomTrackersConfig struct {

// ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled.
func (c CustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) {
return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` +
` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`,
return `The following configuration counts the active series coming from dev and prod namespaces for each tenant` +
` and labels them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`,
map[string]string{
"dev": `{namespace=~"dev-.*"}`,
"prod": `{namespace=~"prod-.*"}`,
Expand Down Expand Up @@ -189,3 +189,45 @@ func NewCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err e
c.string = customTrackersConfigString(c.source)
return c, nil
}

// MergeCustomTrackersConfig returns a new CustomTrackersConfig containing the merge of the two
// CustomTrackersConfig in input. The two configs in input are not manipulated. If a key exists
// in both configs, second config wins over first config.
func MergeCustomTrackersConfig(first, second CustomTrackersConfig) CustomTrackersConfig {
if len(first.config) == 0 && len(second.config) == 0 {
return CustomTrackersConfig{}
}
if len(second.config) == 0 {
return first
}
if len(first.config) == 0 {
return second
}

merged := CustomTrackersConfig{
source: make(map[string]string, len(first.source)+len(second.source)),
config: make(map[string]labelsMatchers, len(first.config)+len(second.config)),
string: "",
}

// Merge source.
for key, value := range first.source {
merged.source[key] = value
}
for key, value := range second.source {
merged.source[key] = value
}

// Merge config.
for key, value := range first.config {
merged.config[key] = value
}
for key, value := range second.config {
merged.config[key] = value
}

// Rebuild the string representation.
merged.string = customTrackersConfigString(merged.source)

return merged
}
101 changes: 101 additions & 0 deletions pkg/ingester/activeseries/model/custom_trackers_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,3 +341,104 @@ func TestCustomTrackersConfig_Equal(t *testing.T) {
})
}
}

func TestMergeCustomTrackersConfig(t *testing.T) {
tests := map[string]struct {
first string
second string
expected string
}{
"both configs are empty": {
first: "",
second: "",
expected: "",
},
"the first config is empty": {
first: "",
second: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
expected: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
},
"the second config is empty": {
first: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
second: "",
expected: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
},
"both configs are non-empty and they both have the same key-value pairs in the same order": {
first: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
second: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
expected: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
},
"both configs are non-empty and they both have the same key-value pairs but in different order": {
first: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
second: `
foo: "{foo='bar'}"
baz: "{baz='bar'}"`,
expected: `
baz: "{baz='bar'}"
foo: "{foo='bar'}"`,
},
"both configs are non-empty and they some but not all overlapping keys": {
first: `
baz: "{baz='first'}"
foo: "{foo='first'}"`,
second: `
foo: "{foo='second'}"
bar: "{bar='second'}"`,
expected: `
bar: "{bar='second'}"
baz: "{baz='first'}"
foo: "{foo='second'}"`,
},
"both configs are non-empty and they no overlapping keys": {
first: `
baz: "{baz='first'}"
foo: "{foo='first'}"`,
second: `
bar: "{bar='second'}"`,
expected: `
bar: "{bar='second'}"
baz: "{baz='first'}"
foo: "{foo='first'}"`,
},
}

for testName, testData := range tests {
t.Run(testName, func(t *testing.T) {
first := mustNewCustomTrackersConfigDeserializedFromYaml(t, testData.first)
second := mustNewCustomTrackersConfigDeserializedFromYaml(t, testData.second)
expected := mustNewCustomTrackersConfigDeserializedFromYaml(t, testData.expected)

merged := MergeCustomTrackersConfig(first, second)
require.Equal(t, expected.source, merged.source)
require.Equal(t, expected.config, merged.config)
require.Equal(t, expected.string, merged.string)

// The original configs should NOT have been changed.
expectedFirst := mustNewCustomTrackersConfigDeserializedFromYaml(t, testData.first)
require.Equal(t, expectedFirst.source, first.source)
require.Equal(t, expectedFirst.config, first.config)
require.Equal(t, expectedFirst.string, first.string)

expectedSecond := mustNewCustomTrackersConfigDeserializedFromYaml(t, testData.second)
require.Equal(t, expectedSecond.source, second.source)
require.Equal(t, expectedSecond.config, second.config)
require.Equal(t, expectedSecond.string, second.string)
})
}
}
18 changes: 9 additions & 9 deletions pkg/ingester/ingester_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9114,7 +9114,7 @@ func TestIngesterActiveSeries(t *testing.T) {
})

activeSeriesTenantOverride := new(TenantLimitsMock)
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesBaseCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})
activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil)

tests := map[string]struct {
Expand Down Expand Up @@ -9404,7 +9404,7 @@ func TestIngesterActiveSeries(t *testing.T) {
cfg.ActiveSeriesMetrics.Enabled = !testData.disableActiveSeries

limits := defaultLimitsTestConfig()
limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig
limits.ActiveSeriesBaseCustomTrackersConfig = activeSeriesDefaultConfig
limits.NativeHistogramsIngestionEnabled = true
overrides, err := validation.NewOverrides(limits, activeSeriesTenantOverride)
require.NoError(t, err)
Expand Down Expand Up @@ -9476,7 +9476,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) {

defaultActiveSeriesTenantOverride := new(TenantLimitsMock)
defaultActiveSeriesTenantOverride.On("ByUserID", userID2).Return(nil)
defaultActiveSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})
defaultActiveSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesBaseCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})

tests := map[string]struct {
test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer)
Expand Down Expand Up @@ -9537,9 +9537,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) {
// Add new runtime configs
activeSeriesTenantOverride := new(TenantLimitsMock)
activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil)
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesBaseCustomTrackersConfig: activeSeriesTenantConfig, NativeHistogramsIngestionEnabled: true})
limits := defaultLimitsTestConfig()
limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig
limits.ActiveSeriesBaseCustomTrackersConfig = activeSeriesDefaultConfig
limits.NativeHistogramsIngestionEnabled = true
override, err := validation.NewOverrides(limits, activeSeriesTenantOverride)
require.NoError(t, err)
Expand Down Expand Up @@ -9670,7 +9670,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) {

// Remove runtime configs
limits := defaultLimitsTestConfig()
limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig
limits.ActiveSeriesBaseCustomTrackersConfig = activeSeriesDefaultConfig
limits.NativeHistogramsIngestionEnabled = true
override, err := validation.NewOverrides(limits, nil)
require.NoError(t, err)
Expand Down Expand Up @@ -9787,14 +9787,14 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) {

// Change runtime configs
activeSeriesTenantOverride := new(TenantLimitsMock)
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{
activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesBaseCustomTrackersConfig: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{
"team_a": `{team="a"}`,
"team_b": `{team="b"}`,
"team_c": `{team="b"}`,
"team_d": `{team="b"}`,
}), NativeHistogramsIngestionEnabled: true})
limits := defaultLimitsTestConfig()
limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig
limits.ActiveSeriesBaseCustomTrackersConfig = activeSeriesDefaultConfig
limits.NativeHistogramsIngestionEnabled = true
override, err := validation.NewOverrides(limits, activeSeriesTenantOverride)
require.NoError(t, err)
Expand Down Expand Up @@ -9926,7 +9926,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) {
cfg.ActiveSeriesMetrics.Enabled = true

limits := defaultLimitsTestConfig()
limits.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig
limits.ActiveSeriesBaseCustomTrackersConfig = testData.activeSeriesConfig
limits.NativeHistogramsIngestionEnabled = true
var overrides *validation.Overrides
var err error
Expand Down
Loading

0 comments on commit 12c09fd

Please sign in to comment.