From 49ca7875bab68e62ed73426c4df20d0fcd5a814b Mon Sep 17 00:00:00 2001 From: Nikos Angelopoulos Date: Fri, 29 Nov 2024 11:29:05 +0100 Subject: [PATCH 1/5] feat(distributor): add experimental `memberlist` kvStore for ha_tracker --- CHANGELOG.md | 1 + .../mimir/configure/configuration-parameters/index.md | 5 ++--- pkg/distributor/ha_tracker.go | 7 +------ pkg/distributor/ha_tracker_test.go | 4 ++-- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d0fc5254e..503151e9d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ * [FEATURE] PromQL: Add experimental `info` function. Experimental functions are disabled by default, but can be enabled setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9879 * [FEATURE] Distributor: Support promotion of OTel resource attributes to labels. #8271 * [FEATURE] Querier: Add experimental `double_exponential_smoothing` PromQL function. Experimental functions are disabled by default, but can be enabled by setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9844 +* [FEATURE] Distributor: Add experimental `memberlist` kvStore for ha_tracker. Memberlist parameters can be changed through `-memberlist-*` flags * [ENHANCEMENT] Query Frontend: Return server-side `bytes_processed` statistics following Server-Timing format. #9645 #9985 * [ENHANCEMENT] mimirtool: Adds bearer token support for mimirtool's analyze ruler/prometheus commands. #9587 * [ENHANCEMENT] Ruler: Support `exclude_alerts` parameter in `/api/v1/rules` endpoint. #9300 diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index b21c2bd6317..c5c68e4e2bf 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -798,9 +798,8 @@ ha_tracker: # CLI flag: -distributor.ha-tracker.failover-timeout [ha_tracker_failover_timeout: | default = 30s] - # Backend storage to use for the ring. Please be aware that memberlist is not - # supported by the HA tracker since gossip propagation is too slow for HA - # purposes. + # Backend storage to use for the ring. Please be aware that memberlist is + # supported by the HA tracker but its experimental. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. diff --git a/pkg/distributor/ha_tracker.go b/pkg/distributor/ha_tracker.go index 154707dee5a..2099008bd8c 100644 --- a/pkg/distributor/ha_tracker.go +++ b/pkg/distributor/ha_tracker.go @@ -34,7 +34,6 @@ import ( var ( errNegativeUpdateTimeoutJitterMax = errors.New("HA tracker max update timeout jitter shouldn't be negative") errInvalidFailoverTimeout = "HA Tracker failover timeout (%v) must be at least 1s greater than update timeout - max jitter (%v)" - errMemberlistUnsupported = errors.New("memberlist is not supported by the HA tracker since gossip propagation is too slow for HA purposes") ) type haTrackerLimits interface { @@ -152,7 +151,7 @@ type HATrackerConfig struct { // more than this duration FailoverTimeout time.Duration `yaml:"ha_tracker_failover_timeout" category:"advanced"` - KVStore kv.Config `yaml:"kvstore" doc:"description=Backend storage to use for the ring. Please be aware that memberlist is not supported by the HA tracker since gossip propagation is too slow for HA purposes."` + KVStore kv.Config `yaml:"kvstore" doc:"description=Backend storage to use for the ring. Please be aware that memberlist is supported by the HA tracker but its experimental."` } // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -180,10 +179,6 @@ func (cfg *HATrackerConfig) Validate() error { return fmt.Errorf(errInvalidFailoverTimeout, cfg.FailoverTimeout, minFailureTimeout) } - if cfg.KVStore.Store == "memberlist" { - return errMemberlistUnsupported - } - return nil } diff --git a/pkg/distributor/ha_tracker_test.go b/pkg/distributor/ha_tracker_test.go index 9438fd5ae3b..68ef61e417b 100644 --- a/pkg/distributor/ha_tracker_test.go +++ b/pkg/distributor/ha_tracker_test.go @@ -430,7 +430,7 @@ func TestHATrackerConfig_Validate(t *testing.T) { }(), expectedErr: nil, }, - "should fail if KV backend is set to memberlist": { + "should pass if KV backend is set to memberlist": { cfg: func() HATrackerConfig { cfg := HATrackerConfig{} flagext.DefaultValues(&cfg) @@ -438,7 +438,7 @@ func TestHATrackerConfig_Validate(t *testing.T) { return cfg }(), - expectedErr: errMemberlistUnsupported, + expectedErr: nil, }, } From 61931312e750a288679b94f8fe925db2459c0553 Mon Sep 17 00:00:00 2001 From: Nikos Angelopoulos Date: Mon, 2 Dec 2024 21:01:13 +0100 Subject: [PATCH 2/5] enable memberlistKV singleton and add ReplicaDesc Codec --- pkg/mimir/modules.go | 2 ++ pkg/mimir/modules_test.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index c66d15c6474..62e136dcd3d 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -1023,6 +1023,7 @@ func (t *Mimir) initMemberlistKV() (services.Service, error) { // Append to the list of codecs instead of overwriting the value to allow third parties to inject their own codecs. t.Cfg.MemberlistKV.Codecs = append(t.Cfg.MemberlistKV.Codecs, ring.GetCodec()) t.Cfg.MemberlistKV.Codecs = append(t.Cfg.MemberlistKV.Codecs, ring.GetPartitionRingCodec()) + t.Cfg.MemberlistKV.Codecs = append(t.Cfg.MemberlistKV.Codecs, distributor.GetReplicaDescCodec()) dnsProviderReg := prometheus.WrapRegistererWithPrefix( "cortex_", @@ -1037,6 +1038,7 @@ func (t *Mimir) initMemberlistKV() (services.Service, error) { // Update the config. t.Cfg.Distributor.DistributorRing.Common.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV + t.Cfg.Distributor.HATrackerConfig.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV t.Cfg.Ingester.IngesterRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV t.Cfg.Ingester.IngesterPartitionRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV t.Cfg.StoreGateway.ShardingRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV diff --git a/pkg/mimir/modules_test.go b/pkg/mimir/modules_test.go index c8ecc7b8daf..d71c8715717 100644 --- a/pkg/mimir/modules_test.go +++ b/pkg/mimir/modules_test.go @@ -193,6 +193,7 @@ func TestMultiKVSetup(t *testing.T) { Distributor: func(t *testing.T, c Config) { require.NotNil(t, c.Distributor.DistributorRing.Common.KVStore.Multi.ConfigProvider) + require.NotNil(t, c.Distributor.HATrackerConfig.KVStore.MemberlistKV) require.NotNil(t, c.Ingester.IngesterRing.KVStore.Multi.ConfigProvider) require.NotNil(t, c.Ingester.IngesterPartitionRing.KVStore.Multi.ConfigProvider) }, From 65a77354a1f29f70c9fc23f148fd9b0a1a171536 Mon Sep 17 00:00:00 2001 From: Nikos Angelopoulos Date: Tue, 3 Dec 2024 09:22:48 +0100 Subject: [PATCH 3/5] docs: add experimental information --- CHANGELOG.md | 2 +- docs/sources/mimir/configure/about-versioning.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 503151e9d4c..a6bb7adbf35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,7 @@ * [FEATURE] PromQL: Add experimental `info` function. Experimental functions are disabled by default, but can be enabled setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9879 * [FEATURE] Distributor: Support promotion of OTel resource attributes to labels. #8271 * [FEATURE] Querier: Add experimental `double_exponential_smoothing` PromQL function. Experimental functions are disabled by default, but can be enabled by setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9844 -* [FEATURE] Distributor: Add experimental `memberlist` kvStore for ha_tracker. Memberlist parameters can be changed through `-memberlist-*` flags +* [FEATURE] Distributor: Add experimental `memberlist` KV store for ha_tracker. It can be enabled using the `-distributor.ha-tracker.kvstore.store` flag. Memberlist parameters can be configured via the `-memberlist-*` flags. #10054 * [ENHANCEMENT] Query Frontend: Return server-side `bytes_processed` statistics following Server-Timing format. #9645 #9985 * [ENHANCEMENT] mimirtool: Adds bearer token support for mimirtool's analyze ruler/prometheus commands. #9587 * [ENHANCEMENT] Ruler: Support `exclude_alerts` parameter in `/api/v1/rules` endpoint. #9300 diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index 892ca5d0296..dceb841034b 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -88,6 +88,8 @@ The following features are currently experimental: - `-distributor.otel-created-timestamp-zero-ingestion-enabled` - Promote a certain set of OTel resource attributes to labels - `-distributor.promote-otel-resource-attributes` + - Add experimental `memberlist` key-value store for ha_tracker. Note that this feature is `experimental`, as the upper limits of propagation times have not yet been validated. + - `-distributor.ha-tracker.kvstore.store` - Hash ring - Disabling ring heartbeat timeouts - `-distributor.ring.heartbeat-timeout=0` From 40bd75aa335d9531e343562380c65ce1baaebc8d Mon Sep 17 00:00:00 2001 From: Nikos Angelopoulos Date: Wed, 4 Dec 2024 09:54:10 +0100 Subject: [PATCH 4/5] docs: address PR comments --- CHANGELOG.md | 2 +- .../sources/mimir/configure/configuration-parameters/index.md | 4 ++-- pkg/distributor/ha_tracker.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6bb7adbf35..be4b17637f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,7 +73,7 @@ * [FEATURE] PromQL: Add experimental `info` function. Experimental functions are disabled by default, but can be enabled setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9879 * [FEATURE] Distributor: Support promotion of OTel resource attributes to labels. #8271 * [FEATURE] Querier: Add experimental `double_exponential_smoothing` PromQL function. Experimental functions are disabled by default, but can be enabled by setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9844 -* [FEATURE] Distributor: Add experimental `memberlist` KV store for ha_tracker. It can be enabled using the `-distributor.ha-tracker.kvstore.store` flag. Memberlist parameters can be configured via the `-memberlist-*` flags. #10054 +* [FEATURE] Distributor: Add experimental `memberlist` KV store for ha_tracker. You can enable it using the `-distributor.ha-tracker.kvstore.store` flag. You can configure Memberlist parameters via the `-memberlist-*` flags. #10054 * [ENHANCEMENT] Query Frontend: Return server-side `bytes_processed` statistics following Server-Timing format. #9645 #9985 * [ENHANCEMENT] mimirtool: Adds bearer token support for mimirtool's analyze ruler/prometheus commands. #9587 * [ENHANCEMENT] Ruler: Support `exclude_alerts` parameter in `/api/v1/rules` endpoint. #9300 diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index c5c68e4e2bf..5cc1628b023 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -798,8 +798,8 @@ ha_tracker: # CLI flag: -distributor.ha-tracker.failover-timeout [ha_tracker_failover_timeout: | default = 30s] - # Backend storage to use for the ring. Please be aware that memberlist is - # supported by the HA tracker but its experimental. + # Backend storage to use for the ring. Note that memberlist support is + # experimental. kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, # inmemory, memberlist, multi. diff --git a/pkg/distributor/ha_tracker.go b/pkg/distributor/ha_tracker.go index 2099008bd8c..1edb93720b1 100644 --- a/pkg/distributor/ha_tracker.go +++ b/pkg/distributor/ha_tracker.go @@ -151,7 +151,7 @@ type HATrackerConfig struct { // more than this duration FailoverTimeout time.Duration `yaml:"ha_tracker_failover_timeout" category:"advanced"` - KVStore kv.Config `yaml:"kvstore" doc:"description=Backend storage to use for the ring. Please be aware that memberlist is supported by the HA tracker but its experimental."` + KVStore kv.Config `yaml:"kvstore" doc:"description=Backend storage to use for the ring. Note that memberlist support is experimental."` } // RegisterFlags adds the flags required to config this to the given FlagSet. From 5cd0b8f65a3f877972e5eab1d7846946d6ed6c6f Mon Sep 17 00:00:00 2001 From: Nikos Angelopoulos Date: Wed, 11 Dec 2024 09:51:43 +0100 Subject: [PATCH 5/5] add information about non provided cleanup --- docs/sources/mimir/configure/about-versioning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index dceb841034b..933da9552ea 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -88,7 +88,7 @@ The following features are currently experimental: - `-distributor.otel-created-timestamp-zero-ingestion-enabled` - Promote a certain set of OTel resource attributes to labels - `-distributor.promote-otel-resource-attributes` - - Add experimental `memberlist` key-value store for ha_tracker. Note that this feature is `experimental`, as the upper limits of propagation times have not yet been validated. + - Add experimental `memberlist` key-value store for ha_tracker. Note that this feature is `experimental`, as the upper limits of propagation times have not yet been validated. Additionally, cleanup operations have not yet been implemented for the memberlist entries. - `-distributor.ha-tracker.kvstore.store` - Hash ring - Disabling ring heartbeat timeouts