partitionOffsetClient: Add topic label to metrics (#10462)

* partitionOffsetClient: Add topic label to metrics

* Update CHANGELOG.md

CHANGELOG.md

@@ -9,6 +9,7 @@
 * [CHANGE] Querier: pass query matchers to queryable `IsApplicable` hook. #10256
 * [CHANGE] Query-frontend: Add `topic` label to `cortex_ingest_storage_strong_consistency_requests_total`, `cortex_ingest_storage_strong_consistency_failures_total`, and `cortex_ingest_storage_strong_consistency_wait_duration_seconds` metrics. #10220
 * [CHANGE] Ruler: cap the rate of retries for remote query evaluation to 170/sec. This is configurable via `-ruler.query-frontend.max-retries-rate`. #10375 #10403
+* [CHANGE] Query-frontend: Add `topic` label to `cortex_ingest_storage_reader_last_produced_offset_requests_total`, `cortex_ingest_storage_reader_last_produced_offset_failures_total`, `cortex_ingest_storage_reader_last_produced_offset_request_duration_seconds`, `cortex_ingest_storage_reader_partition_start_offset_requests_total`, `cortex_ingest_storage_reader_partition_start_offset_failures_total`, `cortex_ingest_storage_reader_partition_start_offset_request_duration_seconds` metrics. #10462
 * [ENHANCEMENT] Query Frontend: Return server-side `samples_processed` statistics. #10103
 * [ENHANCEMENT] Distributor: OTLP receiver now converts also metric metadata. See also https://github.com/prometheus/prometheus/pull/15416. #10168
 * [ENHANCEMENT] Distributor: discard float and histogram samples with duplicated timestamps from each timeseries in a request before the request is forwarded to ingesters. Discarded samples are tracked by `cortex_discarded_samples_total` metrics with the reason `sample_duplicate_timestamp`. #10145 #10430

pkg/storage/ingest/partition_offset_client.go

@@ -43,12 +43,14 @@ func newPartitionOffsetClient(client *kgo.Client, topic string, reg prometheus.R
 		topic:  topic,
 
 		lastProducedOffsetRequestsTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_last_produced_offset_requests_total",
-			Help: "Total number of requests issued to get the last produced offset.",
+			Name:        "cortex_ingest_storage_reader_last_produced_offset_requests_total",
+			Help:        "Total number of requests issued to get the last produced offset.",
+			ConstLabels: prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 		lastProducedOffsetFailuresTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_last_produced_offset_failures_total",
-			Help: "Total number of failed requests to get the last produced offset.",
+			Name:        "cortex_ingest_storage_reader_last_produced_offset_failures_total",
+			Help:        "Total number of failed requests to get the last produced offset.",
+			ConstLabels: prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 		lastProducedOffsetLatency: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
 			Name:                            "cortex_ingest_storage_reader_last_produced_offset_request_duration_seconds",
@@ -57,15 +59,18 @@ func newPartitionOffsetClient(client *kgo.Client, topic string, reg prometheus.R
 			NativeHistogramMaxBucketNumber:  100,
 			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.DefBuckets,
+			ConstLabels:                     prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 
 		partitionStartOffsetRequestsTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_partition_start_offset_requests_total",
-			Help: "Total number of requests issued to get the partition start offset.",
+			Name:        "cortex_ingest_storage_reader_partition_start_offset_requests_total",
+			Help:        "Total number of requests issued to get the partition start offset.",
+			ConstLabels: prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 		partitionStartOffsetFailuresTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_partition_start_offset_failures_total",
-			Help: "Total number of failed requests to get the partition start offset.",
+			Name:        "cortex_ingest_storage_reader_partition_start_offset_failures_total",
+			Help:        "Total number of failed requests to get the partition start offset.",
+			ConstLabels: prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 		partitionStartOffsetLatency: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
 			Name:                            "cortex_ingest_storage_reader_partition_start_offset_request_duration_seconds",
@@ -74,6 +79,7 @@ func newPartitionOffsetClient(client *kgo.Client, topic string, reg prometheus.R
 			NativeHistogramMaxBucketNumber:  100,
 			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.DefBuckets,
+			ConstLabels:                     prometheus.Labels{"topic": topic},
 		}, []string{"partition"}),
 	}
 }

pkg/storage/ingest/partition_offset_client_test.go

@@ -67,11 +67,11 @@ func TestPartitionOffsetClient_FetchPartitionLastProducedOffset(t *testing.T) {
 		assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 			# HELP cortex_ingest_storage_reader_last_produced_offset_failures_total Total number of failed requests to get the last produced offset.
 			# TYPE cortex_ingest_storage_reader_last_produced_offset_failures_total counter
-			cortex_ingest_storage_reader_last_produced_offset_failures_total{partition="0"} 0
+			cortex_ingest_storage_reader_last_produced_offset_failures_total{partition="0",topic="test"} 0
 
 			# HELP cortex_ingest_storage_reader_last_produced_offset_requests_total Total number of requests issued to get the last produced offset.
 			# TYPE cortex_ingest_storage_reader_last_produced_offset_requests_total counter
-			cortex_ingest_storage_reader_last_produced_offset_requests_total{partition="0"} 3
+			cortex_ingest_storage_reader_last_produced_offset_requests_total{partition="0",topic="test"} 3
 		`), "cortex_ingest_storage_reader_last_produced_offset_requests_total",
 			"cortex_ingest_storage_reader_last_produced_offset_failures_total"))
 	})
@@ -220,11 +220,11 @@ func TestPartitionOffsetClient_FetchPartitionStartOffset(t *testing.T) {
 		assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 			# HELP cortex_ingest_storage_reader_partition_start_offset_failures_total Total number of failed requests to get the partition start offset.
 			# TYPE cortex_ingest_storage_reader_partition_start_offset_failures_total counter
-			cortex_ingest_storage_reader_partition_start_offset_failures_total{partition="0"} 0
+			cortex_ingest_storage_reader_partition_start_offset_failures_total{partition="0",topic="test"} 0
 
 			# HELP cortex_ingest_storage_reader_partition_start_offset_requests_total Total number of requests issued to get the partition start offset.
 			# TYPE cortex_ingest_storage_reader_partition_start_offset_requests_total counter
-			cortex_ingest_storage_reader_partition_start_offset_requests_total{partition="0"} 4
+			cortex_ingest_storage_reader_partition_start_offset_requests_total{partition="0",topic="test"} 4
 		`), "cortex_ingest_storage_reader_partition_start_offset_requests_total",
 			"cortex_ingest_storage_reader_partition_start_offset_failures_total"))
 	})
@@ -380,11 +380,11 @@ func TestPartitionOffsetClient_FetchPartitionsLastProducedOffsets(t *testing.T)
 		assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 			# HELP cortex_ingest_storage_reader_last_produced_offset_failures_total Total number of failed requests to get the last produced offset.
 			# TYPE cortex_ingest_storage_reader_last_produced_offset_failures_total counter
-			cortex_ingest_storage_reader_last_produced_offset_failures_total{partition="mixed"} 0
+			cortex_ingest_storage_reader_last_produced_offset_failures_total{partition="mixed",topic="test"} 0
 
 			# HELP cortex_ingest_storage_reader_last_produced_offset_requests_total Total number of requests issued to get the last produced offset.
 			# TYPE cortex_ingest_storage_reader_last_produced_offset_requests_total counter
-			cortex_ingest_storage_reader_last_produced_offset_requests_total{partition="mixed"} 4
+			cortex_ingest_storage_reader_last_produced_offset_requests_total{partition="mixed",topic="test"} 4
 		`), "cortex_ingest_storage_reader_last_produced_offset_requests_total",
 			"cortex_ingest_storage_reader_last_produced_offset_failures_total"))
 	})