Instrument client object refs metrics

Adds a new gauge metric with the cache_key label. E.g: vso_client_factory_client_refs{cache_key="kubernetes-f42f026de00efe6e8aff24"} 1
hashicorp · Jun 12, 2024 · 4d3e597 · 4d3e597
1 parent 3377c25
commit 4d3e597
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 0 deletions.
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -39,6 +39,7 @@ const (
 	NameRequestsTotal         = "requests_total"
 	NameRequestsErrorsTotal   = "requests_errors_total"
 	NameTaintedClients        = "tainted_clients"
+	NameClientRefs            = "client_refs"
 )
 
 var ResourceStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{

diff --git a/internal/vault/client_factory.go b/internal/vault/client_factory.go
@@ -124,6 +124,7 @@ type cachingClientFactory struct {
 	requestCounterVec      *prometheus.CounterVec
 	requestErrorCounterVec *prometheus.CounterVec
 	taintedClientGauge     *prometheus.GaugeVec
+	clientRefGauge         *prometheus.GaugeVec
 	revokeOnEvict          bool
 	pruneStorageOnEvict    bool
 	ctrlClient             ctrlclient.Client
@@ -299,6 +300,9 @@ func (m *cachingClientFactory) onClientEvict(ctx context.Context, client ctrlcli
 	}
 
 	m.removeClientLock(cacheKey)
+	if m.clientRefGauge != nil {
+		m.clientRefGauge.DeleteLabelValues(cacheKey.String())
+	}
 }
 
 // Restore will attempt to restore a Client from storage. If storage is not enabled then no restoration will take place.
@@ -590,6 +594,11 @@ func (m *cachingClientFactory) updateClientStatsAfterGet(ctx context.Context, ca
 				"creationTimestamp", c.Stat().CreationTimestamp(),
 				"reason", decrementReason,
 			)
+			if m.clientRefGauge != nil {
+				m.clientRefGauge.WithLabelValues(cacheKey.String()).Set(
+					float64(c.Stat().RefCount()),
+				)
+			}
 			// send the client to the cache pruner.
 			m.orphanPrunerClientCh <- c
 		}
@@ -604,6 +613,11 @@ func (m *cachingClientFactory) updateClientStatsAfterGet(ctx context.Context, ca
 				"creationTimestamp", c.Stat().CreationTimestamp(),
 				"reason", incrementReason,
 			)
+			if m.clientRefGauge != nil {
+				m.clientRefGauge.WithLabelValues(cacheKey.String()).Set(
+					float64(c.Stat().RefCount()),
+				)
+			}
 		}
 	}
 }
@@ -1004,6 +1018,14 @@ func NewCachingClientFactory(ctx context.Context, client ctrlclient.Client, cach
 				metrics.LabelCacheKey,
 			},
 		),
+		clientRefGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: metricsFQNClientRefs,
+				Help: "Client factory number of object references",
+			}, []string{
+				metrics.LabelCacheKey,
+			},
+		),
 	}
 
 	if config.CollectClientCacheMetrics {
@@ -1016,6 +1038,7 @@ func NewCachingClientFactory(ctx context.Context, client ctrlclient.Client, cach
 			factory.requestCounterVec,
 			factory.requestErrorCounterVec,
 			factory.taintedClientGauge,
+			factory.clientRefGauge,
 			clientFactoryOperationTimes,
 		)
 	}

diff --git a/internal/vault/client_factory_metrics.go b/internal/vault/client_factory_metrics.go
@@ -25,6 +25,9 @@ var (
 	metricsFQNClientFactoryTaintedClients = prometheus.BuildFQName(
 		metrics.Namespace, subsystemClientFactory, metrics.NameTaintedClients)
 
+	metricsFQNClientRefs = prometheus.BuildFQName(
+		metrics.Namespace, subsystemClientFactory, metrics.NameClientRefs)
+
 	// TODO: update to use Native Histograms once it is no longer an experimental Prometheus feature
 	// ref: https://github.com/prometheus/prometheus/milestone/10
 	clientFactoryOperationTimes = prometheus.NewHistogramVec(prometheus.HistogramOpts{