Merge pull request #8 from lyubent/hint_metrics

Add hint metrics

Author: suhailpatel

README.md

@@ -140,6 +140,15 @@ These metrics come from Cassandra's storage service which keeps track of the clu
 | `seastat_storage_tokens` | Number of tokens reported by Cassandra  | Gauge |
 | `seastat_storage_node_status` | Status (`live`, `unreachable`, `joining`, `moving`, `leaving`) of each node in the cluster (tagged by node and status) | Gauge |
 
+## Hint Metrics
+
+These metrics come from the [Storage](https://cassandra.apache.org/doc/latest/operating/metrics.html#storage-metrics) metric which keeps track of hints, node load and storage exceptions.
+
+| Name          | Description   | Type |
+| ------------- | ------------- | ---- |
+| `seastat_total_hints` | Number of hint messages written to this node since [re]start. Includes one entry for each host to be hinted per hint. | Counter |
+| `seastat_total_hints_in_progress` | Number of hints attempting to be sent currently from this node. | Gauge |
+
 ## Scrape Metrics
 
 Seastat also exposes some internal metrics of how long the scrape took and the timestamp of the last scrape

jolokia/client.go

@@ -442,6 +442,27 @@ func (c *jolokiaClient) StorageStats() (StorageStats, error) {
 	return stats, nil
 }
 
+// HintStats gives information on hints being created and handed off in Cassandra
+// Hint metrics are ephemeral and reset when the Cassandra process restarts
+func (c *jolokiaClient) HintStats() (HintStats, error) {
+	v, err := c.read("org.apache.cassandra.metrics", "type=Storage", "name=*")
+	if err != nil {
+		return HintStats{}, fmt.Errorf("err reading CQL stats: %v", err)
+	}
+
+	stats := HintStats{}
+	v.Get("value").GetObject().Visit(func(key []byte, val *fastjson.Value) {
+		attributes := extractAttributes(string(key))
+		switch attributes["name"] {
+		case "TotalHintsInProgress":
+			stats.TotalHintsInProgress = Gauge(val.Get("Count").GetInt64())
+		case "TotalHints":
+			stats.TotalHints = Counter(val.Get("Count").GetInt64())
+		}
+	})
+	return stats, nil
+}
+
 // get makes a GET request to the targetPath and returns the contents of the
 // body as a JSON value ready for items to be plucked. If any part of the
 // request pipeline fails, an err is returned

jolokia/jolokia.go

@@ -84,6 +84,10 @@ type Client interface {
 	// which encapsulates things like number of keyspaces and what nodes
 	// are part of the cluster
 	StorageStats() (StorageStats, error)
+
+	// HintStats gives information on hints being created and handed off in Cassandra
+	// Hint metrics are ephemeral and reset when the Cassandra process restarts
+	HintStats() (HintStats, error)
 }
 
 // Table embeds information about a Keyspace and Table that exists in
@@ -188,3 +192,10 @@ type StorageStats struct {
 	MovingNodes      []string
 	LeavingNodes     []string
 }
+
+// HintStats embeds information gathered from the Storage metric in
+// cassandra such as the number of total hints and hints being handed off
+type HintStats struct {
+	TotalHintsInProgress Gauge
+	TotalHints           Counter
+}

server/collector.go

@@ -84,6 +84,10 @@ func (c *SeastatCollector) Describe(ch chan<- *prometheus.Desc) {
 		PromStorageKeyspaces,
 		PromStorageTokens,
 		PromStorageNodeStatus,
+
+		// HintStats
+		PromTotalHintsInProgress,
+		PromTotalHints,
 	}
 
 	for _, desc := range descs {
@@ -112,6 +116,7 @@ func (c *SeastatCollector) Collect(ch chan<- prometheus.Metric) {
 	addMemoryStats(metrics, ch)
 	addGCStats(metrics, ch)
 	addStorageStats(metrics, ch)
+	addHintStats(metrics, ch)
 }
 
 func addTableStats(metrics ScrapedMetrics, ch chan<- prometheus.Metric) {
@@ -398,3 +403,14 @@ func addStorageStats(metrics ScrapedMetrics, ch chan<- prometheus.Metric) {
 		}
 	}
 }
+
+func addHintStats(metrics ScrapedMetrics, ch chan<- prometheus.Metric) {
+	if metrics.HintStats == nil {
+		return
+	}
+
+	ch <- prometheus.MustNewConstMetric(PromTotalHintsInProgress,
+		prometheus.GaugeValue, float64(metrics.HintStats.TotalHintsInProgress))
+	ch <- prometheus.MustNewConstMetric(PromTotalHints,
+		prometheus.CounterValue, float64(metrics.HintStats.TotalHints))
+}

server/prom_metrics.go

@@ -343,3 +343,17 @@ var (
 		[]string{"node", "status"}, nil,
 	)
 )
+
+var (
+	PromTotalHintsInProgress = prometheus.NewDesc(
+		"seastat_hints_in_progress",
+		"Number of hints attempting to be handed off since Cassandra started",
+		[]string{}, nil,
+	)
+
+	PromTotalHints = prometheus.NewDesc(
+		"seastat_hints_total",
+		"Number of hint messages written to this node since Cassandra started",
+		[]string{}, nil,
+	)
+)

server/scraper.go

@@ -41,6 +41,7 @@ type ScrapedMetrics struct {
 	MemoryStats        *jolokia.MemoryStats
 	GCStats            []jolokia.GCStats
 	StorageStats       *jolokia.StorageStats
+	HintStats          *jolokia.HintStats
 
 	ScrapeDuration time.Duration
 	ScrapeTime     time.Time
@@ -191,6 +192,13 @@ func (s *Scraper) scrapeAllMetrics() ScrapedMetrics {
 		out.StorageStats = &storageStats
 	}
 
+	hintStats, err := s.client.HintStats()
+	if err != nil {
+		logrus.Debugf("🦂 Could not get Hint stats: %v", err)
+	} else {
+		out.HintStats = &hintStats
+	}
+
 	out.ScrapeDuration = time.Since(scrapeStart)
 	out.ScrapeTime = time.Now()
 	return out