feat: custom metrics for check results (#1222)

* feat: custom metrics for check results

* chore: use assignment switch statement for metric collector type inference

* chore: make resources

* chore: convert metrics to array and log errors

* chore: use cel expr for custom metrics

api/external/api.go

@@ -1,7 +1,5 @@
 package external
 
-// +kubebuilder:skip
-
 type Endpointer interface {
 	GetEndpoint() string
 }
@@ -12,6 +10,7 @@ type Describable interface {
 	GetName() string
 	GetLabels() map[string]string
 	GetTransformDeleteStrategy() string
+	GetMetricsSpec() []Metrics
 }
 
 type WithType interface {

api/external/metrics.go

@@ -0,0 +1,9 @@
+package external
+
+// +kubebuilder:object:generate=true
+type Metrics struct {
+	Name   string            `json:"name,omitempty" yaml:"name,omitempty"`
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+	Type   string            `json:"type,omitempty" yaml:"type,omitempty"`
+	Value  string            `json:"value,omitempty" yaml:"value,omitempty"`
+}

api/v1/common.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/flanksource/canary-checker/api/external"
 	"github.com/flanksource/commons/duration"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/gomplate/v3"
@@ -293,6 +294,8 @@ type Description struct {
 	Labels Labels `yaml:"labels,omitempty" json:"labels,omitempty"`
 	// Transformed checks have a delete strategy on deletion they can either be marked healthy, unhealthy or left as is
 	TransformDeleteStrategy string `yaml:"transformDeleteStrategy,omitempty" json:"transformDeleteStrategy,omitempty"`
+	// Metrics to expose from check results
+	Metrics []external.Metrics `json:"metrics,omitempty" yaml:"metrics,omitempty"`
 }
 
 func (d Description) String() string {
@@ -310,6 +313,10 @@ func (d Description) GetIcon() string {
 	return d.Icon
 }
 
+func (d Description) GetMetricsSpec() []external.Metrics {
+	return d.Metrics
+}
+
 func (d Description) GetName() string {
 	return d.Name
 }

checks/metrics.go

@@ -0,0 +1,143 @@
+package checks
+
+import (
+	"encoding/json"
+	"sort"
+	"strconv"
+
+	"github.com/flanksource/canary-checker/api/context"
+	v1 "github.com/flanksource/canary-checker/api/v1"
+	"github.com/flanksource/canary-checker/pkg"
+	"github.com/flanksource/commons/logger"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var collectorMap = make(map[string]prometheus.Collector)
+
+func promLabelsOrderedKeys(labels map[string]string) []string {
+	var keys []string
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func promLabelsOrderedVals(labels map[string]string) []string {
+	var vals []string
+	keys := promLabelsOrderedKeys(labels)
+	for _, k := range keys {
+		vals = append(vals, labels[k])
+	}
+	return vals
+}
+
+func addPrometheusMetric(name, metricType string, labels map[string]string) prometheus.Collector {
+	var collector prometheus.Collector
+	switch metricType {
+	case "histogram":
+		collector = prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{Name: name},
+			promLabelsOrderedKeys(labels),
+		)
+	case "counter":
+		collector = prometheus.NewCounterVec(
+			prometheus.CounterOpts{Name: name},
+			promLabelsOrderedKeys(labels),
+		)
+	case "gauge":
+		collector = prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{Name: name},
+			promLabelsOrderedKeys(labels),
+		)
+	default:
+		return nil
+	}
+
+	collectorMap[name] = collector
+	prometheus.MustRegister(collector)
+	return collector
+}
+
+func exportCheckMetrics(ctx *context.Context, results pkg.Results) {
+	if len(results) == 0 {
+		return
+	}
+
+	for _, r := range results {
+		for _, spec := range r.Check.GetMetricsSpec() {
+			if spec.Name == "" || spec.Value == "" {
+				continue
+			}
+
+			var collector prometheus.Collector
+			var exists bool
+			if collector, exists = collectorMap[spec.Name]; !exists {
+				collector = addPrometheusMetric(spec.Name, spec.Type, spec.Labels)
+				if collector == nil {
+					logger.Errorf("Invalid type for check.metrics %s for check[%s]", spec.Type, r.Check.GetName())
+					continue
+				}
+			}
+
+			// Convert result Data into JSON for templating
+			var rData map[string]any
+			resultBytes, err := json.Marshal(r.Data)
+			if err != nil {
+				logger.Errorf("Error converting check result data into json: %v", err)
+				continue
+			}
+			if err := json.Unmarshal(resultBytes, &rData); err != nil {
+				logger.Errorf("Error converting check result data into json: %v", err)
+				continue
+			}
+
+			tplValue := v1.Template{Expression: spec.Value}
+			templateInput := map[string]any{
+				"result": rData,
+				"check": map[string]any{
+					"name":        r.Check.GetName(),
+					"description": r.Check.GetDescription(),
+					"labels":      r.Check.GetLabels(),
+					"endpoint":    r.Check.GetEndpoint(),
+					"duration":    r.GetDuration(),
+				},
+			}
+
+			valRaw, err := template(ctx.New(templateInput), tplValue)
+			if err != nil {
+				logger.Errorf("Error templating value for check.metrics template %s for check[%s]: %v", spec.Value, r.Check.GetName(), err)
+				continue
+			}
+			val, err := strconv.ParseFloat(valRaw, 64)
+			if err != nil {
+				logger.Errorf("Error converting value %s to float for check.metrics template %s for check[%s]: %v", valRaw, spec.Value, r.Check.GetName(), err)
+				continue
+			}
+			tplLabels := make(map[string]string)
+			for labelKey, labelVal := range spec.Labels {
+				label, err := template(ctx.New(templateInput), v1.Template{Expression: labelVal})
+				if err != nil {
+					logger.Errorf("Error templating label %s:%s for check.metrics for check[%s]: %v", labelKey, labelVal, r.Check.GetName(), err)
+					continue
+				}
+				tplLabels[labelKey] = label
+			}
+			orderedLabelVals := promLabelsOrderedVals(tplLabels)
+
+			switch collector := collector.(type) {
+			case *prometheus.HistogramVec:
+				collector.WithLabelValues(orderedLabelVals...).Observe(val)
+			case *prometheus.GaugeVec:
+				collector.WithLabelValues(orderedLabelVals...).Set(val)
+			case *prometheus.CounterVec:
+				if val <= 0 {
+					continue
+				}
+				collector.WithLabelValues(orderedLabelVals...).Add(val)
+			default:
+				logger.Errorf("Got unknown type for check.metrics %T", collector)
+			}
+		}
+	}
+}

checks/runchecks.go

@@ -81,11 +81,15 @@ func RunChecks(ctx *context.Context) ([]*pkg.CheckResult, error) {
 		if _, ok := disabledChecks[c.Type()]; ok {
 			continue
 		}
-
-		if Checks(checks).Includes(c) {
-			result := c.Run(ctx)
-			results = append(results, transformResults(ctx, result)...)
+		if !Checks(checks).Includes(c) {
+			continue
 		}
+
+		result := c.Run(ctx)
+		transformedResults := transformResults(ctx, result)
+		results = append(results, transformedResults...)
+
+		exportCheckMetrics(ctx, transformedResults)
 	}
 
 	return processResults(ctx, results), nil