Add metrics for API calls

cmd/application-load-balancer-controller/main.go

@@ -2,14 +2,17 @@ package main
 
 import (
 	"flag"
+	"net/http"
 	"os"
 
 	"github.com/stackitcloud/application-load-balancer-controller/pkg/controller/ingress"
+	"github.com/stackitcloud/application-load-balancer-controller/pkg/metrics"
 	albclient "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit"
 	stackitconfig "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit/config"
 	sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config"
 	albsdk "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api"
 	certsdk "github.com/stackitcloud/stackit-sdk-go/services/certificates/v2api"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -32,6 +35,10 @@ type options struct {
 	cloudConfig             string
 }
 
+func init() {
+	ctrlmetrics.Registry.MustRegister(metrics.NewExporter())
+}
+
 // nolint:funlen // This function isn't awfully complex.
 func main() {
 	var opts options
@@ -75,13 +82,15 @@ func main() {
 	}
 	albOpts := []sdkconfig.ConfigurationOption{
 		sdkconfig.WithUserAgent("application-load-balancer-controller"),
+		sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(http.DefaultTransport)),
 	}
 	if config.Global.APIEndpoints.ApplicationLoadBalancerAPI != "" {
 		albOpts = append(albOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerAPI))
 	}
 
 	certOpts := []sdkconfig.ConfigurationOption{
 		sdkconfig.WithUserAgent("application-load-balancer-controller"),
+		sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(http.DefaultTransport)),
 	}
 	if config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI != "" {
 		certOpts = append(certOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI))

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/onsi/ginkgo/v2 v2.32.1
 	github.com/onsi/gomega v1.42.1
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/stackitcloud/stackit-sdk-go/core v0.26.0
 	github.com/stackitcloud/stackit-sdk-go/services/alb v0.17.1
 	github.com/stackitcloud/stackit-sdk-go/services/certificates v1.9.1
@@ -38,13 +40,12 @@ require (
 	github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect

pkg/metrics/http.go

@@ -0,0 +1,67 @@
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func NewInstrumentedHTTPClient(roundTripper http.RoundTripper) *http.Client {
+	return &http.Client{
+		Transport: &InstrumentedRoundTripper{
+			base: roundTripper,
+		},
+	}
+}
+
+type InstrumentedRoundTripper struct {
+	base http.RoundTripper
+}
+
+func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) {
+	startTime := time.Now()
+	response, err := rt.base.RoundTrip(request)
+	duration := time.Since(startTime)
+
+	statusCode := "network_error"
+	if response != nil {
+		statusCode = strconv.Itoa(response.StatusCode)
+	}
+
+	labels := prometheus.Labels{
+		componentLabel: component,
+		hostLabel:      request.URL.Host,
+		methodLabel:    request.Method,
+		pathLabel:      getPath(request),
+		codeLabel:      statusCode,
+	}
+
+	HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds())
+	HTTPRequestCount.With(labels).Inc()
+
+	isHTTPError := response != nil && response.StatusCode >= 400
+	isNetworkError := err != nil
+
+	if isHTTPError || isNetworkError {
+		HTTPErrorCount.With(labels).Inc()
+	}
+
+	return response, err
+}
+
+func getPath(request *http.Request) string {
+	var cleanedPath []string
+	for element := range strings.SplitSeq(request.URL.Path, "/") {
+		if err := uuid.Validate(element); err == nil {
+			cleanedPath = append(cleanedPath, "uuid")
+			continue
+		}
+		cleanedPath = append(cleanedPath, element)
+	}
+
+	return strings.Join(cleanedPath, "/")
+}

pkg/metrics/http_test.go

@@ -0,0 +1,162 @@
+package metrics
+
+//var _ = Describe("Metrics", func() {
+//	DescribeTable("operationFromRequest", func(method, path, expected string) {
+//		requestURL, _ := url.Parse("https://host" + path)
+//		request := &http.Request{
+//			Method: method,
+//			URL:    requestURL,
+//		}
+//		op := operationFromRequest(request)
+//		Expect(op).To(Equal(expected))
+//	},
+//		Entry("post token", "POST", "/token", "post_token"),
+//		Entry("get load-balancers", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers", "get_load-balancers"),
+//		Entry("get load-balancers instance", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers/id", "get_load-balancers_instance"),
+//	)
+//
+//	Describe("InstrumentedRoundTripper", func() {
+//		It("increments HTTPRequestCount for responses", func() {
+//			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+//				w.WriteHeader(http.StatusOK)
+//			}))
+//			defer server.Close()
+//
+//			labels := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    "GET",
+//				operationLabel: "get_request-count-test",
+//				codeLabel:      "200",
+//			}
+//			before := testutil.ToFloat64(HTTPRequestCount.With(labels))
+//
+//			client := NewInstrumentedHTTPClient("test")
+//
+//			response, err := client.Get(server.URL + "/request-count-test")
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response.Body.Close()
+//
+//			after := testutil.ToFloat64(HTTPRequestCount.With(labels))
+//			Expect(after - before).To(Equal(float64(1)))
+//		})
+//
+//		It("records HTTPRequestDurationHistogram observations for responses", func() {
+//			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+//				w.WriteHeader(http.StatusOK)
+//			}))
+//			defer server.Close()
+//
+//			labels := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    "GET",
+//				operationLabel: "get_request-duration-test",
+//				codeLabel:      "200",
+//			}
+//			before := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
+//
+//			client := NewInstrumentedHTTPClient("test")
+//
+//			response, err := client.Get(server.URL + "/request-duration-test")
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response.Body.Close()
+//
+//			after := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
+//			Expect(after - before).To(Equal(uint64(1)))
+//		})
+//
+//		It("increments HTTPErrorCount for error responses (400, 404, 500)", func() {
+//			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+//				if r.Method == http.MethodPost {
+//					w.WriteHeader(http.StatusInternalServerError)
+//					return
+//				}
+//				if r.URL.Path == "/404" {
+//					w.WriteHeader(http.StatusNotFound)
+//					return
+//				}
+//				w.WriteHeader(http.StatusBadRequest)
+//			}))
+//			defer server.Close()
+//
+//			labels400 := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    http.MethodGet,
+//				operationLabel: "get_",
+//				codeLabel:      "400",
+//			}
+//			labels404 := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    http.MethodGet,
+//				operationLabel: "get_404",
+//				codeLabel:      "404",
+//			}
+//			labels500 := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    http.MethodPost,
+//				operationLabel: "post_",
+//				codeLabel:      "500",
+//			}
+//			before400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
+//			before404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
+//			before500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
+//
+//			client := NewInstrumentedHTTPClient("test")
+//
+//			response1, err := client.Get(server.URL)
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response1.Body.Close()
+//
+//			response2, err := client.Get(server.URL + "/404")
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response2.Body.Close()
+//
+//			response3, err := client.Post(server.URL, "application/json", nil)
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response3.Body.Close()
+//
+//			after400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
+//			after404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
+//			after500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
+//
+//			Expect(after400 - before400).To(Equal(float64(1)))
+//			Expect(after404 - before404).To(Equal(float64(1)))
+//			Expect(after500 - before500).To(Equal(float64(1)))
+//			Expect((after400 - before400) + (after404 - before404) + (after500 - before500)).To(Equal(float64(3)))
+//		})
+//
+//		It("does not increment HTTPErrorCount for successful responses", func() {
+//			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+//				w.WriteHeader(http.StatusOK)
+//			}))
+//			defer server.Close()
+//
+//			labels := prometheus.Labels{
+//				apiLabel:       "test",
+//				methodLabel:    http.MethodGet,
+//				operationLabel: "get_",
+//				codeLabel:      "200",
+//			}
+//			before := testutil.ToFloat64(HTTPErrorCount.With(labels))
+//
+//			client := NewInstrumentedHTTPClient("test")
+//
+//			response, err := client.Get(server.URL)
+//			Expect(err).NotTo(HaveOccurred())
+//			defer response.Body.Close()
+//
+//			after := testutil.ToFloat64(HTTPErrorCount.With(labels))
+//			Expect(after - before).To(Equal(float64(0)))
+//		})
+//	})
+//})
+//
+//func histogramSampleCount(observer prometheus.Observer) uint64 {
+//	metric, ok := observer.(prometheus.Metric)
+//	Expect(ok).To(BeTrue())
+//
+//	dtoMetric := &dto.Metric{}
+//	Expect(metric.Write(dtoMetric)).To(Succeed())
+//
+//	return dtoMetric.GetHistogram().GetSampleCount()
+//}
+//

pkg/metrics/metrics.go

@@ -0,0 +1,60 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	metricPrefix   = "stackit_api"
+	component      = "application-load-balancer-controller"
+	componentLabel = "component"
+	hostLabel      = "host"
+	methodLabel    = "method"
+	pathLabel      = "path"
+	codeLabel      = "status_code"
+)
+
+var (
+	HTTPRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   metricPrefix,
+		Name:        "http_requests_total",
+		Help:        "The number of requests to external APIs",
+		ConstLabels: nil,
+	}, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel})
+
+	HTTPErrorCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   metricPrefix,
+		Name:        "http_errors_total",
+		Help:        "Number of HTTP errors returned by external APIs",
+		ConstLabels: nil,
+	}, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel})
+
+	HTTPRequestDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   metricPrefix,
+		Name:        "http_request_duration_seconds",
+		Help:        "The response times of external API requests",
+		ConstLabels: nil,
+		Buckets:     nil,
+	}, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel})
+)
+
+type Exporter struct {
+}
+
+func NewExporter() *Exporter {
+	e := &Exporter{}
+
+	return e
+}
+
+func (e *Exporter) Describe(descs chan<- *prometheus.Desc) {
+	HTTPRequestCount.Describe(descs)
+	HTTPErrorCount.Describe(descs)
+	HTTPRequestDurationHistogram.Describe(descs)
+}
+
+func (e *Exporter) Collect(metrics chan<- prometheus.Metric) {
+	HTTPRequestCount.Collect(metrics)
+	HTTPErrorCount.Collect(metrics)
+	HTTPRequestDurationHistogram.Collect(metrics)
+}

pkg/metrics/suite_test.go

@@ -0,0 +1,13 @@
+package metrics
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMetrics(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Metrics Provider Suite")
+}