diff --git a/cmd/application-load-balancer-controller/main.go b/cmd/application-load-balancer-controller/main.go index 390f4e7..01e4abe 100644 --- a/cmd/application-load-balancer-controller/main.go +++ b/cmd/application-load-balancer-controller/main.go @@ -2,14 +2,17 @@ package main import ( "flag" + "net/http" "os" "github.com/stackitcloud/application-load-balancer-controller/pkg/controller/ingress" + "github.com/stackitcloud/application-load-balancer-controller/pkg/metrics" albclient "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit" stackitconfig "github.com/stackitcloud/application-load-balancer-controller/pkg/stackit/config" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" albsdk "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api" certsdk "github.com/stackitcloud/stackit-sdk-go/services/certificates/v2api" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" @@ -32,6 +35,10 @@ type options struct { cloudConfig string } +func init() { + ctrlmetrics.Registry.MustRegister(metrics.NewExporter()) +} + // nolint:funlen // This function isn't awfully complex. func main() { var opts options @@ -75,6 +82,7 @@ func main() { } albOpts := []sdkconfig.ConfigurationOption{ sdkconfig.WithUserAgent("application-load-balancer-controller"), + sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(http.DefaultTransport)), } if config.Global.APIEndpoints.ApplicationLoadBalancerAPI != "" { albOpts = append(albOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerAPI)) @@ -82,6 +90,7 @@ func main() { certOpts := []sdkconfig.ConfigurationOption{ sdkconfig.WithUserAgent("application-load-balancer-controller"), + sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(http.DefaultTransport)), } if config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI != "" { certOpts = append(certOpts, sdkconfig.WithEndpoint(config.Global.APIEndpoints.ApplicationLoadBalancerCertificateAPI)) diff --git a/go.mod b/go.mod index 6ea1f30..f8ad356 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 github.com/stackitcloud/stackit-sdk-go/services/alb v0.17.1 github.com/stackitcloud/stackit-sdk-go/services/certificates v1.9.1 @@ -38,13 +40,12 @@ require ( github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/spf13/pflag v1.0.9 // indirect diff --git a/pkg/metrics/http.go b/pkg/metrics/http.go new file mode 100644 index 0000000..9e06426 --- /dev/null +++ b/pkg/metrics/http.go @@ -0,0 +1,67 @@ +package metrics + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" +) + +func NewInstrumentedHTTPClient(roundTripper http.RoundTripper) *http.Client { + return &http.Client{ + Transport: &InstrumentedRoundTripper{ + base: roundTripper, + }, + } +} + +type InstrumentedRoundTripper struct { + base http.RoundTripper +} + +func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + startTime := time.Now() + response, err := rt.base.RoundTrip(request) + duration := time.Since(startTime) + + statusCode := "network_error" + if response != nil { + statusCode = strconv.Itoa(response.StatusCode) + } + + labels := prometheus.Labels{ + componentLabel: component, + hostLabel: request.URL.Host, + methodLabel: request.Method, + pathLabel: getPath(request), + codeLabel: statusCode, + } + + HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds()) + HTTPRequestCount.With(labels).Inc() + + isHTTPError := response != nil && response.StatusCode >= 400 + isNetworkError := err != nil + + if isHTTPError || isNetworkError { + HTTPErrorCount.With(labels).Inc() + } + + return response, err +} + +func getPath(request *http.Request) string { + var cleanedPath []string + for element := range strings.SplitSeq(request.URL.Path, "/") { + if err := uuid.Validate(element); err == nil { + cleanedPath = append(cleanedPath, "uuid") + continue + } + cleanedPath = append(cleanedPath, element) + } + + return strings.Join(cleanedPath, "/") +} diff --git a/pkg/metrics/http_test.go b/pkg/metrics/http_test.go new file mode 100644 index 0000000..ab81843 --- /dev/null +++ b/pkg/metrics/http_test.go @@ -0,0 +1,162 @@ +package metrics + +//var _ = Describe("Metrics", func() { +// DescribeTable("operationFromRequest", func(method, path, expected string) { +// requestURL, _ := url.Parse("https://host" + path) +// request := &http.Request{ +// Method: method, +// URL: requestURL, +// } +// op := operationFromRequest(request) +// Expect(op).To(Equal(expected)) +// }, +// Entry("post token", "POST", "/token", "post_token"), +// Entry("get load-balancers", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers", "get_load-balancers"), +// Entry("get load-balancers instance", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers/id", "get_load-balancers_instance"), +// ) +// +// Describe("InstrumentedRoundTripper", func() { +// It("increments HTTPRequestCount for responses", func() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { +// w.WriteHeader(http.StatusOK) +// })) +// defer server.Close() +// +// labels := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: "GET", +// operationLabel: "get_request-count-test", +// codeLabel: "200", +// } +// before := testutil.ToFloat64(HTTPRequestCount.With(labels)) +// +// client := NewInstrumentedHTTPClient("test") +// +// response, err := client.Get(server.URL + "/request-count-test") +// Expect(err).NotTo(HaveOccurred()) +// defer response.Body.Close() +// +// after := testutil.ToFloat64(HTTPRequestCount.With(labels)) +// Expect(after - before).To(Equal(float64(1))) +// }) +// +// It("records HTTPRequestDurationHistogram observations for responses", func() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { +// w.WriteHeader(http.StatusOK) +// })) +// defer server.Close() +// +// labels := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: "GET", +// operationLabel: "get_request-duration-test", +// codeLabel: "200", +// } +// before := histogramSampleCount(HTTPRequestDurationHistogram.With(labels)) +// +// client := NewInstrumentedHTTPClient("test") +// +// response, err := client.Get(server.URL + "/request-duration-test") +// Expect(err).NotTo(HaveOccurred()) +// defer response.Body.Close() +// +// after := histogramSampleCount(HTTPRequestDurationHistogram.With(labels)) +// Expect(after - before).To(Equal(uint64(1))) +// }) +// +// It("increments HTTPErrorCount for error responses (400, 404, 500)", func() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// if r.Method == http.MethodPost { +// w.WriteHeader(http.StatusInternalServerError) +// return +// } +// if r.URL.Path == "/404" { +// w.WriteHeader(http.StatusNotFound) +// return +// } +// w.WriteHeader(http.StatusBadRequest) +// })) +// defer server.Close() +// +// labels400 := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: http.MethodGet, +// operationLabel: "get_", +// codeLabel: "400", +// } +// labels404 := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: http.MethodGet, +// operationLabel: "get_404", +// codeLabel: "404", +// } +// labels500 := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: http.MethodPost, +// operationLabel: "post_", +// codeLabel: "500", +// } +// before400 := testutil.ToFloat64(HTTPErrorCount.With(labels400)) +// before404 := testutil.ToFloat64(HTTPErrorCount.With(labels404)) +// before500 := testutil.ToFloat64(HTTPErrorCount.With(labels500)) +// +// client := NewInstrumentedHTTPClient("test") +// +// response1, err := client.Get(server.URL) +// Expect(err).NotTo(HaveOccurred()) +// defer response1.Body.Close() +// +// response2, err := client.Get(server.URL + "/404") +// Expect(err).NotTo(HaveOccurred()) +// defer response2.Body.Close() +// +// response3, err := client.Post(server.URL, "application/json", nil) +// Expect(err).NotTo(HaveOccurred()) +// defer response3.Body.Close() +// +// after400 := testutil.ToFloat64(HTTPErrorCount.With(labels400)) +// after404 := testutil.ToFloat64(HTTPErrorCount.With(labels404)) +// after500 := testutil.ToFloat64(HTTPErrorCount.With(labels500)) +// +// Expect(after400 - before400).To(Equal(float64(1))) +// Expect(after404 - before404).To(Equal(float64(1))) +// Expect(after500 - before500).To(Equal(float64(1))) +// Expect((after400 - before400) + (after404 - before404) + (after500 - before500)).To(Equal(float64(3))) +// }) +// +// It("does not increment HTTPErrorCount for successful responses", func() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { +// w.WriteHeader(http.StatusOK) +// })) +// defer server.Close() +// +// labels := prometheus.Labels{ +// apiLabel: "test", +// methodLabel: http.MethodGet, +// operationLabel: "get_", +// codeLabel: "200", +// } +// before := testutil.ToFloat64(HTTPErrorCount.With(labels)) +// +// client := NewInstrumentedHTTPClient("test") +// +// response, err := client.Get(server.URL) +// Expect(err).NotTo(HaveOccurred()) +// defer response.Body.Close() +// +// after := testutil.ToFloat64(HTTPErrorCount.With(labels)) +// Expect(after - before).To(Equal(float64(0))) +// }) +// }) +//}) +// +//func histogramSampleCount(observer prometheus.Observer) uint64 { +// metric, ok := observer.(prometheus.Metric) +// Expect(ok).To(BeTrue()) +// +// dtoMetric := &dto.Metric{} +// Expect(metric.Write(dtoMetric)).To(Succeed()) +// +// return dtoMetric.GetHistogram().GetSampleCount() +//} +// diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..c338a99 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,60 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + metricPrefix = "stackit_api" + component = "application-load-balancer-controller" + componentLabel = "component" + hostLabel = "host" + methodLabel = "method" + pathLabel = "path" + codeLabel = "status_code" +) + +var ( + HTTPRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_requests_total", + Help: "The number of requests to external APIs", + ConstLabels: nil, + }, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel}) + + HTTPErrorCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: "http_errors_total", + Help: "Number of HTTP errors returned by external APIs", + ConstLabels: nil, + }, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel}) + + HTTPRequestDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricPrefix, + Name: "http_request_duration_seconds", + Help: "The response times of external API requests", + ConstLabels: nil, + Buckets: nil, + }, []string{componentLabel, hostLabel, methodLabel, pathLabel, codeLabel}) +) + +type Exporter struct { +} + +func NewExporter() *Exporter { + e := &Exporter{} + + return e +} + +func (e *Exporter) Describe(descs chan<- *prometheus.Desc) { + HTTPRequestCount.Describe(descs) + HTTPErrorCount.Describe(descs) + HTTPRequestDurationHistogram.Describe(descs) +} + +func (e *Exporter) Collect(metrics chan<- prometheus.Metric) { + HTTPRequestCount.Collect(metrics) + HTTPErrorCount.Collect(metrics) + HTTPRequestDurationHistogram.Collect(metrics) +} diff --git a/pkg/metrics/suite_test.go b/pkg/metrics/suite_test.go new file mode 100644 index 0000000..42aa781 --- /dev/null +++ b/pkg/metrics/suite_test.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Provider Suite") +}