From d50e9fbe1e7c1ec0903efa8b00a5b02869054516 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 2 Jul 2024 09:41:45 -0400 Subject: [PATCH] feat(stackdriver_exporter): Add ErrorLogger for promhttp (#277) * feat(stackdriver_exporter): Add ErrorLogger for promhttp I had recently experienced #103 and #166 in production and it took quite some time to recognize there was a problem with `stackdriver_exporter` because nothing was logged out to indiciate problems gathering metrics. From my perspective, the pod was healthy and online and I could curl `/metrics` to get results. Grafana Agent however was getting errors when scraping, specifically errors like so: ``` [from Gatherer #2] collected metric "stackdriver_gce_instance_compute_googleapis_com_instance_disk_write_bytes_count" { label:{name:"device_name" value:"REDACTED_FOR_SECURITY"} label:{name:"device_type" value:"permanent"} label:{name:"instance_id" value:"2924941021702260446"} label:{name:"instance_name" value:"REDACTED_FOR_SECURITY"} label:{name:"project_id" value:"REDACTED_FOR_SECURITY"} label:{name:"storage_type" value:"pd-ssd"} label:{name:"unit" value:"By"} label:{name:"zone" value:"us-central1-a"} counter:{value:0} timestamp_ms:1698871080000} was collected before with the same name and label values ``` To help identify the root cause I've added the ability to opt into logging out errors that come from the handler. Specifically, I've created the struct `customPromErrorLogger` that implements the `promhttp.http.Logger` interface. There is a new flag: `monitoring.enable-promhttp-custom-logger` which if it is set to true, then we create an instance of `customPromErrorLogger` and use it as the value for ErrorLogger in `promhttp.Handler{}`. Otherwise, `stackdriver_exporter` works as it did before and does not log out errors collectoing metrics. - refs #103, #166 --------- Signed-off-by: pokom --- stackdriver_exporter.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index fce94d4..8726fc3 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -15,6 +15,7 @@ package main import ( "fmt" + stdlog "log" "net/http" "os" "strings" @@ -238,9 +239,9 @@ func (h *handler) innerHandler(filters map[string]bool) http.Handler { registry, } } - + opts := promhttp.HandlerOpts{ErrorLog: stdlog.New(log.NewStdlibAdapter(level.Error(h.logger)), "", 0)} // Delegate http serving to Prometheus client library, which will call collector.Collect. - return promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{}) + return promhttp.HandlerFor(gatherers, opts) } // filterMetricTypePrefixes filters the initial list of metric type prefixes, with the ones coming from an individual