From c1cb03286b44e7c46044c9acfe80ed143105af62 Mon Sep 17 00:00:00 2001 From: Vasilis Themelis Date: Wed, 29 May 2024 22:22:34 +0100 Subject: [PATCH 1/3] Log when liveness probes fails due to full write queue Signed-off-by: Vasilis Themelis --- storage/diskmetricstore.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/diskmetricstore.go b/storage/diskmetricstore.go index ff1dbe6e..b5940685 100644 --- a/storage/diskmetricstore.go +++ b/storage/diskmetricstore.go @@ -125,7 +125,9 @@ func (dms *DiskMetricStore) Healthy() error { // A pushgateway that cannot be written to should not be // considered as healthy. if len(dms.writeQueue) == cap(dms.writeQueue) { - return fmt.Errorf("write queue is full") + error_msg := "write queue is full" + level.Error(dms.logger).Log("msg", error_msg) + return fmt.Errorf(error_msg) } return nil From 499e5a234363be8f5316adee596f4045ef47ae72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rabenstein?= Date: Thu, 6 Jun 2024 18:46:36 +0200 Subject: [PATCH 2/3] TBS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Rabenstein --- storage/diskmetricstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/diskmetricstore.go b/storage/diskmetricstore.go index b5940685..15a2f711 100644 --- a/storage/diskmetricstore.go +++ b/storage/diskmetricstore.go @@ -125,9 +125,9 @@ func (dms *DiskMetricStore) Healthy() error { // A pushgateway that cannot be written to should not be // considered as healthy. if len(dms.writeQueue) == cap(dms.writeQueue) { - error_msg := "write queue is full" - level.Error(dms.logger).Log("msg", error_msg) - return fmt.Errorf(error_msg) + err := fmt.Errorf("write queue is full") + level.Warn(dms.logger).Log("msg", err) + return err } return nil From 1eea315725f0bdce026ac569e4db6958d8a9f198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rabenstein?= Date: Thu, 6 Jun 2024 18:50:09 +0200 Subject: [PATCH 3/3] TBS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Björn Rabenstein --- storage/diskmetricstore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/diskmetricstore.go b/storage/diskmetricstore.go index 15a2f711..462c9912 100644 --- a/storage/diskmetricstore.go +++ b/storage/diskmetricstore.go @@ -125,9 +125,9 @@ func (dms *DiskMetricStore) Healthy() error { // A pushgateway that cannot be written to should not be // considered as healthy. if len(dms.writeQueue) == cap(dms.writeQueue) { - err := fmt.Errorf("write queue is full") - level.Warn(dms.logger).Log("msg", err) - return err + err := fmt.Errorf("write queue is full") + level.Warn(dms.logger).Log("msg", err) + return err } return nil