feat: metrics for services and checks

internals/daemon/api_metrics.go

@@ -15,6 +15,7 @@
 package daemon
 
 import (
+	"bytes"
 	"net/http"
 
 	"github.com/canonical/pebble/internals/logger"
@@ -37,18 +38,26 @@ type metricsResponse struct {
 }
 
 func (r metricsResponse) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	openTelemetryWriter := metrics.NewOpenTelemetryWriter(w)
+	var buf bytes.Buffer
+	metricsWriter := metrics.NewOpenTelemetryWriter(&buf)
 
-	err := r.svcMgr.WriteMetrics(openTelemetryWriter)
+	err := r.svcMgr.WriteMetrics(metricsWriter)
 	if err != nil {
-		logger.Noticef("Cannot write to HTTP response: %v", err.Error())
+		logger.Noticef("Cannot write service metrics: %v", err)
 		http.Error(w, "# internal server error", http.StatusInternalServerError)
 		return
 	}
 
-	err = r.chkMgr.WriteMetrics(openTelemetryWriter)
+	err = r.chkMgr.WriteMetrics(metricsWriter)
 	if err != nil {
-		logger.Noticef("Cannot write to HTTP response: %v", err.Error())
+		logger.Noticef("Cannot write check metrics: %v", err)
+		http.Error(w, "# internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	_, err = buf.WriteTo(w)
+	if err != nil {
+		logger.Noticef("Cannot write to HTTP response: %v", err)
 		http.Error(w, "# internal server error", http.StatusInternalServerError)
 		return
 	}

internals/daemon/api_metrics_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2025 Canonical Ltd
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 3 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package daemon
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	. "gopkg.in/check.v1"
+
+	"github.com/canonical/pebble/internals/overlord/servstate"
+)
+
+func (s *apiSuite) TestMetrics(c *C) {
+	writeTestLayer(s.pebbleDir, `
+services:
+    test1:
+        override: replace
+        command: sleep 10
+`)
+	d := s.daemon(c)
+	d.overlord.Loop()
+
+	// Start test service.
+	payload := bytes.NewBufferString(`{"action": "start", "services": ["test1"]}`)
+	req, err := http.NewRequest("POST", "/v1/services", payload)
+	c.Assert(err, IsNil)
+	rsp := v1PostServices(apiCmd("/v1/services"), req, nil).(*resp)
+	rec := httptest.NewRecorder()
+	rsp.ServeHTTP(rec, req)
+	c.Check(rec.Result().StatusCode, Equals, 202)
+
+	// Wait for it to be running.
+	serviceMgr := d.overlord.ServiceManager()
+	for i := 0; ; i++ {
+		if i > 50 {
+			c.Fatalf("timed out waiting for service to start")
+		}
+		services, err := serviceMgr.Services([]string{"test1"})
+		c.Assert(err, IsNil)
+		if len(services) == 1 && services[0].Current == servstate.StatusActive {
+			break
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	// Get metrics.
+	metricsCmd := apiCmd("/v1/metrics")
+	metricsReq, err := http.NewRequest("GET", "/v1/metrics", nil)
+	c.Assert(err, IsNil)
+	metricsRec := httptest.NewRecorder()
+	metricsRsp := v1GetMetrics(metricsCmd, metricsReq, nil).(metricsResponse)
+	metricsRsp.ServeHTTP(metricsRec, metricsReq)
+	c.Check(metricsRec.Code, Equals, 200)
+	expected := `
+# HELP pebble_service_start_count Number of times the service has started
+# TYPE pebble_service_start_count counter
+pebble_service_start_count{service="test1"} 1
+# HELP pebble_service_active Whether the service is currently active (1) or not (0)
+# TYPE pebble_service_active gauge
+pebble_service_active{service="test1"} 1
+`[1:]
+	c.Assert(metricsRec.Body.String(), Equals, expected)
+}

internals/daemon/daemon.go

@@ -373,7 +373,6 @@ func (d *Daemon) Init() error {
 	}
 
 	logger.Noticef("Started daemon.")
-
 	return nil
 }
 

internals/metrics/metrics.go

@@ -17,7 +17,6 @@ package metrics
 import (
 	"fmt"
 	"io"
-	"strings"
 )
 
 type MetricType int
@@ -27,6 +26,17 @@ const (
 	TypeGaugeInt
 )
 
+func (mt MetricType) String() string {
+	switch mt {
+	case TypeCounterInt:
+		return "counter"
+	case TypeGaugeInt:
+		return "gauge"
+	default:
+		panic("invalid metric type")
+	}
+}
+
 // Metric represents a single metric.
 type Metric struct {
 	Name       string
@@ -63,28 +73,29 @@ func NewOpenTelemetryWriter(w io.Writer) *OpenTelemetryWriter {
 }
 
 func (otw *OpenTelemetryWriter) Write(m Metric) error {
-	var metricType string
-	switch m.Type {
-	case TypeCounterInt:
-		metricType = "counter"
-	case TypeGaugeInt:
-		metricType = "gauge"
+	if m.Comment != "" {
+		_, err := fmt.Fprintf(otw.w, "# HELP %s %s\n", m.Name, m.Comment)
+		if err != nil {
+			return err
+		}
 	}
 
-	_, err := fmt.Fprintf(otw.w, "# HELP %s %s\n", m.Name, m.Comment)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprintf(otw.w, "# TYPE %s %s\n", m.Name, metricType)
+	_, err := fmt.Fprintf(otw.w, "# TYPE %s %s\n", m.Name, m.Type.String())
 	if err != nil {
 		return err
 	}
 
-	labels := make([]string, len(m.Labels))
-	for i, label := range m.Labels {
-		labels[i] = fmt.Sprintf("%s=%s", label.key, label.value)
+	io.WriteString(otw.w, m.Name)
+	if len(m.Labels) > 0 {
+		io.WriteString(otw.w, "{")
+		for i, label := range m.Labels {
+			if i > 0 {
+				io.WriteString(otw.w, ",")
+			}
+			fmt.Fprintf(otw.w, "%s=%q", label.key, label.value) // Use %q to quote values.
+		}
+		io.WriteString(otw.w, "}")
 	}
-
-	_, err = fmt.Fprintf(otw.w, "%s{%s} %d\n", m.Name, strings.Join(labels, ","), m.ValueInt64)
+	fmt.Fprintf(otw.w, " %d\n", m.ValueInt64)
 	return err
 }

internals/metrics/metrics_test.go

@@ -16,12 +16,17 @@ package metrics_test
 
 import (
 	"bytes"
+	"testing"
 
 	. "gopkg.in/check.v1"
 
 	"github.com/canonical/pebble/internals/metrics"
 )
 
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
 type OpenTelemetryWriterSuite struct{}
 
 var _ = Suite(&OpenTelemetryWriterSuite{})
@@ -44,10 +49,11 @@ func (s *OpenTelemetryWriterSuite) TestOpenTelemetryWriter(c *C) {
 					metrics.NewLabel("key2", "value2"),
 				},
 			},
-			expected: `# HELP my_counter A simple counter
+			expected: `
+# HELP my_counter A simple counter
 # TYPE my_counter counter
-my_counter{key1=value1,key2=value2} 42
-`,
+my_counter{key1="value1",key2="value2"} 42
+`[1:],
 		},
 		{
 			name: "GaugeInt",
@@ -58,10 +64,11 @@ my_counter{key1=value1,key2=value2} 42
 				Comment:    "A simple gauge",
 				Labels:     []metrics.Label{}, // Test with no labels
 			},
-			expected: `# HELP my_gauge A simple gauge
+			expected: `
+# HELP my_gauge A simple gauge
 # TYPE my_gauge gauge
-my_gauge{} 1
-`,
+my_gauge 1
+`[1:],
 		},
 		{
 			name: "NoComment", // Test without comment
@@ -71,10 +78,10 @@ my_gauge{} 1
 				ValueInt64: 42,
 				Labels:     []metrics.Label{metrics.NewLabel("env", "prod")},
 			},
-			expected: `# HELP no_comment_metric 
+			expected: `
 # TYPE no_comment_metric counter
-no_comment_metric{env=prod} 42
-`,
+no_comment_metric{env="prod"} 42
+`[1:],
 		},
 
 		{
@@ -89,10 +96,11 @@ no_comment_metric{env=prod} 42
 					metrics.NewLabel("key-with-dash", "value-with-dash"),
 				},
 			},
-			expected: `# HELP special_chars Metric with special characters
+			expected: `
+# HELP special_chars Metric with special characters
 # TYPE special_chars gauge
-special_chars{key_with_underscore=value_with_underscore,key-with-dash=value-with-dash} 42
-`,
+special_chars{key_with_underscore="value_with_underscore",key-with-dash="value-with-dash"} 42
+`[1:],
 		},
 	}
 

internals/overlord/checkstate/handlers.go

@@ -47,6 +47,7 @@ func (m *CheckManager) doPerformCheck(task *state.Task, tomb *tombpkg.Tomb) erro
 		select {
 		case <-ticker.C:
 			err := runCheck(tomb.Context(nil), chk, config.Timeout.Value)
+			m.incPerformCheckCount(config)
 			if !tomb.Alive() {
 				return checkStopped(config.Name, task.Kind(), tomb.Err())
 			}
@@ -91,7 +92,6 @@ func (m *CheckManager) doPerformCheck(task *state.Task, tomb *tombpkg.Tomb) erro
 				task.Set(checkDetailsAttr, &details)
 				m.state.Unlock()
 			}
-			m.incPerformCheckCount(config)
 
 		case <-tomb.Dying():
 			return checkStopped(config.Name, task.Kind(), tomb.Err())