Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: metrics for services and checks #519

Open
wants to merge 49 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
f76470c
poc: a metrics module for pebble
IronCore864 Nov 13, 2024
6d8ee59
chore: undo unnecessary change
IronCore864 Nov 14, 2024
b4abc9a
chore: undo unnecessary change
IronCore864 Nov 14, 2024
a274276
chore: undo unnecessary change
IronCore864 Nov 14, 2024
a2c07e6
chore: metrics identity basic auth poc
IronCore864 Nov 26, 2024
4ebb633
chore: a poc for metrics with labels
IronCore864 Nov 27, 2024
7468b95
poc: remove adding identities using env vars according to comment in …
IronCore864 Nov 28, 2024
790a8f9
chore: update tests for the metrics lib poc
IronCore864 Nov 28, 2024
272005b
chore: refactor identities and access according to spec review
IronCore864 Dec 9, 2024
5be3e96
feat: use sha512 to verify password
IronCore864 Jan 21, 2025
a6c374d
feat: move the metrics api to /v1/metrics
IronCore864 Jan 21, 2025
1bd54cb
chore: remove Username from apiBasicIdentity
IronCore864 Jan 21, 2025
98ea11e
chore: revert changes on user state
IronCore864 Jan 21, 2025
68c18b7
Merge branch 'master' into poc-custom-metrics-lib
IronCore864 Jan 21, 2025
7fc255e
chore: fix failed identity tests
IronCore864 Jan 21, 2025
31a0617
test: unit tests for basic identity
IronCore864 Jan 22, 2025
306f2d3
feat: add basic identity
IronCore864 Jan 23, 2025
6c53491
chore: update comments
IronCore864 Jan 23, 2025
a57f041
chore: rework the metrics for services
IronCore864 Jan 24, 2025
b7a442f
chore: add metrics for checks, not done
IronCore864 Jan 24, 2025
e49419d
chore: refactor according to review and add more unit tests
IronCore864 Feb 10, 2025
8ebebb8
chore: refactor metrics, add open telemetry writer
IronCore864 Feb 11, 2025
363eaf0
Merge branch 'master' into poc-custom-metrics-lib
IronCore864 Feb 11, 2025
a1db1a6
chore: refactor according to review, fix check counter reset issue
IronCore864 Feb 11, 2025
047cc42
Merge branch 'master' into poc-custom-metrics-lib
IronCore864 Feb 11, 2025
c527344
chore: add a test for check metrics
IronCore864 Feb 12, 2025
5476299
test: add tests for open telemetry writer
IronCore864 Feb 12, 2025
9e5b65a
test: service metrics
IronCore864 Feb 12, 2025
d678766
chore: update tests
IronCore864 Feb 12, 2025
c99fa53
chore: fix linting
IronCore864 Feb 12, 2025
35794e6
Merge branch 'master' into basic-identity
IronCore864 Feb 13, 2025
c8f3aba
chore: prioritize basic type identity, add tests
IronCore864 Feb 13, 2025
af5e0b2
test: add more tests
IronCore864 Feb 13, 2025
edccbac
chore: prioritize basic type
IronCore864 Feb 13, 2025
83c1717
chore: fix some of the comments according to review
IronCore864 Feb 14, 2025
467d8fa
chore: unexport check metrics and update tests
IronCore864 Feb 14, 2025
8bc87f1
chore: use buffer in api metrics
IronCore864 Feb 14, 2025
6963e5e
chore: use wtier for metrics labels
IronCore864 Feb 14, 2025
cfa73a5
test: add test for api metrics
IronCore864 Feb 14, 2025
f37f94f
chore: fix linting
IronCore864 Feb 14, 2025
7369fe3
chore: remove unnecessary changes
IronCore864 Feb 14, 2025
136ae06
chore: refactor according to review
IronCore864 Feb 17, 2025
13aa3b1
chore: create internal checkData
IronCore864 Feb 18, 2025
5fa2d92
chore: change check metrics to success count and failure count
IronCore864 Feb 18, 2025
45f206f
chore: revert unnecessary change
IronCore864 Feb 18, 2025
054b702
Merge branch 'basic-identity' into poc-custom-metrics-lib
IronCore864 Feb 18, 2025
c49f565
chore: refactor after review
IronCore864 Feb 20, 2025
8fe03da
Merge branch 'basic-identity' into poc-custom-metrics-lib
IronCore864 Feb 20, 2025
2bb6414
chore: refactor after review
IronCore864 Feb 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion client/identities.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ type Identity struct {
Access IdentityAccess `json:"access" yaml:"access"`

// One or more of the following type-specific configuration fields must be
// non-nil (currently the only type is "local").
// non-nil (currently the only types are "local" and "basic").
Local *LocalIdentity `json:"local,omitempty" yaml:"local,omitempty"`
Basic *BasicIdentity `json:"basic,omitempty" yaml:"basic,omitempty"`
}

// IdentityAccess defines the access level for an identity.
Expand All @@ -47,6 +48,12 @@ type LocalIdentity struct {
UserID *uint32 `json:"user-id" yaml:"user-id"`
}

// BasicIdentity holds identity configuration specific to the "basic" type
// (for username/password authentication).
type BasicIdentity struct {
Password string `json:"password" yaml:"password"`
}

// For future extension.
type IdentitiesOptions struct{}

Expand Down
3 changes: 3 additions & 0 deletions internals/cli/cmd_identities.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ func (cmd *cmdIdentities) writeText(identities map[string]*client.Identity) erro
if identity.Local != nil {
types = append(types, "local")
}
if identity.Basic != nil {
types = append(types, "basic")
}
sort.Strings(types)
if len(types) == 0 {
types = append(types, "unknown")
Expand Down
19 changes: 17 additions & 2 deletions internals/daemon/access.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (ac AdminAccess) CheckAccess(d *Daemon, r *http.Request, user *UserState) R
if user == nil {
return Unauthorized(accessDenied)
}
if user.Access == state.AdminAccess {
if user.Identity.Access == state.AdminAccess {
return nil
}
// An identity explicitly set to "access: read" or "access: untrusted" isn't allowed.
Expand All @@ -61,10 +61,25 @@ func (ac UserAccess) CheckAccess(d *Daemon, r *http.Request, user *UserState) Re
if user == nil {
return Unauthorized(accessDenied)
}
switch user.Access {
switch user.Identity.Access {
case state.ReadAccess, state.AdminAccess:
return nil
}
// An identity explicitly set to "access: untrusted" isn't allowed.
return Unauthorized(accessDenied)
}

// MetricsAccess allows requests over the UNIX domain socket from any local user
type MetricsAccess struct{}

func (ac MetricsAccess) CheckAccess(d *Daemon, r *http.Request, user *UserState) Response {
if user == nil {
return Unauthorized(accessDenied)
}
switch user.Identity.Access {
case state.MetricsAccess, state.AdminAccess:
return nil
}
// An identity explicitly set to "access: untrusted" isn't allowed.
return Unauthorized(accessDenied)
}
4 changes: 4 additions & 0 deletions internals/daemon/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ var API = []*Command{{
WriteAccess: AdminAccess{},
GET: v1GetIdentities,
POST: v1PostIdentities,
}, {
Path: "/metrics",
ReadAccess: MetricsAccess{},
GET: Metrics,
}}

var (
Expand Down
34 changes: 34 additions & 0 deletions internals/daemon/api_metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright (c) 2024 Canonical Ltd
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License version 3 as
// published by the Free Software Foundation.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package daemon

import (
"net/http"

"github.com/canonical/pebble/internals/metrics"
)

func Metrics(c *Command, r *http.Request, _ *UserState) Response {
return metricsResponse{}
}

// metricsResponse is a Response implementation to serve the metrics in a prometheus metrics format.
type metricsResponse struct{}

func (r metricsResponse) ServeHTTP(w http.ResponseWriter, req *http.Request) {
registry := metrics.GetRegistry()
w.WriteHeader(http.StatusOK)
w.Write([]byte(registry.GatherMetrics()))
}
18 changes: 9 additions & 9 deletions internals/daemon/api_notices.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,16 @@ type addedNotice struct {
func v1GetNotices(c *Command, r *http.Request, user *UserState) Response {
// TODO(benhoyt): the design of notices presumes UIDs; if in future when we
// support identities that aren't UID based, we'll need to fix this.
if user == nil || user.UID == nil {
if user == nil || user.Identity.Local == nil {
return Forbidden("cannot determine UID of request, so cannot retrieve notices")
}

// By default, return notices with the request UID and public notices.
userID := user.UID
userID := &user.Identity.Local.UserID

query := r.URL.Query()
if len(query["user-id"]) > 0 {
if user.Access != state.AdminAccess {
if user.Identity.Access != state.AdminAccess {
return Forbidden(`only admins may use the "user-id" filter`)
}
var err error
Expand All @@ -65,7 +65,7 @@ func v1GetNotices(c *Command, r *http.Request, user *UserState) Response {
}

if len(query["users"]) > 0 {
if user.Access != state.AdminAccess {
if user.Identity.Access != state.AdminAccess {
return Forbidden(`only admins may use the "users" filter`)
}
if len(query["user-id"]) > 0 {
Expand Down Expand Up @@ -179,7 +179,7 @@ func sanitizeTypesFilter(queryTypes []string) ([]state.NoticeType, error) {
}

func v1PostNotices(c *Command, r *http.Request, user *UserState) Response {
if user == nil || user.UID == nil {
if user == nil || user.Identity.Local == nil {
return Forbidden("cannot determine UID of request, so cannot create notice")
}

Expand Down Expand Up @@ -228,7 +228,7 @@ func v1PostNotices(c *Command, r *http.Request, user *UserState) Response {
st.Lock()
defer st.Unlock()

noticeId, err := st.AddNotice(user.UID, state.CustomNotice, payload.Key, &state.AddNoticeOptions{
noticeId, err := st.AddNotice(&user.Identity.Local.UserID, state.CustomNotice, payload.Key, &state.AddNoticeOptions{
Data: data,
RepeatAfter: repeatAfter,
})
Expand All @@ -240,7 +240,7 @@ func v1PostNotices(c *Command, r *http.Request, user *UserState) Response {
}

func v1GetNotice(c *Command, r *http.Request, user *UserState) Response {
if user == nil || user.UID == nil {
if user == nil || user.Identity.Local == nil {
return Forbidden("cannot determine UID of request, so cannot retrieve notice")
}
noticeID := muxVars(r)["id"]
Expand All @@ -263,10 +263,10 @@ func noticeViewableByUser(notice *state.Notice, user *UserState) bool {
// Notice has no UID, so it's viewable by any user (with a UID).
return true
}
if user.Access == state.AdminAccess {
if user.Identity.Access == state.AdminAccess {
// User is admin, they can view anything.
return true
}
// Otherwise user's UID must match notice's UID.
return *user.UID == userID
return user.Identity.Local.UserID == userID
}
59 changes: 47 additions & 12 deletions internals/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os"
Expand All @@ -36,6 +37,7 @@ import (
"gopkg.in/tomb.v2"

"github.com/canonical/pebble/internals/logger"
"github.com/canonical/pebble/internals/metrics"
"github.com/canonical/pebble/internals/osutil"
"github.com/canonical/pebble/internals/osutil/sys"
"github.com/canonical/pebble/internals/overlord"
Expand Down Expand Up @@ -115,8 +117,9 @@ type Daemon struct {

// UserState represents the state of an authenticated API user.
type UserState struct {
Access state.IdentityAccess
UID *uint32
// Access state.IdentityAccess
// UID *uint32
Identity *state.Identity
}

// A ResponseFunc handles one of the individual verbs for a method
Expand Down Expand Up @@ -146,22 +149,21 @@ const (
accessForbidden
)

func userFromRequest(st *state.State, r *http.Request, ucred *Ucrednet) (*UserState, error) {
if ucred == nil {
// No ucred details, no UserState. Currently, "local" (ucred-based) is
// the only type of identity we support.
return nil, nil
func userFromRequest(st *state.State, r *http.Request, ucred *Ucrednet, username, password string) (*UserState, error) {
var userID *uint32
if ucred != nil {
userID = &ucred.Uid
}

st.Lock()
identity := st.IdentityFromInputs(&ucred.Uid)
identity := st.IdentityFromInputs(userID, username, password)
st.Unlock()

if identity == nil {
// No identity that matches these inputs (for now, just UID).
return nil, nil
}
return &UserState{Access: identity.Access, UID: &ucred.Uid}, nil
return &UserState{Identity: identity}, nil
}

func (d *Daemon) Overlord() *overlord.Overlord {
Expand Down Expand Up @@ -212,7 +214,8 @@ func (c *Command) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// not good: https://github.com/canonical/pebble/pull/369
var user *UserState
if _, isOpen := access.(OpenAccess); !isOpen {
user, err = userFromRequest(c.d.state, r, ucred)
basicAuthUsername, basicAuthPassword, _ := r.BasicAuth()
user, err = userFromRequest(c.d.state, r, ucred, basicAuthUsername, basicAuthPassword)
if err != nil {
Forbidden("forbidden").ServeHTTP(w, r)
return
Expand All @@ -223,10 +226,26 @@ func (c *Command) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if user == nil && ucred != nil {
if ucred.Uid == 0 || ucred.Uid == uint32(os.Getuid()) {
// Admin if UID is 0 (root) or the UID the daemon is running as.
user = &UserState{Access: state.AdminAccess, UID: &ucred.Uid}
// user = &UserState{Access: state.AdminAccess, UID: &ucred.Uid}
user = &UserState{
Identity: &state.Identity{
Access: state.AdminAccess,
Local: &state.LocalIdentity{
UserID: ucred.Uid,
},
},
}
} else {
// Regular read access if any other local UID.
user = &UserState{Access: state.ReadAccess, UID: &ucred.Uid}
// user = &UserState{Access: state.ReadAccess, UID: &ucred.Uid}
user = &UserState{
Identity: &state.Identity{
Access: state.ReadAccess,
Local: &state.LocalIdentity{
UserID: ucred.Uid,
},
},
}
}
}

Expand Down Expand Up @@ -366,6 +385,22 @@ func (d *Daemon) Init() error {
}

logger.Noticef("Started daemon.")

registry := metrics.GetRegistry()
myCounter := registry.NewCounterVec("my_counter", "Total number of something processed.", []string{"operation", "status"})
myGauge := registry.NewGaugeVec("my_gauge", "Current value of something.", []string{"sensor"})
// Goroutine to update metrics randomly
go func() {
for {
myCounter.WithLabelValues("read", "success").Inc()
myCounter.WithLabelValues("write", "success").Add(2)
myCounter.WithLabelValues("read", "failed").Inc()
myGauge.WithLabelValues("temperature").Set(20.0 + rand.Float64()*10.0)

time.Sleep(time.Duration(rand.Intn(5)+1) * time.Second) // Random sleep between 1 and 5 seconds
}
}()

return nil
}

Expand Down
Loading
Loading