diff --git a/CHANGELOG-6.md b/CHANGELOG-6.md index 5551cd6094..8d8950e152 100644 --- a/CHANGELOG-6.md +++ b/CHANGELOG-6.md @@ -8,6 +8,11 @@ Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Added `max-silenced-expiry-time-allowed` (in minutes) backend configuration variable to control maximum time an alert can be silenced. +- Added `default-silenced-expiry-time` (in minutes) backend configuration variable to create silenced with a default expiry time if user doesn't set expiry time while creating an silence. + + ### Changed - Upgraded CI Go version to 1.21.3 - Upgraded jwt version to 4.4.3 diff --git a/backend/apid/actions/error.go b/backend/apid/actions/error.go index 533ad29b1c..9b3e3b7aeb 100644 --- a/backend/apid/actions/error.go +++ b/backend/apid/actions/error.go @@ -52,6 +52,9 @@ const ( // the operation has completed successfully. For example, a successful // response from a server could have been delayed long DeadlineExceeded + + // Threshold set in config reached. These settings are done in config of backend.yaml + ThresholdReached ) // Default error messages if not message is provided. @@ -65,6 +68,7 @@ var standardErrorMessages = map[ErrCode]string{ PaymentRequired: "license required", PreconditionFailed: "precondition failed", DeadlineExceeded: "deadline exceeded", + ThresholdReached: "Threshold reached", } // Error describes an issue that ocurred while performing the action. diff --git a/backend/apid/graphql/error.go b/backend/apid/graphql/error.go index a85b85e8b8..e81c66dd50 100644 --- a/backend/apid/graphql/error.go +++ b/backend/apid/graphql/error.go @@ -30,6 +30,8 @@ func newStdErr(input string, err error) stdErr { out.code = schema.ErrCodes.ERR_ALREADY_EXISTS case (*store.ErrNotFound): out.code = schema.ErrCodes.ERR_NOT_FOUND + case (*store.ErrThreshold): + out.code = schema.ErrCodes.ERR_THRESHOLD_REACHED } return out } diff --git a/backend/apid/graphql/schema/errors.gql.go b/backend/apid/graphql/schema/errors.gql.go index c78d38ae39..de1469e57d 100644 --- a/backend/apid/graphql/schema/errors.gql.go +++ b/backend/apid/graphql/schema/errors.gql.go @@ -208,6 +208,7 @@ var ErrCodes = _EnumTypeErrCodeValues{ ERR_INTERNAL: "ERR_INTERNAL", ERR_NOT_FOUND: "ERR_NOT_FOUND", ERR_PERMISSION_DENIED: "ERR_PERMISSION_DENIED", + ERR_THRESHOLD_REACHED: "ERR_THRESHOLD_REACHED", } // ErrCodeType A terse description of an error. @@ -242,6 +243,11 @@ func _EnumTypeErrCodeConfigFn() graphql1.EnumConfig { Description: "Operation was canceled because the authorization token did not have sufficient\npermissions.", Value: "ERR_PERMISSION_DENIED", }, + "ERR_THRESHOLD_REACHED": &graphql1.EnumValueConfig{ + DeprecationReason: "", + Description: "Indicates that set thresholds in configured have reached", + Value: "ERR_THRESHOLD_REACHED", + }, }, } } @@ -267,4 +273,6 @@ type _EnumTypeErrCodeValues struct { permissions. */ ERR_PERMISSION_DENIED ErrCode + // ERR_THRESHOLD_REACHED - Indicates that set thresholds in configured have reached + ERR_THRESHOLD_REACHED ErrCode } diff --git a/backend/apid/graphql/schema/errors.graphql b/backend/apid/graphql/schema/errors.graphql index ef462ca582..29c0887723 100644 --- a/backend/apid/graphql/schema/errors.graphql +++ b/backend/apid/graphql/schema/errors.graphql @@ -51,4 +51,10 @@ enum ErrCode { permissions. """ ERR_PERMISSION_DENIED + + """ + Indicates that set thresholds in configured have reached + """ + ERR_THRESHOLD_REACHED + } diff --git a/backend/backend.go b/backend/backend.go index ab2ae5e95c..c61ac841ce 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -328,6 +328,13 @@ func Initialize(ctx context.Context, config *Config) (*Backend, error) { // Create the store, which lives on top of etcd stor := etcdstore.NewStore(b.Client, config.EtcdName) + + // set config details + scfg := etcdstore.Config{} + scfg.DefaultSilencedExpiryTime = config.DefaultSilencedExpiryTime + scfg.MaxSilencedExpiryTimeAllowed = config.MaxSilencedExpiryTimeAllowed + etcdstore.SetConfig(scfg, stor) + b.Store = stor storv2 := etcdstorev2.NewStore(b.Client) var storev2Proxy storev2.Proxy diff --git a/backend/cmd/start.go b/backend/cmd/start.go index 1d0d28d044..12757449a4 100644 --- a/backend/cmd/start.go +++ b/backend/cmd/start.go @@ -72,6 +72,10 @@ const ( flagLabels = "labels" flagAnnotations = "annotations" + // silenced expiry flags + flagMaxSilencedExpiryTimeAllowed = "max-silenced-expiry-time-allowed" + flagDefaultSilencedExpiryTime = "default-silenced-expiry-time" + // Etcd flag constants flagEtcdClientURLs = "etcd-client-urls" flagEtcdListenClientURLs = "etcd-listen-client-urls" @@ -255,6 +259,9 @@ func StartCommand(initialize InitializeFunc) *cobra.Command { CacheDir: viper.GetString(flagCacheDir), StateDir: viper.GetString(flagStateDir), + DefaultSilencedExpiryTime: viper.GetDuration(flagDefaultSilencedExpiryTime), + MaxSilencedExpiryTimeAllowed: viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), + EtcdAdvertiseClientURLs: viper.GetStringSlice(flagEtcdAdvertiseClientURLs), EtcdListenClientURLs: viper.GetStringSlice(flagEtcdListenClientURLs), EtcdClientURLs: fallbackStringSlice(flagEtcdClientURLs, flagEtcdAdvertiseClientURLs), @@ -448,6 +455,10 @@ func handleConfig(cmd *cobra.Command, arguments []string, server bool) error { viper.SetDefault(flagEventLogBufferSize, 100000) viper.SetDefault(flagEventLogFile, "") viper.SetDefault(flagEventLogParallelEncoders, false) + + // default silenced value are set for 1 day = 1440m + viper.SetDefault(flagMaxSilencedExpiryTimeAllowed, "1440m") + viper.SetDefault(flagDefaultSilencedExpiryTime, "1440m") } // Etcd defaults @@ -583,6 +594,10 @@ func flagSet(server bool) *pflag.FlagSet { flagSet.Duration(flagPlatformMetricsLoggingInterval, viper.GetDuration(flagPlatformMetricsLoggingInterval), "platform metrics logging interval") flagSet.String(flagPlatformMetricsLogFile, viper.GetString(flagPlatformMetricsLogFile), "platform metrics log file path") + // silenced configuration flags + flagSet.Duration(flagDefaultSilencedExpiryTime, viper.GetDuration(flagDefaultSilencedExpiryTime), "Default expiry time for silenced if not set in minutes") + flagSet.Duration(flagMaxSilencedExpiryTimeAllowed, viper.GetDuration(flagMaxSilencedExpiryTimeAllowed), "Maximum expiry time allowed for silenced in minutes") + // Etcd server flags flagSet.StringSlice(flagEtcdPeerURLs, viper.GetStringSlice(flagEtcdPeerURLs), "list of URLs to listen on for peer traffic") _ = flagSet.SetAnnotation(flagEtcdPeerURLs, "categories", []string{"store"}) diff --git a/backend/config.go b/backend/config.go index 1c353f7f9e..87f84dfb16 100644 --- a/backend/config.go +++ b/backend/config.go @@ -132,4 +132,8 @@ type Config struct { EventLogBufferWait time.Duration EventLogFile string EventLogParallelEncoders bool + + // expiry setting for silences + DefaultSilencedExpiryTime time.Duration + MaxSilencedExpiryTimeAllowed time.Duration } diff --git a/backend/store/etcd/silenced_store.go b/backend/store/etcd/silenced_store.go index 7c9127f914..c7fe3319ce 100644 --- a/backend/store/etcd/silenced_store.go +++ b/backend/store/etcd/silenced_store.go @@ -17,6 +17,7 @@ import ( const ( silencedPathPrefix = "silenced" maxTxnOps = 64 // this is half of the etcd default maximum + silencedLimitError = "silenced crossed maximum duration allowed" ) var ( @@ -207,6 +208,13 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc if err := silenced.Validate(); err != nil { return &store.ErrNotValid{Err: err} } + allowedMaxTime := time.Now().Add(s.cfg.MaxSilencedExpiryTimeAllowed).Unix() + + // check for maximum allowed duration for silenced allowed + if silenced.ExpireAt > 0 && (silenced.ExpireAt > allowedMaxTime) { + err := errors.New(silencedLimitError) + return &store.ErrThreshold{Err: err} + } if silenced.ExpireAt == 0 && silenced.Expire > 0 { start := time.Now() @@ -214,6 +222,21 @@ func (s *Store) UpdateSilencedEntry(ctx context.Context, silenced *corev2.Silenc start = time.Unix(silenced.Begin, 0) } silenced.ExpireAt = start.Add(time.Duration(silenced.Expire) * time.Second).Unix() + + // check for maximum allowed duration for silenced allowed + if silenced.ExpireAt > allowedMaxTime { + err := errors.New(silencedLimitError) + return &store.ErrThreshold{Err: err} + } + } + + // set default silenced expiry time configured in backend yaml file + if silenced.Expire <= 0 && silenced.ExpireAt == 0 { + start := time.Now() + if silenced.Begin > 0 { + start = time.Unix(silenced.Begin, 0) + } + silenced.ExpireAt = start.Add(s.cfg.DefaultSilencedExpiryTime).Unix() } silencedBytes, err := proto.Marshal(silenced) diff --git a/backend/store/etcd/silenced_store_test.go b/backend/store/etcd/silenced_store_test.go index 2fd687b0b1..a206806471 100644 --- a/backend/store/etcd/silenced_store_test.go +++ b/backend/store/etcd/silenced_store_test.go @@ -67,8 +67,9 @@ func TestSilencedStorage(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, entry) assert.Equal(t, "subscription:*", entry.Name) - // Entries without expirations should return -1 - assert.Equal(t, int64(-1), entry.Expire) + + // check entries without -1 expiry + assert.NotEqual(t, int64(-1), entry.Expire) // Delete silenced entry by name err = store.DeleteSilencedEntryByName(ctx, silenced.Name) @@ -76,6 +77,7 @@ func TestSilencedStorage(t *testing.T) { // Update a silenced entry's expire time silenced.Expire = 2 + silenced.ExpireAt = 0 err = store.UpdateSilencedEntry(ctx, silenced) assert.NoError(t, err) @@ -100,6 +102,7 @@ func TestSilencedStorageWithExpire(t *testing.T) { silenced := types.FixtureSilenced("subscription:checkname") silenced.Namespace = "default" silenced.Expire = 15 + silenced.ExpireAt = 0 ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) err := store.UpdateSilencedEntry(ctx, silenced) @@ -120,9 +123,9 @@ func TestSilencedStorageWithBegin(t *testing.T) { silenced := types.FixtureSilenced("subscription:checkname") silenced.Namespace = "default" // set a begin time in the future - silenced.Begin = time.Date(1970, 01, 01, 01, 00, 00, 00, time.UTC).Unix() + silenced.Begin = time.Now().Add(time.Duration(1) * time.Second).Unix() // current time is before the start time - currentTime := time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC).Unix() + currentTime := time.Now().Unix() ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) err := store.UpdateSilencedEntry(ctx, silenced) @@ -137,8 +140,11 @@ func TestSilencedStorageWithBegin(t *testing.T) { require.NotNil(t, entry) assert.False(t, entry.Begin < currentTime) + // Wait for begin time to elapse current time. i.e let silencing begin + time.Sleep(3 * time.Second) + // reset current time to be ahead of begin time - currentTime = time.Date(1970, 01, 01, 02, 00, 00, 00, time.UTC).Unix() + currentTime = time.Now().Unix() assert.True(t, entry.Begin < currentTime) }) } @@ -150,7 +156,7 @@ func TestSilencedStorageWithBeginAndExpire(t *testing.T) { silenced.Expire = 15 currentTime := time.Now().UTC().Unix() // set a begin time in the future - silenced.Begin = currentTime + 3600 + silenced.Begin = currentTime + 100 // current time is before the start time ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) @@ -168,3 +174,72 @@ func TestSilencedStorageWithBeginAndExpire(t *testing.T) { assert.Equal(t, entry.Expire, int64(15)) }) } + +func TestSilencedStorageWithMaxAllowedThresholdExpiry(t *testing.T) { + testWithEtcd(t, func(store store.Store) { + silenced := types.FixtureSilenced("subscription:checkname") + silenced.Namespace = "default" + silenced.ExpireAt = time.Now().Add(time.Duration(30000) * time.Second).Unix() + // set a begin time + silenced.Begin = time.Now().Unix() + ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) + + err := store.UpdateSilencedEntry(ctx, silenced) + + // assert that error is thrown for breaching max expiry time allowed + assert.Error(t, err) + + entry, err := store.GetSilencedEntryByName(ctx, silenced.Name) + + // assert that entry is nil + assert.NoError(t, err) + assert.Nil(t, entry) + + }) +} + +func TestSilencedStorageWithMaxAllowedThresholdExpiryWithError(t *testing.T) { + testWithEtcd(t, func(store store.Store) { + silenced := types.FixtureSilenced("subscription:checkname") + silenced.Namespace = "default" + silenced.ExpireAt = 0 + silenced.Expire = 3001 + silenced.Begin = time.Now().Unix() + ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) + + err := store.UpdateSilencedEntry(ctx, silenced) + + // assert that error is thrown for breaching max expiry time allowed + assert.Error(t, err) + + entry, err := store.GetSilencedEntryByName(ctx, silenced.Name) + + // assert that entry is nil + assert.NoError(t, err) + assert.Nil(t, entry) + + }) +} + +func TestSilencedStorageWithMaxAllowedThresholdExpiryAndWithoutError(t *testing.T) { + testWithEtcd(t, func(store store.Store) { + silenced := types.FixtureSilenced("subscription:checkname") + silenced.Namespace = "default" + silenced.ExpireAt = 0 + silenced.Expire = 100 + silenced.Begin = time.Now().Unix() + ctx := context.WithValue(context.Background(), types.NamespaceKey, silenced.Namespace) + + err := store.UpdateSilencedEntry(ctx, silenced) + + // assert that error is nil + assert.Nil(t, err) + + entry, err := store.GetSilencedEntryByName(ctx, silenced.Name) + + // assert that entry is not nil + assert.NoError(t, err) + assert.NotNil(t, entry) + + }) +} diff --git a/backend/store/etcd/store.go b/backend/store/etcd/store.go index f1bd46c22e..618f9c7f3b 100644 --- a/backend/store/etcd/store.go +++ b/backend/store/etcd/store.go @@ -7,6 +7,7 @@ import ( "path" "reflect" "strings" + "time" "github.com/gogo/protobuf/proto" "github.com/sensu/sensu-go/backend/store" @@ -22,10 +23,16 @@ const ( EtcdRoot = "/sensu.io" ) +type Config struct { + DefaultSilencedExpiryTime time.Duration + MaxSilencedExpiryTimeAllowed time.Duration +} + // Store is an implementation of the sensu-go/backend/store.Store iface. type Store struct { client *clientv3.Client keepalivesPath string + cfg Config } // NewStore creates a new Store. @@ -38,6 +45,11 @@ func NewStore(client *clientv3.Client, name string) *Store { return store } +// SetConfig adds Store configurations +func SetConfig(cfg Config, store *Store) { + store.cfg = cfg +} + // Create the given key with the serialized object. func Create(ctx context.Context, client *clientv3.Client, key, namespace string, object interface{}) error { bytes, err := marshal(object) diff --git a/backend/store/etcd/store_test.go b/backend/store/etcd/store_test.go index 58b729c0ff..c11524942a 100644 --- a/backend/store/etcd/store_test.go +++ b/backend/store/etcd/store_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "testing" + "time" "github.com/gogo/protobuf/proto" corev2 "github.com/sensu/core/v2" @@ -28,6 +29,9 @@ func testWithEtcd(t *testing.T, f func(store.Store)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) @@ -42,6 +46,9 @@ func testWithEtcdStore(t *testing.T, f func(*Store)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) @@ -56,6 +63,9 @@ func testWithEtcdClient(t *testing.T, f func(store.Store, *clientv3.Client)) { s := NewStore(client, e.Name()) + s.cfg.MaxSilencedExpiryTimeAllowed = time.Duration(3000 * time.Second) + s.cfg.DefaultSilencedExpiryTime = time.Duration(3000 * time.Second) + // Mock a default namespace require.NoError(t, s.CreateNamespace(context.Background(), types.FixtureNamespace("default"))) diff --git a/backend/store/store.go b/backend/store/store.go index 8953242c89..48b892a1c0 100644 --- a/backend/store/store.go +++ b/backend/store/store.go @@ -60,6 +60,15 @@ func (e *ErrNotFound) Error() string { return fmt.Sprintf("key %s not found", e.Key) } +// ErrThreshold is returned when configured thresholds are reached +type ErrThreshold struct { + Err error +} + +func (e *ErrThreshold) Error() string { + return fmt.Sprintf("Threshold reached: %s", e.Err.Error()) +} + // ErrNotValid is returned when an object failed validation type ErrNotValid struct { Err error