wfe/ra: Periodically load rate limit overrides from the database

cmd/boulder-ra/main.go

@@ -69,9 +69,10 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides passed in this file must be
-			// identical to those in the WFE.
+			// neither this field nor the OverridesFromDB feature flag is set,
+			// all requesters will be subject to the default rate limits.
+			// Overrides passed in this file must be identical to those in the
+			// WFE.
 			//
 			// Note: At this time, only the Failed Authorizations overrides are
 			// necessary in the RA.
@@ -271,8 +272,15 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, scope)
 		limiter, err = ratelimits.NewLimiter(clk, source, scope)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides)
+		if c.RA.Features.OverridesFromDB {
+			saroc := sapb.NewStorageAuthorityReadOnlyClient(saConn)
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.RA.Limiter.Defaults, saroc.GetEnabledRateLimitOverrides, scope, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides, scope, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overrideRefresherShutdown := txnBuilder.NewRefresher()
+		defer overrideRefresherShutdown()
 	}
 
 	rai := ra.NewRegistrationAuthorityImpl(

cmd/boulder-wfe2/main.go

@@ -151,10 +151,10 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides for the Failed Authorizations
-			// overrides passed in this file must be identical to those in the
-			// RA.
+			// neither this field nor the OverridesFromDB feature flag is set,
+			// all requesters will be subject to the default rate limits.
+			// Overrides for the Failed Authorizations overrides passed in this
+			// file must be identical to those in the RA.
 			Overrides string
 		}
 
@@ -326,6 +326,7 @@ func main() {
 	var limiter *ratelimits.Limiter
 	var txnBuilder *ratelimits.TransactionBuilder
 	var limiterRedis *bredis.Ring
+	overridesRefresherShutdown := func() {}
 	if c.WFE.Limiter.Defaults != "" {
 		// Setup rate limiting.
 		limiterRedis, err = bredis.NewRingFromConfig(*c.WFE.Limiter.Redis, stats, logger)
@@ -334,8 +335,13 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides)
+		if c.WFE.Features.OverridesFromDB {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.WFE.Limiter.Defaults, sac.GetEnabledRateLimitOverrides, stats, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides, stats, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overridesRefresherShutdown = txnBuilder.NewRefresher()
 	}
 
 	var accountGetter wfe2.AccountGetter
@@ -413,6 +419,7 @@ func main() {
 	defer func() {
 		ctx, cancel := context.WithTimeout(context.Background(), c.WFE.ShutdownStopTimeout.Duration)
 		defer cancel()
+		overridesRefresherShutdown()
 		_ = srv.Shutdown(ctx)
 		_ = tlsSrv.Shutdown(ctx)
 		limiterRedis.StopLookups()

cmd/sfe/main.go

@@ -216,7 +216,7 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "")
+		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "", stats, logger)
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
 	}
 

features/features.go

@@ -86,6 +86,10 @@ type Config struct {
 	// during certificate issuance. This flag must be set to true in the
 	// RA, VA, and WFE2 services for full functionality.
 	DNSAccount01Enabled bool
+
+	// OverridesFromDB causes the WFE and RA to retrieve rate limit overrides
+	// from the database, instead of from a file.
+	OverridesFromDB bool
 }
 
 var fMu = new(sync.RWMutex)

ra/ra_test.go

@@ -363,7 +363,7 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, sapb.StorageAutho
 	rlSource := ratelimits.NewInmemSource()
 	limiter, err := ratelimits.NewLimiter(fc, rlSource, stats)
 	test.AssertNotError(t, err, "making limiter")
-	txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "")
+	txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "", metrics.NoopRegisterer, log)
 	test.AssertNotError(t, err, "making transaction composer")
 
 	testKeyPolicy, err := goodkey.NewPolicy(nil, nil)
@@ -708,7 +708,7 @@ func TestPerformValidation_FailedValidationsTriggerPauseIdentifiersRatelimit(t *
 			Burst:  1,
 			Count:  1,
 			Period: config.Duration{Duration: time.Hour * 24}},
-	})
+	}, nil, metrics.NoopRegisterer, blog.NewMock())
 	test.AssertNotError(t, err, "making transaction composer")
 	ra.txnBuilder = txnBuilder
 
@@ -967,7 +967,7 @@ func TestDeactivateAuthorization_Pausing(t *testing.T) {
 			Burst:  1,
 			Count:  1,
 			Period: config.Duration{Duration: time.Hour * 24}},
-	})
+	}, nil, metrics.NoopRegisterer, blog.NewMock())
 	test.AssertNotError(t, err, "making transaction composer")
 	ra.txnBuilder = txnBuilder
 

ratelimits/limit.go

@@ -1,6 +1,7 @@
 package ratelimits
 
 import (
+	"context"
 	"encoding/csv"
 	"errors"
 	"fmt"
@@ -9,10 +10,14 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/letsencrypt/boulder/config"
 	"github.com/letsencrypt/boulder/core"
 	"github.com/letsencrypt/boulder/identifier"
+	blog "github.com/letsencrypt/boulder/log"
 	"github.com/letsencrypt/boulder/strictyaml"
 )
 
@@ -105,8 +110,9 @@ func ValidateLimit(l *Limit) error {
 
 type Limits map[string]*Limit
 
-// loadDefaults marshals the defaults YAML file at path into a map of limits.
-func loadDefaults(path string) (LimitConfigs, error) {
+// loadDefaultsFromFile marshals the defaults YAML file at path into a map of
+// limits.
+func loadDefaultsFromFile(path string) (LimitConfigs, error) {
 	lm := make(LimitConfigs)
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -132,8 +138,8 @@ type overrideYAML struct {
 
 type overridesYAML []map[string]overrideYAML
 
-// loadOverrides marshals the YAML file at path into a map of overrides.
-func loadOverrides(path string) (overridesYAML, error) {
+// loadOverridesFromFile marshals the YAML file at path into a map of overrides.
+func loadOverridesFromFile(path string) (overridesYAML, error) {
 	ov := overridesYAML{}
 	data, err := os.ReadFile(path)
 	if err != nil {
@@ -202,6 +208,9 @@ func parseOverrideNameEnumId(key string) (Name, string, error) {
 // formatted as a list of maps, where each map has a single key representing the
 // limit name and a value that is a map containing the limit fields and an
 // additional 'ids' field that is a list of ids that this override applies to.
+//
+// When the OverridesFromDB feature flag is on, hydrateOverrideLimit is used as
+// this method's equivalent.
 func parseOverrideLimits(newOverridesYAML overridesYAML) (Limits, error) {
 	parsed := make(Limits)
 
@@ -263,6 +272,49 @@ func parseOverrideLimits(newOverridesYAML overridesYAML) (Limits, error) {
 	return parsed, nil
 }
 
+// hydrateOverrideLimit validates the limit Name, values, and override bucket
+// key. It returns the correct bucket key to use in-memory. It should be called
+// when loading overrides from the database.
+//
+// When the OverridesFromDB feature flag is off, parseOverrideLimits is used as
+// this method's equivalent.
+func hydrateOverrideLimit(bucketKey string, limit *Limit) (string, error) {
+	if !limit.Name.isValid() {
+		return "", fmt.Errorf("unrecognized limit name %d", limit.Name)
+	}
+
+	err := ValidateLimit(limit)
+	if err != nil {
+		return "", err
+	}
+
+	err = validateIdForName(limit.Name, bucketKey)
+	if err != nil {
+		return "", err
+	}
+
+	// Interpret and compute a new in-memory bucket key for two rate limits,
+	// since their keys aren't nice to store in a config file or database entry.
+	switch limit.Name {
+	case CertificatesPerDomain:
+		// Convert IP addresses to their covering /32 (IPv4) or /64
+		// (IPv6) prefixes in CIDR notation.
+		ip, err := netip.ParseAddr(bucketKey)
+		if err == nil {
+			prefix, err := coveringIPPrefix(limit.Name, ip)
+			if err != nil {
+				return "", fmt.Errorf("computing prefix for IP address %q: %w", bucketKey, err)
+			}
+			bucketKey = prefix.String()
+		}
+	case CertificatesPerFQDNSet:
+		// Compute the hash of a comma-separated list of identifier values.
+		bucketKey = fmt.Sprintf("%x", core.HashIdentifiers(identifier.FromStringSlice(strings.Split(bucketKey, ","))))
+	}
+
+	return bucketKey, nil
+}
+
 // parseDefaultLimits validates a map of default limits and rekeys it by 'Name'.
 func parseDefaultLimits(newDefaultLimits LimitConfigs) (Limits, error) {
 	parsed := make(Limits)
@@ -291,47 +343,24 @@ func parseDefaultLimits(newDefaultLimits LimitConfigs) (Limits, error) {
 	return parsed, nil
 }
 
+type OverridesRefresher func(context.Context, prometheus.Gauge, blog.Logger) (Limits, error)
+
 type limitRegistry struct {
 	// defaults stores default limits by 'name'.
 	defaults Limits
 
 	// overrides stores override limits by 'name:id'.
 	overrides Limits
-}
-
-func newLimitRegistryFromFiles(defaults, overrides string) (*limitRegistry, error) {
-	defaultsData, err := loadDefaults(defaults)
-	if err != nil {
-		return nil, err
-	}
 
-	if overrides == "" {
-		return newLimitRegistry(defaultsData, nil)
-	}
+	// refreshOverrides is a function to refresh override limits.
+	refreshOverrides OverridesRefresher
 
-	overridesData, err := loadOverrides(overrides)
-	if err != nil {
-		return nil, err
-	}
+	stats              prometheus.Registerer
+	overridesTimestamp prometheus.Gauge
+	overridesErrors    prometheus.Gauge
+	overridesPerLimit  map[Name]prometheus.Gauge
 
-	return newLimitRegistry(defaultsData, overridesData)
-}
-
-func newLimitRegistry(defaults LimitConfigs, overrides overridesYAML) (*limitRegistry, error) {
-	regDefaults, err := parseDefaultLimits(defaults)
-	if err != nil {
-		return nil, err
-	}
-
-	regOverrides, err := parseOverrideLimits(overrides)
-	if err != nil {
-		return nil, err
-	}
-
-	return &limitRegistry{
-		defaults:  regDefaults,
-		overrides: regOverrides,
-	}, nil
+	logger blog.Logger
 }
 
 // getLimit returns the limit for the specified by name and bucketKey, name is
@@ -358,13 +387,69 @@ func (l *limitRegistry) getLimit(name Name, bucketKey string) (*Limit, error) {
 	return nil, errLimitDisabled
 }
 
+// loadOverrides replaces this registry's overrides with a new dataset. This is
+// separate from the goroutine in NewRefresher(), for ease of testing.
+func (l *limitRegistry) loadOverrides(ctx context.Context) error {
+	newOverrides, err := l.refreshOverrides(ctx, l.overridesErrors, l.logger)
+	if err != nil {
+		l.logger.Errf("loading overrides: %v", err)
+		return err
+	}
+
+	// If it's an empty set, don't replace any current overrides.
+	if len(newOverrides) < 1 {
+		l.logger.Warning("loading overrides: no valid overrides")
+		return nil
+	}
+
+	newOverridesPerLimit := make(map[Name]float64)
+	for _, override := range newOverrides {
+		newOverridesPerLimit[override.Name]++
+	}
+
+	l.overrides = newOverrides
+	for name := range l.overridesPerLimit {
+		count, ok := newOverridesPerLimit[name]
+		if ok {
+			l.overridesPerLimit[name].Set(count)
+		} else {
+			l.overridesPerLimit[name].Set(0)
+		}
+	}
+	l.overridesTimestamp.SetToCurrentTime()
+
+	return nil
+}
+
+// NewRefresher periodically loads refreshed overrides using this registry's
+// refreshOverrides function.
+func (l *limitRegistry) NewRefresher() context.CancelFunc {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	ticker := time.NewTicker(30 * time.Minute)
+
+	go func() {
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				l.loadOverrides(ctx) //nolint:errcheck // Refreshing overrides is best-effort.
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return cancel
+}
+
 // LoadOverridesByBucketKey loads the overrides YAML at the supplied path,
 // parses it with the existing helpers, and returns the resulting limits map
 // keyed by "<name>:<id>". This function is exported to support admin tooling
 // used during the migration from overrides.yaml to the overrides database
 // table.
 func LoadOverridesByBucketKey(path string) (Limits, error) {
-	ovs, err := loadOverrides(path)
+	ovs, err := loadOverridesFromFile(path)
 	if err != nil {
 		return nil, err
 	}

ratelimits/limit_test.go

@@ -18,7 +18,7 @@ import (
 //
 // TODO(#7901): Update the tests to test these functions individually.
 func loadAndParseDefaultLimits(path string) (Limits, error) {
-	fromFile, err := loadDefaults(path)
+	fromFile, err := loadDefaultsFromFile(path)
 	if err != nil {
 		return nil, err
 	}
@@ -31,7 +31,7 @@ func loadAndParseDefaultLimits(path string) (Limits, error) {
 //
 // TODO(#7901): Update the tests to test these functions individually.
 func loadAndParseOverrideLimits(path string) (Limits, error) {
-	fromFile, err := loadOverrides(path)
+	fromFile, err := loadOverridesFromFile(path)
 	if err != nil {
 		return nil, err
 	}

ratelimits/limiter_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/letsencrypt/boulder/config"
 	berrors "github.com/letsencrypt/boulder/errors"
+	blog "github.com/letsencrypt/boulder/log"
 	"github.com/letsencrypt/boulder/metrics"
 	"github.com/letsencrypt/boulder/test"
 )
@@ -32,7 +33,7 @@ func newTestLimiter(t *testing.T, s Source, clk clock.FakeClock) *Limiter {
 //   - 'NewRegistrationsPerIPAddress' burst: 20 count: 20 period: 1s
 //   - 'NewRegistrationsPerIPAddress:64.112.117.1' burst: 40 count: 40 period: 1s
 func newTestTransactionBuilder(t *testing.T) *TransactionBuilder {
-	c, err := NewTransactionBuilderFromFiles("testdata/working_default.yml", "testdata/working_override.yml")
+	c, err := NewTransactionBuilderFromFiles("testdata/working_default.yml", "testdata/working_override.yml", metrics.NoopRegisterer, blog.NewMock())
 	test.AssertNotError(t, err, "should not error")
 	return c
 }