automod: configurable circuit breaker thresholds (#830)

automod/engine/circuit_breaker_test.go

@@ -44,7 +44,7 @@ func TestTakedownCircuitBreaker(t *testing.T) {
 	p1cbor := p1buf.Bytes()
 
 	// generate double the quote of events; expect to only count the quote worth of actions
-	for i := 0; i < 2*QuotaModTakedownDay; i++ {
+	for i := 0; i < 2*eng.Config.QuotaModTakedownDay; i++ {
 		ident := identity.Identity{
 			DID:    syntax.DID(fmt.Sprintf("did:plc:abc%d", i)),
 			Handle: syntax.Handle("handle.example.com"),
@@ -63,7 +63,7 @@ func TestTakedownCircuitBreaker(t *testing.T) {
 
 	takedowns, err := eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay)
 	assert.NoError(err)
-	assert.Equal(QuotaModTakedownDay, takedowns)
+	assert.Equal(eng.Config.QuotaModTakedownDay, takedowns)
 
 	reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay)
 	assert.NoError(err)
@@ -89,7 +89,7 @@ func TestReportCircuitBreaker(t *testing.T) {
 	p1cbor := p1buf.Bytes()
 
 	// generate double the quota of events; expect to only count the quota worth of actions
-	for i := 0; i < 2*QuotaModReportDay; i++ {
+	for i := 0; i < 2*eng.Config.QuotaModReportDay; i++ {
 		ident := identity.Identity{
 			DID:    syntax.DID(fmt.Sprintf("did:plc:abc%d", i)),
 			Handle: syntax.Handle("handle.example.com"),
@@ -112,5 +112,5 @@ func TestReportCircuitBreaker(t *testing.T) {
 
 	reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay)
 	assert.NoError(err)
-	assert.Equal(QuotaModReportDay, reports)
+	assert.Equal(eng.Config.QuotaModReportDay, reports)
 }

automod/engine/effects.go

@@ -2,18 +2,6 @@ package engine
 
 import (
 	"sync"
-	"time"
-)
-
-var (
-	// time period within which automod will not re-report an account for the same reasonType
-	ReportDupePeriod = 1 * 24 * time.Hour
-	// number of reports automod can file per day, for all subjects and types combined (circuit breaker)
-	QuotaModReportDay = 2000
-	// number of takedowns automod can action per day, for all subjects combined (circuit breaker)
-	QuotaModTakedownDay = 200
-	// number of misc actions automod can do per day, for all subjects combined (circuit breaker)
-	QuotaModActionDay = 1000
 )
 
 type CounterRef struct {

automod/engine/engine.go

@@ -52,6 +52,14 @@ type Engine struct {
 type EngineConfig struct {
 	// if enabled, account metadata is not hydrated for every event by default
 	SkipAccountMeta bool
+	// time period within which automod will not re-report an account for the same reasonType
+	ReportDupePeriod time.Duration
+	// number of reports automod can file per day, for all subjects and types combined (circuit breaker)
+	QuotaModReportDay int
+	// number of takedowns automod can action per day, for all subjects combined (circuit breaker)
+	QuotaModTakedownDay int
+	// number of misc actions automod can do per day, for all subjects combined (circuit breaker)
+	QuotaModActionDay int
 }
 
 // Entrypoint for external code pushing #identity events in to the engine.

automod/engine/persisthelpers.go

@@ -98,7 +98,12 @@ func (eng *Engine) circuitBreakReports(ctx context.Context, reports []ModReport)
 	if err != nil {
 		return nil, fmt.Errorf("checking report action quota: %w", err)
 	}
-	if c >= QuotaModReportDay {
+
+	quotaModReportDay := eng.Config.QuotaModReportDay
+	if quotaModReportDay == 0 {
+		quotaModReportDay = 10000
+	}
+	if c >= quotaModReportDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod reports")
 		return []ModReport{}, nil
 	}
@@ -117,7 +122,11 @@ func (eng *Engine) circuitBreakTakedown(ctx context.Context, takedown bool) (boo
 	if err != nil {
 		return false, fmt.Errorf("checking takedown action quota: %w", err)
 	}
-	if c >= QuotaModTakedownDay {
+	quotaModTakedownDay := eng.Config.QuotaModTakedownDay
+	if quotaModTakedownDay == 0 {
+		quotaModTakedownDay = 200
+	}
+	if c >= quotaModTakedownDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod takedowns")
 		return false, nil
 	}
@@ -137,7 +146,11 @@ func (eng *Engine) circuitBreakModAction(ctx context.Context, action bool) (bool
 	if err != nil {
 		return false, fmt.Errorf("checking mod action quota: %w", err)
 	}
-	if c >= QuotaModActionDay {
+	quotaModActionDay := eng.Config.QuotaModActionDay
+	if quotaModActionDay == 0 {
+		quotaModActionDay = 2000
+	}
+	if c >= quotaModActionDay {
 		eng.Logger.Warn("CIRCUIT BREAKER: automod action")
 		return false, nil
 	}
@@ -191,7 +204,11 @@ func (eng *Engine) createReportIfFresh(ctx context.Context, xrpcc *xrpc.Client,
 		if err != nil {
 			return false, err
 		}
-		if time.Since(created.Time()) > ReportDupePeriod {
+		reportDupePeriod := eng.Config.ReportDupePeriod
+		if reportDupePeriod == 0 {
+			reportDupePeriod = 1 * 24 * time.Hour
+		}
+		if time.Since(created.Time()) > reportDupePeriod {
 			continue
 		}
 
@@ -267,7 +284,11 @@ func (eng *Engine) createRecordReportIfFresh(ctx context.Context, xrpcc *xrpc.Cl
 		if err != nil {
 			return false, err
 		}
-		if time.Since(created.Time()) > ReportDupePeriod {
+		reportDupePeriod := eng.Config.ReportDupePeriod
+		if reportDupePeriod == 0 {
+			reportDupePeriod = 1 * 24 * time.Hour
+		}
+		if time.Since(created.Time()) > reportDupePeriod {
 			continue
 		}
 

cmd/hepa/main.go

@@ -149,6 +149,30 @@ func run(args []string) error {
 			Usage:   "secret token for prescreen server",
 			EnvVars: []string{"HEPA_PRESCREEN_TOKEN"},
 		},
+		&cli.DurationFlag{
+			Name:    "report-dupe-period",
+			Usage:   "time period within which automod will not re-report an account for the same reasonType",
+			EnvVars: []string{"HEPA_REPORT_DUPE_PERIOD"},
+			Value:   1 * 24 * time.Hour,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-report-day",
+			Usage:   "number of reports automod can file per day, for all subjects and types combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_REPORT_DAY"},
+			Value:   10000,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-takedown-day",
+			Usage:   "number of takedowns automod can action per day, for all subjects combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_TAKEDOWN_DAY"},
+			Value:   200,
+		},
+		&cli.IntFlag{
+			Name:    "quota-mod-action-day",
+			Usage:   "number of misc actions automod can do per day, for all subjects combined (circuit breaker)",
+			EnvVars: []string{"HEPA_QUOTA_MOD_ACTION_DAY"},
+			Value:   2000,
+		},
 	}
 
 	app.Commands = []*cli.Command{
@@ -255,6 +279,10 @@ var runCmd = &cli.Command{
 				FirehoseParallelism: cctx.Int("firehose-parallelism"), // DEPRECATED
 				PreScreenHost:       cctx.String("prescreen-host"),
 				PreScreenToken:      cctx.String("prescreen-token"),
+				ReportDupePeriod:    cctx.Duration("report-dupe-period"),
+				QuotaModReportDay:   cctx.Int("quota-mod-report-day"),
+				QuotaModTakedownDay: cctx.Int("quota-mod-takedown-day"),
+				QuotaModActionDay:   cctx.Int("quota-mod-action-day"),
 			},
 		)
 		if err != nil {

cmd/hepa/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/bluesky-social/indigo/automod"
 	"github.com/bluesky-social/indigo/automod/cachestore"
 	"github.com/bluesky-social/indigo/automod/countstore"
+	"github.com/bluesky-social/indigo/automod/engine"
 	"github.com/bluesky-social/indigo/automod/flagstore"
 	"github.com/bluesky-social/indigo/automod/rules"
 	"github.com/bluesky-social/indigo/automod/setstore"
@@ -54,6 +55,10 @@ type Config struct {
 	FirehoseParallelism int // DEPRECATED
 	PreScreenHost       string
 	PreScreenToken      string
+	ReportDupePeriod    time.Duration
+	QuotaModReportDay   int
+	QuotaModTakedownDay int
+	QuotaModActionDay   int
 }
 
 func NewServer(dir identity.Directory, config Config) (*Server, error) {
@@ -219,6 +224,12 @@ func NewServer(dir identity.Directory, config Config) (*Server, error) {
 		OzoneClient: ozoneClient,
 		AdminClient: adminClient,
 		BlobClient:  blobClient,
+		Config: engine.EngineConfig{
+			ReportDupePeriod:    config.ReportDupePeriod,
+			QuotaModReportDay:   config.QuotaModReportDay,
+			QuotaModTakedownDay: config.QuotaModTakedownDay,
+			QuotaModActionDay:   config.QuotaModActionDay,
+		},
 	}
 
 	s := &Server{