diff --git a/automod/engine/account_meta.go b/automod/engine/account_meta.go index 5a5a178a2..e7d4e86ba 100644 --- a/automod/engine/account_meta.go +++ b/automod/engine/account_meta.go @@ -6,6 +6,13 @@ import ( "github.com/bluesky-social/indigo/atproto/identity" ) +var ( + ReviewStateEscalated = "escalated" + ReviewStateOpen = "open" + ReviewStateClosed = "closed" + ReviewStateNone = "none" +) + // information about a repo/account/identity, always pre-populated and relevant to many rules type AccountMeta struct { Identity *identity.Identity @@ -34,4 +41,7 @@ type AccountPrivate struct { EmailConfirmed bool IndexedAt *time.Time AccountTags []string + // ReviewState will be one of ReviewStateEscalated, ReviewStateOpen, ReviewStateClosed, ReviewStateNone, or "" (unknown) + ReviewState string + Appealed bool } diff --git a/automod/engine/context.go b/automod/engine/context.go index cc5b6a5e8..1efff7ae1 100644 --- a/automod/engine/context.go +++ b/automod/engine/context.go @@ -272,6 +272,14 @@ func (c *AccountContext) TakedownAccount() { c.effects.TakedownAccount() } +func (c *AccountContext) EscalateAccount() { + c.effects.EscalateAccount() +} + +func (c *AccountContext) AcknowledgeAccount() { + c.effects.AcknowledgeAccount() +} + func (c *RecordContext) AddRecordFlag(val string) { c.effects.AddRecordFlag(val) } @@ -288,6 +296,14 @@ func (c *RecordContext) TakedownRecord() { c.effects.TakedownRecord() } +func (c *RecordContext) EscalateRecord() { + c.effects.EscalateRecord() +} + +func (c *RecordContext) AcknowledgeRecord() { + c.effects.AcknowledgeRecord() +} + func (c *RecordContext) TakedownBlob(cid string) { c.effects.TakedownBlob(cid) } diff --git a/automod/engine/effects.go b/automod/engine/effects.go index ed5ac2998..57e5a558f 100644 --- a/automod/engine/effects.go +++ b/automod/engine/effects.go @@ -12,6 +12,8 @@ var ( QuotaModReportDay = 2000 // number of takedowns automod can action per day, for all subjects combined (circuit breaker) QuotaModTakedownDay = 200 + // number of misc actions automod can do per day, for all subjects combined (circuit breaker) + QuotaModActionDay = 1000 ) type CounterRef struct { @@ -42,8 +44,12 @@ type Effects struct { AccountFlags []string // Reports which should be filed against this account, as a result of rule execution. AccountReports []ModReport - // If "true", indicates that a rule indicates that the entire account should have a takedown. + // If "true", a rule decided that the entire account should have a takedown. AccountTakedown bool + // If "true", a rule decided that the reported account should be escalated. + AccountEscalate bool + // If "true", a rule decided that the reports on account should be resolved as acknowledged. + AccountAcknowledge bool // Same as "AccountLabels", but at record-level RecordLabels []string // Same as "AccountFlags", but at record-level @@ -52,6 +58,10 @@ type Effects struct { RecordReports []ModReport // Same as "AccountTakedown", but at record-level RecordTakedown bool + // Same as "AccountEscalate", but at record-level + RecordEscalate bool + // Same as "AccountAcknowledge", but at record-level + RecordAcknowledge bool // Set of Blob CIDs to takedown (eg, purge from CDN) when doing a record takedown BlobTakedowns []string // If "true", indicates that a rule indicates that the action causing the event should be blocked or prevented @@ -128,6 +138,16 @@ func (e *Effects) TakedownAccount() { e.AccountTakedown = true } +// Enqueues the account to be "escalated" for mod review at the end of rule processing. +func (e *Effects) EscalateAccount() { + e.AccountEscalate = true +} + +// Enqueues reports on account to be "acknowledged" (closed) at the end of rule processing. +func (e *Effects) AcknowledgeAccount() { + e.AccountAcknowledge = true +} + // Enqueues the provided label (string value) to be added to the record at the end of rule processing. func (e *Effects) AddRecordLabel(val string) { e.mu.Lock() @@ -172,6 +192,16 @@ func (e *Effects) TakedownRecord() { e.RecordTakedown = true } +// Enqueues the record to be "escalated" for mod review at the end of rule processing. +func (e *Effects) EscalateRecord() { + e.RecordEscalate = true +} + +// Enqueues the record to be "escalated" for mod review at the end of rule processing. +func (e *Effects) AcknowledgeRecord() { + e.RecordAcknowledge = true +} + // Enqueues the blob CID to be taken down (aka, CDN purge) as part of any record takedown func (e *Effects) TakedownBlob(cid string) { e.mu.Lock() diff --git a/automod/engine/fetch_account_meta.go b/automod/engine/fetch_account_meta.go index 16dae3f2d..8cc651050 100644 --- a/automod/engine/fetch_account_meta.go +++ b/automod/engine/fetch_account_meta.go @@ -131,7 +131,22 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) ( if rd.Moderation.SubjectStatus.Takendown != nil && *rd.Moderation.SubjectStatus.Takendown == true { am.Takendown = true } + if rd.Moderation.SubjectStatus.Appealed != nil && *rd.Moderation.SubjectStatus.Appealed == true { + ap.Appealed = true + } ap.AccountTags = dedupeStrings(rd.Moderation.SubjectStatus.Tags) + if rd.Moderation.SubjectStatus.ReviewState != nil { + switch *rd.Moderation.SubjectStatus.ReviewState { + case "#reviewOpen": + ap.ReviewState = ReviewStateOpen + case "#reviewEscalated": + ap.ReviewState = ReviewStateEscalated + case "#reviewClosed": + ap.ReviewState = ReviewStateClosed + case "#reviewNonde": + ap.ReviewState = ReviewStateNone + } + } } am.Private = &ap } diff --git a/automod/engine/metrics.go b/automod/engine/metrics.go index bf71197d4..1a08944e7 100644 --- a/automod/engine/metrics.go +++ b/automod/engine/metrics.go @@ -37,7 +37,17 @@ var actionNewReportCount = promauto.NewCounterVec(prometheus.CounterOpts{ var actionNewTakedownCount = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "automod_new_action_takedowns", - Help: "Number of new flags persisted", + Help: "Number of new takedowns", +}, []string{"type"}) + +var actionNewEscalationCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_escalations", + Help: "Number of new subject escalations", +}, []string{"type"}) + +var actionNewAcknowledgeCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_acknowledges", + Help: "Number of new subjects acknowledged", }, []string{"type"}) var accountMetaFetches = promauto.NewCounter(prometheus.CounterOpts{ diff --git a/automod/engine/persist.go b/automod/engine/persist.go index 7d6675bbf..03cd63705 100644 --- a/automod/engine/persist.go +++ b/automod/engine/persist.go @@ -57,8 +57,28 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { if err != nil { return fmt.Errorf("circuit-breaking takedowns: %w", err) } + newEscalation := c.effects.AccountEscalate + if c.Account.Private != nil && c.Account.Private.ReviewState == ReviewStateEscalated { + // de-dupe account escalation + newEscalation = false + } else { + newEscalation, err = eng.circuitBreakModAction(ctx, newEscalation) + if err != nil { + return fmt.Errorf("circuit-breaking escalation: %w", err) + } + } + newAcknowledge := c.effects.AccountAcknowledge + if c.Account.Private != nil && (c.Account.Private.ReviewState == "closed" || c.Account.Private.ReviewState == "none") { + // de-dupe account escalation + newAcknowledge = false + } else { + newAcknowledge, err = eng.circuitBreakModAction(ctx, newAcknowledge) + if err != nil { + return fmt.Errorf("circuit-breaking acknowledge: %w", err) + } + } - anyModActions := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 + anyModActions := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 if anyModActions && eng.Notifier != nil { for _, srv := range dedupeStrings(c.effects.NotifyServices) { if err := eng.Notifier.SendAccount(ctx, srv, c); err != nil { @@ -145,9 +165,56 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error { if err != nil { c.Logger.Error("failed to execute account takedown", "err", err) } + + // we don't want to escalate if there is a takedown + newEscalation = false + } + + if newEscalation { + c.Logger.Warn("account-escalate") + actionNewEscalationCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-escalation" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account escalation", "err", err) + } } - needCachePurge := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || createdReports + if newAcknowledge { + c.Logger.Warn("account-acknowledge") + actionNewAcknowledgeCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-acknowledge" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account acknowledge", "err", err) + } + } + + needCachePurge := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newFlags) > 0 || createdReports if needCachePurge { return eng.PurgeAccountCaches(ctx, c.Account.Identity.DID) } @@ -210,8 +277,18 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { if err != nil { return fmt.Errorf("failed to circuit break takedowns: %w", err) } + // @TODO: should we check for existing escalation? there doesn't seem to be an existing flag for this at record level + newEscalation, err := eng.circuitBreakModAction(ctx, c.effects.RecordEscalate) + if err != nil { + return fmt.Errorf("circuit-breaking escalation: %w", err) + } + // @TODO: should we check if the subject is already acked? there doesn't seem to be an existing flag for this at record level + newAcknowledge, err := eng.circuitBreakModAction(ctx, c.effects.RecordAcknowledge) + if err != nil { + return fmt.Errorf("circuit-breaking acknowledge: %w", err) + } - if newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 { + if newEscalation || newAcknowledge || newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 { if eng.Notifier != nil { for _, srv := range dedupeStrings(c.effects.NotifyServices) { if err := eng.Notifier.SendRecord(ctx, srv, c); err != nil { @@ -231,7 +308,7 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { } // exit early - if !newTakedown && len(newLabels) == 0 && len(newReports) == 0 { + if !newAcknowledge && !newEscalation && !newTakedown && len(newLabels) == 0 && len(newReports) == 0 { return nil } @@ -303,5 +380,45 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error { c.Logger.Error("failed to execute record takedown", "err", err) } } + + if newEscalation { + c.Logger.Warn("record-escalation") + actionNewEscalationCount.WithLabelValues("record").Inc() + comment := "[automod]: automated record-escalation" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to execute record escalation", "err", err) + } + } + + if newAcknowledge { + c.Logger.Warn("record-acknowledge") + actionNewAcknowledgeCount.WithLabelValues("record").Inc() + comment := "[automod]: automated record-acknowledge" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to execute record acknowledge", "err", err) + } + } return nil } diff --git a/automod/engine/persisthelpers.go b/automod/engine/persisthelpers.go index c224cc4ab..f1dd2b039 100644 --- a/automod/engine/persisthelpers.go +++ b/automod/engine/persisthelpers.go @@ -111,6 +111,26 @@ func (eng *Engine) circuitBreakTakedown(ctx context.Context, takedown bool) (boo return takedown, nil } +// Combined circuit breaker for miscellaneous mod actions like: escalate, acknowledge +func (eng *Engine) circuitBreakModAction(ctx context.Context, action bool) (bool, error) { + if !action { + return false, nil + } + c, err := eng.Counters.GetCount(ctx, "automod-quota", "mod-action", countstore.PeriodDay) + if err != nil { + return false, fmt.Errorf("checking mod action quota: %w", err) + } + if c >= QuotaModActionDay { + eng.Logger.Warn("CIRCUIT BREAKER: automod action") + return false, nil + } + err = eng.Counters.Increment(ctx, "automod-quota", "mod-action") + if err != nil { + return false, fmt.Errorf("incrementing mod action quota: %w", err) + } + return action, nil +} + // Creates a moderation report, but checks first if there was a similar recent one, and skips if so. // // Returns a bool indicating if a new report was created. diff --git a/automod/rules/interaction.go b/automod/rules/interaction.go index 61b960528..f8416dc46 100644 --- a/automod/rules/interaction.go +++ b/automod/rules/interaction.go @@ -26,6 +26,7 @@ func InteractionChurnRule(c *automod.RecordContext) error { c.Logger.Info("high-like-churn", "created-today", created, "deleted-today", deleted) c.AddAccountFlag("high-like-churn") c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d likes, %d unlikes today (so far)", created, deleted)) + // c.EscalateAccount() c.Notify("slack") return nil } @@ -38,6 +39,7 @@ func InteractionChurnRule(c *automod.RecordContext) error { c.Logger.Info("high-follow-churn", "created-today", created, "deleted-today", deleted) c.AddAccountFlag("high-follow-churn") c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d follows, %d unfollows today (so far)", created, deleted)) + // c.EscalateAccount() c.Notify("slack") return nil }