:saprkles: Auto escalate accounts for high follow/like churn

automod/engine/account_meta.go

@@ -6,6 +6,13 @@ import (
 	"github.com/bluesky-social/indigo/atproto/identity"
 )
 
+var (
+	ReviewStateEscalated = "escalated"
+	ReviewStateOpen      = "open"
+	ReviewStateClosed    = "closed"
+	ReviewStateNone      = "none"
+)
+
 // information about a repo/account/identity, always pre-populated and relevant to many rules
 type AccountMeta struct {
 	Identity             *identity.Identity
@@ -34,4 +41,7 @@ type AccountPrivate struct {
 	EmailConfirmed bool
 	IndexedAt      *time.Time
 	AccountTags    []string
+	// ReviewState will be one of ReviewStateEscalated, ReviewStateOpen, ReviewStateClosed, ReviewStateNone, or "" (unknown)
+	ReviewState string
+	Appealed    bool
 }

automod/engine/context.go

@@ -272,6 +272,14 @@ func (c *AccountContext) TakedownAccount() {
 	c.effects.TakedownAccount()
 }
 
+func (c *AccountContext) EscalateAccount() {
+	c.effects.EscalateAccount()
+}
+
+func (c *AccountContext) AcknowledgeAccount() {
+	c.effects.AcknowledgeAccount()
+}
+
 func (c *RecordContext) AddRecordFlag(val string) {
 	c.effects.AddRecordFlag(val)
 }
@@ -288,6 +296,14 @@ func (c *RecordContext) TakedownRecord() {
 	c.effects.TakedownRecord()
 }
 
+func (c *RecordContext) EscalateRecord() {
+	c.effects.EscalateRecord()
+}
+
+func (c *RecordContext) AcknowledgeRecord() {
+	c.effects.AcknowledgeRecord()
+}
+
 func (c *RecordContext) TakedownBlob(cid string) {
 	c.effects.TakedownBlob(cid)
 }

automod/engine/effects.go

@@ -12,6 +12,8 @@ var (
 	QuotaModReportDay = 2000
 	// number of takedowns automod can action per day, for all subjects combined (circuit breaker)
 	QuotaModTakedownDay = 200
+	// number of misc actions automod can do per day, for all subjects combined (circuit breaker)
+	QuotaModActionDay = 1000
 )
 
 type CounterRef struct {
@@ -42,8 +44,12 @@ type Effects struct {
 	AccountFlags []string
 	// Reports which should be filed against this account, as a result of rule execution.
 	AccountReports []ModReport
-	// If "true", indicates that a rule indicates that the entire account should have a takedown.
+	// If "true", a rule decided that the entire account should have a takedown.
 	AccountTakedown bool
+	// If "true", a rule decided that the reported account should be escalated.
+	AccountEscalate bool
+	// If "true", a rule decided that the reports on account should be resolved as acknowledged.
+	AccountAcknowledge bool
 	// Same as "AccountLabels", but at record-level
 	RecordLabels []string
 	// Same as "AccountFlags", but at record-level
@@ -52,6 +58,10 @@ type Effects struct {
 	RecordReports []ModReport
 	// Same as "AccountTakedown", but at record-level
 	RecordTakedown bool
+	// Same as "AccountEscalate", but at record-level
+	RecordEscalate bool
+	// Same as "AccountAcknowledge", but at record-level
+	RecordAcknowledge bool
 	// Set of Blob CIDs to takedown (eg, purge from CDN) when doing a record takedown
 	BlobTakedowns []string
 	// If "true", indicates that a rule indicates that the action causing the event should be blocked or prevented
@@ -128,6 +138,16 @@ func (e *Effects) TakedownAccount() {
 	e.AccountTakedown = true
 }
 
+// Enqueues the account to be "escalated" for mod review at the end of rule processing.
+func (e *Effects) EscalateAccount() {
+	e.AccountEscalate = true
+}
+
+// Enqueues reports on account to be "acknowledged" (closed) at the end of rule processing.
+func (e *Effects) AcknowledgeAccount() {
+	e.AccountAcknowledge = true
+}
+
 // Enqueues the provided label (string value) to be added to the record at the end of rule processing.
 func (e *Effects) AddRecordLabel(val string) {
 	e.mu.Lock()
@@ -172,6 +192,16 @@ func (e *Effects) TakedownRecord() {
 	e.RecordTakedown = true
 }
 
+// Enqueues the record to be "escalated" for mod review at the end of rule processing.
+func (e *Effects) EscalateRecord() {
+	e.RecordEscalate = true
+}
+
+// Enqueues the record to be "escalated" for mod review at the end of rule processing.
+func (e *Effects) AcknowledgeRecord() {
+	e.RecordAcknowledge = true
+}
+
 // Enqueues the blob CID to be taken down (aka, CDN purge) as part of any record takedown
 func (e *Effects) TakedownBlob(cid string) {
 	e.mu.Lock()

automod/engine/fetch_account_meta.go

@@ -131,7 +131,22 @@ func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (
 				if rd.Moderation.SubjectStatus.Takendown != nil && *rd.Moderation.SubjectStatus.Takendown == true {
 					am.Takendown = true
 				}
+				if rd.Moderation.SubjectStatus.Appealed != nil && *rd.Moderation.SubjectStatus.Appealed == true {
+					ap.Appealed = true
+				}
 				ap.AccountTags = dedupeStrings(rd.Moderation.SubjectStatus.Tags)
+				if rd.Moderation.SubjectStatus.ReviewState != nil {
+					switch *rd.Moderation.SubjectStatus.ReviewState {
+					case "#reviewOpen":
+						ap.ReviewState = ReviewStateOpen
+					case "#reviewEscalated":
+						ap.ReviewState = ReviewStateEscalated
+					case "#reviewClosed":
+						ap.ReviewState = ReviewStateClosed
+					case "#reviewNonde":
+						ap.ReviewState = ReviewStateNone
+					}
+				}
 			}
 			am.Private = &ap
 		}

automod/engine/metrics.go

@@ -37,7 +37,17 @@ var actionNewReportCount = promauto.NewCounterVec(prometheus.CounterOpts{
 
 var actionNewTakedownCount = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "automod_new_action_takedowns",
-	Help: "Number of new flags persisted",
+	Help: "Number of new takedowns",
+}, []string{"type"})
+
+var actionNewEscalationCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "automod_new_action_escalations",
+	Help: "Number of new subject escalations",
+}, []string{"type"})
+
+var actionNewAcknowledgeCount = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "automod_new_action_acknowledges",
+	Help: "Number of new subjects acknowledged",
 }, []string{"type"})
 
 var accountMetaFetches = promauto.NewCounter(prometheus.CounterOpts{

automod/engine/persist.go

@@ -57,8 +57,28 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error {
 	if err != nil {
 		return fmt.Errorf("circuit-breaking takedowns: %w", err)
 	}
+	newEscalation := c.effects.AccountEscalate
+	if c.Account.Private != nil && c.Account.Private.ReviewState == ReviewStateEscalated {
+		// de-dupe account escalation
+		newEscalation = false
+	} else {
+		newEscalation, err = eng.circuitBreakModAction(ctx, newEscalation)
+		if err != nil {
+			return fmt.Errorf("circuit-breaking escalation: %w", err)
+		}
+	}
+	newAcknowledge := c.effects.AccountAcknowledge
+	if c.Account.Private != nil && (c.Account.Private.ReviewState == "closed" || c.Account.Private.ReviewState == "none") {
+		// de-dupe account escalation
+		newAcknowledge = false
+	} else {
+		newAcknowledge, err = eng.circuitBreakModAction(ctx, newAcknowledge)
+		if err != nil {
+			return fmt.Errorf("circuit-breaking acknowledge: %w", err)
+		}
+	}
 
-	anyModActions := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0
+	anyModActions := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0
 	if anyModActions && eng.Notifier != nil {
 		for _, srv := range dedupeStrings(c.effects.NotifyServices) {
 			if err := eng.Notifier.SendAccount(ctx, srv, c); err != nil {
@@ -145,9 +165,56 @@ func (eng *Engine) persistAccountModActions(c *AccountContext) error {
 		if err != nil {
 			c.Logger.Error("failed to execute account takedown", "err", err)
 		}
+
+		// we don't want to escalate if there is a takedown
+		newEscalation = false
+	}
+
+	if newEscalation {
+		c.Logger.Warn("account-escalate")
+		actionNewEscalationCount.WithLabelValues("account").Inc()
+		comment := "[automod]: auto account-escalation"
+		_, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{
+			CreatedBy: xrpcc.Auth.Did,
+			Event: &toolsozone.ModerationEmitEvent_Input_Event{
+				ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{
+					Comment: &comment,
+				},
+			},
+			Subject: &toolsozone.ModerationEmitEvent_Input_Subject{
+				AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{
+					Did: c.Account.Identity.DID.String(),
+				},
+			},
+		})
+		if err != nil {
+			c.Logger.Error("failed to execute account escalation", "err", err)
+		}
 	}
 
-	needCachePurge := newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || createdReports
+	if newAcknowledge {
+		c.Logger.Warn("account-acknowledge")
+		actionNewAcknowledgeCount.WithLabelValues("account").Inc()
+		comment := "[automod]: auto account-acknowledge"
+		_, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{
+			CreatedBy: xrpcc.Auth.Did,
+			Event: &toolsozone.ModerationEmitEvent_Input_Event{
+				ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{
+					Comment: &comment,
+				},
+			},
+			Subject: &toolsozone.ModerationEmitEvent_Input_Subject{
+				AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{
+					Did: c.Account.Identity.DID.String(),
+				},
+			},
+		})
+		if err != nil {
+			c.Logger.Error("failed to execute account acknowledge", "err", err)
+		}
+	}
+
+	needCachePurge := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newFlags) > 0 || createdReports
 	if needCachePurge {
 		return eng.PurgeAccountCaches(ctx, c.Account.Identity.DID)
 	}
@@ -210,8 +277,18 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error {
 	if err != nil {
 		return fmt.Errorf("failed to circuit break takedowns: %w", err)
 	}
+	// @TODO: should we check for existing escalation? there doesn't seem to be an existing flag for this at record level
+	newEscalation, err := eng.circuitBreakModAction(ctx, c.effects.RecordEscalate)
+	if err != nil {
+		return fmt.Errorf("circuit-breaking escalation: %w", err)
+	}
+	// @TODO: should we check if the subject is already acked? there doesn't seem to be an existing flag for this at record level
+	newAcknowledge, err := eng.circuitBreakModAction(ctx, c.effects.RecordAcknowledge)
+	if err != nil {
+		return fmt.Errorf("circuit-breaking acknowledge: %w", err)
+	}
 
-	if newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 {
+	if newEscalation || newAcknowledge || newTakedown || len(newLabels) > 0 || len(newFlags) > 0 || len(newReports) > 0 {
 		if eng.Notifier != nil {
 			for _, srv := range dedupeStrings(c.effects.NotifyServices) {
 				if err := eng.Notifier.SendRecord(ctx, srv, c); err != nil {
@@ -231,7 +308,7 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error {
 	}
 
 	// exit early
-	if !newTakedown && len(newLabels) == 0 && len(newReports) == 0 {
+	if !newAcknowledge && !newEscalation && !newTakedown && len(newLabels) == 0 && len(newReports) == 0 {
 		return nil
 	}
 
@@ -303,5 +380,45 @@ func (eng *Engine) persistRecordModActions(c *RecordContext) error {
 			c.Logger.Error("failed to execute record takedown", "err", err)
 		}
 	}
+
+	if newEscalation {
+		c.Logger.Warn("record-escalation")
+		actionNewEscalationCount.WithLabelValues("record").Inc()
+		comment := "[automod]: automated record-escalation"
+		_, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{
+			CreatedBy: xrpcc.Auth.Did,
+			Event: &toolsozone.ModerationEmitEvent_Input_Event{
+				ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{
+					Comment: &comment,
+				},
+			},
+			Subject: &toolsozone.ModerationEmitEvent_Input_Subject{
+				RepoStrongRef: &strongRef,
+			},
+		})
+		if err != nil {
+			c.Logger.Error("failed to execute record escalation", "err", err)
+		}
+	}
+
+	if newAcknowledge {
+		c.Logger.Warn("record-acknowledge")
+		actionNewAcknowledgeCount.WithLabelValues("record").Inc()
+		comment := "[automod]: automated record-acknowledge"
+		_, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{
+			CreatedBy: xrpcc.Auth.Did,
+			Event: &toolsozone.ModerationEmitEvent_Input_Event{
+				ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{
+					Comment: &comment,
+				},
+			},
+			Subject: &toolsozone.ModerationEmitEvent_Input_Subject{
+				RepoStrongRef: &strongRef,
+			},
+		})
+		if err != nil {
+			c.Logger.Error("failed to execute record acknowledge", "err", err)
+		}
+	}
 	return nil
 }

automod/engine/persisthelpers.go

@@ -111,6 +111,26 @@ func (eng *Engine) circuitBreakTakedown(ctx context.Context, takedown bool) (boo
 	return takedown, nil
 }
 
+// Combined circuit breaker for miscellaneous mod actions like: escalate, acknowledge
+func (eng *Engine) circuitBreakModAction(ctx context.Context, action bool) (bool, error) {
+	if !action {
+		return false, nil
+	}
+	c, err := eng.Counters.GetCount(ctx, "automod-quota", "mod-action", countstore.PeriodDay)
+	if err != nil {
+		return false, fmt.Errorf("checking mod action quota: %w", err)
+	}
+	if c >= QuotaModActionDay {
+		eng.Logger.Warn("CIRCUIT BREAKER: automod action")
+		return false, nil
+	}
+	err = eng.Counters.Increment(ctx, "automod-quota", "mod-action")
+	if err != nil {
+		return false, fmt.Errorf("incrementing mod action quota: %w", err)
+	}
+	return action, nil
+}
+
 // Creates a moderation report, but checks first if there was a similar recent one, and skips if so.
 //
 // Returns a bool indicating if a new report was created.

automod/rules/interaction.go

@@ -26,6 +26,7 @@ func InteractionChurnRule(c *automod.RecordContext) error {
 			c.Logger.Info("high-like-churn", "created-today", created, "deleted-today", deleted)
 			c.AddAccountFlag("high-like-churn")
 			c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d likes, %d unlikes today (so far)", created, deleted))
+			// c.EscalateAccount()
 			c.Notify("slack")
 			return nil
 		}
@@ -38,6 +39,7 @@ func InteractionChurnRule(c *automod.RecordContext) error {
 			c.Logger.Info("high-follow-churn", "created-today", created, "deleted-today", deleted)
 			c.AddAccountFlag("high-follow-churn")
 			c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d follows, %d unfollows today (so far)", created, deleted))
+			// c.EscalateAccount()
 			c.Notify("slack")
 			return nil
 		}