monitor/finality: add slack alert for fast finality rule violation

monitor/finality_vote.go

@@ -2,6 +2,7 @@ package monitor
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -15,15 +16,17 @@ import (
 const finalityVoteCache = 100
 
 type blockInformation struct {
-	blockHash      common.Hash
-	voterPublicKey []blsCommon.PublicKey
-	voterAddress   []common.Address
+	blockHash           common.Hash
+	voterPublicKey      []blsCommon.PublicKey
+	voterAddress        []common.Address
+	aggregatedSignature blsCommon.Signature
 }
 
 type FinalityVoteMonitor struct {
 	chain         consensus.ChainHeaderReader
 	engine        consensus.FastFinalityPoSA
 	observedVotes *lru.Cache
+	alerter       *slackAlerter
 }
 
 func NewFinalityVoteMonitor(
@@ -38,6 +41,7 @@ func NewFinalityVoteMonitor(
 	return &FinalityVoteMonitor{
 		engine:        engine,
 		observedVotes: observedVotes,
+		alerter:       NewSlackAlert(),
 	}, nil
 }
 
@@ -90,6 +94,7 @@ func (monitor *FinalityVoteMonitor) CheckFinalityVote(block *types.Block) error
 			block.Hash(),
 			voterPublicKey,
 			voterAddress,
+			extraData.AggregatedFinalityVotes,
 		)
 	}
 
@@ -101,14 +106,16 @@ func (monitor *FinalityVoteMonitor) checkSameHeightVote(
 	blockHash common.Hash,
 	voterPublicKey []blsCommon.PublicKey,
 	voterAddress []common.Address,
+	aggregatedSignature blsCommon.Signature,
 ) error {
 	rawBlockInfo, ok := monitor.observedVotes.Get(blockNumber)
 	if !ok {
 		monitor.observedVotes.Add(blockNumber, []blockInformation{
 			{
-				blockHash:      blockHash,
-				voterPublicKey: voterPublicKey,
-				voterAddress:   voterAddress,
+				blockHash:           blockHash,
+				voterPublicKey:      voterPublicKey,
+				voterAddress:        voterAddress,
+				aggregatedSignature: aggregatedSignature,
 			},
 		})
 		return nil
@@ -126,27 +133,48 @@ func (monitor *FinalityVoteMonitor) checkSameHeightVote(
 		for _, cachePublicKey := range block.voterPublicKey {
 			for _, blockPublicKey := range voterPublicKey {
 				if blockPublicKey.Equals(cachePublicKey) {
-					log.Error(
-						"Fast finality rule is violated",
-						"voter public key", common.Bytes2Hex(blockPublicKey.Marshal()),
-						"block number", blockNumber,
-						"block 1 hash", block.blockHash,
-						"block 1 voter public key", prettyPrintPublicKey(block.voterPublicKey),
-						"block 1 voter address", prettyPrintAddress(block.voterAddress),
-						"block 2 hash", blockHash,
-						"block 2 voter public key", prettyPrintPublicKey(voterPublicKey),
-						"block 2 voter address", prettyPrintAddress(voterAddress),
+					alertHeader := "Fast finality rule is violated"
+					alertFormat := "- Voter public key: %s\n" +
+						"- Block number: %d\n" +
+						"- Block 1 hash: %s\n" +
+						"- Block 1 voter public key: %s\n" +
+						"- Block 1 voter address: %s\n" +
+						"- Block 1 aggregated signature: %s\n" +
+						"- Block 2 hash: %s\n" +
+						"- Block 2 voter public key: %s\n" +
+						"- Block 2 voter address: %s\n" +
+						"- Block 2 aggregated signature: %s\n"
+
+					alertBody := fmt.Sprintf(
+						alertFormat,
+						common.Bytes2Hex(blockPublicKey.Marshal()),
+						blockNumber,
+						block.blockHash,
+						prettyPrintPublicKey(block.voterPublicKey),
+						prettyPrintAddress(block.voterAddress),
+						common.Bytes2Hex(block.aggregatedSignature.Marshal()),
+						blockHash,
+						prettyPrintPublicKey(voterPublicKey),
+						prettyPrintAddress(voterAddress),
+						common.Bytes2Hex(aggregatedSignature.Marshal()),
 					)
+
+					if monitor.alerter != nil {
+						monitor.alerter.Alert(alertHeader, alertBody)
+					}
+					log.Error(alertHeader, "message", alertBody)
+
 					violated = true
 				}
 			}
 		}
 	}
 
 	blockInfo = append(blockInfo, blockInformation{
-		blockHash:      blockHash,
-		voterPublicKey: voterPublicKey,
-		voterAddress:   voterAddress,
+		blockHash:           blockHash,
+		voterPublicKey:      voterPublicKey,
+		voterAddress:        voterAddress,
+		aggregatedSignature: aggregatedSignature,
 	})
 
 	monitor.observedVotes.Add(blockNumber, blockInfo)

monitor/finality_vote_test.go

@@ -18,6 +18,7 @@ func TestCheckSameHeightVote(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create bls key, err %s", err)
 	}
+	signature := key1.Sign([]byte{1})
 	address1 := common.Address{0x1}
 
 	key2, err := blst.RandKey()
@@ -28,19 +29,19 @@ func TestCheckSameHeightVote(t *testing.T) {
 
 	voterPublicKey := []blsCommon.PublicKey{key1.PublicKey()}
 	voterAddress := []common.Address{address1}
-	if monitor.checkSameHeightVote(0, common.Hash{0x1}, voterPublicKey, voterAddress) != nil {
+	if monitor.checkSameHeightVote(0, common.Hash{0x1}, voterPublicKey, voterAddress, signature) != nil {
 		t.Fatalf("Expect no error when checkSameHeightVote")
 	}
 
 	voterPublicKey = []blsCommon.PublicKey{key2.PublicKey()}
 	voterAddress = []common.Address{address2}
-	if monitor.checkSameHeightVote(0, common.Hash{0x2}, voterPublicKey, voterAddress) != nil {
+	if monitor.checkSameHeightVote(0, common.Hash{0x2}, voterPublicKey, voterAddress, signature) != nil {
 		t.Fatalf("Expect no error when checkSameHeightVote")
 	}
 
 	voterPublicKey = []blsCommon.PublicKey{key2.PublicKey()}
 	voterAddress = []common.Address{address2}
-	if monitor.checkSameHeightVote(0, common.Hash{0x3}, voterPublicKey, voterAddress) == nil {
+	if monitor.checkSameHeightVote(0, common.Hash{0x3}, voterPublicKey, voterAddress, signature) == nil {
 		t.Fatalf("Expect error when checkSameHeightVote")
 	}
 }

monitor/slack_alerter.go

@@ -0,0 +1,85 @@
+package monitor
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const reponseBuffer = 2048
+
+type slackAlerter struct {
+	url    string
+	client *http.Client
+}
+
+func formatMessage(header, body string) string {
+	headerSection := map[string]interface{}{
+		"type": "header",
+		"text": map[string]interface{}{
+			"type":  "plain_text",
+			"text":  header,
+			"emoji": true,
+		},
+	}
+
+	bodySection := map[string]interface{}{
+		"type": "section",
+		"text": map[string]interface{}{
+			"type":  "plain_text",
+			"text":  body,
+			"emoji": true,
+		},
+	}
+
+	messageBlock := map[string]interface{}{
+		"blocks": []interface{}{
+			headerSection,
+			bodySection,
+		},
+	}
+
+	message, _ := json.Marshal(messageBlock)
+	return string(message)
+}
+
+func (alerter *slackAlerter) Alert(header, body string) {
+	message := formatMessage(header, body)
+
+	request, err := http.NewRequest("POST", alerter.url, strings.NewReader(message))
+	if err != nil {
+		log.Error("Failed to send Slack alert", "err", err)
+		return
+	}
+
+	response, err := alerter.client.Do(request)
+	if err != nil {
+		log.Error("Failed to send HTTP request", "err", err)
+		return
+	}
+
+	if response.StatusCode >= 400 {
+		responseBody := make([]byte, reponseBuffer)
+		response.Body.Read(responseBody)
+		log.Error("Error response from server", "status", response.StatusCode, "body", responseBody)
+		return
+	}
+}
+
+func NewSlackAlert() *slackAlerter {
+	slackUrl := os.Getenv("SLACK_WEBHOOK_URL")
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return errors.New("invalid redirect")
+		},
+	}
+
+	return &slackAlerter{
+		url:    slackUrl,
+		client: client,
+	}
+}