Skip to content

Commit

Permalink
monitor/finality: add slack alert for fast finality rule violation (a…
Browse files Browse the repository at this point in the history
…xieinfinity#356)

The slack webhook URL for alerting is input via the environment variable
SLACK_WEBHOOK_URL.
  • Loading branch information
minh-bq committed Oct 3, 2023
1 parent 6b6843a commit 42acc10
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 22 deletions.
66 changes: 47 additions & 19 deletions monitor/finality_vote.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package monitor

import (
"errors"
"fmt"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus"
Expand All @@ -15,15 +16,17 @@ import (
const finalityVoteCache = 100

type blockInformation struct {
blockHash common.Hash
voterPublicKey []blsCommon.PublicKey
voterAddress []common.Address
blockHash common.Hash
voterPublicKey []blsCommon.PublicKey
voterAddress []common.Address
aggregatedSignature blsCommon.Signature
}

type FinalityVoteMonitor struct {
chain consensus.ChainHeaderReader
engine consensus.FastFinalityPoSA
observedVotes *lru.Cache
alerter *slackAlerter
}

func NewFinalityVoteMonitor(
Expand All @@ -38,6 +41,7 @@ func NewFinalityVoteMonitor(
return &FinalityVoteMonitor{
engine: engine,
observedVotes: observedVotes,
alerter: NewSlackAlert(),
}, nil
}

Expand Down Expand Up @@ -90,6 +94,7 @@ func (monitor *FinalityVoteMonitor) CheckFinalityVote(block *types.Block) error
block.Hash(),
voterPublicKey,
voterAddress,
extraData.AggregatedFinalityVotes,
)
}

Expand All @@ -101,14 +106,16 @@ func (monitor *FinalityVoteMonitor) checkSameHeightVote(
blockHash common.Hash,
voterPublicKey []blsCommon.PublicKey,
voterAddress []common.Address,
aggregatedSignature blsCommon.Signature,
) error {
rawBlockInfo, ok := monitor.observedVotes.Get(blockNumber)
if !ok {
monitor.observedVotes.Add(blockNumber, []blockInformation{
{
blockHash: blockHash,
voterPublicKey: voterPublicKey,
voterAddress: voterAddress,
blockHash: blockHash,
voterPublicKey: voterPublicKey,
voterAddress: voterAddress,
aggregatedSignature: aggregatedSignature,
},
})
return nil
Expand All @@ -126,27 +133,48 @@ func (monitor *FinalityVoteMonitor) checkSameHeightVote(
for _, cachePublicKey := range block.voterPublicKey {
for _, blockPublicKey := range voterPublicKey {
if blockPublicKey.Equals(cachePublicKey) {
log.Error(
"Fast finality rule is violated",
"voter public key", common.Bytes2Hex(blockPublicKey.Marshal()),
"block number", blockNumber,
"block 1 hash", block.blockHash,
"block 1 voter public key", prettyPrintPublicKey(block.voterPublicKey),
"block 1 voter address", prettyPrintAddress(block.voterAddress),
"block 2 hash", blockHash,
"block 2 voter public key", prettyPrintPublicKey(voterPublicKey),
"block 2 voter address", prettyPrintAddress(voterAddress),
alertHeader := "Fast finality rule is violated"
alertFormat := "- Voter public key: %s\n" +
"- Block number: %d\n" +
"- Block 1 hash: %s\n" +
"- Block 1 voter public key: %s\n" +
"- Block 1 voter address: %s\n" +
"- Block 1 aggregated signature: %s\n" +
"- Block 2 hash: %s\n" +
"- Block 2 voter public key: %s\n" +
"- Block 2 voter address: %s\n" +
"- Block 2 aggregated signature: %s\n"

alertBody := fmt.Sprintf(
alertFormat,
common.Bytes2Hex(blockPublicKey.Marshal()),
blockNumber,
block.blockHash,
prettyPrintPublicKey(block.voterPublicKey),
prettyPrintAddress(block.voterAddress),
common.Bytes2Hex(block.aggregatedSignature.Marshal()),
blockHash,
prettyPrintPublicKey(voterPublicKey),
prettyPrintAddress(voterAddress),
common.Bytes2Hex(aggregatedSignature.Marshal()),
)

if monitor.alerter != nil {
monitor.alerter.Alert(alertHeader, alertBody)
}
log.Error(alertHeader, "message", alertBody)

violated = true
}
}
}
}

blockInfo = append(blockInfo, blockInformation{
blockHash: blockHash,
voterPublicKey: voterPublicKey,
voterAddress: voterAddress,
blockHash: blockHash,
voterPublicKey: voterPublicKey,
voterAddress: voterAddress,
aggregatedSignature: aggregatedSignature,
})

monitor.observedVotes.Add(blockNumber, blockInfo)
Expand Down
7 changes: 4 additions & 3 deletions monitor/finality_vote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func TestCheckSameHeightVote(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create bls key, err %s", err)
}
signature := key1.Sign([]byte{1})
address1 := common.Address{0x1}

key2, err := blst.RandKey()
Expand All @@ -28,19 +29,19 @@ func TestCheckSameHeightVote(t *testing.T) {

voterPublicKey := []blsCommon.PublicKey{key1.PublicKey()}
voterAddress := []common.Address{address1}
if monitor.checkSameHeightVote(0, common.Hash{0x1}, voterPublicKey, voterAddress) != nil {
if monitor.checkSameHeightVote(0, common.Hash{0x1}, voterPublicKey, voterAddress, signature) != nil {
t.Fatalf("Expect no error when checkSameHeightVote")
}

voterPublicKey = []blsCommon.PublicKey{key2.PublicKey()}
voterAddress = []common.Address{address2}
if monitor.checkSameHeightVote(0, common.Hash{0x2}, voterPublicKey, voterAddress) != nil {
if monitor.checkSameHeightVote(0, common.Hash{0x2}, voterPublicKey, voterAddress, signature) != nil {
t.Fatalf("Expect no error when checkSameHeightVote")
}

voterPublicKey = []blsCommon.PublicKey{key2.PublicKey()}
voterAddress = []common.Address{address2}
if monitor.checkSameHeightVote(0, common.Hash{0x3}, voterPublicKey, voterAddress) == nil {
if monitor.checkSameHeightVote(0, common.Hash{0x3}, voterPublicKey, voterAddress, signature) == nil {
t.Fatalf("Expect error when checkSameHeightVote")
}
}
85 changes: 85 additions & 0 deletions monitor/slack_alerter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package monitor

import (
"encoding/json"
"errors"
"net/http"
"os"
"strings"

"github.com/ethereum/go-ethereum/log"
)

const reponseBuffer = 2048

type slackAlerter struct {
url string
client *http.Client
}

func formatMessage(header, body string) string {
headerSection := map[string]interface{}{
"type": "header",
"text": map[string]interface{}{
"type": "plain_text",
"text": header,
"emoji": true,
},
}

bodySection := map[string]interface{}{
"type": "section",
"text": map[string]interface{}{
"type": "plain_text",
"text": body,
"emoji": true,
},
}

messageBlock := map[string]interface{}{
"blocks": []interface{}{
headerSection,
bodySection,
},
}

message, _ := json.Marshal(messageBlock)
return string(message)
}

func (alerter *slackAlerter) Alert(header, body string) {
message := formatMessage(header, body)

request, err := http.NewRequest("POST", alerter.url, strings.NewReader(message))
if err != nil {
log.Error("Failed to send Slack alert", "err", err)
return
}

response, err := alerter.client.Do(request)
if err != nil {
log.Error("Failed to send HTTP request", "err", err)
return
}

if response.StatusCode >= 400 {
responseBody := make([]byte, reponseBuffer)
response.Body.Read(responseBody)
log.Error("Error response from server", "status", response.StatusCode, "body", responseBody)
return
}
}

func NewSlackAlert() *slackAlerter {
slackUrl := os.Getenv("SLACK_WEBHOOK_URL")
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return errors.New("invalid redirect")
},
}

return &slackAlerter{
url: slackUrl,
client: client,
}
}

0 comments on commit 42acc10

Please sign in to comment.