From 5f39b4ca39e0d9fefe5a33a46236993556d65815 Mon Sep 17 00:00:00 2001
From: bylingo <790653865@qq.com>
Date: Wed, 3 Jul 2024 15:17:25 +0800
Subject: [PATCH] add failure counter metrics

---
 aggregator/aggregator_xlayer.go | 31 +++++++++++++++++++++++++++++--
 aggregator/metrics/metrics.go   | 31 +++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/aggregator/aggregator_xlayer.go b/aggregator/aggregator_xlayer.go
index 2b90cb5e96..335c192954 100644
--- a/aggregator/aggregator_xlayer.go
+++ b/aggregator/aggregator_xlayer.go
@@ -14,6 +14,13 @@ import (
 	"github.com/0xPolygonHermez/zkevm-node/state"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/jackc/pgx/v4"
+	"github.com/0xPolygonHermez/zkevm-node/aggregator/metrics"
+)
+
+var (
+	nowSendTxStartBatchNumber uint64 = 0
+	nowWaitMinedStartBatchNumber uint64 = 0
+	initStartBatchNumber bool = false
 )
 
 func (a *Aggregator) settleDirect(
@@ -96,13 +103,26 @@ func (a *Aggregator) settleWithAggLayer(
 		return false
 	}
 
+	if !initStartBatchNumber {
+		nowSendTxStartBatchNumber = proof.BatchNumber
+		nowWaitMinedStartBatchNumber = proof.BatchNumber
+		initStartBatchNumber = true
+	}
+
 	log.Debug("final proof signedTx: ", signedTx.Tx.ZKP.Proof.Hex())
 	txHash, err := a.AggLayerClient.SendTx(*signedTx)
 	if err != nil {
 		log.Errorf("failed to send tx to the interop: %v", err)
 		a.handleFailureToSendToAggLayer(ctx, proof)
-
+		// Monitor Failure with metrics
+		if nowSendTxStartBatchNumber == proof.BatchNumber {
+			metrics.SendTxFailedInc()
+		}
 		return false
+	} else {
+		// reset counter when success
+		metrics.SendTxFailedReset()
+		nowSendTxStartBatchNumber = proof.BatchNumberFinal + 1
 	}
 
 	log.Infof("tx %s sent to agglayer, waiting to be mined", txHash.Hex())
@@ -112,8 +132,15 @@ func (a *Aggregator) settleWithAggLayer(
 	if err := a.AggLayerClient.WaitTxToBeMined(txHash, waitCtx); err != nil {
 		log.Errorf("interop didn't mine the tx: %v", err)
 		a.handleFailureToSendToAggLayer(ctx, proof)
-
+		// Monitor Failure with metrics
+		if nowWaitMinedStartBatchNumber == proof.BatchNumber {
+			metrics.WaitMinedFailedInc()
+		}
 		return false
+	}else {
+		// reset counter when success
+		metrics.WaitMinedFailedReset()
+		nowWaitMinedStartBatchNumber = proof.BatchNumberFinal + 1
 	}
 
 	// TODO: wait for synchronizer to catch up
diff --git a/aggregator/metrics/metrics.go b/aggregator/metrics/metrics.go
index ea8bed4242..e547c62f6c 100644
--- a/aggregator/metrics/metrics.go
+++ b/aggregator/metrics/metrics.go
@@ -9,6 +9,8 @@ const (
 	prefix                      = "aggregator_"
 	currentConnectedProversName = prefix + "current_connected_provers"
 	currentWorkingProversName   = prefix + "current_working_provers"
+	sendTxFailedCount = prefix + "send_tx_failed_count"
+	waitTxToBeMinedFailedCount = prefix + "wait_tx_mined_failed_count"
 )
 
 // Register the metrics for the sequencer package.
@@ -22,6 +24,14 @@ func Register() {
 			Name: currentWorkingProversName,
 			Help: "[AGGREGATOR] current working provers",
 		},
+		{
+			Name: sendTxFailedCount,
+			Help: "[AGGREGATOR] agglayerClient.SendTx failed counter",
+		},
+		{
+			Name: waitTxToBeMinedFailedCount,
+			Help: "[AGGREGATOR] agglayerClient.WaitTxToBeMined failed counter",
+		},
 	}
 
 	metrics.RegisterGauges(gauges...)
@@ -49,3 +59,24 @@ func WorkingProver() {
 func IdlingProver() {
 	metrics.GaugeDec(currentWorkingProversName)
 }
+
+// SendTxFailedInc increment the gauge for sendTx Failed Counter
+func SendTxFailedInc() {
+	metrics.GaugeInc(sendTxFailedCount)
+}
+
+// SendTxFailedReset when a proof is successfully sent to agglayer
+func SendTxFailedReset() {
+	metrics.GaugeSet(sendTxFailedCount, 0)
+}
+
+// WaitMinedFailedInc increment the gauge for sendTx Failed Counter
+func WaitMinedFailedInc() {
+	metrics.GaugeInc(waitTxToBeMinedFailedCount)
+}
+
+// WaitMinedFailedReset when a proof is successfully sent to agglayer
+func WaitMinedFailedReset() {
+	metrics.GaugeSet(waitTxToBeMinedFailedCount, 0)
+}
+