Add metrics

claimtxman/claimtxman_xlayer.go

@@ -434,6 +434,7 @@ func (tm *ClaimTxManager) monitorTxsXLayer(ctx context.Context) error {
 		return fmt.Errorf("failed to get created monitored txs: %v", err)
 	}
 	mLog.Infof("found %v monitored tx to process", len(mTxs))
+	metrics.RecordPendingMonitoredTxsCount(len(mTxs))
 
 	isResetNonce := false // it will reset the nonce in one cycle
 	for _, mTx := range mTxs {
@@ -454,6 +455,8 @@ func (tm *ClaimTxManager) monitorTxsXLayer(ctx context.Context) error {
 			if err != nil {
 				mTxLog.Errorf("failed to update tx status to confirmed: %v", err)
 			}
+			metrics.RecordMonitoredTxsResult(string(mTx.Status))
+			metrics.RecordAutoClaimDuration(time.Since(mTx.CreatedAt))
 			continue
 		}
 
@@ -509,6 +512,8 @@ func (tm *ClaimTxManager) monitorTxsXLayer(ctx context.Context) error {
 				if err != nil {
 					mTxLog.Errorf("failed to update monitored tx when confirmed: %v", err)
 				}
+				metrics.RecordMonitoredTxsResult(string(mTx.Status))
+				metrics.RecordAutoClaimDuration(time.Since(mTx.CreatedAt))
 				break
 			}
 
@@ -534,6 +539,7 @@ func (tm *ClaimTxManager) monitorTxsXLayer(ctx context.Context) error {
 			if err != nil {
 				mTxLog.Errorf("failed to update monitored tx when max history size limit reached: %v", err)
 			}
+			metrics.RecordMonitoredTxsResult(string(mTx.Status))
 
 			// Notify FE that tx is pending user claim
 			go func() {

metrics/consts.go

@@ -21,10 +21,22 @@ const (
 	labelIsSuccess       = "is_success"
 	labelMethod          = "method"
 
-	prefixOrder            = prefix + "order_"
-	metricOrderCount       = prefixOrder + "count"
-	metricOrderTotalAmount = prefixOrder + "total_amount"
-	metricOrderWaitTime    = prefixOrder + "wait_time_sec"
-	labelNetworkID         = "network_id"
-	labelToken             = "token"
+	prefixOrder             = prefix + "order_"
+	metricOrderCount        = prefixOrder + "count"
+	metricOrderTotalAmount  = prefixOrder + "total_amount"
+	metricOrderWaitTime     = prefixOrder + "wait_time_sec"
+	labelNetworkID          = "network_id"
+	labelToken              = "token"
+	labelTokenAddress       = "token_address"
+	labelTokenOriginNetwork = "token_origin_network"
+
+	prefixMonitoredTxs             = prefix + "monitored_txs_"
+	metricMonitoredTxsPendingCount = prefixMonitoredTxs + "pending_count"
+	metricMonitoredTxsResultCount  = prefixMonitoredTxs + "result_count"
+	metricMonitoredTxsDuration     = prefixMonitoredTxs + "duration_sec"
+	labelStatus                    = "status"
+
+	prefixSynchronizer           = prefix + "synchronizer_"
+	metricSynchronizerEventCount = prefixSynchronizer + "event_count"
+	labelEventType               = "type"
 )

metrics/metrics.go

@@ -23,9 +23,13 @@ func initMetrics() {
 
 	registerCounter(prometheus.CounterOpts{Name: metricRequestCount}, labelMethod, labelIsSuccess)
 	registerHistogram(prometheus.HistogramOpts{Name: metricRequestLatency}, labelMethod, labelIsSuccess)
-	registerCounter(prometheus.CounterOpts{Name: metricOrderCount}, labelNetworkID, labelToken)
-	registerCounter(prometheus.CounterOpts{Name: metricOrderTotalAmount}, labelNetworkID, labelToken)
+	registerCounter(prometheus.CounterOpts{Name: metricOrderCount}, labelNetworkID, labelToken, labelTokenAddress, labelTokenOriginNetwork)
+	registerCounter(prometheus.CounterOpts{Name: metricOrderTotalAmount}, labelNetworkID, labelToken, labelTokenAddress, labelTokenOriginNetwork)
 	registerHistogram(prometheus.HistogramOpts{Name: metricOrderWaitTime}, labelNetworkID)
+	registerGauge(prometheus.GaugeOpts{Name: metricMonitoredTxsPendingCount})
+	registerCounter(prometheus.CounterOpts{Name: metricMonitoredTxsResultCount}, labelStatus)
+	registerHistogram(prometheus.HistogramOpts{Name: metricMonitoredTxsDuration})
+	registerCounter(prometheus.CounterOpts{Name: metricSynchronizerEventCount}, labelNetworkID, labelEventType)
 }
 
 // RecordRequest increments the request count for the method
@@ -42,7 +46,7 @@ func RecordRequestLatency(method string, latency time.Duration, isSuccess bool)
 // RecordOrder records one bridge order, increase the order count and add the amount to the total order amount
 // networkID is the "from" network of the transaction
 func RecordOrder(networkID, tokenOriginNetwork uint32, tokenAddress common.Address, amount *big.Int) {
-	tokenSymbol := ""
+	tokenSymbol := "unknown"
 	if coinsCache := localcache.GetDefaultCache(); coinsCache != nil {
 		coinInfo, err := coinsCache.GetCoinInfoByAddress(context.Background(), tokenOriginNetwork, tokenAddress)
 		if err == nil {
@@ -56,12 +60,38 @@ func RecordOrder(networkID, tokenOriginNetwork uint32, tokenAddress common.Addre
 		log.Warnf("cannot convert [%v] to float", amount.String())
 	}
 
-	labels := map[string]string{labelNetworkID: strconv.Itoa(int(networkID)), labelToken: tokenSymbol}
+	labels := map[string]string{
+		labelNetworkID:          strconv.Itoa(int(networkID)),
+		labelToken:              tokenSymbol,
+		labelTokenAddress:       tokenAddress.String(),
+		labelTokenOriginNetwork: strconv.Itoa(int(tokenOriginNetwork)),
+	}
 
 	counterInc(metricOrderCount, labels)
 	counterAdd(metricOrderTotalAmount, floatAmount, labels)
 }
 
+// RecordOrderWaitTime records the waiting time (seconds) of a bridge order, from order creation (deposit time) to ready_for_claim time
 func RecordOrderWaitTime(networkID uint32, dur time.Duration) {
 	histogramObserve(metricOrderWaitTime, float64(dur/time.Second), map[string]string{labelNetworkID: strconv.Itoa(int(networkID))})
 }
+
+// RecordPendingMonitoredTxsCount records the current number of pending monitored txs (status == "created")
+func RecordPendingMonitoredTxsCount(count int) {
+	gaugeSet(metricMonitoredTxsPendingCount, float64(count), map[string]string{})
+}
+
+// RecordMonitoredTxsResult records the final result of a monitored tx (confirmed or failed)
+func RecordMonitoredTxsResult(status string) {
+	counterInc(metricMonitoredTxsResultCount, map[string]string{labelStatus: status})
+}
+
+// RecordAutoClaimDuration records the duration (seconds) of a confirmed monitored tx (from creation time to confirm time)
+func RecordAutoClaimDuration(dur time.Duration) {
+	histogramObserve(metricMonitoredTxsDuration, float64(dur/time.Second), map[string]string{})
+}
+
+// RecordSynchronizerEvent records an event log consumed by the synchronizer
+func RecordSynchronizerEvent(networkID uint32, eventType string) {
+	counterInc(metricSynchronizerEventCount, map[string]string{labelNetworkID: strconv.Itoa(int(networkID)), labelEventType: eventType})
+}

metrics/prometheus.go

@@ -110,57 +110,57 @@ func gaugeSet(name string, value float64, labelValues map[string]string) {
 	c.With(labelValues).Set(value)
 }
 
-func gaugeInc(name string, labelValues map[string]string) {
-	if !initialized {
-		return
-	}
-
-	c, ok := gauges[name]
-	if !ok {
-		getLogger(name, typeGauge).Errorf("collector not found")
-		return
-	}
-	c.With(labelValues).Inc()
-}
-
-func gaugeDec(name string, labelValues map[string]string) {
-	if !initialized {
-		return
-	}
-
-	c, ok := gauges[name]
-	if !ok {
-		getLogger(name, typeGauge).Errorf("collector not found")
-		return
-	}
-	c.With(labelValues).Dec()
-}
-
-func gaugeAdd(name string, value float64, labelValues map[string]string) {
-	if !initialized {
-		return
-	}
-
-	c, ok := gauges[name]
-	if !ok {
-		getLogger(name, typeGauge).Errorf("collector not found")
-		return
-	}
-	c.With(labelValues).Add(value)
-}
-
-func gaugeSub(name string, value float64, labelValues map[string]string) {
-	if !initialized {
-		return
-	}
-
-	c, ok := gauges[name]
-	if !ok {
-		getLogger(name, typeGauge).Errorf("collector not found")
-		return
-	}
-	c.With(labelValues).Sub(value)
-}
+//func gaugeInc(name string, labelValues map[string]string) {
+//	if !initialized {
+//		return
+//	}
+//
+//	c, ok := gauges[name]
+//	if !ok {
+//		getLogger(name, typeGauge).Errorf("collector not found")
+//		return
+//	}
+//	c.With(labelValues).Inc()
+//}
+//
+//func gaugeDec(name string, labelValues map[string]string) {
+//	if !initialized {
+//		return
+//	}
+//
+//	c, ok := gauges[name]
+//	if !ok {
+//		getLogger(name, typeGauge).Errorf("collector not found")
+//		return
+//	}
+//	c.With(labelValues).Dec()
+//}
+//
+//func gaugeAdd(name string, value float64, labelValues map[string]string) {
+//	if !initialized {
+//		return
+//	}
+//
+//	c, ok := gauges[name]
+//	if !ok {
+//		getLogger(name, typeGauge).Errorf("collector not found")
+//		return
+//	}
+//	c.With(labelValues).Add(value)
+//}
+//
+//func gaugeSub(name string, value float64, labelValues map[string]string) {
+//	if !initialized {
+//		return
+//	}
+//
+//	c, ok := gauges[name]
+//	if !ok {
+//		getLogger(name, typeGauge).Errorf("collector not found")
+//		return
+//	}
+//	c.With(labelValues).Sub(value)
+//}
 
 /*
  * -------------------- Counter functions --------------------

synchronizer/synchronizer.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/0xPolygonHermez/zkevm-bridge-service/etherman"
 	"github.com/0xPolygonHermez/zkevm-bridge-service/messagepush"
+	"github.com/0xPolygonHermez/zkevm-bridge-service/metrics"
 	"github.com/0xPolygonHermez/zkevm-bridge-service/redisstorage"
 	"github.com/0xPolygonHermez/zkevm-bridge-service/utils/gerror"
 	"github.com/0xPolygonHermez/zkevm-node/log"
@@ -394,6 +395,8 @@ func (s *ClientSynchronizer) processBlockRange(blocks []etherman.Block, order ma
 				// this is activated when the bridge detects the CreateNewRollup or the AddExistingRollup event from the rollupManager
 				log.Info("Event received. Activating LxLyEtrog...")
 			}
+
+			metrics.RecordSynchronizerEvent(uint32(s.networkID), string(element.Name))
 		}
 		log.Infof("NetworkID: %d. block %d element number %d", s.networkID, blocks[i].BlockNumber, counter)
 		err = s.storage.Commit(s.ctx, dbTx)