[WIP] feat(boost): add initial prometheus metrics

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.67-bullseye AS chef
+FROM rust:1.70-bullseye AS chef
 RUN cargo install cargo-chef
 WORKDIR /app
 

mev-boost-rs/Cargo.toml

@@ -23,6 +23,7 @@ ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus"
 beacon-api-client = { git = "https://github.com/ralexstokes/beacon-api-client" }
 
 mev-rs = { path = "../mev-rs" }
+prometheus = "0.13.3"
 
 [dev-dependencies]
 rand = "0.8.5"

mev-boost-rs/src/lib.rs

@@ -1,3 +1,4 @@
+mod metrics;
 mod relay;
 mod relay_mux;
 mod service;

mev-boost-rs/src/metrics.rs

@@ -0,0 +1,122 @@
+use std::sync::{Once, OnceLock};
+
+use ethereum_consensus::primitives::BlsPublicKey;
+use prometheus::{
+    register_histogram_vec, register_int_counter_vec, HistogramOpts, HistogramVec, IntCounterVec,
+    Opts, DEFAULT_BUCKETS,
+};
+
+const NAMESPACE: &str = "boost";
+const SUBSYSTEM: &str = "builder";
+
+const API_METHOD_LABEL: &str = "method";
+const RELAY_LABEL: &str = "relay";
+
+pub static API_REQUESTS_COUNTER: OnceLock<IntCounterVec> = OnceLock::new();
+pub static API_TIMEOUT_COUNTER: OnceLock<IntCounterVec> = OnceLock::new();
+pub static API_REQUEST_DURATION_SECONDS: OnceLock<HistogramVec> = OnceLock::new();
+
+pub static AUCTION_INVALID_BIDS_COUNTER: OnceLock<IntCounterVec> = OnceLock::new();
+
+static INIT: Once = Once::new();
+
+pub(crate) fn init() {
+    INIT.call_once(|| {
+        API_REQUESTS_COUNTER
+            .set(
+                register_int_counter_vec!(
+                    Opts::new("api_requests_total", "total number of builder API requests")
+                        .namespace(NAMESPACE)
+                        .subsystem(SUBSYSTEM),
+                    &[API_METHOD_LABEL, RELAY_LABEL]
+                )
+                .unwrap(),
+            )
+            .unwrap();
+
+        API_TIMEOUT_COUNTER
+            .set(
+                register_int_counter_vec!(
+                    Opts::new("api_timeouts_total", "total number of builder API timeouts")
+                        .namespace(NAMESPACE)
+                        .subsystem(SUBSYSTEM),
+                    &[API_METHOD_LABEL, RELAY_LABEL]
+                )
+                .unwrap(),
+            )
+            .unwrap();
+        API_REQUEST_DURATION_SECONDS
+            .set(
+                register_histogram_vec!(
+                    HistogramOpts {
+                        common_opts: Opts::new(
+                            "api_request_duration_seconds",
+                            "duration (in seconds) of builder API timeouts"
+                        )
+                        .namespace(NAMESPACE)
+                        .subsystem(SUBSYSTEM),
+                        buckets: DEFAULT_BUCKETS.to_vec(),
+                    },
+                    &[API_METHOD_LABEL, RELAY_LABEL]
+                )
+                .unwrap(),
+            )
+            .unwrap();
+
+        AUCTION_INVALID_BIDS_COUNTER
+            .set(
+                register_int_counter_vec!(
+                    Opts::new("auction_invalid_bids_total", "total number of invalid builder bids")
+                        .namespace(NAMESPACE)
+                        .subsystem(SUBSYSTEM),
+                    &[RELAY_LABEL]
+                )
+                .unwrap(),
+            )
+            .unwrap();
+    });
+}
+
+pub fn inc_api_int_counter_vec(
+    counter_vec: &OnceLock<IntCounterVec>,
+    meth: ApiMethod,
+    relay: &BlsPublicKey,
+) {
+    if let Some(counter) = counter_vec.get() {
+        counter.with_label_values(&[meth.as_str(), &relay.to_string()]).inc();
+    }
+}
+
+pub fn observe_api_histogram_vec(
+    hist_vec: &OnceLock<HistogramVec>,
+    meth: ApiMethod,
+    relay: &BlsPublicKey,
+    obs: f64,
+) {
+    if let Some(hist) = hist_vec.get() {
+        hist.with_label_values(&[meth.as_str(), &relay.to_string()]).observe(obs);
+    }
+}
+
+pub fn inc_auction_int_counter_vec(counter_vec: &OnceLock<IntCounterVec>, relay: &BlsPublicKey) {
+    if let Some(counter) = counter_vec.get() {
+        counter.with_label_values(&[&relay.to_string()]).inc();
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub enum ApiMethod {
+    Register,
+    GetHeader,
+    GetPayload,
+}
+
+impl ApiMethod {
+    pub const fn as_str(&self) -> &str {
+        match self {
+            Self::Register => "register",
+            Self::GetHeader => "get_header",
+            Self::GetPayload => "get_payload",
+        }
+    }
+}

mev-boost-rs/src/relay_mux.rs

@@ -1,4 +1,10 @@
-use crate::relay::Relay;
+use crate::{
+    metrics::{
+        self, API_REQUESTS_COUNTER, API_REQUEST_DURATION_SECONDS, API_TIMEOUT_COUNTER,
+        AUCTION_INVALID_BIDS_COUNTER,
+    },
+    relay::Relay,
+};
 use async_trait::async_trait;
 use ethereum_consensus::{
     primitives::{BlsPublicKey, Slot, U256},
@@ -14,7 +20,12 @@ use mev_rs::{
 };
 use parking_lot::Mutex;
 use rand::prelude::*;
-use std::{collections::HashMap, ops::Deref, sync::Arc, time::Duration};
+use std::{
+    collections::HashMap,
+    ops::Deref,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 
 // See note in the `mev-relay-rs::Relay` about this constant.
 // TODO likely drop this feature...
@@ -98,15 +109,28 @@ impl BlindedBlockProvider for RelayMux {
         let registrations = &registrations;
         let responses = stream::iter(self.relays.iter().cloned())
             .map(|relay| async move {
+                let start = Instant::now();
                 let response = relay.register_validators(registrations).await;
-                (relay.public_key, response)
+                (relay.public_key, start.elapsed(), response)
             })
             .buffer_unordered(self.relays.len())
             .collect::<Vec<_>>()
             .await;
 
         let mut num_failures = 0;
-        for (relay, response) in responses {
+        for (relay, duration, response) in responses {
+            metrics::inc_api_int_counter_vec(
+                &API_REQUESTS_COUNTER,
+                metrics::ApiMethod::Register,
+                &relay,
+            );
+            metrics::observe_api_histogram_vec(
+                &API_REQUEST_DURATION_SECONDS,
+                metrics::ApiMethod::Register,
+                &relay,
+                duration.as_secs_f64(),
+            );
+
             if let Err(err) = response {
                 num_failures += 1;
                 tracing::warn!("failed to register with relay {relay}: {err}");
@@ -124,31 +148,59 @@ impl BlindedBlockProvider for RelayMux {
         let responses = stream::iter(self.relays.iter().cloned())
             .enumerate()
             .map(|(index, relay)| async move {
+                let start = Instant::now();
                 let response = tokio::time::timeout(
                     Duration::from_secs(FETCH_BEST_BID_TIME_OUT_SECS),
                     relay.fetch_best_bid(bid_request),
                 )
                 .await;
-                (index, response)
+                (index, start.elapsed(), response)
             })
             .buffer_unordered(self.relays.len())
             .collect::<Vec<_>>()
             .await;
 
         let mut bids = Vec::with_capacity(responses.len());
-        for (relay_index, response) in responses {
+        for (relay_index, duration, response) in responses {
             let relay_public_key = &self.relays[relay_index].public_key;
 
+            metrics::inc_api_int_counter_vec(
+                &API_REQUESTS_COUNTER,
+                metrics::ApiMethod::GetHeader,
+                relay_public_key,
+            );
+            metrics::observe_api_histogram_vec(
+                &API_REQUEST_DURATION_SECONDS,
+                metrics::ApiMethod::GetHeader,
+                relay_public_key,
+                duration.as_secs_f64(),
+            );
+
             match response {
                 Ok(Ok(mut bid)) => {
                     if let Err(err) = validate_bid(&mut bid, relay_public_key, &self.context) {
-                        tracing::warn!("invalid signed builder bid from relay {relay_public_key}: {err}");
+                        tracing::warn!(
+                            "invalid signed builder bid from relay {relay_public_key}: {err}"
+                        );
+                        metrics::inc_auction_int_counter_vec(
+                            &AUCTION_INVALID_BIDS_COUNTER,
+                            relay_public_key,
+                        );
                     } else {
                         bids.push((bid, relay_index));
                     }
                 }
-                Ok(Err(err)) => tracing::warn!("failed to get a bid from relay {relay_public_key}: {err}"),
-                Err(..) => tracing::warn!("failed to get bid from relay {relay_public_key} within {FETCH_BEST_BID_TIME_OUT_SECS}s timeout"),
+                Ok(Err(err)) => {
+                    tracing::warn!("failed to get a bid from relay {relay_public_key}: {err}")
+                }
+                Err(..) => {
+                    tracing::warn!("failed to get bid from relay {relay_public_key} within {FETCH_BEST_BID_TIME_OUT_SECS}s timeout");
+                    metrics::inc_api_int_counter_vec(
+                        &API_TIMEOUT_COUNTER,
+                        metrics::ApiMethod::GetHeader,
+                        relay_public_key,
+                    );
+                }
             }
         }
 
@@ -197,15 +249,28 @@ impl BlindedBlockProvider for RelayMux {
         let relays = relay_indices.into_iter().map(|i| self.relays[i].clone());
         let responses = stream::iter(relays)
             .map(|relay| async move {
+                let start = Instant::now();
                 let response = relay.open_bid(signed_block).await;
-                (relay.public_key, response)
+                (relay.public_key, start.elapsed(), response)
             })
             .buffer_unordered(self.relays.len())
             .collect::<Vec<_>>()
             .await;
 
         let expected_block_hash = signed_block.block_hash();
-        for (relay, response) in responses.into_iter() {
+        for (relay, duration, response) in responses.into_iter() {
+            metrics::inc_api_int_counter_vec(
+                &API_REQUESTS_COUNTER,
+                metrics::ApiMethod::GetPayload,
+                &relay,
+            );
+            metrics::observe_api_histogram_vec(
+                &API_REQUEST_DURATION_SECONDS,
+                metrics::ApiMethod::GetPayload,
+                &relay,
+                duration.as_secs_f64(),
+            );
+
             match response {
                 Ok(payload) => {
                     let block_hash = payload.block_hash();

mev-boost-rs/src/service.rs

@@ -1,4 +1,5 @@
 use crate::{
+    metrics,
     relay::{Relay, RelayEndpoint},
     relay_mux::RelayMux,
 };
@@ -65,6 +66,8 @@ impl Service {
 
     /// Spawns a new [`RelayMux`] and [`BlindedBlockProviderServer`] task
     pub fn spawn(self, context: Option<Context>) -> Result<ServiceHandle, Error> {
+        metrics::init();
+
         let Self { host, port, relays, network } = self;
         let context =
             if let Some(context) = context { context } else { Context::try_from(&network)? };

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.67"
+channel = "1.70"