diff --git a/Cargo.lock b/Cargo.lock index ec4afab1..2265b1f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1508,6 +1508,7 @@ dependencies = [ "mev-rs", "parking_lot", "pin-project", + "prometheus", "rand", "serde", "serde_json", @@ -1954,6 +1955,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.21" diff --git a/Dockerfile b/Dockerfile index a0465fb0..9e3b2b01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.67-bullseye AS chef +FROM rust:1.70-bullseye AS chef RUN cargo install cargo-chef WORKDIR /app diff --git a/mev-boost-rs/Cargo.toml b/mev-boost-rs/Cargo.toml index 56faa685..1c85cb0a 100644 --- a/mev-boost-rs/Cargo.toml +++ b/mev-boost-rs/Cargo.toml @@ -23,6 +23,7 @@ ethereum-consensus = { git = "https://github.com/ralexstokes/ethereum-consensus" beacon-api-client = { git = "https://github.com/ralexstokes/beacon-api-client" } mev-rs = { path = "../mev-rs" } +prometheus = "0.13.3" [dev-dependencies] rand = "0.8.5" diff --git a/mev-boost-rs/src/lib.rs b/mev-boost-rs/src/lib.rs index 2c964373..19cf9776 100644 --- a/mev-boost-rs/src/lib.rs +++ b/mev-boost-rs/src/lib.rs @@ -1,3 +1,4 @@ +mod metrics; mod relay; mod relay_mux; mod service; diff --git a/mev-boost-rs/src/metrics.rs b/mev-boost-rs/src/metrics.rs new file mode 100644 index 00000000..7e86777a --- /dev/null +++ b/mev-boost-rs/src/metrics.rs @@ -0,0 +1,122 @@ +use std::sync::{Once, OnceLock}; + +use ethereum_consensus::primitives::BlsPublicKey; +use prometheus::{ + register_histogram_vec, register_int_counter_vec, HistogramOpts, HistogramVec, IntCounterVec, + Opts, DEFAULT_BUCKETS, +}; + +const NAMESPACE: &str = "boost"; +const SUBSYSTEM: &str = "builder"; + +const API_METHOD_LABEL: &str = "method"; +const RELAY_LABEL: &str = "relay"; + +pub static API_REQUESTS_COUNTER: OnceLock = OnceLock::new(); +pub static API_TIMEOUT_COUNTER: OnceLock = OnceLock::new(); +pub static API_REQUEST_DURATION_SECONDS: OnceLock = OnceLock::new(); + +pub static AUCTION_INVALID_BIDS_COUNTER: OnceLock = OnceLock::new(); + +static INIT: Once = Once::new(); + +pub(crate) fn init() { + INIT.call_once(|| { + API_REQUESTS_COUNTER + .set( + register_int_counter_vec!( + Opts::new("api_requests_total", "total number of builder API requests") + .namespace(NAMESPACE) + .subsystem(SUBSYSTEM), + &[API_METHOD_LABEL, RELAY_LABEL] + ) + .unwrap(), + ) + .unwrap(); + + API_TIMEOUT_COUNTER + .set( + register_int_counter_vec!( + Opts::new("api_timeouts_total", "total number of builder API timeouts") + .namespace(NAMESPACE) + .subsystem(SUBSYSTEM), + &[API_METHOD_LABEL, RELAY_LABEL] + ) + .unwrap(), + ) + .unwrap(); + API_REQUEST_DURATION_SECONDS + .set( + register_histogram_vec!( + HistogramOpts { + common_opts: Opts::new( + "api_request_duration_seconds", + "duration (in seconds) of builder API timeouts" + ) + .namespace(NAMESPACE) + .subsystem(SUBSYSTEM), + buckets: DEFAULT_BUCKETS.to_vec(), + }, + &[API_METHOD_LABEL, RELAY_LABEL] + ) + .unwrap(), + ) + .unwrap(); + + AUCTION_INVALID_BIDS_COUNTER + .set( + register_int_counter_vec!( + Opts::new("auction_invalid_bids_total", "total number of invalid builder bids") + .namespace(NAMESPACE) + .subsystem(SUBSYSTEM), + &[RELAY_LABEL] + ) + .unwrap(), + ) + .unwrap(); + }); +} + +pub fn inc_api_int_counter_vec( + counter_vec: &OnceLock, + meth: ApiMethod, + relay: &BlsPublicKey, +) { + if let Some(counter) = counter_vec.get() { + counter.with_label_values(&[meth.as_str(), &relay.to_string()]).inc(); + } +} + +pub fn observe_api_histogram_vec( + hist_vec: &OnceLock, + meth: ApiMethod, + relay: &BlsPublicKey, + obs: f64, +) { + if let Some(hist) = hist_vec.get() { + hist.with_label_values(&[meth.as_str(), &relay.to_string()]).observe(obs); + } +} + +pub fn inc_auction_int_counter_vec(counter_vec: &OnceLock, relay: &BlsPublicKey) { + if let Some(counter) = counter_vec.get() { + counter.with_label_values(&[&relay.to_string()]).inc(); + } +} + +#[derive(Copy, Clone, Debug)] +pub enum ApiMethod { + Register, + GetHeader, + GetPayload, +} + +impl ApiMethod { + pub const fn as_str(&self) -> &str { + match self { + Self::Register => "register", + Self::GetHeader => "get_header", + Self::GetPayload => "get_payload", + } + } +} diff --git a/mev-boost-rs/src/relay_mux.rs b/mev-boost-rs/src/relay_mux.rs index 3a87288c..9b4284de 100644 --- a/mev-boost-rs/src/relay_mux.rs +++ b/mev-boost-rs/src/relay_mux.rs @@ -1,4 +1,10 @@ -use crate::relay::Relay; +use crate::{ + metrics::{ + self, API_REQUESTS_COUNTER, API_REQUEST_DURATION_SECONDS, API_TIMEOUT_COUNTER, + AUCTION_INVALID_BIDS_COUNTER, + }, + relay::Relay, +}; use async_trait::async_trait; use ethereum_consensus::{ primitives::{BlsPublicKey, Slot, U256}, @@ -14,7 +20,12 @@ use mev_rs::{ }; use parking_lot::Mutex; use rand::prelude::*; -use std::{collections::HashMap, ops::Deref, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + ops::Deref, + sync::Arc, + time::{Duration, Instant}, +}; // See note in the `mev-relay-rs::Relay` about this constant. // TODO likely drop this feature... @@ -98,15 +109,28 @@ impl BlindedBlockProvider for RelayMux { let registrations = ®istrations; let responses = stream::iter(self.relays.iter().cloned()) .map(|relay| async move { + let start = Instant::now(); let response = relay.register_validators(registrations).await; - (relay.public_key, response) + (relay.public_key, start.elapsed(), response) }) .buffer_unordered(self.relays.len()) .collect::>() .await; let mut num_failures = 0; - for (relay, response) in responses { + for (relay, duration, response) in responses { + metrics::inc_api_int_counter_vec( + &API_REQUESTS_COUNTER, + metrics::ApiMethod::Register, + &relay, + ); + metrics::observe_api_histogram_vec( + &API_REQUEST_DURATION_SECONDS, + metrics::ApiMethod::Register, + &relay, + duration.as_secs_f64(), + ); + if let Err(err) = response { num_failures += 1; tracing::warn!("failed to register with relay {relay}: {err}"); @@ -124,31 +148,59 @@ impl BlindedBlockProvider for RelayMux { let responses = stream::iter(self.relays.iter().cloned()) .enumerate() .map(|(index, relay)| async move { + let start = Instant::now(); let response = tokio::time::timeout( Duration::from_secs(FETCH_BEST_BID_TIME_OUT_SECS), relay.fetch_best_bid(bid_request), ) .await; - (index, response) + (index, start.elapsed(), response) }) .buffer_unordered(self.relays.len()) .collect::>() .await; let mut bids = Vec::with_capacity(responses.len()); - for (relay_index, response) in responses { + for (relay_index, duration, response) in responses { let relay_public_key = &self.relays[relay_index].public_key; + metrics::inc_api_int_counter_vec( + &API_REQUESTS_COUNTER, + metrics::ApiMethod::GetHeader, + relay_public_key, + ); + metrics::observe_api_histogram_vec( + &API_REQUEST_DURATION_SECONDS, + metrics::ApiMethod::GetHeader, + relay_public_key, + duration.as_secs_f64(), + ); + match response { Ok(Ok(mut bid)) => { if let Err(err) = validate_bid(&mut bid, relay_public_key, &self.context) { - tracing::warn!("invalid signed builder bid from relay {relay_public_key}: {err}"); + tracing::warn!( + "invalid signed builder bid from relay {relay_public_key}: {err}" + ); + metrics::inc_auction_int_counter_vec( + &AUCTION_INVALID_BIDS_COUNTER, + relay_public_key, + ); } else { bids.push((bid, relay_index)); } } - Ok(Err(err)) => tracing::warn!("failed to get a bid from relay {relay_public_key}: {err}"), - Err(..) => tracing::warn!("failed to get bid from relay {relay_public_key} within {FETCH_BEST_BID_TIME_OUT_SECS}s timeout"), + Ok(Err(err)) => { + tracing::warn!("failed to get a bid from relay {relay_public_key}: {err}") + } + Err(..) => { + tracing::warn!("failed to get bid from relay {relay_public_key} within {FETCH_BEST_BID_TIME_OUT_SECS}s timeout"); + metrics::inc_api_int_counter_vec( + &API_TIMEOUT_COUNTER, + metrics::ApiMethod::GetHeader, + relay_public_key, + ); + } } } @@ -197,15 +249,28 @@ impl BlindedBlockProvider for RelayMux { let relays = relay_indices.into_iter().map(|i| self.relays[i].clone()); let responses = stream::iter(relays) .map(|relay| async move { + let start = Instant::now(); let response = relay.open_bid(signed_block).await; - (relay.public_key, response) + (relay.public_key, start.elapsed(), response) }) .buffer_unordered(self.relays.len()) .collect::>() .await; let expected_block_hash = signed_block.block_hash(); - for (relay, response) in responses.into_iter() { + for (relay, duration, response) in responses.into_iter() { + metrics::inc_api_int_counter_vec( + &API_REQUESTS_COUNTER, + metrics::ApiMethod::GetPayload, + &relay, + ); + metrics::observe_api_histogram_vec( + &API_REQUEST_DURATION_SECONDS, + metrics::ApiMethod::GetPayload, + &relay, + duration.as_secs_f64(), + ); + match response { Ok(payload) => { let block_hash = payload.block_hash(); diff --git a/mev-boost-rs/src/service.rs b/mev-boost-rs/src/service.rs index 19ed8a07..a5c570ce 100644 --- a/mev-boost-rs/src/service.rs +++ b/mev-boost-rs/src/service.rs @@ -1,4 +1,5 @@ use crate::{ + metrics, relay::{Relay, RelayEndpoint}, relay_mux::RelayMux, }; @@ -65,6 +66,8 @@ impl Service { /// Spawns a new [`RelayMux`] and [`BlindedBlockProviderServer`] task pub fn spawn(self, context: Option) -> Result { + metrics::init(); + let Self { host, port, relays, network } = self; let context = if let Some(context) = context { context } else { Context::try_from(&network)? }; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5a39a277..f400973c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.67" +channel = "1.70"