From e2a661668ebe154f8b84eb9c5102f398fae72a13 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 12 Dec 2024 16:38:39 +0100 Subject: [PATCH 001/136] historical fee service: working on calculating SMAs --- packages/services/src/historical_fees.rs | 301 ++++++++++++++++++ packages/services/src/lib.rs | 1 + packages/services/src/state_committer.rs | 2 + .../src/state_committer/fee_optimization.rs | 78 +++++ packages/services/tests/historical_fees.rs | 27 ++ 5 files changed, 409 insertions(+) create mode 100644 packages/services/src/historical_fees.rs create mode 100644 packages/services/src/state_committer/fee_optimization.rs create mode 100644 packages/services/tests/historical_fees.rs diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs new file mode 100644 index 00000000..66d77f58 --- /dev/null +++ b/packages/services/src/historical_fees.rs @@ -0,0 +1,301 @@ +pub mod port { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct BlockFees { + pub height: u64, + pub fees: Fees, + } + + pub mod l1 { + use std::ops::RangeInclusive; + + use itertools::Itertools; + use nonempty::NonEmpty; + + use super::BlockFees; + + #[derive(Debug)] + pub struct SequentialBlockFees { + fees: Vec, + } + + impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } + } + + // Cannot be empty + #[allow(clippy::len_without_is_empty)] + impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } + } + + #[derive(Debug)] + pub struct InvalidSequence(String); + + impl std::error::Error for InvalidSequence {} + + impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } + } + + impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + if !is_sequential { + return Err(InvalidSequence( + "blocks are not sequential by height".to_string(), + )); + } + + Ok(Self { fees }) + } + } + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait FeesProvider { + async fn fees(&self, height_range: RangeInclusive) -> NonEmpty; + async fn current_block_height(&self) -> u64; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use nonempty::NonEmpty; + + use crate::{ + historical_fees::port::{BlockFees, Fees}, + types::CollectNonEmpty, + }; + + use super::FeesProvider; + + pub struct TestFeesProvider { + fees: BTreeMap, + } + + impl FeesProvider for TestFeesProvider { + async fn current_block_height(&self) -> u64 { + *self.fees.keys().last().unwrap() + } + + async fn fees(&self, height_range: RangeInclusive) -> NonEmpty { + self.fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_nonempty() + .unwrap() + } + } + + impl TestFeesProvider { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128, + reward: i as u128, + base_fee_per_blob_gas: i as u128, + }, + ) + }) + .collect() + } + } + } + + pub mod service { + use super::{l1::FeesProvider, Fees}; + + pub struct HistoricalFeesProvider

{ + fees_provider: P, + } + impl

HistoricalFeesProvider

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } + } + + impl HistoricalFeesProvider

{ + pub async fn calculate_sma(&self, last_n_blocks: u32) -> Fees { + let fees = self.fees_provider.fees(0..=last_n_blocks as u64).await; + + eprintln!("got fees: {:?}", fees); + + let a = fees.last().fees; + eprintln!("got fees: {:?}", a); + a + } + } + } +} + +#[cfg(test)] +mod tests { + use port::{l1::SequentialBlockFees, BlockFees, Fees}; + + use super::*; + + #[test] + fn given_sequential_block_fees_when_valid_then_creation_succeeds() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn given_sequential_block_fees_when_empty_then_creation_fails() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn given_sequential_block_fees_when_non_sequential_then_creation_fails() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height\")" + ); + } + + #[test] + fn given_sequential_block_fees_when_valid_then_can_iterate_over_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + assert_eq!( + iterated_fees, block_fees, + "Expected iterator to yield the same block fees" + ); + } +} diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 0e1391a1..29b18fbf 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -3,6 +3,7 @@ pub mod block_committer; pub mod block_importer; pub mod cost_reporter; pub mod health_reporter; +pub mod historical_fees; pub mod state_committer; pub mod state_listener; pub mod state_pruner; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index ca2dcf3f..4395b606 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,3 +1,5 @@ +mod fee_optimization; + pub mod service { use std::{num::NonZeroUsize, time::Duration}; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs new file mode 100644 index 00000000..e8a6b3f9 --- /dev/null +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -0,0 +1,78 @@ +use std::{ops::RangeInclusive, time::Duration}; + +use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider}; +use nonempty::NonEmpty; +use rayon::range_inclusive; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub short_term_sma_num_blocks: u32, + pub long_term_sma_num_blocks: u32, +} + +pub struct SendOrWaitDecider

{ + price_service: HistoricalFeesProvider

, + config: Config, +} + +impl

SendOrWaitDecider

{ + pub fn new(price_service: HistoricalFeesProvider

, config: Config) -> Self { + Self { + price_service, + config, + } + } +} + +impl SendOrWaitDecider

{ + pub async fn should_send_blob_tx(&self) -> bool { + let short_term_sma = self + .price_service + .calculate_sma(self.config.short_term_sma_num_blocks) + .await; + + let long_term_sma = self + .price_service + .calculate_sma(self.config.long_term_sma_num_blocks) + .await; + + return short_term_sma.base_fee_per_gas < long_term_sma.base_fee_per_gas; + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + time::Duration, + }; + + use crate::historical_fees::port::{ + l1::testing::{incrementing_fees, TestFeesProvider}, + BlockFees, Fees, + }; + + use crate::types::CollectNonEmpty; + + use super::*; + + #[tokio::test] + async fn should_send_if_shortterm_sma_lower_than_longterm_sma() { + // given + let config = Config { + short_term_sma_num_blocks: 5, + long_term_sma_num_blocks: 50, + }; + + let fees_provider = TestFeesProvider::new(incrementing_fees(50)); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let sut = SendOrWaitDecider::new(price_service, config); + + // when + let decision = sut.should_send_blob_tx().await; + + // then + assert!(decision, "Should have sent"); + } +} diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs new file mode 100644 index 00000000..125fd01c --- /dev/null +++ b/packages/services/tests/historical_fees.rs @@ -0,0 +1,27 @@ +use std::{collections::BTreeMap, ops::RangeInclusive}; + +use nonempty::NonEmpty; +use services::{ + historical_fees::port::{ + l1::{testing, FeesProvider}, + service::HistoricalFeesProvider, + BlockFees, Fees, + }, + types::CollectNonEmpty, +}; + +#[tokio::test] +async fn calculates_sma_correctly_for_last_1_lock() { + // given + let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let price_service = HistoricalFeesProvider::new(fees_provider); + let last_n_blocks = 1; + + // when + let sma = price_service.calculate_sma(last_n_blocks).await; + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); +} From 8eb3243a5432948aeeca499797063ba47498a68b Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 13:37:21 +0100 Subject: [PATCH 002/136] starting to add decision logic based on sma price --- packages/services/src/historical_fees.rs | 53 ++- .../src/state_committer/fee_optimization.rs | 379 +++++++++++++++++- packages/services/tests/historical_fees.rs | 19 +- 3 files changed, 428 insertions(+), 23 deletions(-) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs index 66d77f58..bc43b279 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/historical_fees.rs @@ -1,5 +1,5 @@ pub mod port { - #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] pub struct Fees { pub base_fee_per_gas: u128, pub reward: u128, @@ -134,9 +134,9 @@ pub mod port { ( i, Fees { - base_fee_per_gas: i as u128, - reward: i as u128, - base_fee_per_blob_gas: i as u128, + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, }, ) }) @@ -146,7 +146,11 @@ pub mod port { } pub mod service { - use super::{l1::FeesProvider, Fees}; + use std::ops::RangeInclusive; + + use nonempty::NonEmpty; + + use super::{l1::FeesProvider, BlockFees, Fees}; pub struct HistoricalFeesProvider

{ fees_provider: P, @@ -158,14 +162,39 @@ pub mod port { } impl HistoricalFeesProvider

{ - pub async fn calculate_sma(&self, last_n_blocks: u32) -> Fees { - let fees = self.fees_provider.fees(0..=last_n_blocks as u64).await; - - eprintln!("got fees: {:?}", fees); + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, last_n_blocks: u64) -> Fees { + let current_height = self.fees_provider.current_block_height().await; + + let starting_block = current_height.saturating_sub(last_n_blocks.saturating_sub(1)); + let fees = self + .fees_provider + .fees(starting_block..=current_height) + .await; + + Self::mean(&fees) + } - let a = fees.last().fees; - eprintln!("got fees: {:?}", a); - a + fn mean(fees: &NonEmpty) -> Fees { + let total = fees + .iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + let count = fees.len() as u128; + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } } } } diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index e8a6b3f9..0a49d98e 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,13 +1,14 @@ use std::{ops::RangeInclusive, time::Duration}; -use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider}; +use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; +use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use nonempty::NonEmpty; use rayon::range_inclusive; #[derive(Debug, Clone, Copy)] pub struct Config { - pub short_term_sma_num_blocks: u32, - pub long_term_sma_num_blocks: u32, + pub short_term_sma_num_blocks: u64, + pub long_term_sma_num_blocks: u64, } pub struct SendOrWaitDecider

{ @@ -25,7 +26,7 @@ impl

SendOrWaitDecider

{ } impl SendOrWaitDecider

{ - pub async fn should_send_blob_tx(&self) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self .price_service .calculate_sma(self.config.short_term_sma_num_blocks) @@ -36,7 +37,25 @@ impl SendOrWaitDecider

{ .calculate_sma(self.config.long_term_sma_num_blocks) .await; - return short_term_sma.base_fee_per_gas < long_term_sma.base_fee_per_gas; + let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + eprintln!( + "Short term: {:?}, Long term: {:?}, Short term tx price: {}, Long term tx price: {}", + short_term_sma, long_term_sma, short_term_tx_price, long_term_tx_price + ); + + short_term_tx_price < long_term_tx_price + } + + // TODO: Segfault maybe dont leak so much eth abstractions + fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; + let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; + + base_fee + blob_fee + fees.reward } } @@ -57,22 +76,362 @@ mod tests { use super::*; #[tokio::test] - async fn should_send_if_shortterm_sma_lower_than_longterm_sma() { + async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { // given let config = Config { - short_term_sma_num_blocks: 5, - long_term_sma_num_blocks: 50, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ]; - let fees_provider = TestFeesProvider::new(incrementing_fees(50)); + let fees_provider = TestFeesProvider::new(fees); let price_service = HistoricalFeesProvider::new(fees_provider); + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 40, + base_fee_per_blob_gas: 40 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20 + } + ); + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; // when - let decision = sut.should_send_blob_tx().await; + let decision = sut.should_send_blob_tx(num_blobs).await; // then assert!(decision, "Should have sent"); } + + #[tokio::test] + async fn wont_send_because_normal_gas_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 3366, + reward: 40, + base_fee_per_blob_gas: 40 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } + + #[tokio::test] + async fn wont_send_because_blob_gas_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 40, + base_fee_per_blob_gas: 366 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } + + #[tokio::test] + async fn wont_send_because_rewards_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 33333366, + base_fee_per_blob_gas: 40, + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } } diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index 125fd01c..a8317561 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -11,7 +11,7 @@ use services::{ }; #[tokio::test] -async fn calculates_sma_correctly_for_last_1_lock() { +async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); let price_service = HistoricalFeesProvider::new(fees_provider); @@ -25,3 +25,20 @@ async fn calculates_sma_correctly_for_last_1_lock() { assert_eq!(sma.reward, 5); assert_eq!(sma.base_fee_per_blob_gas, 5); } + +#[tokio::test] +async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let price_service = HistoricalFeesProvider::new(fees_provider); + let last_n_blocks = 5; + + // when + let sma = price_service.calculate_sma(last_n_blocks).await; + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); +} From f7cf478e4338937f01df33cdd069c1df1b369336 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 13:50:26 +0100 Subject: [PATCH 003/136] cleanup tests --- packages/services/src/historical_fees.rs | 2 +- .../src/state_committer/fee_optimization.rs | 351 ++++-------------- packages/services/tests/historical_fees.rs | 12 +- 3 files changed, 76 insertions(+), 289 deletions(-) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs index bc43b279..416e9902 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/historical_fees.rs @@ -146,7 +146,7 @@ pub mod port { } pub mod service { - use std::ops::RangeInclusive; + use nonempty::NonEmpty; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 0a49d98e..d9a0bbeb 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,9 +1,4 @@ -use std::{ops::RangeInclusive, time::Duration}; - use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; -use alloy::eips::eip4844::DATA_GAS_PER_BLOB; -use nonempty::NonEmpty; -use rayon::range_inclusive; #[derive(Debug, Clone, Copy)] pub struct Config { @@ -39,10 +34,6 @@ impl SendOrWaitDecider

{ let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - eprintln!( - "Short term: {:?}, Long term: {:?}, Short term tx price: {}, Long term tx price: {}", - short_term_sma, long_term_sma, short_term_tx_price, long_term_tx_price - ); short_term_tx_price < long_term_tx_price } @@ -61,20 +52,22 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { - use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - time::Duration, - }; - - use crate::historical_fees::port::{ - l1::testing::{incrementing_fees, TestFeesProvider}, - BlockFees, Fees, - }; - use crate::types::CollectNonEmpty; + use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; use super::*; + fn constant_fees(num_blocks: u64, fees: Fees) -> Vec<(u64, Fees)> { + (0..=num_blocks).map(|height| (height, fees)).collect() + } + + fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { + fees.iter_mut() + .rev() + .take(num_latest as usize) + .for_each(|(_, f)| *f = updated_fees); + } + #[tokio::test] async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { // given @@ -82,79 +75,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ]; - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 40, - base_fee_per_blob_gas: 40 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -172,79 +115,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ]; - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 3366, - reward: 40, - base_fee_per_blob_gas: 40 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 10000, reward: 20, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -262,79 +155,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, - }, - ), - ]; - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 40, - base_fee_per_blob_gas: 366 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, - base_fee_per_blob_gas: 1000 - } + base_fee_per_blob_gas: 1000, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -352,79 +195,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ), - ]; - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 33333366, - base_fee_per_blob_gas: 40, - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 100_000_000, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index a8317561..b5fb6ba4 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -1,14 +1,8 @@ -use std::{collections::BTreeMap, ops::RangeInclusive}; -use nonempty::NonEmpty; -use services::{ - historical_fees::port::{ - l1::{testing, FeesProvider}, +use services::historical_fees::port::{ + l1::testing, service::HistoricalFeesProvider, - BlockFees, Fees, - }, - types::CollectNonEmpty, -}; + }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { From b3ec73b4b997a1aa6c9de1c59b722595d6b142f0 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:11:58 +0100 Subject: [PATCH 004/136] shorten tests --- .../src/state_committer/fee_optimization.rs | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index d9a0bbeb..0bf719c6 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -61,6 +61,20 @@ mod tests { (0..=num_blocks).map(|height| (height, fees)).collect() } + fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (config.long_term_sma_num_blocks - config.short_term_sma_num_blocks) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, config.short_term_sma_num_blocks as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() + } + fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { fees.iter_mut() .rev() @@ -76,18 +90,13 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, @@ -116,22 +125,17 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, + base_fee_per_gas: 100, + reward: 100_000_000, + base_fee_per_blob_gas: 100, }, ); @@ -156,22 +160,17 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, + reward: 100_000_000, + base_fee_per_blob_gas: 100, }, ); @@ -196,18 +195,13 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 100_000_000, From 6599fcfaa9b312896e817c9813120fd4131c6f10 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:17:38 +0100 Subject: [PATCH 005/136] parameterized tests --- .../src/state_committer/fee_optimization.rs | 168 ++++-------------- 1 file changed, 33 insertions(+), 135 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 0bf719c6..a05f5624 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -21,6 +21,7 @@ impl

SendOrWaitDecider

{ } impl SendOrWaitDecider

{ + // TODO: segfault validate blob number pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self .price_service @@ -52,14 +53,9 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { - - use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; - use super::*; - - fn constant_fees(num_blocks: u64, fees: Fees) -> Vec<(u64, Fees)> { - (0..=num_blocks).map(|height| (height, fees)).collect() - } + use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; + use test_case::test_case; fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( @@ -75,35 +71,39 @@ mod tests { .collect() } - fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { - fees.iter_mut() - .rev() - .take(num_latest as usize) - .for_each(|(_, f)| *f = updated_fees); - } - #[tokio::test] - async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, + true; "Should send because short-term prices lower than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, + false; "Wont send because normal gas too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, + false; "Wont send because blob gas too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, + false; "Wont send because rewards too expensive" + )] + async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + expected_decision: bool, + ) { // given let config = Config { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ); - + let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); let price_service = HistoricalFeesProvider::new(fees_provider); @@ -111,114 +111,12 @@ mod tests { let num_blobs = 6; // when - let decision = sut.should_send_blob_tx(num_blobs).await; + let should_send = sut.should_send_blob_tx(num_blobs).await; // then - assert!(decision, "Should have sent"); - } - - #[tokio::test] - async fn wont_send_because_normal_gas_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 100, - reward: 100_000_000, - base_fee_per_blob_gas: 100, - }, + assert_eq!( + should_send, expected_decision, + "Expected decision: {expected_decision}, got: {should_send}", ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); - } - - #[tokio::test] - async fn wont_send_because_blob_gas_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 100, - }, - ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); - } - - #[tokio::test] - async fn wont_send_because_rewards_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); } } From 281086b76c57f38a1a74f80d22144431814e6b62 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:31:00 +0100 Subject: [PATCH 006/136] add more tests --- .../src/state_committer/fee_optimization.rs | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index a05f5624..a854ff09 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -73,28 +73,65 @@ mod tests { #[tokio::test] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, - true; "Should send because short-term prices lower than long-term" + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, // Old fees + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, // New fees + 6, // num_blobs + true; + "Should send because all short-term prices lower than long-term" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, - false; "Wont send because normal gas too expensive" + 6, + false; + "Wont send because normal gas too expensive" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, - false; "Wont send because blob gas too expensive" + 6, + false; + "Wont send because blob gas too expensive" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, - false; "Wont send because rewards too expensive" + 6, + false; + "Wont send because rewards too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + 1, + false; + "Fees identical" + )] + #[test_case( + Fees { base_fee_per_gas: 500, reward: 500, base_fee_per_blob_gas: 500 }, + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + 6, + true; + "6 blobs significantly cheaper in short-term" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 70 }, + 6, + false; + "6 blobs slightly more expensive blob gas in short-term" + )] + #[test_case( + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + 5, + false; + "Five blobs but identical fees" )] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, + num_blobs: u32, expected_decision: bool, ) { // given @@ -108,7 +145,6 @@ mod tests { let price_service = HistoricalFeesProvider::new(fees_provider); let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; // when let should_send = sut.should_send_blob_tx(num_blobs).await; @@ -116,7 +152,7 @@ mod tests { // then assert_eq!( should_send, expected_decision, - "Expected decision: {expected_decision}, got: {should_send}", + "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", ); } } From 8b74db3dfeed473bfcd3fc58086c2cac5ad14773 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:35:12 +0100 Subject: [PATCH 007/136] more edge cases --- .../src/state_committer/fee_optimization.rs | 105 +++++++++++------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index a854ff09..67c77e59 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -57,6 +57,7 @@ mod tests { use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; + // Function to generate historical fees data fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, @@ -73,60 +74,88 @@ mod tests { #[tokio::test] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, // Old fees - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, // New fees - 6, // num_blobs + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees + 10, // num_blobs true; - "Should send because all short-term prices lower than long-term" + "Should send because all short-term fees are lower than long-term" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, - 6, - false; - "Wont send because normal gas too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + 5, + true; + "Should send because short-term base_fee_per_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, - 6, - false; - "Wont send because blob gas too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + 5, + false; + "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, - 6, - false; - "Wont send because rewards too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees + 5, + true; + "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - 1, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees + 5, false; - "Fees identical" + "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 500, reward: 500, base_fee_per_blob_gas: 500 }, - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - 6, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees + 5, true; - "6 blobs significantly cheaper in short-term" + "Should send because short-term reward is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 70 }, - 6, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees + 5, false; - "6 blobs slightly more expensive blob gas in short-term" + "Should not send because short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - 5, + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees + 10, + true; + "Should send because multiple short-term fees are lower" + )] + #[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees + 10, false; - "Five blobs but identical fees" + "Should not send because all fees are identical" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees + 0, + true; + "Zero blobs but short-term base_fee_per_gas and reward are lower" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees + 0, + false; + "Zero blobs but short-term reward is higher" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees + 0, + true; + "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] async fn parameterized_send_or_wait_tests( old_fees: Fees, @@ -134,7 +163,7 @@ mod tests { num_blobs: u32, expected_decision: bool, ) { - // given + // Given let config = Config { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, @@ -146,10 +175,10 @@ mod tests { let sut = SendOrWaitDecider::new(price_service, config); - // when + // When let should_send = sut.should_send_blob_tx(num_blobs).await; - // then + // Then assert_eq!( should_send, expected_decision, "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", From cc451cfe643763c0aa480d64df2f6ecfd84343d1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:42:16 +0100 Subject: [PATCH 008/136] use sequential fees struct, rename price service to fee analytics --- .../{historical_fees.rs => fee_analytics.rs} | 66 +++++++++++-------- packages/services/src/lib.rs | 2 +- .../src/state_committer/fee_optimization.rs | 18 ++--- packages/services/tests/historical_fees.rs | 14 ++-- 4 files changed, 52 insertions(+), 48 deletions(-) rename packages/services/src/{historical_fees.rs => fee_analytics.rs} (89%) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/fee_analytics.rs similarity index 89% rename from packages/services/src/historical_fees.rs rename to packages/services/src/fee_analytics.rs index 416e9902..50ae41c0 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/fee_analytics.rs @@ -80,7 +80,7 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> NonEmpty; + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees; async fn current_block_height(&self) -> u64; } @@ -88,14 +88,15 @@ pub mod port { pub mod testing { use std::{collections::BTreeMap, ops::RangeInclusive}; + use itertools::Itertools; use nonempty::NonEmpty; use crate::{ - historical_fees::port::{BlockFees, Fees}, + fee_analytics::port::{BlockFees, Fees}, types::CollectNonEmpty, }; - use super::FeesProvider; + use super::{FeesProvider, SequentialBlockFees}; pub struct TestFeesProvider { fees: BTreeMap, @@ -106,8 +107,9 @@ pub mod port { *self.fees.keys().last().unwrap() } - async fn fees(&self, height_range: RangeInclusive) -> NonEmpty { - self.fees + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + let fees = self + .fees .iter() .skip_while(|(height, _)| !height_range.contains(height)) .take_while(|(height, _)| height_range.contains(height)) @@ -115,8 +117,9 @@ pub mod port { height: *height, fees: *fees, }) - .collect_nonempty() - .unwrap() + .collect_vec(); + + fees.try_into().unwrap() } } @@ -146,22 +149,24 @@ pub mod port { } pub mod service { - use nonempty::NonEmpty; - use super::{l1::FeesProvider, BlockFees, Fees}; + use super::{ + l1::{FeesProvider, SequentialBlockFees}, + BlockFees, Fees, + }; - pub struct HistoricalFeesProvider

{ + pub struct FeeAnalytics

{ fees_provider: P, } - impl

HistoricalFeesProvider

{ + impl

FeeAnalytics

{ pub fn new(fees_provider: P) -> Self { Self { fees_provider } } } - impl HistoricalFeesProvider

{ + impl FeeAnalytics

{ // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background @@ -174,12 +179,14 @@ pub mod port { .fees(starting_block..=current_height) .await; - Self::mean(&fees) + Self::mean(fees) } - fn mean(fees: &NonEmpty) -> Fees { + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + let total = fees - .iter() + .into_iter() .map(|bf| bf.fees) .fold(Fees::default(), |acc, f| Fees { base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, @@ -187,8 +194,6 @@ pub mod port { base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, }); - let count = fees.len() as u128; - // TODO: segfault should we round to nearest here? Fees { base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), @@ -207,7 +212,7 @@ mod tests { use super::*; #[test] - fn given_sequential_block_fees_when_valid_then_creation_succeeds() { + fn can_create_valid_sequential_fees() { // Given let block_fees = vec![ BlockFees { @@ -241,7 +246,7 @@ mod tests { } #[test] - fn given_sequential_block_fees_when_empty_then_creation_fails() { + fn sequential_fees_cannot_be_empty() { // Given let block_fees: Vec = vec![]; @@ -260,7 +265,7 @@ mod tests { } #[test] - fn given_sequential_block_fees_when_non_sequential_then_creation_fails() { + fn fees_must_be_sequential() { // Given let block_fees = vec![ BlockFees { @@ -295,18 +300,13 @@ mod tests { ); } + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested #[test] - fn given_sequential_block_fees_when_valid_then_can_iterate_over_fees() { + fn produced_iterator_gives_correct_values() { // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, BlockFees { height: 2, fees: Fees { @@ -315,6 +315,14 @@ mod tests { base_fee_per_blob_gas: 15, }, }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, ]; let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 29b18fbf..69e04048 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,8 +2,8 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; +pub mod fee_analytics; pub mod health_reporter; -pub mod historical_fees; pub mod state_committer; pub mod state_listener; pub mod state_pruner; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 67c77e59..37b4abc8 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,4 +1,4 @@ -use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; +use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; #[derive(Debug, Clone, Copy)] pub struct Config { @@ -7,14 +7,14 @@ pub struct Config { } pub struct SendOrWaitDecider

{ - price_service: HistoricalFeesProvider

, + fee_analytics: FeeAnalytics

, config: Config, } impl

SendOrWaitDecider

{ - pub fn new(price_service: HistoricalFeesProvider

, config: Config) -> Self { + pub fn new(fee_analytics: FeeAnalytics

, config: Config) -> Self { Self { - price_service, + fee_analytics, config, } } @@ -24,12 +24,12 @@ impl SendOrWaitDecider

{ // TODO: segfault validate blob number pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self - .price_service + .fee_analytics .calculate_sma(self.config.short_term_sma_num_blocks) .await; let long_term_sma = self - .price_service + .fee_analytics .calculate_sma(self.config.long_term_sma_num_blocks) .await; @@ -54,7 +54,7 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; + use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; // Function to generate historical fees data @@ -171,9 +171,9 @@ mod tests { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); + let analytics_service = FeeAnalytics::new(fees_provider); - let sut = SendOrWaitDecider::new(price_service, config); + let sut = SendOrWaitDecider::new(analytics_service, config); // When let should_send = sut.should_send_blob_tx(num_blobs).await; diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index b5fb6ba4..8f6c9d48 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -1,18 +1,14 @@ - -use services::historical_fees::port::{ - l1::testing, - service::HistoricalFeesProvider, - }; +use services::fee_analytics::port::{l1::testing, service::FeeAnalytics}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); - let price_service = HistoricalFeesProvider::new(fees_provider); + let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 1; // when - let sma = price_service.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(last_n_blocks).await; // then assert_eq!(sma.base_fee_per_gas, 5); @@ -24,11 +20,11 @@ async fn calculates_sma_correctly_for_last_1_block() { async fn calculates_sma_correctly_for_last_5_blocks() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); - let price_service = HistoricalFeesProvider::new(fees_provider); + let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 5; // when - let sma = price_service.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(last_n_blocks).await; // then let mean = (5 + 4 + 3 + 2 + 1) / 5; From fea814ae3edfa1fdcd4ba0620f6d1fa4eda4f5eb Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:53:40 +0100 Subject: [PATCH 009/136] add activation fee threshold --- .../src/state_committer/fee_optimization.rs | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 37b4abc8..870b7470 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -2,6 +2,7 @@ use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; #[derive(Debug, Clone, Copy)] pub struct Config { + pub sma_activation_fee_treshold: u128, pub short_term_sma_num_blocks: u64, pub long_term_sma_num_blocks: u64, } @@ -34,6 +35,10 @@ impl SendOrWaitDecider

{ .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + if short_term_tx_price < self.config.sma_activation_fee_treshold { + return true; + } + let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); short_term_tx_price < long_term_tx_price @@ -72,18 +77,35 @@ mod tests { .collect() } - #[tokio::test] #[test_case( Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees - 10, // num_blobs + 6, + 0, true; "Should send because all short-term fees are lower than long-term" )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + 6, + 0, + false; + "Should not send because all short-term fees are higher than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + 6, + 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + true; + "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" + )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, true; "Should send because short-term base_fee_per_gas is lower" )] @@ -91,6 +113,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, false; "Should not send because short-term base_fee_per_gas is higher" )] @@ -98,6 +121,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees 5, + 0, true; "Should send because short-term base_fee_per_blob_gas is lower" )] @@ -105,6 +129,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees 5, + 0, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] @@ -112,6 +137,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, true; "Should send because short-term reward is lower" )] @@ -119,20 +145,23 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, false; "Should not send because short-term reward is higher" )] #[test_case( Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees - 10, + 6, + 0, true; "Should send because multiple short-term fees are lower" )] #[test_case( Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees - 10, + 6, + 0, false; "Should not send because all fees are identical" )] @@ -140,6 +169,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees 0, + 0, true; "Zero blobs but short-term base_fee_per_gas and reward are lower" )] @@ -147,6 +177,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees 0, + 0, false; "Zero blobs but short-term reward is higher" )] @@ -154,17 +185,21 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees 0, + 0, true; "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] + #[tokio::test] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, num_blobs: u32, + activation_fee_treshold: u128, expected_decision: bool, ) { // Given let config = Config { + sma_activation_fee_treshold: activation_fee_treshold, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; From 3bfcca0afe4228ea0a2a936aa9fe9884a94a9f3c Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 15:19:35 +0100 Subject: [PATCH 010/136] add comparison strategy --- .../src/state_committer/fee_optimization.rs | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 870b7470..ff604efc 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,10 +1,26 @@ use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; +// TODO: segfault validate percentages +#[derive(Debug, Clone, Copy)] +pub enum ComparisonStrategy { + /// Short-term fee must be at most (1 - percentage) of the long-term fee. + /// For example, if percentage = 0.1 (10%), then: + /// short_term ≤ long_term * 0.9. + StrictlyLessOrEqualByPercent(f64), + + /// Short-term fee may be more expensive, but not by more than the given percentage. + /// For example, if percentage = 0.1 (10%), then: + /// short_term ≤ long_term * 1.1. + /// Short-term can be cheaper by any amount. + WithinVicinityOfPriceByPercent(f64), +} + #[derive(Debug, Clone, Copy)] pub struct Config { pub sma_activation_fee_treshold: u128, pub short_term_sma_num_blocks: u64, pub long_term_sma_num_blocks: u64, + pub comparison_strategy: ComparisonStrategy, } pub struct SendOrWaitDecider

{ @@ -35,13 +51,30 @@ impl SendOrWaitDecider

{ .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + if short_term_tx_price < self.config.sma_activation_fee_treshold { return true; } let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - short_term_tx_price < long_term_tx_price + let percentage = match self.config.comparison_strategy { + ComparisonStrategy::StrictlyLessOrEqualByPercent(p) => 1.0 - p, + ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, + }; + + // TODO: segfault proper type conversions + let allowed_max = (long_term_tx_price as f64 * percentage) as u128; + + eprintln!( + "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", + short_term_tx_price, + long_term_tx_price, + allowed_max, + short_term_tx_price.saturating_sub(allowed_max) + ); + + short_term_tx_price <= allowed_max } // TODO: Segfault maybe dont leak so much eth abstractions @@ -82,7 +115,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees 6, 0, - true; + true; "Should send because all short-term fees are lower than long-term" )] #[test_case( @@ -90,7 +123,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees 6, 0, - false; + false; "Should not send because all short-term fees are higher than long-term" )] #[test_case( @@ -98,7 +131,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees 6, 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - true; + true; "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" )] #[test_case( @@ -202,6 +235,7 @@ mod tests { sma_activation_fee_treshold: activation_fee_treshold, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0), }; let fees = generate_fees(config, old_fees, new_fees); From 7e5769f927514063d59706b629636e09b381b2ce Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 15:26:19 +0100 Subject: [PATCH 011/136] add tests for treshold strategy --- .../src/state_committer/fee_optimization.rs | 247 +++++++++++++----- 1 file changed, 188 insertions(+), 59 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index ff604efc..06a9ee7c 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -63,18 +63,18 @@ impl SendOrWaitDecider

{ ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, }; - // TODO: segfault proper type conversions - let allowed_max = (long_term_tx_price as f64 * percentage) as u128; + // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta + let treshold = (long_term_tx_price as f64 * percentage) as u128; eprintln!( "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", short_term_tx_price, long_term_tx_price, - allowed_max, - short_term_tx_price.saturating_sub(allowed_max) + treshold, + short_term_tx_price.saturating_sub(treshold) ); - short_term_tx_price <= allowed_max + short_term_tx_price < treshold } // TODO: Segfault maybe dont leak so much eth abstractions @@ -111,146 +111,275 @@ mod tests { } #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because all short-term fees are lower than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because all short-term fees are higher than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + Config { + sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term base_fee_per_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term reward is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because multiple short-term fees are lower" )] #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because all fees are identical" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Zero blobs but short-term base_fee_per_gas and reward are lower" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Zero blobs but short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] + // New Tests (Introducing other Comparison Strategies) + #[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) + }, + true; + "Should send (cheaper) with StrictlyLessOrEqualByPercent(10%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + // Strictly less or equal by 0% means must be cheaper or equal, which it's not + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, + false; + "Should not send (more expensive) with StrictlyLessOrEqualByPercent(0%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + // Below threshold means we send anyway + sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + }, + true; + "Below activation threshold, send anyway for WithinVicinityOfPriceByPercent(10%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + }, + true; + "Within vicinity of price by 10%: short_term slightly more expensive but allowed" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) + }, + false; + "Within vicinity with 0% means must not exceed long-term, not sending" + )] #[tokio::test] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, num_blobs: u32, - activation_fee_treshold: u128, + config: Config, expected_decision: bool, ) { - // Given - let config = Config { - sma_activation_fee_treshold: activation_fee_treshold, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0), - }; - let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); - // When let should_send = sut.should_send_blob_tx(num_blobs).await; - // Then assert_eq!( should_send, expected_decision, - "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", + "For num_blobs={num_blobs}, config={:?}: Expected decision: {}, got: {}", + config, expected_decision, should_send ); } } From 855ac2603bf21ddeb19985d39c2e9625497d69fa Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 16:59:14 +0100 Subject: [PATCH 012/136] l1 adapter for historical fees, about to test out algo on historical data --- Cargo.lock | 1 + packages/adapters/eth/Cargo.toml | 1 + packages/adapters/eth/src/lib.rs | 134 +++++++++++++++++- packages/adapters/eth/src/websocket.rs | 11 +- .../adapters/eth/src/websocket/connection.rs | 20 ++- .../websocket/health_tracking_middleware.rs | 18 ++- .../{historical_fees.rs => fee_analytics.rs} | 0 7 files changed, 177 insertions(+), 8 deletions(-) rename packages/services/tests/{historical_fees.rs => fee_analytics.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 6b99e671..9b64b27d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2673,6 +2673,7 @@ dependencies = [ "serde", "serde_json", "services", + "static_assertions", "test-case", "thiserror 1.0.69", "tokio", diff --git a/packages/adapters/eth/Cargo.toml b/packages/adapters/eth/Cargo.toml index 6c18d2c9..0ab31d3b 100644 --- a/packages/adapters/eth/Cargo.toml +++ b/packages/adapters/eth/Cargo.toml @@ -21,6 +21,7 @@ alloy = { workspace = true, features = [ "rpc-types", "reqwest-rustls-tls", ] } +static_assertions = { workspace = true } async-trait = { workspace = true } aws-config = { workspace = true, features = ["default"] } aws-sdk-kms = { workspace = true, features = ["default"] } diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 38e069fc..2836fb33 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -1,13 +1,19 @@ -use std::num::{NonZeroU32, NonZeroUsize}; +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroUsize}, + ops::RangeInclusive, +}; use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, + providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE, }; use delegate::delegate; -use itertools::{izip, Itertools}; +use itertools::{izip, zip, Itertools}; use services::{ + fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -23,6 +29,7 @@ mod websocket; pub use alloy::primitives::Address; pub use aws::*; use fuel_block_committer_encoding::blob::{self, generate_sidecar}; +use static_assertions::const_assert; pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; #[derive(Debug, Copy, Clone)] @@ -186,13 +193,101 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } } +impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + const REWARD_PERCENTILE: f64 = + alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; + + // so that a alloy version bump doesn't surprise us + const_assert!(REWARD_PERCENTILE == 20.0,); + + let mut fees = vec![]; + + // TODO: segfault see when this can be None + // TODO: check edgecases + let mut current_height = height_range.clone().min().unwrap(); + while current_height <= *height_range.end() { + // There is a comment in alloy about not doing more than 1024 blocks at a time + const RPC_LIMIT: u64 = 1024; + + let upper_bound = min( + current_height.saturating_add(RPC_LIMIT), + *height_range.end(), + ); + + let history = self + .fees( + current_height..=upper_bound, + std::slice::from_ref(&REWARD_PERCENTILE), + ) + .await + .unwrap(); + + fees.push(history); + + current_height = upper_bound.saturating_add(1); + } + + let new_fees = fees + .into_iter() + .flat_map(|fees| { + // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe + // for block count 0? + eprintln!("received {fees:?}"); + let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); + let rewards = fees + .reward + .unwrap() + .into_iter() + .map(|mut perc| perc.pop().unwrap()) + .collect_vec(); + + let oldest_block = fees.oldest_block; + + debug_assert_eq!(rewards.len(), number_of_blocks); + + izip!( + (oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + }) + .collect_vec(); + + eprintln!("converted into {new_fees:?}"); + + new_fees.try_into().unwrap() + } + + async fn current_block_height(&self) -> u64 { + self._get_block_number().await.unwrap() + } +} + #[cfg(test)] mod test { + use std::time::Duration; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::block_bundler::port::l1::FragmentEncoder; + use services::{ + block_bundler::port::l1::FragmentEncoder, block_committer::port::l1::Api, + fee_analytics::port::l1::FeesProvider, + }; - use crate::BlobEncoder; + use crate::{BlobEncoder, Signer, Signers}; #[test] fn gas_usage_correctly_calculated() { @@ -207,4 +302,35 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } + + #[tokio::test] + async fn can_connect_to_eth_mainnet() { + let signers = Signers::for_keys(crate::L1Keys { + main: crate::L1Key::Private( + "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), + ), + blob: None, + }) + .await + .unwrap(); + + let client = crate::WebsocketClient::connect( + "wss://ethereum-rpc.publicnode.com".parse().unwrap(), + Default::default(), + signers, + 10, + crate::TxConfig { + tx_max_fee: u128::MAX, + send_tx_request_timeout: Duration::MAX, + }, + ) + .await + .unwrap(); + + let current_height = client._get_block_number().await.unwrap(); + + let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; + + panic!("{:?}", fees); + } } diff --git a/packages/adapters/eth/src/websocket.rs b/packages/adapters/eth/src/websocket.rs index 9ccd8091..8b64038b 100644 --- a/packages/adapters/eth/src/websocket.rs +++ b/packages/adapters/eth/src/websocket.rs @@ -1,10 +1,11 @@ -use std::{num::NonZeroU32, str::FromStr, time::Duration}; +use std::{num::NonZeroU32, ops::RangeInclusive, str::FromStr, time::Duration}; use ::metrics::{prometheus::core::Collector, HealthChecker, RegistersMetrics}; use alloy::{ consensus::SignableTransaction, network::TxSigner, primitives::{Address, ChainId, B256}, + rpc::types::FeeHistory, signers::{local::PrivateKeySigner, Signature}, }; use serde::Deserialize; @@ -225,6 +226,14 @@ impl WebsocketClient { Ok(self.inner.get_transaction_response(tx_hash).await?) } + pub(crate) async fn fees( + &self, + height_range: RangeInclusive, + rewards_percentile: &[f64], + ) -> Result { + Ok(self.inner.fees(height_range, rewards_percentile).await?) + } + pub(crate) async fn is_squeezed_out(&self, tx_hash: [u8; 32]) -> Result { Ok(self.inner.is_squeezed_out(tx_hash).await?) } diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index 8a718b9f..65318b49 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -1,6 +1,7 @@ use std::{ cmp::{max, min}, num::NonZeroU32, + ops::RangeInclusive, time::Duration, }; @@ -14,7 +15,7 @@ use alloy::{ primitives::{Address, U256}, providers::{utils::Eip1559Estimation, Provider, ProviderBuilder, SendableTx, WsConnect}, pubsub::PubSubFrontend, - rpc::types::{TransactionReceipt, TransactionRequest}, + rpc::types::{FeeHistory, TransactionReceipt, TransactionRequest}, sol, }; use itertools::Itertools; @@ -212,6 +213,19 @@ impl EthApi for WsConnection { Ok(submission_tx) } + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result { + let max = *height_range.end(); + let count = height_range.clone().count() as u64; + Ok(self + .provider + .get_fee_history(count, BlockNumberOrTag::Number(max), reward_percentiles) + .await?) + } + async fn get_block_number(&self) -> Result { let response = self.provider.get_block_number().await?; Ok(response) @@ -385,7 +399,9 @@ impl WsConnection { let contract_address = Address::from_slice(contract_address.as_ref()); let contract = FuelStateContract::new(contract_address, provider.clone()); - let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; + // TODO: segfault revert this + // let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; + let interval_u256 = 1u32; let commit_interval = u32::try_from(interval_u256) .map_err(|e| Error::Other(e.to_string())) diff --git a/packages/adapters/eth/src/websocket/health_tracking_middleware.rs b/packages/adapters/eth/src/websocket/health_tracking_middleware.rs index 21f5c3aa..93045485 100644 --- a/packages/adapters/eth/src/websocket/health_tracking_middleware.rs +++ b/packages/adapters/eth/src/websocket/health_tracking_middleware.rs @@ -1,8 +1,9 @@ -use std::num::NonZeroU32; +use std::{num::NonZeroU32, ops::RangeInclusive}; use ::metrics::{ prometheus::core::Collector, ConnectionHealthTracker, HealthChecker, RegistersMetrics, }; +use alloy::rpc::types::FeeHistory; use delegate::delegate; use services::types::{Address, BlockSubmissionTx, Fragment, NonEmpty, TransactionResponse, U256}; @@ -15,6 +16,11 @@ use crate::{ #[async_trait::async_trait] pub trait EthApi { async fn submit(&self, hash: [u8; 32], height: u32) -> Result; + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result; async fn get_block_number(&self) -> Result; async fn balance(&self, address: Address) -> Result; fn commit_interval(&self) -> NonZeroU32; @@ -117,6 +123,16 @@ where response } + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result { + let response = self.adapter.fees(height_range, reward_percentiles).await; + self.note_network_status(&response); + response + } + async fn is_squeezed_out(&self, tx_hash: [u8; 32]) -> Result { let response = self.adapter.is_squeezed_out(tx_hash).await; self.note_network_status(&response); diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/fee_analytics.rs similarity index 100% rename from packages/services/tests/historical_fees.rs rename to packages/services/tests/fee_analytics.rs From c2f160143ab5bf51ffa94197fa803b1192c276a5 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 19:19:32 +0100 Subject: [PATCH 013/136] historical testing of algo --- Cargo.lock | 22 ++++ packages/adapters/eth/src/lib.rs | 73 ++++++------ packages/services/Cargo.toml | 1 + packages/services/src/fee_analytics.rs | 20 ++-- packages/services/src/state_committer.rs | 2 +- .../src/state_committer/fee_optimization.rs | 27 +++-- packages/services/tests/fee_analytics.rs | 111 +++++++++++++++++- 7 files changed, 197 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b64b27d..582429a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2146,6 +2146,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -5762,6 +5783,7 @@ dependencies = [ "async-trait", "bytesize", "clock", + "csv", "delegate", "eth", "fuel-block-committer-encoding", diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 2836fb33..17cdc6e4 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -2,6 +2,7 @@ use std::{ cmp::min, num::{NonZeroU32, NonZeroUsize}, ops::RangeInclusive, + time::Duration, }; use alloy::{ @@ -35,6 +36,29 @@ pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; #[derive(Debug, Copy, Clone)] pub struct BlobEncoder; +pub async fn make_pub_eth_client() -> WebsocketClient { + let signers = Signers::for_keys(crate::L1Keys { + main: crate::L1Key::Private( + "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), + ), + blob: None, + }) + .await + .unwrap(); + + crate::WebsocketClient::connect( + "wss://ethereum-rpc.publicnode.com".parse().unwrap(), + Default::default(), + signers, + 10, + crate::TxConfig { + tx_max_fee: u128::MAX, + send_tx_request_timeout: Duration::MAX, + }, + ) + .await + .unwrap() +} impl BlobEncoder { #[cfg(feature = "test-helpers")] pub const FRAGMENT_SIZE: usize = BYTES_PER_BLOB; @@ -211,7 +235,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { const RPC_LIMIT: u64 = 1024; let upper_bound = min( - current_height.saturating_add(RPC_LIMIT), + current_height.saturating_add(RPC_LIMIT).saturating_sub(1), *height_range.end(), ); @@ -223,6 +247,11 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .await .unwrap(); + assert_eq!( + history.reward.as_ref().unwrap().len(), + (current_height..=upper_bound).count() + ); + fees.push(history); current_height = upper_bound.saturating_add(1); @@ -233,7 +262,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .flat_map(|fees| { // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe // for block count 0? - eprintln!("received {fees:?}"); + // eprintln!("received {fees:?}"); let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); let rewards = fees .reward @@ -266,7 +295,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { }) .collect_vec(); - eprintln!("converted into {new_fees:?}"); + // eprintln!("converted into {new_fees:?}"); new_fees.try_into().unwrap() } @@ -303,34 +332,12 @@ mod test { assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - #[tokio::test] - async fn can_connect_to_eth_mainnet() { - let signers = Signers::for_keys(crate::L1Keys { - main: crate::L1Key::Private( - "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), - ), - blob: None, - }) - .await - .unwrap(); - - let client = crate::WebsocketClient::connect( - "wss://ethereum-rpc.publicnode.com".parse().unwrap(), - Default::default(), - signers, - 10, - crate::TxConfig { - tx_max_fee: u128::MAX, - send_tx_request_timeout: Duration::MAX, - }, - ) - .await - .unwrap(); - - let current_height = client._get_block_number().await.unwrap(); - - let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; - - panic!("{:?}", fees); - } + // #[tokio::test] + // async fn can_connect_to_eth_mainnet() { + // let current_height = client._get_block_number().await.unwrap(); + // + // let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; + // + // panic!("{:?}", fees); + // } } diff --git a/packages/services/Cargo.toml b/packages/services/Cargo.toml index d23a2313..194ecc6a 100644 --- a/packages/services/Cargo.toml +++ b/packages/services/Cargo.toml @@ -46,6 +46,7 @@ tai64 = { workspace = true } tokio = { workspace = true, features = ["macros"] } test-helpers = { workspace = true } rand = { workspace = true, features = ["small_rng", "std", "std_rng"] } +csv = "1.3" [features] test-helpers = ["dep:mockall", "dep:rand"] diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 50ae41c0..1894cae5 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -66,10 +66,11 @@ pub mod port { .tuple_windows() .all(|(l, r)| l.height + 1 == r.height); + let heights = fees.iter().map(|f| f.height).collect::>(); if !is_sequential { - return Err(InvalidSequence( - "blocks are not sequential by height".to_string(), - )); + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); } Ok(Self { fees }) @@ -98,6 +99,7 @@ pub mod port { use super::{FeesProvider, SequentialBlockFees}; + #[derive(Debug, Clone)] pub struct TestFeesProvider { fees: BTreeMap, } @@ -150,6 +152,8 @@ pub mod port { pub mod service { + use std::ops::RangeInclusive; + use nonempty::NonEmpty; use super::{ @@ -170,14 +174,8 @@ pub mod port { // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, last_n_blocks: u64) -> Fees { - let current_height = self.fees_provider.current_block_height().await; - - let starting_block = current_height.saturating_sub(last_n_blocks.saturating_sub(1)); - let fees = self - .fees_provider - .fees(starting_block..=current_height) - .await; + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { + let fees = self.fees_provider.fees(block_range).await; Self::mean(fees) } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 4395b606..5bdfd3e5 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,4 +1,4 @@ -mod fee_optimization; +pub mod fee_optimization; pub mod service { use std::{num::NonZeroUsize, time::Duration}; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 06a9ee7c..3c691b5d 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -39,15 +39,17 @@ impl

SendOrWaitDecider

{ impl SendOrWaitDecider

{ // TODO: segfault validate blob number - pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32, at_block_height: u64) -> bool { let short_term_sma = self .fee_analytics - .calculate_sma(self.config.short_term_sma_num_blocks) + .calculate_sma( + at_block_height - self.config.short_term_sma_num_blocks..=at_block_height, + ) .await; let long_term_sma = self .fee_analytics - .calculate_sma(self.config.long_term_sma_num_blocks) + .calculate_sma(at_block_height - self.config.long_term_sma_num_blocks..=at_block_height) .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -66,13 +68,13 @@ impl SendOrWaitDecider

{ // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta let treshold = (long_term_tx_price as f64 * percentage) as u128; - eprintln!( - "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", - short_term_tx_price, - long_term_tx_price, - treshold, - short_term_tx_price.saturating_sub(treshold) - ); + // eprintln!( + // "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", + // short_term_tx_price, + // long_term_tx_price, + // treshold, + // short_term_tx_price.saturating_sub(treshold) + // ); short_term_tx_price < treshold } @@ -370,11 +372,14 @@ mod tests { ) { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); + let current_block_height = fees_provider.current_block_height().await; let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); - let should_send = sut.should_send_blob_tx(num_blobs).await; + let should_send = sut + .should_send_blob_tx(num_blobs, current_block_height) + .await; assert_eq!( should_send, expected_decision, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 8f6c9d48..342994fd 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,4 +1,17 @@ -use services::fee_analytics::port::{l1::testing, service::FeeAnalytics}; +use std::{collections::HashMap, path::PathBuf}; + +use eth::make_pub_eth_client; +use services::{ + fee_analytics::{ + self, + port::{ + l1::testing::{self, TestFeesProvider}, + service::FeeAnalytics, + BlockFees, Fees, + }, + }, + state_committer::fee_optimization::SendOrWaitDecider, +}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { @@ -8,7 +21,7 @@ async fn calculates_sma_correctly_for_last_1_block() { let last_n_blocks = 1; // when - let sma = fee_analytics.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(4..=4).await; // then assert_eq!(sma.base_fee_per_gas, 5); @@ -24,7 +37,7 @@ async fn calculates_sma_correctly_for_last_5_blocks() { let last_n_blocks = 5; // when - let sma = fee_analytics.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(0..=4).await; // then let mean = (5 + 4 + 3 + 2 + 1) / 5; @@ -32,3 +45,95 @@ async fn calculates_sma_correctly_for_last_5_blocks() { assert_eq!(sma.reward, mean); assert_eq!(sma.base_fee_per_blob_gas, mean); } + +fn calculate_tx_fee(fees: &Fees) -> u128 { + 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 +} + +fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + let mut csv_writer = + csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)).unwrap(); + csv_writer + .write_record(["height", "tx_fee"].iter()) + .unwrap(); + for (height, fee) in tx_fees { + csv_writer + .write_record([height.to_string(), fee.to_string()]) + .unwrap(); + } + csv_writer.flush().unwrap(); +} + +#[tokio::test] +async fn something() { + let client = make_pub_eth_client().await; + use services::fee_analytics::port::l1::FeesProvider; + + let current_block_height = 21402042; + let starting_block_height = current_block_height - 24 * 3600 / 12; + let data = client + .fees(starting_block_height..=current_block_height) + .await + .into_iter() + .collect::>(); + + let fee_lookup = data + .iter() + .map(|b| (b.height, b.fees)) + .collect::>(); + + let short_sma = 25u64; + let long_sma = 900; + + let current_tx_fees = data + .iter() + .map(|b| (b.height, calculate_tx_fee(&b.fees))) + .collect::>(); + + save_tx_fees(¤t_tx_fees, "current_fees.csv"); + + let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + let fee_analytics = FeeAnalytics::new(local_client.clone()); + + let mut short_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - short_sma..=height) + .await; + + let tx_fee = calculate_tx_fee(&fees); + + short_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + + let decider = SendOrWaitDecider::new( + FeeAnalytics::new(local_client.clone()), + services::state_committer::fee_optimization::Config { + sma_activation_fee_treshold: 1000, + short_term_sma_num_blocks: short_sma, + long_term_sma_num_blocks: long_sma, + comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) + }, + ); + + let mut decisions = vec![]; + let mut long_sma_tx_fees = vec![]; + + for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - long_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + long_sma_tx_fees.push((height, tx_fee)); + + if decider.should_send_blob_tx(6, height).await { + let current_fees = fee_lookup.get(&height).unwrap(); + let current_tx_fee = calculate_tx_fee(current_fees); + decisions.push((height, current_tx_fee)); + } + } + + save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + save_tx_fees(&decisions, "decisions.csv"); +} From c732cd7f3d7abfe1cc203e56ea848d896c2eeb30 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 13:37:21 +0100 Subject: [PATCH 014/136] adding in the fee acceptance scaling for being late with posting --- .../src/state_committer/fee_optimization.rs | 156 +++++++++++------- packages/services/tests/fee_analytics.rs | 24 ++- 2 files changed, 118 insertions(+), 62 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 3c691b5d..899fb8b8 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,26 +1,27 @@ +use std::cmp::min; + use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; -// TODO: segfault validate percentages #[derive(Debug, Clone, Copy)] -pub enum ComparisonStrategy { - /// Short-term fee must be at most (1 - percentage) of the long-term fee. - /// For example, if percentage = 0.1 (10%), then: - /// short_term ≤ long_term * 0.9. - StrictlyLessOrEqualByPercent(f64), - - /// Short-term fee may be more expensive, but not by more than the given percentage. - /// For example, if percentage = 0.1 (10%), then: - /// short_term ≤ long_term * 1.1. - /// Short-term can be cheaper by any amount. - WithinVicinityOfPriceByPercent(f64), +pub struct SmaBlockNumPeriods { + pub short: u64, + pub long: u64, +} + +// TODO: segfault validate start discount is less than end premium and both are positive +#[derive(Debug, Clone, Copy)] +pub struct Feethresholds { + // TODO: segfault validate not 0 + pub max_l2_blocks_behind: u64, + pub start_discount_percentage: f64, + pub end_premium_percentage: f64, + pub always_acceptable_fee: u128, } #[derive(Debug, Clone, Copy)] pub struct Config { - pub sma_activation_fee_treshold: u128, - pub short_term_sma_num_blocks: u64, - pub long_term_sma_num_blocks: u64, - pub comparison_strategy: ComparisonStrategy, + pub sma_periods: SmaBlockNumPeriods, + pub fee_thresholds: Feethresholds, } pub struct SendOrWaitDecider

{ @@ -37,46 +38,75 @@ impl

SendOrWaitDecider

{ } } +#[derive(Debug, Clone, Copy)] +pub struct Context { + pub num_l2_blocks_behind: u64, + pub at_l1_height: u64, +} + impl SendOrWaitDecider

{ // TODO: segfault validate blob number - pub async fn should_send_blob_tx(&self, num_blobs: u32, at_block_height: u64) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { + let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; + let short_term_sma = self .fee_analytics - .calculate_sma( - at_block_height - self.config.short_term_sma_num_blocks..=at_block_height, - ) + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await; let long_term_sma = self .fee_analytics - .calculate_sma(at_block_height - self.config.long_term_sma_num_blocks..=at_block_height) + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await; - let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + + let fee_always_acceptable = + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; - if short_term_tx_price < self.config.sma_activation_fee_treshold { + // TODO: segfault test this + let too_far_behind = + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + + if fee_always_acceptable || too_far_behind { return true; } - let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + + short_term_tx_fee < max_upper_tx_fee + } + + // TODO: segfault test this + fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { + // Define percentages in Parts Per Million (PPM) for precision + // 1 PPM = 0.0001% + const PPM: u128 = 1_000_000; + let start_discount_ppm = + (self.config.fee_thresholds.start_discount_percentage * PPM as f64) as u128; + let end_premium_ppm = + (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; + + let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind as u128; - let percentage = match self.config.comparison_strategy { - ComparisonStrategy::StrictlyLessOrEqualByPercent(p) => 1.0 - p, - ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, - }; + let blocks_behind = context.num_l2_blocks_behind; - // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta - let treshold = (long_term_tx_price as f64 * percentage) as u128; + // TODO: segfault rename possibly + let ratio_ppm = (blocks_behind as u128 * PPM) / max_blocks_behind; - // eprintln!( - // "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", - // short_term_tx_price, - // long_term_tx_price, - // treshold, - // short_term_tx_price.saturating_sub(treshold) - // ); + let initial_multiplier = PPM.saturating_sub(start_discount_ppm); - short_term_tx_price < treshold + let effect_of_being_late = (start_discount_ppm + end_premium_ppm) + .saturating_mul(ratio_ppm) + .saturating_div(PPM); + + let multiplier_ppm = initial_multiplier.saturating_add(effect_of_being_late); + + // TODO: segfault, for now just in case, but this should never happen + let multiplier_ppm = min(PPM + end_premium_ppm, multiplier_ppm); + + fee.saturating_mul(multiplier_ppm).saturating_div(PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -101,9 +131,9 @@ mod tests { fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, - (config.long_term_sma_num_blocks - config.short_term_sma_num_blocks) as usize, + (config.sma_periods.long - config.sma_periods.short) as usize, ); - let newer_fees = std::iter::repeat_n(new_fees, config.short_term_sma_num_blocks as usize); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); older_fees .chain(newer_fees) @@ -117,7 +147,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -130,7 +160,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -143,7 +173,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -156,7 +186,7 @@ mod tests { Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -169,7 +199,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -182,7 +212,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -195,7 +225,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -208,7 +238,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -221,7 +251,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -234,7 +264,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -247,7 +277,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -260,7 +290,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -273,7 +303,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -286,7 +316,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -300,7 +330,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) @@ -313,7 +343,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, // Strictly less or equal by 0% means must be cheaper or equal, which it's not @@ -328,7 +358,7 @@ mod tests { 6, Config { // Below threshold means we send anyway - sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) @@ -341,7 +371,7 @@ mod tests { Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) @@ -354,7 +384,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) @@ -378,7 +408,13 @@ mod tests { let sut = SendOrWaitDecider::new(analytics_service, config); let should_send = sut - .should_send_blob_tx(num_blobs, current_block_height) + .should_send_blob_tx( + num_blobs, + Context { + at_l1_height: current_block_height, + num_l2_blocks_behind: 0, + }, + ) .await; assert_eq!( diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 342994fd..d9cac8fc 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -10,7 +10,7 @@ use services::{ BlockFees, Fees, }, }, - state_committer::fee_optimization::SendOrWaitDecider, + state_committer::fee_optimization::{Context, SendOrWaitDecider}, }; #[tokio::test] @@ -83,6 +83,7 @@ async fn something() { .collect::>(); let short_sma = 25u64; + let middle_sma = 300; let long_sma = 900; let current_tx_fees = data @@ -107,6 +108,16 @@ async fn something() { } save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + let mut middle_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - middle_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + middle_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); + let decider = SendOrWaitDecider::new( FeeAnalytics::new(local_client.clone()), services::state_committer::fee_optimization::Config { @@ -127,7 +138,16 @@ async fn something() { let tx_fee = calculate_tx_fee(&fees); long_sma_tx_fees.push((height, tx_fee)); - if decider.should_send_blob_tx(6, height).await { + if decider + .should_send_blob_tx( + 6, + Context { + at_l1_height: height, + num_l2_blocks_behind: 0, + }, + ) + .await + { let current_fees = fee_lookup.get(&height).unwrap(); let current_tx_fee = calculate_tx_fee(current_fees); decisions.push((height, current_tx_fee)); From 3a2506479c62ef7c12fc757537fede10ba6cffb4 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 14:34:11 +0100 Subject: [PATCH 015/136] checked tests validity, tweaked values to better represent the test scenario --- .../src/state_committer/fee_optimization.rs | 330 +++++++++++------- packages/services/tests/fee_analytics.rs | 186 +++++----- 2 files changed, 305 insertions(+), 211 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 899fb8b8..54a91824 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -63,11 +63,14 @@ impl SendOrWaitDecider

{ let fee_always_acceptable = short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; + eprintln!("fee always acceptable: {}", fee_always_acceptable); // TODO: segfault test this let too_far_behind = context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + eprintln!("too far behind: {}", too_far_behind); + if fee_always_acceptable || too_far_behind { return true; } @@ -75,6 +78,32 @@ impl SendOrWaitDecider

{ let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + let long_vs_max_delta_perc = + ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) + .abs(); + + let short_vs_max_delta_perc = ((max_upper_tx_fee as f64 - short_term_tx_fee as f64) + / short_term_tx_fee as f64 + * 100.) + .abs(); + + if long_term_tx_fee <= max_upper_tx_fee { + eprintln!("The max upper fee({max_upper_tx_fee}) is above the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + } else { + eprintln!("The max upper fee({max_upper_tx_fee}) is below the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + } + + if short_term_tx_fee <= max_upper_tx_fee { + eprintln!("The short term fee({short_term_tx_fee}) is below the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); + } else { + eprintln!("The short term fee({short_term_tx_fee}) is above the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); + } + + eprintln!( + "Short-term fee: {}, Long-term fee: {}, Max upper fee: {}", + short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee + ); + short_term_tx_fee < max_upper_tx_fee } @@ -126,8 +155,8 @@ mod tests { use super::*; use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; + use tokio; - // Function to generate historical fees data fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, @@ -147,11 +176,15 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, }, + 0, // not behind at all true; "Should send because all short-term fees are lower than long-term" )] @@ -160,11 +193,15 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, }, + 0, false; "Should not send because all short-term fees are higher than long-term" )] @@ -173,24 +210,32 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + } }, + 0, true; - "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" + "Should send since short-term fee < always_acceptable_fee" )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term base_fee_per_gas is lower" )] @@ -199,37 +244,49 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] @@ -238,11 +295,15 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term reward is lower" )] @@ -251,24 +312,33 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term reward is higher" )] #[test_case( + // Multiple short-term fees are lower Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because multiple short-term fees are lower" )] @@ -277,120 +347,144 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; - "Should not send because all fees are identical" + "Should not send because all fees are identical and no tolerance" )] #[test_case( + // Zero blobs scenario: blob fee differences don't matter Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; - "Zero blobs but short-term base_fee_per_gas and reward are lower" + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" )] #[test_case( + // Zero blobs but short-term reward is higher Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; - "Zero blobs but short-term reward is higher" + "Zero blobs: short-term reward is higher, don't send" )] #[test_case( + // Zero blobs don't care about higher short-term base_fee_per_blob_gas Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) - }, - true; - "Zero blobs dont care about higher short-term base_fee_per_blob_gas" - )] - // New Tests (Introducing other Comparison Strategies) - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - 6, - Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; - "Should send (cheaper) with StrictlyLessOrEqualByPercent(10%)" + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" )] + // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - // Strictly less or equal by 0% means must be cheaper or equal, which it's not - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, }, + 0, false; - "Should not send (more expensive) with StrictlyLessOrEqualByPercent(0%)" + "Early: short-term expensive, not send" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - // Below threshold means we send anyway - sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + } }, + 100, true; - "Below activation threshold, send anyway for WithinVicinityOfPriceByPercent(10%)" + "Later: after max wait, send regardless" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, - 6, + // Partway: at 80 blocks behind, tolerance might have increased enough to accept + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, }, + 65, true; - "Within vicinity of price by 10%: short_term slightly more expensive but allowed" + "Mid-wait: increased tolerance allows acceptance" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, + Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 26_000_000_000, + }, }, - false; - "Within vicinity with 0% means must not exceed long-term, not sending" + 0, + true; + "Always acceptable fee triggers immediate send" )] #[tokio::test] async fn parameterized_send_or_wait_tests( @@ -398,6 +492,7 @@ mod tests { new_fees: Fees, num_blobs: u32, config: Config, + num_l2_blocks_behind: u64, expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); @@ -412,15 +507,14 @@ mod tests { num_blobs, Context { at_l1_height: current_block_height, - num_l2_blocks_behind: 0, + num_l2_blocks_behind, }, ) .await; assert_eq!( should_send, expected_decision, - "For num_blobs={num_blobs}, config={:?}: Expected decision: {}, got: {}", - config, expected_decision, should_send + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", ); } } diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index d9cac8fc..ffa5acc2 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -64,96 +64,96 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -#[tokio::test] -async fn something() { - let client = make_pub_eth_client().await; - use services::fee_analytics::port::l1::FeesProvider; - - let current_block_height = 21402042; - let starting_block_height = current_block_height - 24 * 3600 / 12; - let data = client - .fees(starting_block_height..=current_block_height) - .await - .into_iter() - .collect::>(); - - let fee_lookup = data - .iter() - .map(|b| (b.height, b.fees)) - .collect::>(); - - let short_sma = 25u64; - let middle_sma = 300; - let long_sma = 900; - - let current_tx_fees = data - .iter() - .map(|b| (b.height, calculate_tx_fee(&b.fees))) - .collect::>(); - - save_tx_fees(¤t_tx_fees, "current_fees.csv"); - - let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - let fee_analytics = FeeAnalytics::new(local_client.clone()); - - let mut short_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - short_sma..=height) - .await; - - let tx_fee = calculate_tx_fee(&fees); - - short_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - - let mut middle_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - middle_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - middle_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); - - let decider = SendOrWaitDecider::new( - FeeAnalytics::new(local_client.clone()), - services::state_committer::fee_optimization::Config { - sma_activation_fee_treshold: 1000, - short_term_sma_num_blocks: short_sma, - long_term_sma_num_blocks: long_sma, - comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) - }, - ); - - let mut decisions = vec![]; - let mut long_sma_tx_fees = vec![]; - - for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - long_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - long_sma_tx_fees.push((height, tx_fee)); - - if decider - .should_send_blob_tx( - 6, - Context { - at_l1_height: height, - num_l2_blocks_behind: 0, - }, - ) - .await - { - let current_fees = fee_lookup.get(&height).unwrap(); - let current_tx_fee = calculate_tx_fee(current_fees); - decisions.push((height, current_tx_fee)); - } - } - - save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - save_tx_fees(&decisions, "decisions.csv"); -} +// #[tokio::test] +// async fn something() { +// let client = make_pub_eth_client().await; +// use services::fee_analytics::port::l1::FeesProvider; +// +// let current_block_height = 21402042; +// let starting_block_height = current_block_height - 24 * 3600 / 12; +// let data = client +// .fees(starting_block_height..=current_block_height) +// .await +// .into_iter() +// .collect::>(); +// +// let fee_lookup = data +// .iter() +// .map(|b| (b.height, b.fees)) +// .collect::>(); +// +// let short_sma = 25u64; +// let middle_sma = 300; +// let long_sma = 900; +// +// let current_tx_fees = data +// .iter() +// .map(|b| (b.height, calculate_tx_fee(&b.fees))) +// .collect::>(); +// +// save_tx_fees(¤t_tx_fees, "current_fees.csv"); +// +// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); +// let fee_analytics = FeeAnalytics::new(local_client.clone()); +// +// let mut short_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - short_sma..=height) +// .await; +// +// let tx_fee = calculate_tx_fee(&fees); +// +// short_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); +// +// let mut middle_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - middle_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// middle_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); +// +// let decider = SendOrWaitDecider::new( +// FeeAnalytics::new(local_client.clone()), +// services::state_committer::fee_optimization::Config { +// sma_activation_fee_treshold: 1000, +// short_term_sma_num_blocks: short_sma, +// long_term_sma_num_blocks: long_sma, +// comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) +// }, +// ); +// +// let mut decisions = vec![]; +// let mut long_sma_tx_fees = vec![]; +// +// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - long_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// long_sma_tx_fees.push((height, tx_fee)); +// +// if decider +// .should_send_blob_tx( +// 6, +// Context { +// at_l1_height: height, +// num_l2_blocks_behind: 0, +// }, +// ) +// .await +// { +// let current_fees = fee_lookup.get(&height).unwrap(); +// let current_tx_fee = calculate_tx_fee(current_fees); +// decisions.push((height, current_tx_fee)); +// } +// } +// +// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); +// save_tx_fees(&decisions, "decisions.csv"); +// } From c1d482496d5270b43e84d04647bcfcfdbce1c37e Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 14:51:52 +0100 Subject: [PATCH 016/136] analyzing algo config --- packages/services/tests/fee_analytics.rs | 183 +++++++++++------------ 1 file changed, 89 insertions(+), 94 deletions(-) diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index ffa5acc2..f8c00e0a 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -10,7 +10,7 @@ use services::{ BlockFees, Fees, }, }, - state_committer::fee_optimization::{Context, SendOrWaitDecider}, + state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, }; #[tokio::test] @@ -64,96 +64,91 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -// #[tokio::test] -// async fn something() { -// let client = make_pub_eth_client().await; -// use services::fee_analytics::port::l1::FeesProvider; -// -// let current_block_height = 21402042; -// let starting_block_height = current_block_height - 24 * 3600 / 12; -// let data = client -// .fees(starting_block_height..=current_block_height) -// .await -// .into_iter() -// .collect::>(); -// -// let fee_lookup = data -// .iter() -// .map(|b| (b.height, b.fees)) -// .collect::>(); -// -// let short_sma = 25u64; -// let middle_sma = 300; -// let long_sma = 900; -// -// let current_tx_fees = data -// .iter() -// .map(|b| (b.height, calculate_tx_fee(&b.fees))) -// .collect::>(); -// -// save_tx_fees(¤t_tx_fees, "current_fees.csv"); -// -// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); -// let fee_analytics = FeeAnalytics::new(local_client.clone()); -// -// let mut short_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - short_sma..=height) -// .await; -// -// let tx_fee = calculate_tx_fee(&fees); -// -// short_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); -// -// let mut middle_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - middle_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// middle_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); -// -// let decider = SendOrWaitDecider::new( -// FeeAnalytics::new(local_client.clone()), -// services::state_committer::fee_optimization::Config { -// sma_activation_fee_treshold: 1000, -// short_term_sma_num_blocks: short_sma, -// long_term_sma_num_blocks: long_sma, -// comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) -// }, -// ); -// -// let mut decisions = vec![]; -// let mut long_sma_tx_fees = vec![]; -// -// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - long_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// long_sma_tx_fees.push((height, tx_fee)); -// -// if decider -// .should_send_blob_tx( -// 6, -// Context { -// at_l1_height: height, -// num_l2_blocks_behind: 0, -// }, -// ) -// .await -// { -// let current_fees = fee_lookup.get(&height).unwrap(); -// let current_tx_fee = calculate_tx_fee(current_fees); -// decisions.push((height, current_tx_fee)); -// } -// } -// -// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); -// save_tx_fees(&decisions, "decisions.csv"); -// } +#[tokio::test] +async fn something() { + let client = make_pub_eth_client().await; + use services::fee_analytics::port::l1::FeesProvider; + + let current_block_height = 21408300; + let starting_block_height = current_block_height - 48 * 3600 / 12; + let data = client + .fees(starting_block_height..=current_block_height) + .await + .into_iter() + .collect::>(); + + let fee_lookup = data + .iter() + .map(|b| (b.height, b.fees)) + .collect::>(); + + let short_sma = 25u64; + let long_sma = 900; + + let current_tx_fees = data + .iter() + .map(|b| (b.height, calculate_tx_fee(&b.fees))) + .collect::>(); + + save_tx_fees(¤t_tx_fees, "current_fees.csv"); + + let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + let fee_analytics = FeeAnalytics::new(local_client.clone()); + + let mut short_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - short_sma..=height) + .await; + + let tx_fee = calculate_tx_fee(&fees); + + short_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + + let decider = SendOrWaitDecider::new( + FeeAnalytics::new(local_client.clone()), + services::state_committer::fee_optimization::Config { + sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + short: short_sma, + long: long_sma, + }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 43200 * 3, + start_discount_percentage: 0.2, + end_premium_percentage: 0.2, + always_acceptable_fee: 1000000000000000u128, + }, + }, + ); + + let mut decisions = vec![]; + let mut long_sma_tx_fees = vec![]; + + for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - long_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + long_sma_tx_fees.push((height, tx_fee)); + + if decider + .should_send_blob_tx( + 6, + Context { + at_l1_height: height, + num_l2_blocks_behind: (height - starting_block_height) * 12, + }, + ) + .await + { + let current_fees = fee_lookup.get(&height).unwrap(); + let current_tx_fee = calculate_tx_fee(current_fees); + decisions.push((height, current_tx_fee)); + } + } + + save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + save_tx_fees(&decisions, "decisions.csv"); +} From 6d74ce3fe597ff968f71ad3ef40ca353926a811d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 19:23:16 +0100 Subject: [PATCH 017/136] committer compiles, unit tests fixed --- committer/src/main.rs | 4 + committer/src/setup.rs | 3 + packages/services/src/fee_analytics.rs | 89 ++++----- packages/services/src/state_committer.rs | 15 +- .../src/state_committer/fee_optimization.rs | 9 +- packages/services/tests/fee_analytics.rs | 178 +++++++++--------- packages/services/tests/state_committer.rs | 8 + packages/services/tests/state_listener.rs | 3 + packages/test-helpers/src/lib.rs | 4 + 9 files changed, 176 insertions(+), 137 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 0473d0c7..3e1714f4 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,6 +7,7 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; +use services::fee_analytics; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; @@ -72,12 +73,15 @@ async fn main() -> Result<()> { &metrics_registry, ); + let fee_analytics = fee_analytics::service::FeeAnalytics::new(ethereum_rpc.clone()); + let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, + fee_analytics, ); let state_importer_handle = diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 3101e8fe..96cec1c4 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,6 +9,7 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, + fee_analytics::{self, service::FeeAnalytics}, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -117,6 +118,7 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, + fee_analytics: FeeAnalytics, ) -> tokio::task::JoinHandle<()> { let state_committer = services::StateCommitter::new( l1, @@ -130,6 +132,7 @@ pub fn state_committer( tx_max_fee: config.app.tx_max_fee as u128, }, SystemClock, + fee_analytics, ); schedule_polling( diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 1894cae5..f1e88a8a 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -149,55 +149,55 @@ pub mod port { } } } +} - pub mod service { +pub mod service { - use std::ops::RangeInclusive; + use std::ops::RangeInclusive; - use nonempty::NonEmpty; + use nonempty::NonEmpty; - use super::{ - l1::{FeesProvider, SequentialBlockFees}, - BlockFees, Fees, - }; + use super::port::{ + l1::{FeesProvider, SequentialBlockFees}, + Fees, + }; - pub struct FeeAnalytics

{ - fees_provider: P, - } - impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } + pub struct FeeAnalytics

{ + fees_provider: P, + } + impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } } + } - impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { - let fees = self.fees_provider.fees(block_range).await; + impl FeeAnalytics

{ + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { + let fees = self.fees_provider.fees(block_range).await; - Self::mean(fees) - } + Self::mean(fees) + } - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), } } } @@ -205,6 +205,7 @@ pub mod port { #[cfg(test)] mod tests { + use itertools::Itertools; use port::{l1::SequentialBlockFees, BlockFees, Fees}; use super::*; @@ -294,7 +295,7 @@ mod tests { ); assert_eq!( result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height\")" + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" ); } @@ -328,8 +329,12 @@ mod tests { let iterated_fees: Vec = sequential_fees.into_iter().collect(); // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); assert_eq!( - iterated_fees, block_fees, + iterated_fees, expectation, "Expected iterator to yield the same block fees" ); } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5bdfd3e5..ad86c1de 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -4,6 +4,7 @@ pub mod service { use std::{num::NonZeroUsize, time::Duration}; use crate::{ + fee_analytics::service::FeeAnalytics, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, }; @@ -35,16 +36,17 @@ pub mod service { } /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, + fee_analytics: FeeAnalytics, } - impl StateCommitter + impl StateCommitter where Clock: crate::state_committer::port::Clock, { @@ -55,6 +57,7 @@ pub mod service { storage: Db, config: Config, clock: Clock, + fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); Self { @@ -64,16 +67,18 @@ pub mod service { config, clock, startup_time, + fee_analytics, } } } - impl StateCommitter + impl StateCommitter where L1: crate::state_committer::port::l1::Api, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, + FeeProvider: crate::fee_analytics::port::l1::FeesProvider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -234,12 +239,14 @@ pub mod service { } } - impl Runner for StateCommitter + impl Runner + for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, + FeeProvider: crate::fee_analytics::port::l1::FeesProvider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 54a91824..19859ac4 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,6 +1,9 @@ use std::cmp::min; -use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; +use crate::fee_analytics::{ + port::{l1::FeesProvider, Fees}, + service::FeeAnalytics, +}; #[derive(Debug, Clone, Copy)] pub struct SmaBlockNumPeriods { @@ -46,6 +49,8 @@ pub struct Context { impl SendOrWaitDecider

{ // TODO: segfault validate blob number + // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes + // (once that is implemented) pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; @@ -479,7 +484,7 @@ mod tests { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, - always_acceptable_fee: 26_000_000_000, + always_acceptable_fee: 1_781_000_000_000 }, }, 0, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index f8c00e0a..c8ddec6a 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -6,9 +6,9 @@ use services::{ self, port::{ l1::testing::{self, TestFeesProvider}, - service::FeeAnalytics, BlockFees, Fees, }, + service::FeeAnalytics, }, state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, }; @@ -64,91 +64,91 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -#[tokio::test] -async fn something() { - let client = make_pub_eth_client().await; - use services::fee_analytics::port::l1::FeesProvider; - - let current_block_height = 21408300; - let starting_block_height = current_block_height - 48 * 3600 / 12; - let data = client - .fees(starting_block_height..=current_block_height) - .await - .into_iter() - .collect::>(); - - let fee_lookup = data - .iter() - .map(|b| (b.height, b.fees)) - .collect::>(); - - let short_sma = 25u64; - let long_sma = 900; - - let current_tx_fees = data - .iter() - .map(|b| (b.height, calculate_tx_fee(&b.fees))) - .collect::>(); - - save_tx_fees(¤t_tx_fees, "current_fees.csv"); - - let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - let fee_analytics = FeeAnalytics::new(local_client.clone()); - - let mut short_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - short_sma..=height) - .await; - - let tx_fee = calculate_tx_fee(&fees); - - short_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - - let decider = SendOrWaitDecider::new( - FeeAnalytics::new(local_client.clone()), - services::state_committer::fee_optimization::Config { - sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { - short: short_sma, - long: long_sma, - }, - fee_thresholds: Feethresholds { - max_l2_blocks_behind: 43200 * 3, - start_discount_percentage: 0.2, - end_premium_percentage: 0.2, - always_acceptable_fee: 1000000000000000u128, - }, - }, - ); - - let mut decisions = vec![]; - let mut long_sma_tx_fees = vec![]; - - for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - long_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - long_sma_tx_fees.push((height, tx_fee)); - - if decider - .should_send_blob_tx( - 6, - Context { - at_l1_height: height, - num_l2_blocks_behind: (height - starting_block_height) * 12, - }, - ) - .await - { - let current_fees = fee_lookup.get(&height).unwrap(); - let current_tx_fee = calculate_tx_fee(current_fees); - decisions.push((height, current_tx_fee)); - } - } - - save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - save_tx_fees(&decisions, "decisions.csv"); -} +// #[tokio::test] +// async fn something() { +// let client = make_pub_eth_client().await; +// use services::fee_analytics::port::l1::FeesProvider; +// +// let current_block_height = 21408300; +// let starting_block_height = current_block_height - 48 * 3600 / 12; +// let data = client +// .fees(starting_block_height..=current_block_height) +// .await +// .into_iter() +// .collect::>(); +// +// let fee_lookup = data +// .iter() +// .map(|b| (b.height, b.fees)) +// .collect::>(); +// +// let short_sma = 25u64; +// let long_sma = 900; +// +// let current_tx_fees = data +// .iter() +// .map(|b| (b.height, calculate_tx_fee(&b.fees))) +// .collect::>(); +// +// save_tx_fees(¤t_tx_fees, "current_fees.csv"); +// +// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); +// let fee_analytics = FeeAnalytics::new(local_client.clone()); +// +// let mut short_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - short_sma..=height) +// .await; +// +// let tx_fee = calculate_tx_fee(&fees); +// +// short_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); +// +// let decider = SendOrWaitDecider::new( +// FeeAnalytics::new(local_client.clone()), +// services::state_committer::fee_optimization::Config { +// sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { +// short: short_sma, +// long: long_sma, +// }, +// fee_thresholds: Feethresholds { +// max_l2_blocks_behind: 43200 * 3, +// start_discount_percentage: 0.2, +// end_premium_percentage: 0.2, +// always_acceptable_fee: 1000000000000000u128, +// }, +// }, +// ); +// +// let mut decisions = vec![]; +// let mut long_sma_tx_fees = vec![]; +// +// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - long_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// long_sma_tx_fees.push((height, tx_fee)); +// +// if decider +// .should_send_blob_tx( +// 6, +// Context { +// at_l1_height: height, +// num_l2_blocks_behind: (height - starting_block_height) * 12, +// }, +// ) +// .await +// { +// let current_fees = fee_lookup.get(&height).unwrap(); +// let current_tx_fee = calculate_tx_fee(current_fees); +// decisions.push((height, current_tx_fee)); +// } +// } +// +// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); +// save_tx_fees(&decisions, "decisions.csv"); +// } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 831dbeb5..26acb9d1 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,4 +1,5 @@ use services::{ + fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -33,6 +34,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // when @@ -73,6 +75,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time beyond the timeout @@ -108,6 +111,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time less than the timeout @@ -150,6 +154,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // when @@ -193,6 +198,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time to exceed the timeout since last finalized fragment @@ -236,6 +242,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time beyond the timeout from startup @@ -291,6 +298,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Submit the initial fragments diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index b60460cc..35f02e65 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,6 +3,7 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ + fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -447,6 +448,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Orig tx @@ -544,6 +546,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index f73fc475..52788924 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,6 +8,8 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; +use services::fee_analytics::port::l1::testing::TestFeesProvider; +use services::fee_analytics::service::FeeAnalytics; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -550,6 +552,7 @@ impl Setup { tx_max_fee: 1_000_000_000, }, self.test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ) .run() .await @@ -584,6 +587,7 @@ impl Setup { tx_max_fee: 1_000_000_000, }, self.test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); committer.run().await.unwrap(); From f7b44cfd8a18d51428dc37441fc76ce776293ccc Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:32:05 +0100 Subject: [PATCH 018/136] tracks fragment age, tests fixed --- ...e42fee96b0925eb2c30a0b98cf9f79c6ed76.json} | 10 +++- ...f99a2bfb27dd1e51d0f877f939e29b7f3a52.json} | 10 +++- committer/src/setup.rs | 2 +- packages/adapters/eth/src/lib.rs | 4 ++ .../adapters/storage/src/mappings/tables.rs | 13 ++++- packages/adapters/storage/src/postgres.rs | 16 +++--- packages/services/src/fee_analytics.rs | 31 +++++++++-- packages/services/src/state_committer.rs | 50 ++++++++++++++++-- .../src/state_committer/fee_optimization.rs | 44 ++++++++-------- packages/services/src/types/storage.rs | 1 + packages/services/tests/fee_analytics.rs | 8 +-- packages/services/tests/state_committer.rs | 52 ++++++++++++++----- packages/services/tests/state_listener.rs | 28 ++++++++-- packages/test-helpers/src/lib.rs | 38 +++++++++----- 14 files changed, 227 insertions(+), 80 deletions(-) rename .sqlx/{query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json => query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json} (58%) rename .sqlx/{query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json => query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json} (51%) diff --git a/.sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json b/.sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json similarity index 58% rename from .sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json rename to .sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json index 0b8b6451..86a298a0 100644 --- a/.sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json +++ b/.sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT f.*\n FROM l1_fragments f\n JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE t.hash = $1\n ", + "query": "\n SELECT\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n JOIN bundles b ON b.id = f.bundle_id\n WHERE t.hash = $1\n ", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "bundle_id", "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "start_height", + "type_info": "Int8" } ], "parameters": { @@ -45,8 +50,9 @@ false, false, false, + false, false ] }, - "hash": "126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147" + "hash": "ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76" } diff --git a/.sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json b/.sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json similarity index 51% rename from .sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json rename to .sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json index 2fd79840..9fe76b57 100644 --- a/.sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json +++ b/.sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n sub.id,\n sub.idx,\n sub.bundle_id,\n sub.data,\n sub.unused_bytes,\n sub.total_bytes\n FROM (\n SELECT DISTINCT ON (f.id)\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN bundles b ON b.id = f.bundle_id\n WHERE\n b.end_height >= $2\n AND NOT EXISTS (\n SELECT 1\n FROM l1_transaction_fragments tf\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE tf.fragment_id = f.id\n AND t.state <> $1\n )\n ORDER BY\n f.id,\n b.start_height ASC,\n f.idx ASC\n ) AS sub\n ORDER BY\n sub.start_height ASC,\n sub.idx ASC\n LIMIT $3;\n", + "query": "SELECT\n sub.id,\n sub.idx,\n sub.bundle_id,\n sub.data,\n sub.unused_bytes,\n sub.total_bytes,\n sub.start_height\n FROM (\n SELECT DISTINCT ON (f.id)\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN bundles b ON b.id = f.bundle_id\n WHERE\n b.end_height >= $2\n AND NOT EXISTS (\n SELECT 1\n FROM l1_transaction_fragments tf\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE tf.fragment_id = f.id\n AND t.state <> $1\n )\n ORDER BY\n f.id,\n b.start_height ASC,\n f.idx ASC\n ) AS sub\n ORDER BY\n sub.start_height ASC,\n sub.idx ASC\n LIMIT $3;\n", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "total_bytes", "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "start_height", + "type_info": "Int8" } ], "parameters": { @@ -47,8 +52,9 @@ false, false, false, + false, false ] }, - "hash": "11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40" + "hash": "ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52" } diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 96cec1c4..209d049f 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -129,7 +129,7 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - tx_max_fee: config.app.tx_max_fee as u128, + price_algo: todo!(), }, SystemClock, fee_analytics, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 17cdc6e4..f5113eec 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -206,6 +206,10 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } impl services::state_committer::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + delegate! { to (*self) { async fn submit_state_fragments( diff --git a/packages/adapters/storage/src/mappings/tables.rs b/packages/adapters/storage/src/mappings/tables.rs index 9729f382..566e68ae 100644 --- a/packages/adapters/storage/src/mappings/tables.rs +++ b/packages/adapters/storage/src/mappings/tables.rs @@ -1,4 +1,4 @@ -use std::num::NonZeroU32; +use std::{i64, num::NonZeroU32}; use num_bigint::BigInt; use services::types::{ @@ -193,6 +193,7 @@ pub struct BundleFragment { pub data: Vec, pub unused_bytes: i64, pub total_bytes: i64, + pub start_height: i64, } impl TryFrom for services::types::storage::BundleFragment { @@ -261,10 +262,18 @@ impl TryFrom for services::types::storage::BundleFragment { total_bytes, }; + let start_height = value.start_height.try_into().map_err(|e| { + crate::error::Error::Conversion(format!( + "Invalid db `start_height` ({}). Reason: {e}", + value.start_height + )) + })?; + Ok(Self { id, - idx, bundle_id, + idx, + oldest_block_in_bundle: start_height, fragment, }) } diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index 9d09a945..b836e4d9 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -277,7 +277,8 @@ impl Postgres { sub.bundle_id, sub.data, sub.unused_bytes, - sub.total_bytes + sub.total_bytes, + sub.start_height FROM ( SELECT DISTINCT ON (f.id) f.*, @@ -323,11 +324,14 @@ impl Postgres { let fragments = sqlx::query_as!( tables::BundleFragment, r#" - SELECT f.* - FROM l1_fragments f - JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id - JOIN l1_blob_transaction t ON t.id = tf.transaction_id - WHERE t.hash = $1 + SELECT + f.*, + b.start_height + FROM l1_fragments f + JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id + JOIN l1_blob_transaction t ON t.id = tf.transaction_id + JOIN bundles b ON b.id = f.bundle_id + WHERE t.hash = $1 "#, tx_hash.as_slice() ) diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index f1e88a8a..79bef51f 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -99,12 +99,37 @@ pub mod port { use super::{FeesProvider, SequentialBlockFees}; + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeesProvider { + fees: Fees, + } + + impl ConstantFeesProvider { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl FeesProvider for ConstantFeesProvider { + async fn fees(&self, _height_range: RangeInclusive) -> SequentialBlockFees { + let fees = BlockFees { + height: self.current_block_height().await, + fees: self.fees, + }; + + vec![fees].try_into().unwrap() + } + async fn current_block_height(&self) -> u64 { + 0 + } + } + #[derive(Debug, Clone)] - pub struct TestFeesProvider { + pub struct PreconfiguredFeesProvider { fees: BTreeMap, } - impl FeesProvider for TestFeesProvider { + impl FeesProvider for PreconfiguredFeesProvider { async fn current_block_height(&self) -> u64 { *self.fees.keys().last().unwrap() } @@ -125,7 +150,7 @@ pub mod port { } } - impl TestFeesProvider { + impl PreconfiguredFeesProvider { pub fn new(blocks: impl IntoIterator) -> Self { Self { fees: blocks.into_iter().collect(), diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index ad86c1de..569ceacc 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -5,12 +5,15 @@ pub mod service { use crate::{ fee_analytics::service::FeeAnalytics, + state_committer::fee_optimization::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, + Error, Result, Runner, }; use itertools::Itertools; use tracing::info; + use super::fee_optimization::{FeeThresholds, SendOrWaitDecider, SmaBlockNumPeriods}; + // src/config.rs #[derive(Debug, Clone)] pub struct Config { @@ -19,7 +22,7 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub tx_max_fee: u128, + pub price_algo: crate::state_committer::fee_optimization::Config, } #[cfg(feature = "test-helpers")] @@ -30,7 +33,15 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + price_algo: crate::state_committer::fee_optimization::Config { + sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0., + end_premium_percentage: 0., + always_acceptable_fee: u128::MAX, + }, + }, } } } @@ -43,7 +54,7 @@ pub mod service { config: Config, clock: Clock, startup_time: DateTime, - fee_analytics: FeeAnalytics, + decider: SendOrWaitDecider, } impl StateCommitter @@ -60,6 +71,7 @@ pub mod service { fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); + let price_algo = config.price_algo; Self { l1_adapter, fuel_api, @@ -67,7 +79,7 @@ pub mod service { config, clock, startup_time, - fee_analytics, + decider: SendOrWaitDecider::new(fee_analytics, price_algo), } } } @@ -104,6 +116,33 @@ pub mod service { ) -> Result<()> { info!("about to send at most {} fragments", fragments.len()); + // TODO: segfault proper type conversion + let l1_height = self.l1_adapter.current_height().await?; + // TODO: segfault test this + let l2_height = self.fuel_api.latest_height().await?; + + let oldest_l2_block_in_fragments = fragments + .maximum_by_key(|b| b.oldest_block_in_bundle) + .oldest_block_in_bundle; + + let behind_on_l2 = l2_height.saturating_sub(oldest_l2_block_in_fragments); + + let should_send = self + .decider + .should_send_blob_tx( + fragments.len() as u32, + Context { + num_l2_blocks_behind: behind_on_l2 as u64, + at_l1_height: l1_height, + }, + ) + .await; + + if !should_send { + // TODO: segfault log here + return Ok(()); + } + let data = fragments.clone().map(|f| f.fragment); match self @@ -287,6 +326,7 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait Api { + async fn current_height(&self) -> Result; async fn submit_state_fragments( &self, fragments: NonEmpty, diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 19859ac4..cfb51eb7 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -13,7 +13,7 @@ pub struct SmaBlockNumPeriods { // TODO: segfault validate start discount is less than end premium and both are positive #[derive(Debug, Clone, Copy)] -pub struct Feethresholds { +pub struct FeeThresholds { // TODO: segfault validate not 0 pub max_l2_blocks_behind: u64, pub start_discount_percentage: f64, @@ -24,7 +24,7 @@ pub struct Feethresholds { #[derive(Debug, Clone, Copy)] pub struct Config { pub sma_periods: SmaBlockNumPeriods, - pub fee_thresholds: Feethresholds, + pub fee_thresholds: FeeThresholds, } pub struct SendOrWaitDecider

{ @@ -158,7 +158,7 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; + use crate::fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}; use test_case::test_case; use tokio; @@ -182,7 +182,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -199,7 +199,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -216,7 +216,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100, start_discount_percentage: 0.0, @@ -233,7 +233,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -250,7 +250,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -267,7 +267,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -284,7 +284,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -301,7 +301,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -318,7 +318,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -336,7 +336,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -353,7 +353,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -371,7 +371,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -389,7 +389,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -407,7 +407,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -426,7 +426,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -444,7 +444,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -462,7 +462,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -480,7 +480,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -501,7 +501,7 @@ mod tests { expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); - let fees_provider = TestFeesProvider::new(fees); + let fees_provider = PreconfiguredFeesProvider::new(fees); let current_block_height = fees_provider.current_block_height().await; let analytics_service = FeeAnalytics::new(fees_provider); diff --git a/packages/services/src/types/storage.rs b/packages/services/src/types/storage.rs index 3f93e8ca..368c24f3 100644 --- a/packages/services/src/types/storage.rs +++ b/packages/services/src/types/storage.rs @@ -16,6 +16,7 @@ pub struct BundleFragment { pub id: NonNegative, pub idx: NonNegative, pub bundle_id: NonNegative, + pub oldest_block_in_bundle: u32, pub fragment: Fragment, } diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index c8ddec6a..7435f329 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -5,18 +5,18 @@ use services::{ fee_analytics::{ self, port::{ - l1::testing::{self, TestFeesProvider}, + l1::testing::{self, PreconfiguredFeesProvider}, BlockFees, Fees, }, service::FeeAnalytics, }, - state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, + state_committer::fee_optimization::{Context, SendOrWaitDecider}, }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given - let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 1; @@ -32,7 +32,7 @@ async fn calculates_sma_correctly_for_last_1_block() { #[tokio::test] async fn calculates_sma_correctly_for_last_5_blocks() { // given - let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 5; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 26acb9d1..8d17a148 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,5 +1,9 @@ use services::{ - fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, + fee_analytics::{ + port::{l1::testing::ConstantFeesProvider, Fees}, + service::FeeAnalytics, + }, + state_committer::port::l1::Api, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -13,7 +17,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fragments = setup.insert_fragments(0, 4).await; let tx_hash = [0; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -21,6 +25,9 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -34,7 +41,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -54,7 +61,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { let fragments = setup.insert_fragments(0, 5).await; // Only 5 fragments, less than required let tx_hash = [1; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -63,6 +70,9 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( l1_mock_submit, @@ -75,7 +85,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout @@ -111,7 +121,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time less than the timeout @@ -133,7 +143,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fragments = setup.insert_fragments(0, 5).await; let tx_hash = [3; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments).unwrap()), L1Tx { hash: tx_hash, @@ -141,6 +151,9 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -154,7 +167,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -177,7 +190,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fragments_to_submit = setup.insert_fragments(1, 2).await; let tx_hash = [4; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments_to_submit).unwrap()), L1Tx { hash: tx_hash, @@ -185,6 +198,9 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(1) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( @@ -198,7 +214,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time to exceed the timeout since last finalized fragment @@ -221,7 +237,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> let fragments = setup.insert_fragments(0, 5).await; // Only 5 fragments, less than required let tx_hash = [5; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -231,6 +247,9 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> )]); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( l1_mock_submit, fuel_mock, @@ -242,7 +261,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout from startup @@ -266,7 +285,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let tx_hash_1 = [6; 32]; let tx_hash_2 = [7; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([ + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([ ( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { @@ -285,6 +304,11 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ), ]); + l1_mock_submit.expect_current_height().returning(|| { + eprintln!("I was called"); + Box::pin(async { Ok(0) }) + }); + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( l1_mock_submit, @@ -298,7 +322,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Submit the initial fragments diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 35f02e65..9ddfe409 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,7 +3,13 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, + fee_analytics::{ + port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, + service::FeeAnalytics, + }, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -439,8 +445,14 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { nonce, ..Default::default() }; + let mut l1_mock = + mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); + let mut committer = StateCommitter::new( - mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -448,7 +460,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx @@ -537,8 +549,14 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }; + let mut l1_mock = + mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); + let mut committer = StateCommitter::new( - mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -546,7 +564,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 52788924..34538031 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,8 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::TestFeesProvider; +use services::fee_analytics::port::l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}; +use services::fee_analytics::port::Fees; use services::fee_analytics::service::FeeAnalytics; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, @@ -533,15 +534,20 @@ impl Setup { } pub async fn send_fragments(&self, eth_tx: [u8; 32], eth_nonce: u32) { + let mut l1_mock = mocks::l1::expects_state_submissions(vec![( + None, + L1Tx { + hash: eth_tx, + nonce: eth_nonce, + ..Default::default() + }, + )]); + l1_mock + .expect_current_height() + .return_once(move || Box::pin(async { Ok(0) })); + StateCommitter::new( - mocks::l1::expects_state_submissions(vec![( - None, - L1Tx { - hash: eth_tx, - nonce: eth_nonce, - ..Default::default() - }, - )]), + l1_mock, mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -549,10 +555,10 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ) .run() .await @@ -566,7 +572,7 @@ impl Setup { pub async fn commit_block_bundle(&self, eth_tx: [u8; 32], eth_nonce: u32, height: u32) { self.insert_fragments(height, 6).await; - let l1_mock = mocks::l1::expects_state_submissions(vec![( + let mut l1_mock = mocks::l1::expects_state_submissions(vec![( None, L1Tx { hash: eth_tx, @@ -574,6 +580,10 @@ impl Setup { ..Default::default() }, )]); + l1_mock + .expect_current_height() + .return_once(move || Box::pin(async { Ok(0) })); + let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( l1_mock, @@ -584,10 +594,10 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); committer.run().await.unwrap(); From ac72a02237bfd2593513e8438924c9d2bc5e893a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:41:32 +0100 Subject: [PATCH 019/136] pull up the config to state committer level so it can be exported out without revealing the fee algo --- committer/src/setup.rs | 4 +- packages/adapters/eth/src/lib.rs | 12 +- packages/services/src/fee_analytics.rs | 11 +- packages/services/src/state_committer.rs | 42 +++++-- .../{fee_optimization.rs => fee_algo.rs} | 111 ++++++++---------- packages/services/tests/fee_analytics.rs | 14 +-- packages/services/tests/state_committer.rs | 1 - packages/services/tests/state_listener.rs | 2 +- packages/test-helpers/src/lib.rs | 2 +- 9 files changed, 96 insertions(+), 103 deletions(-) rename packages/services/src/state_committer/{fee_optimization.rs => fee_algo.rs} (89%) diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 209d049f..f493f227 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,7 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_analytics::{self, service::FeeAnalytics}, + fee_analytics::service::FeeAnalytics, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -129,7 +129,7 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - price_algo: todo!(), + fee_algo: todo!(), }, SystemClock, fee_analytics, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index f5113eec..fa609e1e 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -9,10 +9,9 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, - providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE, }; use delegate::delegate; -use itertools::{izip, zip, Itertools}; +use itertools::{izip, Itertools}; use services::{ fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, types::{ @@ -311,16 +310,13 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { #[cfg(test)] mod test { - use std::time::Duration; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::{ - block_bundler::port::l1::FragmentEncoder, block_committer::port::l1::Api, - fee_analytics::port::l1::FeesProvider, - }; + use services::block_bundler::port::l1::FragmentEncoder; - use crate::{BlobEncoder, Signer, Signers}; + use crate::BlobEncoder; #[test] fn gas_usage_correctly_calculated() { diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 79bef51f..206896f5 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -16,7 +16,7 @@ pub mod port { use std::ops::RangeInclusive; use itertools::Itertools; - use nonempty::NonEmpty; + use super::BlockFees; @@ -90,12 +90,9 @@ pub mod port { use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; - use nonempty::NonEmpty; + - use crate::{ - fee_analytics::port::{BlockFees, Fees}, - types::CollectNonEmpty, - }; + use crate::fee_analytics::port::{BlockFees, Fees}; use super::{FeesProvider, SequentialBlockFees}; @@ -180,7 +177,7 @@ pub mod service { use std::ops::RangeInclusive; - use nonempty::NonEmpty; + use super::port::{ l1::{FeesProvider, SequentialBlockFees}, diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 569ceacc..48026c42 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,18 +1,42 @@ -pub mod fee_optimization; +mod fee_algo; pub mod service { - use std::{num::NonZeroUsize, time::Duration}; + use std::{ + num::{NonZeroU64, NonZeroUsize}, + time::Duration, + }; use crate::{ fee_analytics::service::FeeAnalytics, - state_committer::fee_optimization::Context, + state_committer::fee_algo::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, + Result, Runner, }; use itertools::Itertools; use tracing::info; - use super::fee_optimization::{FeeThresholds, SendOrWaitDecider, SmaBlockNumPeriods}; + use super::fee_algo::SendOrWaitDecider; + + #[derive(Debug, Clone, Copy)] + pub struct SmaBlockNumPeriods { + pub short: u64, + pub long: u64, + } + + // TODO: segfault validate start discount is less than end premium and both are positive + #[derive(Debug, Clone, Copy)] + pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU64, + pub start_discount_percentage: f64, + pub end_premium_percentage: f64, + pub always_acceptable_fee: u128, + } + + #[derive(Debug, Clone, Copy)] + pub struct FeeAlgoConfig { + pub sma_periods: SmaBlockNumPeriods, + pub fee_thresholds: FeeThresholds, + } // src/config.rs #[derive(Debug, Clone)] @@ -22,7 +46,7 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub price_algo: crate::state_committer::fee_optimization::Config, + pub fee_algo: FeeAlgoConfig, } #[cfg(feature = "test-helpers")] @@ -33,10 +57,10 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - price_algo: crate::state_committer::fee_optimization::Config { + fee_algo: FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0., end_premium_percentage: 0., always_acceptable_fee: u128::MAX, @@ -71,7 +95,7 @@ pub mod service { fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); - let price_algo = config.price_algo; + let price_algo = config.fee_algo; Self { l1_adapter, fuel_api, diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_algo.rs similarity index 89% rename from packages/services/src/state_committer/fee_optimization.rs rename to packages/services/src/state_committer/fee_algo.rs index cfb51eb7..bb2a5b37 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -5,35 +5,15 @@ use crate::fee_analytics::{ service::FeeAnalytics, }; -#[derive(Debug, Clone, Copy)] -pub struct SmaBlockNumPeriods { - pub short: u64, - pub long: u64, -} - -// TODO: segfault validate start discount is less than end premium and both are positive -#[derive(Debug, Clone, Copy)] -pub struct FeeThresholds { - // TODO: segfault validate not 0 - pub max_l2_blocks_behind: u64, - pub start_discount_percentage: f64, - pub end_premium_percentage: f64, - pub always_acceptable_fee: u128, -} - -#[derive(Debug, Clone, Copy)] -pub struct Config { - pub sma_periods: SmaBlockNumPeriods, - pub fee_thresholds: FeeThresholds, -} +use super::service::FeeAlgoConfig; pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, - config: Config, + config: FeeAlgoConfig, } impl

SendOrWaitDecider

{ - pub fn new(fee_analytics: FeeAnalytics

, config: Config) -> Self { + pub fn new(fee_analytics: FeeAnalytics

, config: FeeAlgoConfig) -> Self { Self { fee_analytics, config, @@ -72,7 +52,7 @@ impl SendOrWaitDecider

{ // TODO: segfault test this let too_far_behind = - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); eprintln!("too far behind: {}", too_far_behind); @@ -122,7 +102,7 @@ impl SendOrWaitDecider

{ let end_premium_ppm = (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; - let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind as u128; + let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind.get() as u128; let blocks_behind = context.num_l2_blocks_behind; @@ -158,11 +138,14 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}; + use crate::{ + fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, + state_committer::service::{FeeThresholds, SmaBlockNumPeriods}, + }; use test_case::test_case; use tokio; - fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + fn generate_fees(config: FeeAlgoConfig, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, (config.sma_periods.long - config.sma_periods.short) as usize, @@ -180,10 +163,10 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -197,10 +180,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -214,11 +197,11 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, } @@ -231,10 +214,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -248,10 +231,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -265,10 +248,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -282,10 +265,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -299,10 +282,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -316,10 +299,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -334,10 +317,10 @@ mod tests { Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -351,10 +334,10 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -369,10 +352,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -387,10 +370,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -405,10 +388,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -424,10 +407,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -442,10 +425,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -460,10 +443,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -478,10 +461,10 @@ mod tests { Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 1_781_000_000_000 @@ -496,7 +479,7 @@ mod tests { old_fees: Fees, new_fees: Fees, num_blobs: u32, - config: Config, + config: FeeAlgoConfig, num_l2_blocks_behind: u64, expected_decision: bool, ) { diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 7435f329..39513e98 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,17 +1,11 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::path::PathBuf; -use eth::make_pub_eth_client; -use services::{ - fee_analytics::{ - self, +use services::fee_analytics::{ port::{ - l1::testing::{self, PreconfiguredFeesProvider}, - BlockFees, Fees, + l1::testing::{self}, Fees, }, service::FeeAnalytics, - }, - state_committer::fee_optimization::{Context, SendOrWaitDecider}, -}; + }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 8d17a148..5647c481 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -3,7 +3,6 @@ use services::{ port::{l1::testing::ConstantFeesProvider, Fees}, service::FeeAnalytics, }, - state_committer::port::l1::Api, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 9ddfe409..b62108bf 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -5,7 +5,7 @@ use mockall::predicate::eq; use services::{ fee_analytics::{ port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + l1::testing::ConstantFeesProvider, Fees, }, service::FeeAnalytics, diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 34538031..ac357e66 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}; +use services::fee_analytics::port::l1::testing::ConstantFeesProvider; use services::fee_analytics::port::Fees; use services::fee_analytics::service::FeeAnalytics; use services::types::{ From b8d66319fc6d29205ea6d8288ed7cadbaaa81b72 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:56:27 +0100 Subject: [PATCH 020/136] added tests to state committer showing that the price algo is consulted --- packages/services/tests/state_committer.rs | 441 ++++++++++++++++++++- 1 file changed, 439 insertions(+), 2 deletions(-) diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 5647c481..d29649a1 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,16 @@ use services::{ fee_analytics::{ - port::{l1::testing::ConstantFeesProvider, Fees}, + port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, service::FeeAnalytics, }, + state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaBlockNumPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use std::time::Duration; +use std::{num::NonZeroU64, time::Duration}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -338,3 +342,436 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { // Mocks validate that the fragments have been sent again Ok(()) } + +#[tokio::test] +async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { + // Given + let setup = test_helpers::Setup::init().await; + + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // When + state_committer.run().await?; + + // Then + // Mocks validate that the fragments have been sent + Ok(()) +} + +#[tokio::test] +async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + // Define fee sequence: last 2 blocks have higher fees than the long-term average + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let _fragments = setup.insert_fragments(0, 6).await; + + let mut l1_mock = test_helpers::mocks::l1::expects_state_submissions([]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); + let mut state_committer = StateCommitter::new( + l1_mock, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that no fragments have been sent + Ok(()) +} + +#[tokio::test] +async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + // Define fee sequence with high fees to ensure that without the behind condition, it wouldn't send + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission despite high fees because blocks behind exceed max + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that the fragments have been sent despite high fees + Ok(()) +} + +#[tokio::test] +async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_tolerance() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + let fee_sequence = vec![ + ( + 95, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 96, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 97, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 98, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 99, + Fees { + base_fee_per_gas: 5800, + reward: 5800, + base_fee_per_blob_gas: 5800, + }, + ), + ( + 100, + Fees { + base_fee_per_gas: 5800, + reward: 5800, + base_fee_per_blob_gas: 5800, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, + }; + + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission due to nearing max blocks behind and increased tolerance + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(100) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); + + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind + Ok(()) +} From fb6c016bdca0956d8c01dc829491e5d9ba4bc2b7 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 18:45:58 +0100 Subject: [PATCH 021/136] all tests passing, including e2e --- committer/src/config.rs | 26 ++++++++++++- committer/src/setup.rs | 18 ++++++++- e2e/src/committer.rs | 10 +++++ e2e/src/whole_stack.rs | 2 +- .../adapters/eth/src/websocket/connection.rs | 4 +- packages/services/src/state_committer.rs | 6 +-- .../services/src/state_committer/fee_algo.rs | 38 +++++++++---------- packages/services/tests/state_committer.rs | 10 ++--- run_tests.sh | 2 +- 9 files changed, 81 insertions(+), 35 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index fbc6cf6b..734e92a6 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, str::FromStr, time::Duration, }; @@ -111,6 +111,30 @@ pub struct App { /// How often to run state pruner #[serde(deserialize_with = "human_readable_duration")] pub state_pruner_run_interval: Duration, + /// Configuration for the fee algorithm used by the StateCommitter + pub fee_algo: FeeAlgoConfig, +} + +/// Configuration for the fee algorithm used by the StateCommitter +#[derive(Debug, Clone, Deserialize)] +pub struct FeeAlgoConfig { + /// Short-term period for Simple Moving Average (SMA) in block numbers + pub short_sma_blocks: u64, + + /// Long-term period for Simple Moving Average (SMA) in block numbers + pub long_sma_blocks: u64, + + /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees + pub max_l2_blocks_behind: NonZeroU64, + + /// Starting discount percentage applied we try to achieve if we're 0 l2 blocks behind + pub start_discount_percentage: f64, + + /// Premium percentage we're willing to pay if we're max_l2_blocks_behind - 1 blocks behind + pub end_premium_percentage: f64, + + /// A fee that is always acceptable regardless of other conditions + pub always_acceptable_fee: u64, } /// Configuration settings for managing fuel block bundling and fragment submission operations. diff --git a/committer/src/setup.rs b/committer/src/setup.rs index f493f227..b320ee40 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -10,7 +10,10 @@ use metrics::{ use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, fee_analytics::service::FeeAnalytics, - state_committer::port::Storage, + state_committer::{ + port::Storage, + service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, + }, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -129,7 +132,18 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - fee_algo: todo!(), + fee_algo: FeeAlgoConfig { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config.app.fee_algo.start_discount_percentage, + end_premium_percentage: config.app.fee_algo.end_premium_percentage, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, }, SystemClock, fee_analytics, diff --git a/e2e/src/committer.rs b/e2e/src/committer.rs index ada77606..4cdc21e2 100644 --- a/e2e/src/committer.rs +++ b/e2e/src/committer.rs @@ -121,6 +121,16 @@ impl Committer { "COMMITTER__APP__STATE_PRUNER_RUN_INTERVAL", get_field!(state_pruner_run_interval), ) + .env("COMMITTER__APP__FEE_ALGO__SHORT_SMA_BLOCKS", "1") + .env("COMMITTER__APP__FEE_ALGO__LONG_SMA_BLOCKS", "1") + .env("COMMITTER__APP__FEE_ALGO__MAX_L2_BLOCKS_BEHIND", "1") + .env("COMMITTER__APP__FEE_ALGO__START_DISCOUNT_PERCENTAGE", "0") + .env("COMMITTER__APP__FEE_ALGO__END_PREMIUM_PERCENTAGE", "0") + // we're basically disabling the fee algo here + .env( + "COMMITTER__APP__FEE_ALGO__ALWAYS_ACCEPTABLE_FEE", + u64::MAX.to_string(), + ) .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap()) .kill_on_drop(true); diff --git a/e2e/src/whole_stack.rs b/e2e/src/whole_stack.rs index 59e8e779..1a3e257d 100644 --- a/e2e/src/whole_stack.rs +++ b/e2e/src/whole_stack.rs @@ -60,7 +60,7 @@ impl WholeStack { let db = start_db().await?; let committer = start_committer( - logs, + true, blob_support, db.clone(), ð_node, diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index 65318b49..f6729d21 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -399,9 +399,7 @@ impl WsConnection { let contract_address = Address::from_slice(contract_address.as_ref()); let contract = FuelStateContract::new(contract_address, provider.clone()); - // TODO: segfault revert this - // let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; - let interval_u256 = 1u32; + let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; let commit_interval = u32::try_from(interval_u256) .map_err(|e| Error::Other(e.to_string())) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 48026c42..38387532 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -18,7 +18,7 @@ pub mod service { use super::fee_algo::SendOrWaitDecider; #[derive(Debug, Clone, Copy)] - pub struct SmaBlockNumPeriods { + pub struct SmaPeriods { pub short: u64, pub long: u64, } @@ -34,7 +34,7 @@ pub mod service { #[derive(Debug, Clone, Copy)] pub struct FeeAlgoConfig { - pub sma_periods: SmaBlockNumPeriods, + pub sma_periods: SmaPeriods, pub fee_thresholds: FeeThresholds, } @@ -58,7 +58,7 @@ pub mod service { fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), fee_algo: FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, + sma_periods: SmaPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0., diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index bb2a5b37..edc3186f 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -140,7 +140,7 @@ mod tests { use super::*; use crate::{ fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, - state_committer::service::{FeeThresholds, SmaBlockNumPeriods}, + state_committer::service::{FeeThresholds, SmaPeriods}, }; use test_case::test_case; use tokio; @@ -164,7 +164,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -181,7 +181,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -198,7 +198,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), @@ -215,7 +215,7 @@ mod tests { Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -232,7 +232,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -249,7 +249,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -266,7 +266,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -283,7 +283,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -300,7 +300,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -318,7 +318,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -335,7 +335,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -353,7 +353,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -371,7 +371,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -389,7 +389,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -408,7 +408,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -426,7 +426,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -444,7 +444,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -462,7 +462,7 @@ mod tests { Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index d29649a1..fcd50a98 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -6,7 +6,7 @@ use services::{ }, service::FeeAnalytics, }, - state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaBlockNumPeriods}, + state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -403,7 +403,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.0, @@ -514,7 +514,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.0, @@ -616,7 +616,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), start_discount_percentage: 0.0, @@ -726,7 +726,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.20, diff --git a/run_tests.sh b/run_tests.sh index 3280743e..9c074973 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,4 +9,4 @@ cargo test --manifest-path "$workspace_cargo_manifest" --workspace --exclude e2e # So that we may have a binary in `target/release` cargo build --release --manifest-path "$workspace_cargo_manifest" --bin fuel-block-committer -PATH="$script_location/target/release:$PATH" cargo test --manifest-path "$workspace_cargo_manifest" --package e2e -- --test-threads=1 +PATH="$script_location/target/release:$PATH" cargo test --manifest-path "$workspace_cargo_manifest" --package e2e -- --test-threads=1 --nocapture From c7aa8fa52cadcd1615241cb6e7e0b3cd7af22897 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 19:58:31 +0100 Subject: [PATCH 022/136] cleanup --- Cargo.lock | 1 + packages/services/Cargo.toml | 1 + .../services/src/state_committer/fee_algo.rs | 79 ++++++++++++++----- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 582429a1..84196586 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5796,6 +5796,7 @@ dependencies = [ "mockall", "nonempty", "pretty_assertions", + "proptest", "rand", "rayon", "serde", diff --git a/packages/services/Cargo.toml b/packages/services/Cargo.toml index 194ecc6a..644c6b78 100644 --- a/packages/services/Cargo.toml +++ b/packages/services/Cargo.toml @@ -47,6 +47,7 @@ tokio = { workspace = true, features = ["macros"] } test-helpers = { workspace = true } rand = { workspace = true, features = ["small_rng", "std", "std_rng"] } csv = "1.3" +proptest = { workspace = true, features = ["default"] } [features] test-helpers = ["dep:mockall", "dep:rand"] diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index edc3186f..9a3c1d40 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -94,33 +94,63 @@ impl SendOrWaitDecider

{ // TODO: segfault test this fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { - // Define percentages in Parts Per Million (PPM) for precision - // 1 PPM = 0.0001% - const PPM: u128 = 1_000_000; - let start_discount_ppm = - (self.config.fee_thresholds.start_discount_percentage * PPM as f64) as u128; - let end_premium_ppm = - (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; + const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind.get() as u128; + let max_blocks_behind = u128::from(self.config.fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(context.num_l2_blocks_behind); - let blocks_behind = context.num_l2_blocks_behind; + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({})", + blocks_behind, + max_blocks_behind + ); - // TODO: segfault rename possibly - let ratio_ppm = (blocks_behind as u128 * PPM) / max_blocks_behind; + let start_discount_ppm = + percentage_to_ppm(self.config.fee_thresholds.start_discount_percentage); + let end_premium_ppm = percentage_to_ppm(self.config.fee_thresholds.end_premium_percentage); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = self.calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); - let initial_multiplier = PPM.saturating_sub(start_discount_ppm); + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + PPM + end_premium_ppm, + ); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + } - let effect_of_being_late = (start_discount_ppm + end_premium_ppm) - .saturating_mul(ratio_ppm) - .saturating_div(PPM); + fn calculate_premium_increment( + &self, + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + const PPM: u128 = 1_000_000; // 100% in PPM - let multiplier_ppm = initial_multiplier.saturating_add(effect_of_being_late); + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - // TODO: segfault, for now just in case, but this should never happen - let multiplier_ppm = min(PPM + end_premium_ppm, multiplier_ppm); + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(PPM) + .saturating_div(max_blocks_behind) + }; - fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + total_ppm.saturating_mul(proportion).saturating_div(PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -135,11 +165,20 @@ impl SendOrWaitDecider

{ } } +fn percentage_to_ppm(percentage: f64) -> u128 { + (percentage * 1_000_000.0) as u128 +} + #[cfg(test)] mod tests { + use std::num::NonZeroU64; + use super::*; use crate::{ - fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, + fee_analytics::port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, state_committer::service::{FeeThresholds, SmaPeriods}, }; use test_case::test_case; From 79ee93629b9c78353dc67e23c6e443ee094a343d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 20:27:18 +0100 Subject: [PATCH 023/136] add tests for max fee calculation --- .../services/src/state_committer/fee_algo.rs | 136 +++++++++++++++--- 1 file changed, 118 insertions(+), 18 deletions(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 9a3c1d40..629015fa 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -5,7 +5,7 @@ use crate::fee_analytics::{ service::FeeAnalytics, }; -use super::service::FeeAlgoConfig; +use super::service::{FeeAlgoConfig, FeeThresholds}; pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, @@ -61,7 +61,8 @@ impl SendOrWaitDecider

{ } let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + let max_upper_tx_fee = + Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); let long_vs_max_delta_perc = ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) @@ -92,11 +93,14 @@ impl SendOrWaitDecider

{ short_term_tx_fee < max_upper_tx_fee } - // TODO: segfault test this - fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + context: Context, + ) -> u128 { const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = u128::from(self.config.fee_thresholds.max_l2_blocks_behind.get()); + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); let blocks_behind = u128::from(context.num_l2_blocks_behind); debug_assert!( @@ -106,15 +110,14 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = - percentage_to_ppm(self.config.fee_thresholds.start_discount_percentage); - let end_premium_ppm = percentage_to_ppm(self.config.fee_thresholds.end_premium_percentage); + let start_discount_ppm = percentage_to_ppm(fee_thresholds.start_discount_percentage); + let end_premium_ppm = percentage_to_ppm(fee_thresholds.end_premium_percentage); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% let base_multiplier = PPM.saturating_sub(start_discount_ppm); // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = self.calculate_premium_increment( + let premium_increment = Self::calculate_premium_increment( start_discount_ppm, end_premium_ppm, blocks_behind, @@ -132,7 +135,6 @@ impl SendOrWaitDecider

{ } fn calculate_premium_increment( - &self, start_discount_ppm: u128, end_premium_ppm: u128, blocks_behind: u128, @@ -171,16 +173,13 @@ fn percentage_to_ppm(percentage: f64) -> u128 { #[cfg(test)] mod tests { - use std::num::NonZeroU64; - use super::*; - use crate::{ - fee_analytics::port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, - }, - state_committer::service::{FeeThresholds, SmaPeriods}, + use crate::fee_analytics::port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, }; + use crate::state_committer::service::{FeeThresholds, SmaPeriods}; + use std::num::NonZeroU64; use test_case::test_case; use tokio; @@ -544,4 +543,105 @@ mod tests { "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", ); } + + /// Helper function to convert a percentage to Parts Per Million (PPM) + fn percentage_to_ppm_test_helper(percentage: f64) -> u128 { + (percentage * 1_000_000.0) as u128 + } + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 0, + at_l1_height: 0, + }, + 1000; + "No blocks behind, multiplier should be 100%" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20, + end_premium_percentage: 0.25, + always_acceptable_fee: 0, + }, + 2000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 2050; + "Half blocks behind with discount and premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + 800, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 700; + "Start discount only, no premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.30, + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 1150; + "End premium only, no discount" + )] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10, // 100,000 PPM + end_premium_percentage: 0.20, // 200,000 PPM + always_acceptable_fee: 0, + }, + 10_000, + Context { + num_l2_blocks_behind: 99, + at_l1_height: 0, + }, + 11970; + "High fee with premium" + )] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + context: Context, + expected_max_upper_fee: u128, + ) { + let max_upper_fee = SendOrWaitDecider::::calculate_max_upper_fee( + &fee_thresholds, + fee, + context, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } } From d4785a0158425def99d7415a7d8c70000b4a1e81 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 20:55:32 +0100 Subject: [PATCH 024/136] some todos --- committer/src/config.rs | 2 +- packages/adapters/eth/src/lib.rs | 22 ++------- packages/services/src/fee_analytics.rs | 46 +++++++++++-------- packages/services/src/state_committer.rs | 37 ++++++++------- .../services/src/state_committer/fee_algo.rs | 24 ++++++---- packages/services/tests/fee_analytics.rs | 17 ++++--- packages/services/tests/state_committer.rs | 8 ++-- 7 files changed, 77 insertions(+), 79 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index 734e92a6..06d8a94d 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -125,7 +125,7 @@ pub struct FeeAlgoConfig { pub long_sma_blocks: u64, /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees - pub max_l2_blocks_behind: NonZeroU64, + pub max_l2_blocks_behind: NonZeroU32, /// Starting discount percentage applied we try to achieve if we're 0 l2 blocks behind pub start_discount_percentage: f64, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index fa609e1e..27bba643 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -221,7 +221,7 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -232,7 +232,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { // TODO: segfault see when this can be None // TODO: check edgecases - let mut current_height = height_range.clone().min().unwrap(); + let mut current_height = *height_range.start(); while current_height <= *height_range.end() { // There is a comment in alloy about not doing more than 1024 blocks at a time const RPC_LIMIT: u64 = 1024; @@ -298,19 +298,16 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { }) .collect_vec(); - // eprintln!("converted into {new_fees:?}"); - - new_fees.try_into().unwrap() + Ok(new_fees.try_into().unwrap()) } - async fn current_block_height(&self) -> u64 { - self._get_block_number().await.unwrap() + async fn current_block_height(&self) -> Result { + self._get_block_number().await } } #[cfg(test)] mod test { - use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; @@ -331,13 +328,4 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - - // #[tokio::test] - // async fn can_connect_to_eth_mainnet() { - // let current_height = client._get_block_number().await.unwrap(); - // - // let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; - // - // panic!("{:?}", fees); - // } } diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 206896f5..79426574 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -16,7 +16,6 @@ pub mod port { use std::ops::RangeInclusive; use itertools::Itertools; - use super::BlockFees; @@ -81,8 +80,11 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees; - async fn current_block_height(&self) -> u64; + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + async fn current_block_height(&self) -> crate::Result; } #[cfg(feature = "test-helpers")] @@ -90,7 +92,6 @@ pub mod port { use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; - use crate::fee_analytics::port::{BlockFees, Fees}; @@ -108,16 +109,20 @@ pub mod port { } impl FeesProvider for ConstantFeesProvider { - async fn fees(&self, _height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees( + &self, + _height_range: RangeInclusive, + ) -> crate::Result { let fees = BlockFees { - height: self.current_block_height().await, + height: self.current_block_height().await?, fees: self.fees, }; - vec![fees].try_into().unwrap() + Ok(vec![fees].try_into().unwrap()) } - async fn current_block_height(&self) -> u64 { - 0 + + async fn current_block_height(&self) -> crate::Result { + Ok(0) } } @@ -127,11 +132,18 @@ pub mod port { } impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> u64 { - *self.fees.keys().last().unwrap() + async fn current_block_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) } - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { let fees = self .fees .iter() @@ -143,7 +155,7 @@ pub mod port { }) .collect_vec(); - fees.try_into().unwrap() + Ok(fees.try_into().expect("block fees not sequential")) } } @@ -177,8 +189,6 @@ pub mod service { use std::ops::RangeInclusive; - - use super::port::{ l1::{FeesProvider, SequentialBlockFees}, Fees, @@ -197,10 +207,8 @@ pub mod service { // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { - let fees = self.fees_provider.fees(block_range).await; - - Self::mean(fees) + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + self.fees_provider.fees(block_range).await.map(Self::mean) } fn mean(fees: SequentialBlockFees) -> Fees { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 38387532..7b9f226c 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -2,7 +2,7 @@ mod fee_algo; pub mod service { use std::{ - num::{NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, time::Duration, }; @@ -26,7 +26,7 @@ pub mod service { // TODO: segfault validate start discount is less than end premium and both are positive #[derive(Debug, Clone, Copy)] pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU64, + pub max_l2_blocks_behind: NonZeroU32, pub start_discount_percentage: f64, pub end_premium_percentage: f64, pub always_acceptable_fee: u128, @@ -133,40 +133,39 @@ pub mod service { Ok(std_elapsed >= self.config.fragment_accumulation_timeout) } - async fn submit_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<()> { - info!("about to send at most {} fragments", fragments.len()); - - // TODO: segfault proper type conversion + async fn should_send_tx(&self, fragments: &NonEmpty) -> Result { let l1_height = self.l1_adapter.current_height().await?; - // TODO: segfault test this let l2_height = self.fuel_api.latest_height().await?; let oldest_l2_block_in_fragments = fragments .maximum_by_key(|b| b.oldest_block_in_bundle) .oldest_block_in_bundle; - let behind_on_l2 = l2_height.saturating_sub(oldest_l2_block_in_fragments); + let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - let should_send = self - .decider + self.decider .should_send_blob_tx( - fragments.len() as u32, + u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), Context { - num_l2_blocks_behind: behind_on_l2 as u64, + num_l2_blocks_behind, at_l1_height: l1_height, }, ) - .await; + .await + } - if !should_send { - // TODO: segfault log here + async fn submit_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<()> { + if !self.should_send_tx(&fragments).await? { + info!("decided against sending fragments"); return Ok(()); } + info!("about to send at most {} fragments", fragments.len()); + let data = fragments.clone().map(|f| f.fragment); match self diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 629015fa..c6043b97 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -23,7 +23,7 @@ impl

SendOrWaitDecider

{ #[derive(Debug, Clone, Copy)] pub struct Context { - pub num_l2_blocks_behind: u64, + pub num_l2_blocks_behind: u32, pub at_l1_height: u64, } @@ -31,18 +31,22 @@ impl SendOrWaitDecider

{ // TODO: segfault validate blob number // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes // (once that is implemented) - pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { + pub async fn should_send_blob_tx( + &self, + num_blobs: u32, + context: Context, + ) -> crate::Result { let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; let short_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await; + .await?; let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await; + .await?; let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -50,14 +54,13 @@ impl SendOrWaitDecider

{ short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; eprintln!("fee always acceptable: {}", fee_always_acceptable); - // TODO: segfault test this let too_far_behind = context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); eprintln!("too far behind: {}", too_far_behind); if fee_always_acceptable || too_far_behind { - return true; + return Ok(true); } let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); @@ -90,7 +93,7 @@ impl SendOrWaitDecider

{ short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee ); - short_term_tx_fee < max_upper_tx_fee + Ok(short_term_tx_fee < max_upper_tx_fee) } fn calculate_max_upper_fee( @@ -518,12 +521,12 @@ mod tests { new_fees: Fees, num_blobs: u32, config: FeeAlgoConfig, - num_l2_blocks_behind: u64, + num_l2_blocks_behind: u32, expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = PreconfiguredFeesProvider::new(fees); - let current_block_height = fees_provider.current_block_height().await; + let current_block_height = fees_provider.current_block_height().await.unwrap(); let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); @@ -536,7 +539,8 @@ mod tests { num_l2_blocks_behind, }, ) - .await; + .await + .unwrap(); assert_eq!( should_send, expected_decision, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 39513e98..a89f52ce 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,21 +1,21 @@ use std::path::PathBuf; use services::fee_analytics::{ - port::{ - l1::testing::{self}, Fees, - }, - service::FeeAnalytics, - }; + port::{ + l1::testing::{self}, + Fees, + }, + service::FeeAnalytics, +}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); - let last_n_blocks = 1; // when - let sma = fee_analytics.calculate_sma(4..=4).await; + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); // then assert_eq!(sma.base_fee_per_gas, 5); @@ -28,10 +28,9 @@ async fn calculates_sma_correctly_for_last_5_blocks() { // given let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); - let last_n_blocks = 5; // when - let sma = fee_analytics.calculate_sma(0..=4).await; + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); // then let mean = (5 + 4 + 3 + 2 + 1) / 5; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index fcd50a98..501ccbf4 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -405,7 +405,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -516,7 +516,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -618,7 +618,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), + max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -728,7 +728,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, From 7326055ff9c7d1abb85988777db5c886b29a2fed Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 09:59:31 +0100 Subject: [PATCH 025/136] add tests for converting fees --- committer/src/config.rs | 2 +- packages/adapters/eth/src/lib.rs | 600 ++++++++++++++++-- .../adapters/storage/src/mappings/tables.rs | 2 +- packages/services/src/lib.rs | 2 +- 4 files changed, 534 insertions(+), 72 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index 06d8a94d..5b3b6b03 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroUsize}, str::FromStr, time::Duration, }; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 27bba643..99c1d94c 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -1,5 +1,4 @@ use std::{ - cmp::min, num::{NonZeroU32, NonZeroUsize}, ops::RangeInclusive, time::Duration, @@ -9,8 +8,10 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, + rpc::types::FeeHistory, }; use delegate::delegate; +use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, @@ -224,96 +225,132 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; - // so that a alloy version bump doesn't surprise us const_assert!(REWARD_PERCENTILE == 20.0,); - let mut fees = vec![]; + // There is a comment in alloy about not doing more than 1024 blocks at a time + const RPC_LIMIT: u64 = 1024; - // TODO: segfault see when this can be None - // TODO: check edgecases - let mut current_height = *height_range.start(); - while current_height <= *height_range.end() { - // There is a comment in alloy about not doing more than 1024 blocks at a time - const RPC_LIMIT: u64 = 1024; + let fees: Vec = stream::iter(chunk_range_inclusive(height_range, RPC_LIMIT)) + .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) + .try_collect() + .await?; - let upper_bound = min( - current_height.saturating_add(RPC_LIMIT).saturating_sub(1), - *height_range.end(), - ); + let mut unpacked_fees = vec![]; + for fee in fees { + unpacked_fees.extend(unpack_fee_history(fee)?); + } - let history = self - .fees( - current_height..=upper_bound, - std::slice::from_ref(&REWARD_PERCENTILE), - ) - .await - .unwrap(); + unpacked_fees + .try_into() + .map_err(|e| services::Error::Other(format!("{e}"))) + } - assert_eq!( - history.reward.as_ref().unwrap().len(), - (current_height..=upper_bound).count() - ); + async fn current_block_height(&self) -> Result { + self._get_block_number().await + } +} - fees.push(history); +fn unpack_fee_history(fees: FeeHistory) -> Result> { + let number_of_blocks = if fees.base_fee_per_gas.is_empty() { + 0 + } else { + // We subtract 1 because the last element is the expected fee for the next block + fees.base_fee_per_gas + .len() + .checked_sub(1) + .expect("checked not 0") + }; + + if number_of_blocks == 0 { + return Ok(vec![]); + } - current_height = upper_bound.saturating_add(1); - } + let Some(nested_rewards) = fees.reward.as_ref() else { + return Err(services::Error::Other(format!( + "missing rewards field: {fees:?}" + ))); + }; + + if number_of_blocks != nested_rewards.len() + || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 + { + return Err(services::Error::Other(format!( + "discrepancy in lengths of fee fields: {fees:?}" + ))); + } - let new_fees = fees - .into_iter() - .flat_map(|fees| { - // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe - // for block count 0? - // eprintln!("received {fees:?}"); - let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); - let rewards = fees - .reward - .unwrap() - .into_iter() - .map(|mut perc| perc.pop().unwrap()) - .collect_vec(); - - let oldest_block = fees.oldest_block; - - debug_assert_eq!(rewards.len(), number_of_blocks); - - izip!( - (oldest_block..), - fees.base_fee_per_gas.into_iter(), - fees.base_fee_per_blob_gas.into_iter(), - rewards - ) - .take(number_of_blocks) - .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { - height, - fees: Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - }, - }, + let rewards: Vec<_> = nested_rewards + .iter() + .map(|perc| { + perc.last().copied().ok_or_else(|| { + crate::error::Error::Other( + "should have had at least one reward percentile".to_string(), ) }) - .collect_vec(); + }) + .try_collect()?; + + let values = izip!( + (fees.oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + .collect(); + + Ok(values) +} - Ok(new_fees.try_into().unwrap()) +fn chunk_range_inclusive( + initial_range: RangeInclusive, + chunk_size: u64, +) -> Vec> { + let mut ranges = Vec::new(); + + if chunk_size == 0 { + return ranges; } - async fn current_block_height(&self) -> Result { - self._get_block_number().await + let start = *initial_range.start(); + let end = *initial_range.end(); + + let mut current = start; + while current <= end { + // Calculate the end of the current chunk. + let chunk_end = (current + chunk_size - 1).min(end); + + ranges.push(current..=chunk_end); + + current = chunk_end + 1; } + + ranges } #[cfg(test)] mod test { - - use alloy::eips::eip4844::DATA_GAS_PER_BLOB; + use super::chunk_range_inclusive; + use alloy::{eips::eip4844::DATA_GAS_PER_BLOB, rpc::types::FeeHistory}; use fuel_block_committer_encoding::blob; - use services::block_bundler::port::l1::FragmentEncoder; + use services::{ + block_bundler::port::l1::FragmentEncoder, + fee_analytics::port::{BlockFees, Fees}, + }; + use std::ops::RangeInclusive; - use crate::BlobEncoder; + use crate::{unpack_fee_history, BlobEncoder}; #[test] fn gas_usage_correctly_calculated() { @@ -328,4 +365,429 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } + + #[test] + fn test_chunk_size_zero() { + // given + let initial_range = 1..=10; + let chunk_size = 0; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected: Vec> = vec![]; + assert_eq!( + result, expected, + "Expected empty vector when chunk_size is zero" + ); + } + + #[test] + fn test_chunk_size_larger_than_range() { + // given + let initial_range = 1..=5; + let chunk_size = 10; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=5]; + assert_eq!( + result, expected, + "Expected single chunk when chunk_size exceeds range length" + ); + } + + #[test] + fn test_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 2; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; + assert_eq!(result, expected, "Chunks should exactly divide the range"); + } + + #[test] + fn test_non_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 3; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; + assert_eq!( + result, expected, + "Last chunk should contain the remaining elements" + ); + } + + #[test] + fn test_single_element_range() { + // given + let initial_range = 5..=5; + let chunk_size = 1; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![5..=5]; + assert_eq!( + result, expected, + "Single element range should return one chunk with that element" + ); + } + + #[test] + fn test_start_equals_end_with_large_chunk_size() { + // given + let initial_range = 100..=100; + let chunk_size = 50; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![100..=100]; + assert_eq!( + result, expected, + "Single element range should return one chunk regardless of chunk_size" + ); + } + + #[test] + fn test_chunk_size_one() { + // given + let initial_range = 10..=15; + let chunk_size = 1; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; + assert_eq!( + result, expected, + "Each number should be its own chunk when chunk_size is one" + ); + } + + #[test] + fn test_full_range_chunk() { + // given + let initial_range = 20..=30; + let chunk_size = 11; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![20..=30]; + assert_eq!( + result, expected, + "Whole range should be a single chunk when chunk_size equals range size" + ); + } + + #[test] + fn test_unpack_fee_history_empty_base_fee() { + // given + let fees = FeeHistory { + oldest_block: 100, + base_fee_per_gas: vec![], + base_fee_per_blob_gas: vec![], + reward: Some(vec![]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected: Vec = vec![]; + assert_eq!( + result.unwrap(), + expected, + "Expected empty vector when base_fee_per_gas is empty" + ); + } + + #[test] + fn test_unpack_fee_history_missing_rewards() { + // given + let fees = FeeHistory { + oldest_block: 200, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: None, + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to missing rewards field" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { + // given + let fees = FeeHistory { + oldest_block: 300, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in lengths of fee fields" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { + // given + let fees = FeeHistory { + oldest_block: 400, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in base_fee_per_blob_gas lengths" + ); + } + + #[test] + fn test_unpack_fee_history_empty_reward_percentile() { + // given + let fees = FeeHistory { + oldest_block: 500, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![]]), // Empty percentile + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other("should have had at least one reward percentile".to_string()); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to empty reward percentile" + ); + } + + #[test] + fn test_unpack_fee_history_single_block() { + // given + let fees = FeeHistory { + oldest_block: 600, + base_fee_per_gas: vec![100, 200], // number_of_blocks =1 + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![10]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![BlockFees { + height: 600, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }]; + assert_eq!( + result.unwrap(), + expected, + "Expected one BlockFees entry for a single block" + ); + } + + #[test] + fn test_unpack_fee_history_multiple_blocks() { + // given + let fees = FeeHistory { + oldest_block: 700, + base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 + base_fee_per_blob_gas: vec![150, 250, 350, 450], + reward: Some(vec![vec![10], vec![20], vec![30]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 700, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }, + BlockFees { + height: 701, + fees: Fees { + base_fee_per_gas: 200, + reward: 20, + base_fee_per_blob_gas: 250, + }, + }, + BlockFees { + height: 702, + fees: Fees { + base_fee_per_gas: 300, + reward: 30, + base_fee_per_blob_gas: 350, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected three BlockFees entries for three blocks" + ); + } + + #[test] + fn test_unpack_fee_history_large_values() { + // given + let fees = FeeHistory { + oldest_block: u64::MAX - 2, + base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], + base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], + reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected = vec![ + BlockFees { + height: u64::MAX - 2, + fees: Fees { + base_fee_per_gas: u128::MAX - 2, + reward: u128::MAX - 4, + base_fee_per_blob_gas: u128::MAX - 3, + }, + }, + BlockFees { + height: u64::MAX - 1, + fees: Fees { + base_fee_per_gas: u128::MAX - 1, + reward: u128::MAX - 3, + base_fee_per_blob_gas: u128::MAX - 2, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries with large u64 values" + ); + } + + #[test] + fn test_unpack_fee_history_full_range_chunk() { + // given + let fees = FeeHistory { + oldest_block: 800, + base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 + base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], + reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 800, + fees: Fees { + base_fee_per_gas: 500, + reward: 50, + base_fee_per_blob_gas: 550, + }, + }, + BlockFees { + height: 801, + fees: Fees { + base_fee_per_gas: 600, + reward: 60, + base_fee_per_blob_gas: 650, + }, + }, + BlockFees { + height: 802, + fees: Fees { + base_fee_per_gas: 700, + reward: 70, + base_fee_per_blob_gas: 750, + }, + }, + BlockFees { + height: 803, + fees: Fees { + base_fee_per_gas: 800, + reward: 80, + base_fee_per_blob_gas: 850, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries matching the full range chunk" + ); + } } diff --git a/packages/adapters/storage/src/mappings/tables.rs b/packages/adapters/storage/src/mappings/tables.rs index 566e68ae..ea419785 100644 --- a/packages/adapters/storage/src/mappings/tables.rs +++ b/packages/adapters/storage/src/mappings/tables.rs @@ -1,4 +1,4 @@ -use std::{i64, num::NonZeroU32}; +use std::num::NonZeroU32; use num_bigint::BigInt; use services::types::{ diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 69e04048..aa807be8 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -23,7 +23,7 @@ pub use block_bundler::{ pub use state_committer::service::{Config as StateCommitterConfig, StateCommitter}; use types::InvalidL1Height; -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, PartialEq)] pub enum Error { #[error("{0}")] Other(String), From c123862b801ee6874730a164794653224c99f096 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 10:03:16 +0100 Subject: [PATCH 026/136] move fee conversion to separate mod --- packages/adapters/eth/src/fee_conversion.rs | 532 +++++++++++++++++ packages/adapters/eth/src/lib.rs | 540 +----------------- packages/services/src/state_committer.rs | 2 +- .../services/src/state_committer/fee_algo.rs | 2 +- packages/services/tests/state_committer.rs | 2 +- 5 files changed, 548 insertions(+), 530 deletions(-) create mode 100644 packages/adapters/eth/src/fee_conversion.rs diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs new file mode 100644 index 00000000..eeb80995 --- /dev/null +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -0,0 +1,532 @@ +use std::ops::RangeInclusive; + +use alloy::rpc::types::FeeHistory; +use itertools::{izip, Itertools}; +use services::Result; + +use services::fee_analytics::port::Fees; + +use services::fee_analytics::port::BlockFees; + +pub fn unpack_fee_history(fees: FeeHistory) -> Result> { + let number_of_blocks = if fees.base_fee_per_gas.is_empty() { + 0 + } else { + // We subtract 1 because the last element is the expected fee for the next block + fees.base_fee_per_gas + .len() + .checked_sub(1) + .expect("checked not 0") + }; + + if number_of_blocks == 0 { + return Ok(vec![]); + } + + let Some(nested_rewards) = fees.reward.as_ref() else { + return Err(services::Error::Other(format!( + "missing rewards field: {fees:?}" + ))); + }; + + if number_of_blocks != nested_rewards.len() + || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 + { + return Err(services::Error::Other(format!( + "discrepancy in lengths of fee fields: {fees:?}" + ))); + } + + let rewards: Vec<_> = nested_rewards + .iter() + .map(|perc| { + perc.last().copied().ok_or_else(|| { + crate::error::Error::Other( + "should have had at least one reward percentile".to_string(), + ) + }) + }) + .try_collect()?; + + let values = izip!( + (fees.oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + .collect(); + + Ok(values) +} + +pub fn chunk_range_inclusive( + initial_range: RangeInclusive, + chunk_size: u64, +) -> Vec> { + let mut ranges = Vec::new(); + + if chunk_size == 0 { + return ranges; + } + + let start = *initial_range.start(); + let end = *initial_range.end(); + + let mut current = start; + while current <= end { + // Calculate the end of the current chunk. + let chunk_end = (current + chunk_size - 1).min(end); + + ranges.push(current..=chunk_end); + + current = chunk_end + 1; + } + + ranges +} + +#[cfg(test)] +mod test { + use alloy::rpc::types::FeeHistory; + + use services::fee_analytics::port::{BlockFees, Fees}; + use std::ops::RangeInclusive; + + use crate::fee_conversion::{self}; + + #[test] + fn test_chunk_size_zero() { + // given + let initial_range = 1..=10; + let chunk_size = 0; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected: Vec> = vec![]; + assert_eq!( + result, expected, + "Expected empty vector when chunk_size is zero" + ); + } + + #[test] + fn test_chunk_size_larger_than_range() { + // given + let initial_range = 1..=5; + let chunk_size = 10; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=5]; + assert_eq!( + result, expected, + "Expected single chunk when chunk_size exceeds range length" + ); + } + + #[test] + fn test_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 2; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; + assert_eq!(result, expected, "Chunks should exactly divide the range"); + } + + #[test] + fn test_non_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 3; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; + assert_eq!( + result, expected, + "Last chunk should contain the remaining elements" + ); + } + + #[test] + fn test_single_element_range() { + // given + let initial_range = 5..=5; + let chunk_size = 1; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![5..=5]; + assert_eq!( + result, expected, + "Single element range should return one chunk with that element" + ); + } + + #[test] + fn test_start_equals_end_with_large_chunk_size() { + // given + let initial_range = 100..=100; + let chunk_size = 50; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![100..=100]; + assert_eq!( + result, expected, + "Single element range should return one chunk regardless of chunk_size" + ); + } + + #[test] + fn test_chunk_size_one() { + // given + let initial_range = 10..=15; + let chunk_size = 1; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; + assert_eq!( + result, expected, + "Each number should be its own chunk when chunk_size is one" + ); + } + + #[test] + fn test_full_range_chunk() { + // given + let initial_range = 20..=30; + let chunk_size = 11; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![20..=30]; + assert_eq!( + result, expected, + "Whole range should be a single chunk when chunk_size equals range size" + ); + } + + #[test] + fn test_unpack_fee_history_empty_base_fee() { + // given + let fees = FeeHistory { + oldest_block: 100, + base_fee_per_gas: vec![], + base_fee_per_blob_gas: vec![], + reward: Some(vec![]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected: Vec = vec![]; + assert_eq!( + result.unwrap(), + expected, + "Expected empty vector when base_fee_per_gas is empty" + ); + } + + #[test] + fn test_unpack_fee_history_missing_rewards() { + // given + let fees = FeeHistory { + oldest_block: 200, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: None, + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to missing rewards field" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { + // given + let fees = FeeHistory { + oldest_block: 300, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in lengths of fee fields" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { + // given + let fees = FeeHistory { + oldest_block: 400, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in base_fee_per_blob_gas lengths" + ); + } + + #[test] + fn test_unpack_fee_history_empty_reward_percentile() { + // given + let fees = FeeHistory { + oldest_block: 500, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![]]), // Empty percentile + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other("should have had at least one reward percentile".to_string()); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to empty reward percentile" + ); + } + + #[test] + fn test_unpack_fee_history_single_block() { + // given + let fees = FeeHistory { + oldest_block: 600, + base_fee_per_gas: vec![100, 200], // number_of_blocks =1 + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![10]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![BlockFees { + height: 600, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }]; + assert_eq!( + result.unwrap(), + expected, + "Expected one BlockFees entry for a single block" + ); + } + + #[test] + fn test_unpack_fee_history_multiple_blocks() { + // given + let fees = FeeHistory { + oldest_block: 700, + base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 + base_fee_per_blob_gas: vec![150, 250, 350, 450], + reward: Some(vec![vec![10], vec![20], vec![30]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 700, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }, + BlockFees { + height: 701, + fees: Fees { + base_fee_per_gas: 200, + reward: 20, + base_fee_per_blob_gas: 250, + }, + }, + BlockFees { + height: 702, + fees: Fees { + base_fee_per_gas: 300, + reward: 30, + base_fee_per_blob_gas: 350, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected three BlockFees entries for three blocks" + ); + } + + #[test] + fn test_unpack_fee_history_large_values() { + // given + let fees = FeeHistory { + oldest_block: u64::MAX - 2, + base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], + base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], + reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected = vec![ + BlockFees { + height: u64::MAX - 2, + fees: Fees { + base_fee_per_gas: u128::MAX - 2, + reward: u128::MAX - 4, + base_fee_per_blob_gas: u128::MAX - 3, + }, + }, + BlockFees { + height: u64::MAX - 1, + fees: Fees { + base_fee_per_gas: u128::MAX - 1, + reward: u128::MAX - 3, + base_fee_per_blob_gas: u128::MAX - 2, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries with large u64 values" + ); + } + + #[test] + fn test_unpack_fee_history_full_range_chunk() { + // given + let fees = FeeHistory { + oldest_block: 800, + base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 + base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], + reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 800, + fees: Fees { + base_fee_per_gas: 500, + reward: 50, + base_fee_per_blob_gas: 550, + }, + }, + BlockFees { + height: 801, + fees: Fees { + base_fee_per_gas: 600, + reward: 60, + base_fee_per_blob_gas: 650, + }, + }, + BlockFees { + height: 802, + fees: Fees { + base_fee_per_gas: 700, + reward: 70, + base_fee_per_blob_gas: 750, + }, + }, + BlockFees { + height: 803, + fees: Fees { + base_fee_per_gas: 800, + reward: 80, + base_fee_per_blob_gas: 850, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries matching the full range chunk" + ); + } +} diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 99c1d94c..e0df5cd7 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, + fee_analytics::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -24,6 +24,7 @@ use services::{ mod aws; mod error; +mod fee_conversion; mod metrics; mod websocket; @@ -231,14 +232,17 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { // There is a comment in alloy about not doing more than 1024 blocks at a time const RPC_LIMIT: u64 = 1024; - let fees: Vec = stream::iter(chunk_range_inclusive(height_range, RPC_LIMIT)) - .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) - .try_collect() - .await?; + let fees: Vec = stream::iter(fee_conversion::chunk_range_inclusive( + height_range, + RPC_LIMIT, + )) + .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) + .try_collect() + .await?; let mut unpacked_fees = vec![]; for fee in fees { - unpacked_fees.extend(unpack_fee_history(fee)?); + unpacked_fees.extend(fee_conversion::unpack_fee_history(fee)?); } unpacked_fees @@ -251,106 +255,13 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { } } -fn unpack_fee_history(fees: FeeHistory) -> Result> { - let number_of_blocks = if fees.base_fee_per_gas.is_empty() { - 0 - } else { - // We subtract 1 because the last element is the expected fee for the next block - fees.base_fee_per_gas - .len() - .checked_sub(1) - .expect("checked not 0") - }; - - if number_of_blocks == 0 { - return Ok(vec![]); - } - - let Some(nested_rewards) = fees.reward.as_ref() else { - return Err(services::Error::Other(format!( - "missing rewards field: {fees:?}" - ))); - }; - - if number_of_blocks != nested_rewards.len() - || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 - { - return Err(services::Error::Other(format!( - "discrepancy in lengths of fee fields: {fees:?}" - ))); - } - - let rewards: Vec<_> = nested_rewards - .iter() - .map(|perc| { - perc.last().copied().ok_or_else(|| { - crate::error::Error::Other( - "should have had at least one reward percentile".to_string(), - ) - }) - }) - .try_collect()?; - - let values = izip!( - (fees.oldest_block..), - fees.base_fee_per_gas.into_iter(), - fees.base_fee_per_blob_gas.into_iter(), - rewards - ) - .take(number_of_blocks) - .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { - height, - fees: Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - }, - }, - ) - .collect(); - - Ok(values) -} - -fn chunk_range_inclusive( - initial_range: RangeInclusive, - chunk_size: u64, -) -> Vec> { - let mut ranges = Vec::new(); - - if chunk_size == 0 { - return ranges; - } - - let start = *initial_range.start(); - let end = *initial_range.end(); - - let mut current = start; - while current <= end { - // Calculate the end of the current chunk. - let chunk_end = (current + chunk_size - 1).min(end); - - ranges.push(current..=chunk_end); - - current = chunk_end + 1; - } - - ranges -} - #[cfg(test)] mod test { - use super::chunk_range_inclusive; - use alloy::{eips::eip4844::DATA_GAS_PER_BLOB, rpc::types::FeeHistory}; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::{ - block_bundler::port::l1::FragmentEncoder, - fee_analytics::port::{BlockFees, Fees}, - }; - use std::ops::RangeInclusive; + use services::block_bundler::port::l1::FragmentEncoder; - use crate::{unpack_fee_history, BlobEncoder}; + use crate::BlobEncoder; #[test] fn gas_usage_correctly_calculated() { @@ -365,429 +276,4 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - - #[test] - fn test_chunk_size_zero() { - // given - let initial_range = 1..=10; - let chunk_size = 0; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected: Vec> = vec![]; - assert_eq!( - result, expected, - "Expected empty vector when chunk_size is zero" - ); - } - - #[test] - fn test_chunk_size_larger_than_range() { - // given - let initial_range = 1..=5; - let chunk_size = 10; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=5]; - assert_eq!( - result, expected, - "Expected single chunk when chunk_size exceeds range length" - ); - } - - #[test] - fn test_exact_multiples() { - // given - let initial_range = 1..=10; - let chunk_size = 2; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; - assert_eq!(result, expected, "Chunks should exactly divide the range"); - } - - #[test] - fn test_non_exact_multiples() { - // given - let initial_range = 1..=10; - let chunk_size = 3; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; - assert_eq!( - result, expected, - "Last chunk should contain the remaining elements" - ); - } - - #[test] - fn test_single_element_range() { - // given - let initial_range = 5..=5; - let chunk_size = 1; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![5..=5]; - assert_eq!( - result, expected, - "Single element range should return one chunk with that element" - ); - } - - #[test] - fn test_start_equals_end_with_large_chunk_size() { - // given - let initial_range = 100..=100; - let chunk_size = 50; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![100..=100]; - assert_eq!( - result, expected, - "Single element range should return one chunk regardless of chunk_size" - ); - } - - #[test] - fn test_chunk_size_one() { - // given - let initial_range = 10..=15; - let chunk_size = 1; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; - assert_eq!( - result, expected, - "Each number should be its own chunk when chunk_size is one" - ); - } - - #[test] - fn test_full_range_chunk() { - // given - let initial_range = 20..=30; - let chunk_size = 11; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![20..=30]; - assert_eq!( - result, expected, - "Whole range should be a single chunk when chunk_size equals range size" - ); - } - - #[test] - fn test_unpack_fee_history_empty_base_fee() { - // given - let fees = FeeHistory { - oldest_block: 100, - base_fee_per_gas: vec![], - base_fee_per_blob_gas: vec![], - reward: Some(vec![]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected: Vec = vec![]; - assert_eq!( - result.unwrap(), - expected, - "Expected empty vector when base_fee_per_gas is empty" - ); - } - - #[test] - fn test_unpack_fee_history_missing_rewards() { - // given - let fees = FeeHistory { - oldest_block: 200, - base_fee_per_gas: vec![100, 200], - base_fee_per_blob_gas: vec![150, 250], - reward: None, - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to missing rewards field" - ); - } - - #[test] - fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { - // given - let fees = FeeHistory { - oldest_block: 300, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 250, 350], - reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to discrepancy in lengths of fee fields" - ); - } - - #[test] - fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { - // given - let fees = FeeHistory { - oldest_block: 400, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements - reward: Some(vec![vec![10], vec![20]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to discrepancy in base_fee_per_blob_gas lengths" - ); - } - - #[test] - fn test_unpack_fee_history_empty_reward_percentile() { - // given - let fees = FeeHistory { - oldest_block: 500, - base_fee_per_gas: vec![100, 200], - base_fee_per_blob_gas: vec![150, 250], - reward: Some(vec![vec![]]), // Empty percentile - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other("should have had at least one reward percentile".to_string()); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to empty reward percentile" - ); - } - - #[test] - fn test_unpack_fee_history_single_block() { - // given - let fees = FeeHistory { - oldest_block: 600, - base_fee_per_gas: vec![100, 200], // number_of_blocks =1 - base_fee_per_blob_gas: vec![150, 250], - reward: Some(vec![vec![10]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![BlockFees { - height: 600, - fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, - }, - }]; - assert_eq!( - result.unwrap(), - expected, - "Expected one BlockFees entry for a single block" - ); - } - - #[test] - fn test_unpack_fee_history_multiple_blocks() { - // given - let fees = FeeHistory { - oldest_block: 700, - base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 - base_fee_per_blob_gas: vec![150, 250, 350, 450], - reward: Some(vec![vec![10], vec![20], vec![30]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![ - BlockFees { - height: 700, - fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, - }, - }, - BlockFees { - height: 701, - fees: Fees { - base_fee_per_gas: 200, - reward: 20, - base_fee_per_blob_gas: 250, - }, - }, - BlockFees { - height: 702, - fees: Fees { - base_fee_per_gas: 300, - reward: 30, - base_fee_per_blob_gas: 350, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected three BlockFees entries for three blocks" - ); - } - - #[test] - fn test_unpack_fee_history_large_values() { - // given - let fees = FeeHistory { - oldest_block: u64::MAX - 2, - base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], - base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], - reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected = vec![ - BlockFees { - height: u64::MAX - 2, - fees: Fees { - base_fee_per_gas: u128::MAX - 2, - reward: u128::MAX - 4, - base_fee_per_blob_gas: u128::MAX - 3, - }, - }, - BlockFees { - height: u64::MAX - 1, - fees: Fees { - base_fee_per_gas: u128::MAX - 1, - reward: u128::MAX - 3, - base_fee_per_blob_gas: u128::MAX - 2, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected BlockFees entries with large u64 values" - ); - } - - #[test] - fn test_unpack_fee_history_full_range_chunk() { - // given - let fees = FeeHistory { - oldest_block: 800, - base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 - base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], - reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![ - BlockFees { - height: 800, - fees: Fees { - base_fee_per_gas: 500, - reward: 50, - base_fee_per_blob_gas: 550, - }, - }, - BlockFees { - height: 801, - fees: Fees { - base_fee_per_gas: 600, - reward: 60, - base_fee_per_blob_gas: 650, - }, - }, - BlockFees { - height: 802, - fees: Fees { - base_fee_per_gas: 700, - reward: 70, - base_fee_per_blob_gas: 750, - }, - }, - BlockFees { - height: 803, - fees: Fees { - base_fee_per_gas: 800, - reward: 80, - base_fee_per_blob_gas: 850, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected BlockFees entries matching the full range chunk" - ); - } } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 7b9f226c..5ce37565 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -2,7 +2,7 @@ mod fee_algo; pub mod service { use std::{ - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroUsize}, time::Duration, }; diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index c6043b97..ad255d76 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -182,7 +182,7 @@ mod tests { Fees, }; use crate::state_committer::service::{FeeThresholds, SmaPeriods}; - use std::num::NonZeroU64; + use test_case::test_case; use tokio; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 501ccbf4..5f655702 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -10,7 +10,7 @@ use services::{ types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use std::{num::NonZeroU64, time::Duration}; +use std::time::Duration; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { From 4506b80944eedd015f4b37298cdd0866dc747907 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 19:45:35 +0100 Subject: [PATCH 027/136] removing fee analytics as a service, making it a helper --- committer/src/main.rs | 4 - committer/src/setup.rs | 3 - packages/adapters/eth/src/fee_conversion.rs | 11 +- packages/adapters/eth/src/lib.rs | 9 +- packages/services/src/fee_analytics.rs | 371 ------------- packages/services/src/lib.rs | 1 - packages/services/src/state_committer.rs | 101 +++- .../services/src/state_committer/fee_algo.rs | 33 +- .../src/state_committer/fee_analytics.rs | 503 ++++++++++++++++++ packages/services/tests/fee_analytics.rs | 147 ----- packages/services/tests/state_committer.rs | 55 +- packages/services/tests/state_listener.rs | 14 +- packages/test-helpers/src/lib.rs | 11 +- 13 files changed, 630 insertions(+), 633 deletions(-) delete mode 100644 packages/services/src/fee_analytics.rs create mode 100644 packages/services/src/state_committer/fee_analytics.rs delete mode 100644 packages/services/tests/fee_analytics.rs diff --git a/committer/src/main.rs b/committer/src/main.rs index 3e1714f4..0473d0c7 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,7 +7,6 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; -use services::fee_analytics; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; @@ -73,15 +72,12 @@ async fn main() -> Result<()> { &metrics_registry, ); - let fee_analytics = fee_analytics::service::FeeAnalytics::new(ethereum_rpc.clone()); - let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, - fee_analytics, ); let state_importer_handle = diff --git a/committer/src/setup.rs b/committer/src/setup.rs index b320ee40..b09449c6 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,6 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_analytics::service::FeeAnalytics, state_committer::{ port::Storage, service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, @@ -121,7 +120,6 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, - fee_analytics: FeeAnalytics, ) -> tokio::task::JoinHandle<()> { let state_committer = services::StateCommitter::new( l1, @@ -146,7 +144,6 @@ pub fn state_committer( }, }, SystemClock, - fee_analytics, ); schedule_polling( diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index eeb80995..3063b321 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -2,11 +2,10 @@ use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; -use services::Result; - -use services::fee_analytics::port::Fees; - -use services::fee_analytics::port::BlockFees; +use services::{ + state_committer::port::l1::{BlockFees, Fees}, + Result, +}; pub fn unpack_fee_history(fees: FeeHistory) -> Result> { let number_of_blocks = if fees.base_fee_per_gas.is_empty() { @@ -99,8 +98,8 @@ pub fn chunk_range_inclusive( #[cfg(test)] mod test { use alloy::rpc::types::FeeHistory; + use services::state_committer::port::l1::{BlockFees, Fees}; - use services::fee_analytics::port::{BlockFees, Fees}; use std::ops::RangeInclusive; use crate::fee_conversion::{self}; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index e0df5cd7..2cb7fc31 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_analytics::port::l1::SequentialBlockFees, + state_committer::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -220,9 +220,6 @@ impl services::state_committer::port::l1::Api for WebsocketClient { ) -> Result<(L1Tx, FragmentsSubmitted)>; } } -} - -impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -249,10 +246,6 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .try_into() .map_err(|e| services::Error::Other(format!("{e}"))) } - - async fn current_block_height(&self) -> Result { - self._get_block_number().await - } } #[cfg(test)] diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs deleted file mode 100644 index 79426574..00000000 --- a/packages/services/src/fee_analytics.rs +++ /dev/null @@ -1,371 +0,0 @@ -pub mod port { - #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] - pub struct Fees { - pub base_fee_per_gas: u128, - pub reward: u128, - pub base_fee_per_blob_gas: u128, - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub struct BlockFees { - pub height: u64, - pub fees: Fees, - } - - pub mod l1 { - use std::ops::RangeInclusive; - - use itertools::Itertools; - - use super::BlockFees; - - #[derive(Debug)] - pub struct SequentialBlockFees { - fees: Vec, - } - - impl IntoIterator for SequentialBlockFees { - type Item = BlockFees; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.fees.into_iter() - } - } - - // Cannot be empty - #[allow(clippy::len_without_is_empty)] - impl SequentialBlockFees { - pub fn len(&self) -> usize { - self.fees.len() - } - } - - #[derive(Debug)] - pub struct InvalidSequence(String); - - impl std::error::Error for InvalidSequence {} - - impl std::fmt::Display for InvalidSequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } - } - - impl TryFrom> for SequentialBlockFees { - type Error = InvalidSequence; - fn try_from(mut fees: Vec) -> Result { - if fees.is_empty() { - return Err(InvalidSequence("Input cannot be empty".to_string())); - } - - fees.sort_by_key(|f| f.height); - - let is_sequential = fees - .iter() - .tuple_windows() - .all(|(l, r)| l.height + 1 == r.height); - - let heights = fees.iter().map(|f| f.height).collect::>(); - if !is_sequential { - return Err(InvalidSequence(format!( - "blocks are not sequential by height: {heights:?}" - ))); - } - - Ok(Self { fees }) - } - } - - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - #[cfg_attr(feature = "test-helpers", mockall::automock)] - pub trait FeesProvider { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result; - async fn current_block_height(&self) -> crate::Result; - } - - #[cfg(feature = "test-helpers")] - pub mod testing { - use std::{collections::BTreeMap, ops::RangeInclusive}; - - use itertools::Itertools; - - use crate::fee_analytics::port::{BlockFees, Fees}; - - use super::{FeesProvider, SequentialBlockFees}; - - #[derive(Debug, Clone, Copy)] - pub struct ConstantFeesProvider { - fees: Fees, - } - - impl ConstantFeesProvider { - pub fn new(fees: Fees) -> Self { - Self { fees } - } - } - - impl FeesProvider for ConstantFeesProvider { - async fn fees( - &self, - _height_range: RangeInclusive, - ) -> crate::Result { - let fees = BlockFees { - height: self.current_block_height().await?, - fees: self.fees, - }; - - Ok(vec![fees].try_into().unwrap()) - } - - async fn current_block_height(&self) -> crate::Result { - Ok(0) - } - } - - #[derive(Debug, Clone)] - pub struct PreconfiguredFeesProvider { - fees: BTreeMap, - } - - impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> crate::Result { - Ok(*self - .fees - .keys() - .last() - .expect("no fees registered with PreconfiguredFeesProvider")) - } - - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = self - .fees - .iter() - .skip_while(|(height, _)| !height_range.contains(height)) - .take_while(|(height, _)| height_range.contains(height)) - .map(|(height, fees)| BlockFees { - height: *height, - fees: *fees, - }) - .collect_vec(); - - Ok(fees.try_into().expect("block fees not sequential")) - } - } - - impl PreconfiguredFeesProvider { - pub fn new(blocks: impl IntoIterator) -> Self { - Self { - fees: blocks.into_iter().collect(), - } - } - } - - pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { - (0..num_blocks) - .map(|i| { - ( - i, - Fees { - base_fee_per_gas: i as u128 + 1, - reward: i as u128 + 1, - base_fee_per_blob_gas: i as u128 + 1, - }, - ) - }) - .collect() - } - } - } -} - -pub mod service { - - use std::ops::RangeInclusive; - - use super::port::{ - l1::{FeesProvider, SequentialBlockFees}, - Fees, - }; - - pub struct FeeAnalytics

{ - fees_provider: P, - } - impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } - } - - impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - self.fees_provider.fees(block_range).await.map(Self::mean) - } - - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } - } - } -} - -#[cfg(test)] -mod tests { - use itertools::Itertools; - use port::{l1::SequentialBlockFees, BlockFees, Fees}; - - use super::*; - - #[test] - fn can_create_valid_sequential_fees() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees.clone()); - - // Then - assert!( - result.is_ok(), - "Expected SequentialBlockFees creation to succeed" - ); - let sequential_fees = result.unwrap(); - assert_eq!(sequential_fees.len(), block_fees.len()); - } - - #[test] - fn sequential_fees_cannot_be_empty() { - // Given - let block_fees: Vec = vec![]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for empty input" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"Input cannot be empty\")" - ); - } - - #[test] - fn fees_must_be_sequential() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 3, // Non-sequential height - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for non-sequential heights" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" - ); - } - - // TODO: segfault add more tests so that the in-order iteration invariant is properly tested - #[test] - fn produced_iterator_gives_correct_values() { - // Given - // notice the heights are out of order so that we validate that the returned sequence is in - // order - let block_fees = vec![ - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); - - // When - let iterated_fees: Vec = sequential_fees.into_iter().collect(); - - // Then - let expectation = block_fees - .into_iter() - .sorted_by_key(|b| b.height) - .collect_vec(); - assert_eq!( - iterated_fees, expectation, - "Expected iterator to yield the same block fees" - ); - } -} diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index aa807be8..c6ceb616 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,7 +2,6 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; -pub mod fee_analytics; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5ce37565..5d53539f 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,4 +1,5 @@ mod fee_algo; +mod fee_analytics; pub mod service { use std::{ @@ -7,7 +8,6 @@ pub mod service { }; use crate::{ - fee_analytics::service::FeeAnalytics, state_committer::fee_algo::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, @@ -71,18 +71,19 @@ pub mod service { } /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider, + decider: SendOrWaitDecider, } - impl StateCommitter + impl StateCommitter where + L1: Clone + Send + Sync, Clock: crate::state_committer::port::Clock, { /// Creates a new `StateCommitter`. @@ -92,29 +93,27 @@ pub mod service { storage: Db, config: Config, clock: Clock, - fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); let price_algo = config.fee_algo; Self { - l1_adapter, + l1_adapter: l1_adapter.clone(), fuel_api, storage, config, clock, startup_time, - decider: SendOrWaitDecider::new(fee_analytics, price_algo), + decider: SendOrWaitDecider::new(l1_adapter, price_algo), } } } - impl StateCommitter + impl StateCommitter where - L1: crate::state_committer::port::l1::Api, + L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, - FeeProvider: crate::fee_analytics::port::l1::FeesProvider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -301,14 +300,12 @@ pub mod service { } } - impl Runner - for StateCommitter + impl Runner for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, - FeeProvider: crate::fee_analytics::port::l1::FeesProvider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -331,13 +328,15 @@ pub mod port { }; pub mod l1 { + use std::ops::RangeInclusive; + use nonempty::NonEmpty; + pub use crate::state_committer::fee_analytics::{BlockFees, Fees, SequentialBlockFees}; use crate::{ types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, Result, }; - #[allow(async_fn_in_trait)] #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] @@ -355,6 +354,80 @@ pub mod port { fragments: NonEmpty, previous_tx: Option, ) -> Result<(L1Tx, FragmentsSubmitted)>; + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + use std::{ops::RangeInclusive, sync::Arc}; + + use nonempty::NonEmpty; + + use crate::{ + state_committer::fee_analytics::{ + testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + FeesProvider, + }, + types::{FragmentsSubmitted, L1Tx}, + }; + + use super::{Api, Fees, MockApi, SequentialBlockFees}; + + #[derive(Clone)] + pub struct ApiMockWFees { + pub api: Arc, + fee_provider: Fees, + } + + impl ApiMockWFees { + pub fn new(api: MockApi) -> Self { + Self { + api: Arc::new(api), + fee_provider: ConstantFeesProvider::new(Fees::default()), + } + } + } + + impl ApiMockWFees { + pub fn w_preconfigured_fees( + self, + fees: impl IntoIterator, + ) -> ApiMockWFees { + ApiMockWFees { + api: self.api, + fee_provider: PreconfiguredFeesProvider::new(fees), + } + } + } + + impl Api for ApiMockWFees + where + T: FeesProvider + Send + Sync, + { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + FeesProvider::fees(&self.fee_provider, height_range).await + } + + async fn current_height(&self) -> crate::Result { + self.api.current_height().await + } + + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> crate::Result<(L1Tx, FragmentsSubmitted)> { + self.api + .submit_state_fragments(fragments, previous_tx) + .await + } + } } } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index ad255d76..a247e670 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,21 +1,20 @@ use std::cmp::min; -use crate::fee_analytics::{ - port::{l1::FeesProvider, Fees}, - service::FeeAnalytics, +use super::{ + fee_analytics::{FeeAnalytics, FeesProvider}, + port::l1::Fees, + service::{FeeAlgoConfig, FeeThresholds}, }; -use super::service::{FeeAlgoConfig, FeeThresholds}; - pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, config: FeeAlgoConfig, } impl

SendOrWaitDecider

{ - pub fn new(fee_analytics: FeeAnalytics

, config: FeeAlgoConfig) -> Self { + pub fn new(fee_provider: P, config: FeeAlgoConfig) -> Self { Self { - fee_analytics, + fee_analytics: FeeAnalytics::new(fee_provider), config, } } @@ -28,7 +27,6 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault validate blob number // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes // (once that is implemented) pub async fn should_send_blob_tx( @@ -36,6 +34,8 @@ impl SendOrWaitDecider

{ num_blobs: u32, context: Context, ) -> crate::Result { + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; let short_term_sma = self @@ -177,12 +177,11 @@ fn percentage_to_ppm(percentage: f64) -> u128 { #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, + use crate::state_committer::{ + fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + service::{FeeThresholds, SmaPeriods}, }; - use crate::state_committer::service::{FeeThresholds, SmaPeriods}; - + use test_case::test_case; use tokio; @@ -527,9 +526,8 @@ mod tests { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = PreconfiguredFeesProvider::new(fees); let current_block_height = fees_provider.current_block_height().await.unwrap(); - let analytics_service = FeeAnalytics::new(fees_provider); - let sut = SendOrWaitDecider::new(analytics_service, config); + let sut = SendOrWaitDecider::new(fees_provider, config); let should_send = sut .should_send_blob_tx( @@ -548,11 +546,6 @@ mod tests { ); } - /// Helper function to convert a percentage to Parts Per Million (PPM) - fn percentage_to_ppm_test_helper(percentage: f64) -> u128 { - (percentage * 1_000_000.0) as u128 - } - #[test_case( // Test Case 1: No blocks behind, no discount or premium FeeThresholds { diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs new file mode 100644 index 00000000..29633b26 --- /dev/null +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -0,0 +1,503 @@ +use std::ops::RangeInclusive; + +use itertools::Itertools; + +use crate::state_committer::port::l1::Api; + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockFees { + pub height: u64, + pub fees: Fees, +} + +#[derive(Debug)] +pub struct SequentialBlockFees { + fees: Vec, +} + +#[allow(async_fn_in_trait)] +#[trait_variant::make(Send)] +#[cfg_attr(feature = "test-helpers", mockall::automock)] +pub trait FeesProvider { + async fn fees(&self, height_range: RangeInclusive) -> crate::Result; + async fn current_block_height(&self) -> crate::Result; +} + +impl FeesProvider for T { + async fn fees(&self, height_range: RangeInclusive) -> crate::Result { + Api::fees(self, height_range).await + } + + async fn current_block_height(&self) -> crate::Result { + Api::current_height(self).await + } +} + +impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } +} + +// Cannot be empty +#[allow(clippy::len_without_is_empty)] +impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } +} + +#[derive(Debug)] +pub struct InvalidSequence(String); + +impl std::error::Error for InvalidSequence {} + +impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + let heights = fees.iter().map(|f| f.height).collect::>(); + if !is_sequential { + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); + } + + Ok(Self { fees }) + } +} + +#[cfg(feature = "test-helpers")] +pub mod testing { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use itertools::Itertools; + + use crate::state_committer::port::l1::{BlockFees, Fees}; + + use super::{FeesProvider, SequentialBlockFees}; + + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeesProvider { + fees: Fees, + } + + impl ConstantFeesProvider { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl FeesProvider for ConstantFeesProvider { + async fn fees( + &self, + _height_range: RangeInclusive, + ) -> crate::Result { + let fees = BlockFees { + height: self.current_block_height().await?, + fees: self.fees, + }; + + Ok(vec![fees].try_into().unwrap()) + } + + async fn current_block_height(&self) -> crate::Result { + Ok(0) + } + } + + #[derive(Debug, Clone)] + pub struct PreconfiguredFeesProvider { + fees: BTreeMap, + } + + impl FeesProvider for PreconfiguredFeesProvider { + async fn current_block_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) + } + + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = self + .fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_vec(); + + Ok(fees.try_into().expect("block fees not sequential")) + } + } + + impl PreconfiguredFeesProvider { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, + }, + ) + }) + .collect() + } +} + +pub struct FeeAnalytics

{ + fees_provider: P, +} +impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } +} + +impl FeeAnalytics

{ + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + self.fees_provider.fees(block_range).await.map(Self::mean) + } + + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } + } +} + +#[cfg(test)] +mod tests { + use itertools::Itertools; + + use super::*; + + #[test] + fn can_create_valid_sequential_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn sequential_fees_cannot_be_empty() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn fees_must_be_sequential() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" + ); + } + + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested + #[test] + fn produced_iterator_gives_correct_values() { + // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order + let block_fees = vec![ + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); + assert_eq!( + iterated_fees, expectation, + "Expected iterator to yield the same block fees" + ); + } + use std::path::PathBuf; + + #[tokio::test] + async fn calculates_sma_correctly_for_last_1_block() { + // given + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); + } + + #[tokio::test] + async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); + } + + fn calculate_tx_fee(fees: &Fees) -> u128 { + 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + } + + fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + let mut csv_writer = + csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + .unwrap(); + csv_writer + .write_record(["height", "tx_fee"].iter()) + .unwrap(); + for (height, fee) in tx_fees { + csv_writer + .write_record([height.to_string(), fee.to_string()]) + .unwrap(); + } + csv_writer.flush().unwrap(); + } + + // #[tokio::test] + // async fn something() { + // let client = make_pub_eth_client().await; + // use services::fee_analytics::port::l1::FeesProvider; + // + // let current_block_height = 21408300; + // let starting_block_height = current_block_height - 48 * 3600 / 12; + // let data = client + // .fees(starting_block_height..=current_block_height) + // .await + // .into_iter() + // .collect::>(); + // + // let fee_lookup = data + // .iter() + // .map(|b| (b.height, b.fees)) + // .collect::>(); + // + // let short_sma = 25u64; + // let long_sma = 900; + // + // let current_tx_fees = data + // .iter() + // .map(|b| (b.height, calculate_tx_fee(&b.fees))) + // .collect::>(); + // + // save_tx_fees(¤t_tx_fees, "current_fees.csv"); + // + // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + // let fee_analytics = FeeAnalytics::new(local_client.clone()); + // + // let mut short_sma_tx_fees = vec![]; + // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - short_sma..=height) + // .await; + // + // let tx_fee = calculate_tx_fee(&fees); + // + // short_sma_tx_fees.push((height, tx_fee)); + // } + // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + // + // let decider = SendOrWaitDecider::new( + // FeeAnalytics::new(local_client.clone()), + // services::state_committer::fee_optimization::Config { + // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + // short: short_sma, + // long: long_sma, + // }, + // fee_thresholds: Feethresholds { + // max_l2_blocks_behind: 43200 * 3, + // start_discount_percentage: 0.2, + // end_premium_percentage: 0.2, + // always_acceptable_fee: 1000000000000000u128, + // }, + // }, + // ); + // + // let mut decisions = vec![]; + // let mut long_sma_tx_fees = vec![]; + // + // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - long_sma..=height) + // .await; + // let tx_fee = calculate_tx_fee(&fees); + // long_sma_tx_fees.push((height, tx_fee)); + // + // if decider + // .should_send_blob_tx( + // 6, + // Context { + // at_l1_height: height, + // num_l2_blocks_behind: (height - starting_block_height) * 12, + // }, + // ) + // .await + // { + // let current_fees = fee_lookup.get(&height).unwrap(); + // let current_tx_fee = calculate_tx_fee(current_fees); + // decisions.push((height, current_tx_fee)); + // } + // } + // + // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + // save_tx_fees(&decisions, "decisions.csv"); + // } +} diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs deleted file mode 100644 index a89f52ce..00000000 --- a/packages/services/tests/fee_analytics.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::path::PathBuf; - -use services::fee_analytics::{ - port::{ - l1::testing::{self}, - Fees, - }, - service::FeeAnalytics, -}; - -#[tokio::test] -async fn calculates_sma_correctly_for_last_1_block() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); - - // then - assert_eq!(sma.base_fee_per_gas, 5); - assert_eq!(sma.reward, 5); - assert_eq!(sma.base_fee_per_blob_gas, 5); -} - -#[tokio::test] -async fn calculates_sma_correctly_for_last_5_blocks() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); - - // then - let mean = (5 + 4 + 3 + 2 + 1) / 5; - assert_eq!(sma.base_fee_per_gas, mean); - assert_eq!(sma.reward, mean); - assert_eq!(sma.base_fee_per_blob_gas, mean); -} - -fn calculate_tx_fee(fees: &Fees) -> u128 { - 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 -} - -fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - let mut csv_writer = - csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)).unwrap(); - csv_writer - .write_record(["height", "tx_fee"].iter()) - .unwrap(); - for (height, fee) in tx_fees { - csv_writer - .write_record([height.to_string(), fee.to_string()]) - .unwrap(); - } - csv_writer.flush().unwrap(); -} - -// #[tokio::test] -// async fn something() { -// let client = make_pub_eth_client().await; -// use services::fee_analytics::port::l1::FeesProvider; -// -// let current_block_height = 21408300; -// let starting_block_height = current_block_height - 48 * 3600 / 12; -// let data = client -// .fees(starting_block_height..=current_block_height) -// .await -// .into_iter() -// .collect::>(); -// -// let fee_lookup = data -// .iter() -// .map(|b| (b.height, b.fees)) -// .collect::>(); -// -// let short_sma = 25u64; -// let long_sma = 900; -// -// let current_tx_fees = data -// .iter() -// .map(|b| (b.height, calculate_tx_fee(&b.fees))) -// .collect::>(); -// -// save_tx_fees(¤t_tx_fees, "current_fees.csv"); -// -// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); -// let fee_analytics = FeeAnalytics::new(local_client.clone()); -// -// let mut short_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - short_sma..=height) -// .await; -// -// let tx_fee = calculate_tx_fee(&fees); -// -// short_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); -// -// let decider = SendOrWaitDecider::new( -// FeeAnalytics::new(local_client.clone()), -// services::state_committer::fee_optimization::Config { -// sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { -// short: short_sma, -// long: long_sma, -// }, -// fee_thresholds: Feethresholds { -// max_l2_blocks_behind: 43200 * 3, -// start_discount_percentage: 0.2, -// end_premium_percentage: 0.2, -// always_acceptable_fee: 1000000000000000u128, -// }, -// }, -// ); -// -// let mut decisions = vec![]; -// let mut long_sma_tx_fees = vec![]; -// -// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - long_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// long_sma_tx_fees.push((height, tx_fee)); -// -// if decider -// .should_send_blob_tx( -// 6, -// Context { -// at_l1_height: height, -// num_l2_blocks_behind: (height - starting_block_height) * 12, -// }, -// ) -// .await -// { -// let current_fees = fee_lookup.get(&height).unwrap(); -// let current_tx_fee = calculate_tx_fee(current_fees); -// decisions.push((height, current_tx_fee)); -// } -// } -// -// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); -// save_tx_fees(&decisions, "decisions.csv"); -// } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 5f655702..8f207a21 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,8 @@ use services::{ - fee_analytics::{ - port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, - }, - service::FeeAnalytics, + state_committer::{ + port::l1::{testing::ApiMockWFees, Fees}, + service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, }, - state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -34,7 +30,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -44,7 +40,6 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -78,7 +73,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -88,7 +83,6 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout @@ -114,7 +108,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -124,7 +118,6 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time less than the timeout @@ -160,7 +153,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -170,7 +163,6 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -207,7 +199,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -217,7 +209,6 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time to exceed the timeout since last finalized fragment @@ -254,7 +245,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> .expect_current_height() .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -264,7 +255,6 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout from startup @@ -314,7 +304,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -325,7 +315,6 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Submit the initial fragments @@ -399,9 +388,6 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -431,7 +417,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -442,7 +428,6 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ..Default::default() }, setup.test_clock(), - fee_analytics, ); // When @@ -510,9 +495,6 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -533,7 +515,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -544,7 +526,6 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when @@ -612,9 +593,6 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -644,7 +622,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -655,7 +633,6 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when @@ -722,9 +699,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -754,7 +728,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -765,7 +739,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index b62108bf..20ac4b38 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,13 +3,7 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - fee_analytics::{ - port::{ - l1::testing::ConstantFeesProvider, - Fees, - }, - service::FeeAnalytics, - }, + state_committer::port::l1::testing::ApiMockWFees, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -452,7 +446,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -460,7 +454,6 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx @@ -556,7 +549,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -564,7 +557,6 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index ac357e66..c248ae64 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -1,5 +1,6 @@ #![deny(unused_crate_dependencies)] +use std::sync::Arc; use std::{ops::RangeInclusive, time::Duration}; use clock::TestClock; @@ -8,9 +9,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::ConstantFeesProvider; -use services::fee_analytics::port::Fees; -use services::fee_analytics::service::FeeAnalytics; +use services::state_committer::port::l1::testing::ApiMockWFees; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -547,7 +546,7 @@ impl Setup { .return_once(move || Box::pin(async { Ok(0) })); StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -558,7 +557,6 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ) .run() .await @@ -586,7 +584,7 @@ impl Setup { let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), fuel_mock, self.db(), services::StateCommitterConfig { @@ -597,7 +595,6 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); committer.run().await.unwrap(); From 4c42cb7453870031a9b801b246966a2950ddfae2 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 20:03:29 +0100 Subject: [PATCH 028/136] add check whether provider reported all requested fees --- .../src/state_committer/fee_analytics.rs | 73 ++++++++++++++----- packages/services/tests/state_committer.rs | 28 +++---- 2 files changed, 68 insertions(+), 33 deletions(-) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 29633b26..2761ead3 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -54,6 +54,12 @@ impl SequentialBlockFees { pub fn len(&self) -> usize { self.fees.len() } + + pub fn height_range(&self) -> RangeInclusive { + let start = self.fees.first().expect("not empty").height; + let end = self.fees.last().expect("not empty").height; + start..=end + } } #[derive(Debug)] @@ -199,10 +205,18 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background + // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - self.fees_provider.fees(block_range).await.map(Self::mean) + let fees = self.fees_provider.fees(block_range.clone()).await?; + + let received_height_range = fees.height_range(); + if received_height_range != block_range { + return Err(crate::Error::from(format!( + "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" + ))); + } + + Ok(Self::mean(fees)) } fn mean(fees: SequentialBlockFees) -> Fees { @@ -393,25 +407,46 @@ mod tests { assert_eq!(sma.base_fee_per_blob_gas, mean); } - fn calculate_tx_fee(fees: &Fees) -> u128 { - 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 - } + #[tokio::test] + async fn errors_out_if_returned_fees_are_not_complete() { + // given + let mut fees = testing::incrementing_fees(5); + fees.remove(&4); + let fees_provider = testing::PreconfiguredFeesProvider::new(fees); + let fee_analytics = FeeAnalytics::new(fees_provider); - fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - let mut csv_writer = - csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) - .unwrap(); - csv_writer - .write_record(["height", "tx_fee"].iter()) - .unwrap(); - for (height, fee) in tx_fees { - csv_writer - .write_record([height.to_string(), fee.to_string()]) - .unwrap(); - } - csv_writer.flush().unwrap(); + // when + let err = fee_analytics + .calculate_sma(0..=4) + .await + .expect_err("should have failed because returned fees are not complete"); + + // then + assert_eq!( + err.to_string(), + "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" + ); } + // fn calculate_tx_fee(fees: &Fees) -> u128 { + // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + // } + // + // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + // let mut csv_writer = + // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + // .unwrap(); + // csv_writer + // .write_record(["height", "tx_fee"].iter()) + // .unwrap(); + // for (height, fee) in tx_fees { + // csv_writer + // .write_record([height.to_string(), fee.to_string()]) + // .unwrap(); + // } + // csv_writer.flush().unwrap(); + // } + // #[tokio::test] // async fn something() { // let client = make_pub_eth_client().await; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 8f207a21..0c548651 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -339,7 +339,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -347,7 +347,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 2, + 1, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -355,7 +355,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 3, + 2, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -363,7 +363,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 4, + 3, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -371,7 +371,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 5, + 4, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -379,7 +379,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 6, + 5, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -413,7 +413,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { )]); l1_mock_submit .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( @@ -446,7 +446,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( // Define fee sequence: last 2 blocks have higher fees than the long-term average let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -454,7 +454,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 2, + 1, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -462,7 +462,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 3, + 2, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -470,7 +470,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 4, + 3, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -478,7 +478,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 5, + 4, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -486,7 +486,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 6, + 5, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -511,7 +511,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let mut l1_mock = test_helpers::mocks::l1::expects_state_submissions([]); l1_mock .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( From 61c6fd35af29c39f172af9f5b7abbbae3afbdfdd Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 20:55:34 +0100 Subject: [PATCH 029/136] add caching and tests --- .../src/state_committer/fee_analytics.rs | 259 +++++++++++++++++- 1 file changed, 256 insertions(+), 3 deletions(-) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 2761ead3..e29e4242 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -1,6 +1,7 @@ -use std::ops::RangeInclusive; +use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; +use tokio::sync::RwLock; use crate::state_committer::port::l1::Api; @@ -17,7 +18,7 @@ pub struct BlockFees { pub fees: Fees, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct SequentialBlockFees { fees: Vec, } @@ -30,6 +31,90 @@ pub trait FeesProvider { async fn current_block_height(&self) -> crate::Result; } +#[derive(Debug)] +pub struct CachingFeesProvider

{ + fees_provider: P, + cache: RwLock>, + cache_limit: usize, +} + +impl

CachingFeesProvider

{ + pub fn new(fees_provider: P, cache_limit: usize) -> Self { + Self { + fees_provider, + cache: RwLock::new(BTreeMap::new()), + cache_limit, + } + } +} + +impl FeesProvider for CachingFeesProvider

{ + async fn fees(&self, height_range: RangeInclusive) -> crate::Result { + self.get_fees(height_range).await + } + + async fn current_block_height(&self) -> crate::Result { + self.fees_provider.current_block_height().await + } +} + +impl CachingFeesProvider

{ + pub async fn get_fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let mut missing_heights = vec![]; + + // Mind the scope to release the read lock + { + let cache = self.cache.read().await; + for height in height_range.clone() { + if !cache.contains_key(&height) { + missing_heights.push(height); + } + } + } + + if !missing_heights.is_empty() { + let fetched_fees = self + .fees_provider + .fees( + *missing_heights.first().expect("not empty") + ..=*missing_heights.last().expect("not empty"), + ) + .await?; + + let mut cache = self.cache.write().await; + for block_fee in fetched_fees { + cache.insert(block_fee.height, block_fee.fees); + } + } + + let fees: Vec<_> = { + let cache = self.cache.read().await; + height_range + .filter_map(|h| { + cache.get(&h).map(|f| BlockFees { + height: h, + fees: *f, + }) + }) + .collect() + }; + + self.shrink_cache().await; + + SequentialBlockFees::try_from(fees).map_err(|e| crate::Error::Other(e.to_string())) + } + + async fn shrink_cache(&self) { + let mut cache = self.cache.write().await; + while cache.len() > self.cache_limit { + cache.pop_first(); + } + } +} + impl FeesProvider for T { async fn fees(&self, height_range: RangeInclusive) -> crate::Result { Api::fees(self, height_range).await @@ -194,6 +279,7 @@ pub mod testing { } } +#[derive(Debug, Clone)] pub struct FeeAnalytics

{ fees_provider: P, } @@ -204,7 +290,6 @@ impl

FeeAnalytics

{ } impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { let fees = self.fees_provider.fees(block_range.clone()).await?; @@ -243,6 +328,7 @@ impl FeeAnalytics

{ #[cfg(test)] mod tests { use itertools::Itertools; + use mockall::{predicate::eq, Sequence}; use super::*; @@ -428,6 +514,173 @@ mod tests { ); } + #[tokio::test] + async fn caching_provider_avoids_duplicate_requests() { + // given + let mut mock_provider = MockFeesProvider::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // mock validates no extra calls made + } + + #[tokio::test] + async fn caching_provider_fetches_only_missing_blocks() { + // Given: A mock FeesProvider + let mut mock_provider = MockFeesProvider::new(); + + // Expectation: The provider will fetch blocks 3..=5, since 0..=2 are cached + let mut sequence = Sequence::new(); + mock_provider + .expect_fees() + .with(eq(0..=2)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + mock_provider + .expect_fees() + .with(eq(3..=5)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=2).await.unwrap(); + + // when + let _ = provider.get_fees(2..=5).await.unwrap(); + + // then + // not called for the overlapping area + } + + fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { + SequentialBlockFees::try_from( + height_range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap() + } + + #[tokio::test] + async fn caching_provider_evicts_oldest_blocks() { + // given + let mut mock_provider = MockFeesProvider::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .times(2) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + mock_provider + .expect_fees() + .with(eq(5..=9)) + .times(1) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + let _ = provider.get_fees(5..=9).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // will refetch 0..=4 due to eviction + } + + #[tokio::test] + async fn caching_provider_handles_request_larger_than_cache() { + use mockall::predicate::*; + + // given + let mut mock_provider = MockFeesProvider::new(); + + let cache_limit = 5; + + mock_provider + .expect_fees() + .with(eq(0..=9)) + .times(1) + .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); + + let provider = CachingFeesProvider::new(mock_provider, cache_limit); + + // when + let result = provider.get_fees(0..=9).await.unwrap(); + + assert_eq!(result, generate_sequential_fees(0..=9)); + } + // fn calculate_tx_fee(fees: &Fees) -> u128 { // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 // } From 8b6993b920b2f6dc385b81f27e3cdb21d8031fb8 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 22:36:00 +0100 Subject: [PATCH 030/136] changed the last n block calculation, need to fix tests --- packages/services/src/state_committer.rs | 20 +++++++++++++++---- .../services/src/state_committer/fee_algo.rs | 11 ++++++---- .../src/state_committer/fee_analytics.rs | 6 ++++++ packages/services/tests/state_committer.rs | 19 ++++++++++-------- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5d53539f..7cd45e14 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -8,7 +8,7 @@ pub mod service { }; use crate::{ - state_committer::fee_algo::Context, + state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, }; @@ -78,7 +78,7 @@ pub mod service { config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider, + decider: SendOrWaitDecider>, } impl StateCommitter @@ -96,14 +96,20 @@ pub mod service { ) -> Self { let startup_time = clock.now(); let price_algo = config.fee_algo; + + // TODO: segfault, configure this cache + let decider = SendOrWaitDecider::new( + CachingFeesProvider::new(l1_adapter.clone(), 24 * 3600 / 12), + price_algo, + ); Self { - l1_adapter: l1_adapter.clone(), + l1_adapter, fuel_api, storage, config, clock, startup_time, - decider: SendOrWaitDecider::new(l1_adapter, price_algo), + decider, } } } @@ -141,6 +147,10 @@ pub mod service { .oldest_block_in_bundle; let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); + eprintln!( + "deciding whether to send tx with {} fragments", + fragments.len() + ); self.decider .should_send_blob_tx( @@ -159,9 +169,11 @@ pub mod service { previous_tx: Option, ) -> Result<()> { if !self.should_send_tx(&fragments).await? { + eprintln!("decided against sending fragments"); info!("decided against sending fragments"); return Ok(()); } + eprintln!("decided to send fragments"); info!("about to send at most {} fragments", fragments.len()); diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index a247e670..56c92806 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -36,17 +36,21 @@ impl SendOrWaitDecider

{ ) -> crate::Result { // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs - let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; + let last_n_blocks = |n: u64| { + context.at_l1_height.saturating_sub(n.saturating_sub(1))..=context.at_l1_height + }; let short_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await?; + eprintln!("short term sma: {:?}", short_term_sma); let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; + eprintln!("long term sma: {:?}", long_term_sma); let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -479,7 +483,6 @@ mod tests { "Later: after max wait, send regardless" )] #[test_case( - // Partway: at 80 blocks behind, tolerance might have increased enough to accept Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, @@ -492,7 +495,7 @@ mod tests { always_acceptable_fee: 0, }, }, - 65, + 80, true; "Mid-wait: increased tolerance allows acceptance" )] @@ -507,7 +510,7 @@ mod tests { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, - always_acceptable_fee: 1_781_000_000_000 + always_acceptable_fee: 2_700_000_000_000 }, }, 0, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index e29e4242..36b0f14a 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -292,15 +292,21 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; + eprintln!("fees received"); + eprintln!("checking if fees are complete"); let received_height_range = fees.height_range(); + eprintln!("received height range: {:?}", received_height_range); if received_height_range != block_range { + eprintln!("not equeal {received_height_range:?} != {block_range:?}",); return Err(crate::Error::from(format!( "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" ))); } + eprintln!("calculating mean"); Ok(Self::mean(fees)) } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 0c548651..e407783e 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -544,7 +544,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { // Define fee sequence with high fees to ensure that without the behind condition, it wouldn't send let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -552,7 +552,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 2, + 1, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -560,7 +560,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 3, + 2, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -568,7 +568,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 4, + 3, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -576,7 +576,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 5, + 4, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -584,7 +584,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 6, + 5, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -594,7 +594,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, @@ -700,7 +700,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -727,6 +727,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); + eprintln!("about to contrust the committer"); let mut state_committer = StateCommitter::new( ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, @@ -740,9 +741,11 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran }, setup.test_clock(), ); + eprintln!("constructed the committer"); // when state_committer.run().await?; + eprintln!("ran the committer"); // then // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind From 94c89797b74ef8072ffe547e2f88bdc581201325 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 08:48:18 +0100 Subject: [PATCH 031/136] fixed tests --- packages/services/src/state_committer.rs | 10 +++++----- .../services/src/state_committer/fee_analytics.rs | 15 +++++++++------ packages/services/tests/state_committer.rs | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 7cd45e14..eef5b386 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -389,9 +389,9 @@ pub mod port { use super::{Api, Fees, MockApi, SequentialBlockFees}; #[derive(Clone)] - pub struct ApiMockWFees { + pub struct ApiMockWFees

{ pub api: Arc, - fee_provider: Fees, + fee_provider: P, } impl ApiMockWFees { @@ -403,7 +403,7 @@ pub mod port { } } - impl ApiMockWFees { + impl

ApiMockWFees

{ pub fn w_preconfigured_fees( self, fees: impl IntoIterator, @@ -415,9 +415,9 @@ pub mod port { } } - impl Api for ApiMockWFees + impl

Api for ApiMockWFees

where - T: FeesProvider + Send + Sync, + P: FeesProvider + Send + Sync, { async fn fees( &self, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 36b0f14a..6f6efdfd 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -207,14 +207,17 @@ pub mod testing { impl FeesProvider for ConstantFeesProvider { async fn fees( &self, - _height_range: RangeInclusive, + height_range: RangeInclusive, ) -> crate::Result { - let fees = BlockFees { - height: self.current_block_height().await?, - fees: self.fees, - }; + let fees = height_range + .into_iter() + .map(|height| BlockFees { + height, + fees: self.fees, + }) + .collect_vec(); - Ok(vec![fees].try_into().unwrap()) + Ok(fees.try_into().unwrap()) } async fn current_block_height(&self) -> crate::Result { diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index e407783e..63d307dc 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -594,7 +594,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 5 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, @@ -618,7 +618,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { )]); l1_mock_submit .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( From 004be103d8adae60b3a12bd2b3c303f96632e3de Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:39:04 +0100 Subject: [PATCH 032/136] validate percentages --- committer/src/main.rs | 2 +- committer/src/setup.rs | 18 ++- packages/services/src/state_committer.rs | 53 ++++++-- .../services/src/state_committer/fee_algo.rs | 113 ++++++++---------- .../src/state_committer/fee_analytics.rs | 1 - packages/services/tests/state_committer.rs | 13 +- packages/test-helpers/src/lib.rs | 1 - 7 files changed, 114 insertions(+), 87 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 0473d0c7..5810df75 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -78,7 +78,7 @@ async fn main() -> Result<()> { storage.clone(), cancel_token.clone(), &config, - ); + )?; let state_importer_handle = setup::block_importer(fuel_adapter, storage.clone(), cancel_token.clone(), &config); diff --git a/committer/src/setup.rs b/committer/src/setup.rs index b09449c6..8f9178dc 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -120,7 +120,7 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, -) -> tokio::task::JoinHandle<()> { +) -> Result> { let state_committer = services::StateCommitter::new( l1, fuel, @@ -137,8 +137,16 @@ pub fn state_committer( }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config.app.fee_algo.start_discount_percentage, - end_premium_percentage: config.app.fee_algo.end_premium_percentage, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config + .app + .fee_algo + .end_premium_percentage + .try_into()?, always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, }, }, @@ -146,12 +154,12 @@ pub fn state_committer( SystemClock, ); - schedule_polling( + Ok(schedule_polling( config.app.tx_finalization_check_interval, state_committer, "State Committer", cancel_token, - ) + )) } pub fn block_importer( diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index eef5b386..f4565d3e 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -10,7 +10,7 @@ pub mod service { use crate::{ state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, + Error, Result, Runner, }; use itertools::Itertools; use tracing::info; @@ -23,15 +23,56 @@ pub mod service { pub long: u64, } - // TODO: segfault validate start discount is less than end premium and both are positive + #[derive(Default, Copy, Clone, Debug, PartialEq)] + pub struct Percentage(f64); + + impl TryFrom for Percentage { + type Error = crate::Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } + } + + impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } + } + + impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } + } + #[derive(Debug, Clone, Copy)] pub struct FeeThresholds { pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: f64, - pub end_premium_percentage: f64, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, pub always_acceptable_fee: u128, } + #[cfg(feature = "test-helpers")] + impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } + } + #[derive(Debug, Clone, Copy)] pub struct FeeAlgoConfig { pub sma_periods: SmaPeriods, @@ -61,9 +102,7 @@ pub mod service { sma_periods: SmaPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0., - end_premium_percentage: 0., - always_acceptable_fee: u128::MAX, + ..FeeThresholds::default() }, }, } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 56c92806..c099a51d 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,5 +1,7 @@ use std::cmp::min; +use crate::state_committer::service::Percentage; + use super::{ fee_analytics::{FeeAnalytics, FeesProvider}, port::l1::Fees, @@ -105,8 +107,6 @@ impl SendOrWaitDecider

{ fee: u128, context: Context, ) -> u128 { - const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); let blocks_behind = u128::from(context.num_l2_blocks_behind); @@ -117,11 +117,11 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = percentage_to_ppm(fee_thresholds.start_discount_percentage); - let end_premium_ppm = percentage_to_ppm(fee_thresholds.end_premium_percentage); + let start_discount_ppm = fee_thresholds.start_discount_percentage.ppm(); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = PPM.saturating_sub(start_discount_ppm); + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); // 2. How late are we: eg. late enough to add 25% to our base multiplier let premium_increment = Self::calculate_premium_increment( @@ -134,11 +134,12 @@ impl SendOrWaitDecider

{ // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% let multiplier_ppm = min( base_multiplier.saturating_add(premium_increment), - PPM + end_premium_ppm, + Percentage::PPM + end_premium_ppm, ); // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) } fn calculate_premium_increment( @@ -147,19 +148,19 @@ impl SendOrWaitDecider

{ blocks_behind: u128, max_blocks_behind: u128, ) -> u128 { - const PPM: u128 = 1_000_000; // 100% in PPM - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); let proportion = if max_blocks_behind == 0 { 0 } else { blocks_behind - .saturating_mul(PPM) + .saturating_mul(Percentage::PPM) .saturating_div(max_blocks_behind) }; - total_ppm.saturating_mul(proportion).saturating_div(PPM) + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -174,16 +175,15 @@ impl SendOrWaitDecider

{ } } -fn percentage_to_ppm(percentage: f64) -> u128 { - (percentage * 1_000_000.0) as u128 -} - #[cfg(test)] mod tests { use super::*; - use crate::state_committer::{ - fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - service::{FeeThresholds, SmaPeriods}, + use crate::{ + state_committer::{ + fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + service::{FeeThresholds, Percentage, SmaPeriods}, + }, + types::NonNegative, }; use test_case::test_case; @@ -211,9 +211,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }, 0, // not behind at all @@ -228,9 +227,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }, 0, @@ -246,8 +244,7 @@ mod tests { fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, + ..Default::default() } }, 0, @@ -262,9 +259,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -279,9 +275,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -296,9 +291,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -313,9 +307,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -330,9 +323,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -347,9 +339,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -365,9 +356,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -382,9 +372,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -400,9 +389,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -418,9 +406,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -436,9 +423,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -455,8 +441,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), always_acceptable_fee: 0, }, }, @@ -473,8 +459,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, } }, @@ -490,8 +476,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, }, @@ -508,8 +494,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 2_700_000_000_000 }, }, @@ -553,9 +539,8 @@ mod tests { // Test Case 1: No blocks behind, no discount or premium FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, 1000, Context { @@ -568,8 +553,8 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.25, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), always_acceptable_fee: 0, }, 2000, @@ -583,9 +568,9 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25, - end_premium_percentage: 0.0, + start_discount_percentage: 0.25.try_into().unwrap(), always_acceptable_fee: 0, + ..Default::default() }, 800, Context { @@ -598,9 +583,9 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.30, + end_premium_percentage: 0.30.try_into().unwrap(), always_acceptable_fee: 0, + ..Default::default() }, 1000, Context { @@ -614,8 +599,8 @@ mod tests { // Test Case 8: High fee with premium FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10, // 100,000 PPM - end_premium_percentage: 0.20, // 200,000 PPM + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, 10_000, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 6f6efdfd..2c611226 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -293,7 +293,6 @@ impl

FeeAnalytics

{ } impl FeeAnalytics

{ - // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 63d307dc..9b38c639 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -392,9 +392,8 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -499,9 +498,8 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -597,9 +595,8 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -703,8 +700,8 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, }; diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index c248ae64..3f8c7cdc 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -1,6 +1,5 @@ #![deny(unused_crate_dependencies)] -use std::sync::Arc; use std::{ops::RangeInclusive, time::Duration}; use clock::TestClock; From 963422e2a1b0a943737f1348c83bcf44e081b0c1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:47:39 +0100 Subject: [PATCH 033/136] add capping for discount percentage at 100% --- .../services/src/state_committer/fee_algo.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index c099a51d..11565c98 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -117,7 +117,10 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = fee_thresholds.start_discount_percentage.ppm(); + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% @@ -611,6 +614,21 @@ mod tests { 11970; "High fee with premium" )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 1, + at_l1_height: 0, + }, + 12; + "Discount exceeds 100%, should be capped to 100%" +)] fn test_calculate_max_upper_fee( fee_thresholds: FeeThresholds, fee: u128, From 80b9cef3afad8f2d2fc849b84ef6543d9e4c6b9f Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:57:01 +0100 Subject: [PATCH 034/136] if too far back, makes decision even with a faulty fee provider --- .../services/src/state_committer/fee_algo.rs | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 11565c98..58c5e267 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -29,13 +29,16 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes - // (once that is implemented) + // TODO: segfault logging pub async fn should_send_blob_tx( &self, num_blobs: u32, context: Context, ) -> crate::Result { + if self.too_far_behind(context) { + return Ok(true); + } + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs let last_n_blocks = |n: u64| { @@ -56,16 +59,7 @@ impl SendOrWaitDecider

{ let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); - let fee_always_acceptable = - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; - eprintln!("fee always acceptable: {}", fee_always_acceptable); - - let too_far_behind = - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); - - eprintln!("too far behind: {}", too_far_behind); - - if fee_always_acceptable || too_far_behind { + if self.fee_always_acceptable(short_term_tx_fee) { return Ok(true); } @@ -102,6 +96,14 @@ impl SendOrWaitDecider

{ Ok(short_term_tx_fee < max_upper_tx_fee) } + fn too_far_behind(&self, context: Context) -> bool { + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + fn calculate_max_upper_fee( fee_thresholds: &FeeThresholds, fee: u128, @@ -647,4 +649,37 @@ mod tests { expected_max_upper_fee, max_upper_fee ); } + #[tokio::test] + async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = FeeAlgoConfig { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let fee_provider = PreconfiguredFeesProvider::new(vec![]); + let sut = SendOrWaitDecider::new(fee_provider, config); + + let context = Context { + num_l2_blocks_behind: 20, + at_l1_height: 100, + }; + + // when + let should_send = sut + .should_send_blob_tx(1, context) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); + } } From 7fa623dae5fa430fa33dc572c44e6b4a9b08ba26 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 10:04:17 +0100 Subject: [PATCH 035/136] add logging, remove print statements --- packages/services/src/state_committer.rs | 9 +--- .../services/src/state_committer/fee_algo.rs | 41 +++++++------------ .../src/state_committer/fee_analytics.rs | 6 --- packages/services/tests/state_committer.rs | 10 ++--- 4 files changed, 19 insertions(+), 47 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index f4565d3e..22c6b436 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -186,10 +186,6 @@ pub mod service { .oldest_block_in_bundle; let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - eprintln!( - "deciding whether to send tx with {} fragments", - fragments.len() - ); self.decider .should_send_blob_tx( @@ -208,12 +204,9 @@ pub mod service { previous_tx: Option, ) -> Result<()> { if !self.should_send_tx(&fragments).await? { - eprintln!("decided against sending fragments"); - info!("decided against sending fragments"); + info!("decided against sending fragments due to high fees"); return Ok(()); } - eprintln!("decided to send fragments"); - info!("about to send at most {} fragments", fragments.len()); let data = fragments.clone().map(|f| f.fragment); diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 58c5e267..5e6cc4fa 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,5 +1,7 @@ use std::cmp::min; +use tracing::info; + use crate::state_committer::service::Percentage; use super::{ @@ -29,13 +31,13 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault logging pub async fn should_send_blob_tx( &self, num_blobs: u32, context: Context, ) -> crate::Result { if self.too_far_behind(context) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", context.num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); return Ok(true); } @@ -49,17 +51,16 @@ impl SendOrWaitDecider

{ .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await?; - eprintln!("short term sma: {:?}", short_term_sma); let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; - eprintln!("long term sma: {:?}", long_term_sma); let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); return Ok(true); } @@ -67,33 +68,21 @@ impl SendOrWaitDecider

{ let max_upper_tx_fee = Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); - let long_vs_max_delta_perc = - ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) - .abs(); - - let short_vs_max_delta_perc = ((max_upper_tx_fee as f64 - short_term_tx_fee as f64) - / short_term_tx_fee as f64 - * 100.) - .abs(); + let should_send = short_term_tx_fee < max_upper_tx_fee; - if long_term_tx_fee <= max_upper_tx_fee { - eprintln!("The max upper fee({max_upper_tx_fee}) is above the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); } else { - eprintln!("The max upper fee({max_upper_tx_fee}) is below the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); } - if short_term_tx_fee <= max_upper_tx_fee { - eprintln!("The short term fee({short_term_tx_fee}) is below the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); - } else { - eprintln!("The short term fee({short_term_tx_fee}) is above the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); - } - - eprintln!( - "Short-term fee: {}, Long-term fee: {}, Max upper fee: {}", - short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee - ); - - Ok(short_term_tx_fee < max_upper_tx_fee) + Ok(should_send) } fn too_far_behind(&self, context: Context) -> bool { diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 2c611226..e7db5702 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -294,21 +294,15 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; - eprintln!("fees received"); - eprintln!("checking if fees are complete"); let received_height_range = fees.height_range(); - eprintln!("received height range: {:?}", received_height_range); if received_height_range != block_range { - eprintln!("not equeal {received_height_range:?} != {block_range:?}",); return Err(crate::Error::from(format!( "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" ))); } - eprintln!("calculating mean"); Ok(Self::mean(fees)) } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 9b38c639..4ada71e7 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -297,10 +297,9 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ), ]); - l1_mock_submit.expect_current_height().returning(|| { - eprintln!("I was called"); - Box::pin(async { Ok(0) }) - }); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -724,7 +723,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); - eprintln!("about to contrust the committer"); let mut state_committer = StateCommitter::new( ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, @@ -738,11 +736,9 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran }, setup.test_clock(), ); - eprintln!("constructed the committer"); // when state_committer.run().await?; - eprintln!("ran the committer"); // then // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind From 3ab2e934213354b85af909946d26d22eba97c238 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 10:10:58 +0100 Subject: [PATCH 036/136] added fees at height --- .../src/state_committer/fee_analytics.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index e7db5702..548223cb 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -306,6 +306,18 @@ impl FeeAnalytics

{ Ok(Self::mean(fees)) } + pub async fn fees_at_height(&self, height: u64) -> crate::Result { + let fee = self + .fees_provider + .fees(height..=height) + .await? + .into_iter() + .next() + .expect("sequential fees guaranteed not empty"); + + Ok(fee.fees) + } + fn mean(fees: SequentialBlockFees) -> Fees { let count = fees.len() as u128; @@ -331,6 +343,7 @@ impl FeeAnalytics

{ mod tests { use itertools::Itertools; use mockall::{predicate::eq, Sequence}; + use testing::{incrementing_fees, PreconfiguredFeesProvider}; use super::*; @@ -683,6 +696,29 @@ mod tests { assert_eq!(result, generate_sequential_fees(0..=9)); } + #[tokio::test] + async fn price_at_height_returns_correct_fee() { + // given + let fees_map = incrementing_fees(5); + let fees_provider = PreconfiguredFeesProvider::new(fees_map.clone()); + let fee_analytics = FeeAnalytics::new(fees_provider); + let height = 2; + + // when + let fee = fee_analytics.fees_at_height(height).await.unwrap(); + + // then + let expected_fee = Fees { + base_fee_per_gas: 3, + reward: 3, + base_fee_per_blob_gas: 3, + }; + assert_eq!( + fee, expected_fee, + "Fee at height {height} should be {expected_fee:?}" + ); + } + // fn calculate_tx_fee(fees: &Fees) -> u128 { // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 // } From 62f6f3844829f2cece33c7f608644694df7653a0 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:33:47 +0100 Subject: [PATCH 037/136] moving fee tracking into its own service due to a need for regular prices for metrics --- committer/src/setup.rs | 47 +- packages/adapters/eth/src/fee_conversion.rs | 4 +- packages/adapters/eth/src/lib.rs | 29 +- packages/services/src/fee_tracker.rs | 4 + .../services/src/fee_tracker/fee_analytics.rs | 386 ++++++++ packages/services/src/fee_tracker/port.rs | 463 ++++++++++ packages/services/src/fee_tracker/service.rs | 370 ++++++++ packages/services/src/fee_tracker/testing.rs | 2 + packages/services/src/lib.rs | 1 + packages/services/src/state_committer.rs | 189 +--- .../services/src/state_committer/fee_algo.rs | 674 -------------- .../src/state_committer/fee_analytics.rs | 829 ------------------ packages/services/tests/fee_tracker.rs | 379 ++++++++ packages/services/tests/state_committer.rs | 52 +- packages/services/tests/state_listener.rs | 12 +- packages/test-helpers/src/lib.rs | 15 +- 16 files changed, 1719 insertions(+), 1737 deletions(-) create mode 100644 packages/services/src/fee_tracker.rs create mode 100644 packages/services/src/fee_tracker/fee_analytics.rs create mode 100644 packages/services/src/fee_tracker/port.rs create mode 100644 packages/services/src/fee_tracker/service.rs create mode 100644 packages/services/src/fee_tracker/testing.rs delete mode 100644 packages/services/src/state_committer/fee_algo.rs delete mode 100644 packages/services/src/state_committer/fee_analytics.rs create mode 100644 packages/services/tests/fee_tracker.rs diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 8f9178dc..fac00a21 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,10 +9,8 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - state_committer::{ - port::Storage, - service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, - }, + fee_tracker::service::{FeeThresholds, FeeTracker, SmaPeriods}, + state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -121,6 +119,26 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, ) -> Result> { + let fee_tracker = FeeTracker::new( + l1.clone(), + services::fee_tracker::service::Config { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, + ); + let state_committer = services::StateCommitter::new( l1, fuel, @@ -130,28 +148,9 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - fee_algo: FeeAlgoConfig { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config - .app - .fee_algo - .start_discount_percentage - .try_into()?, - end_premium_percentage: config - .app - .fee_algo - .end_premium_percentage - .try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, - }, }, SystemClock, + fee_tracker, ); Ok(schedule_polling( diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 3063b321..0d34073c 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -3,7 +3,7 @@ use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; use services::{ - state_committer::port::l1::{BlockFees, Fees}, + fee_tracker::port::l1::{BlockFees, Fees}, Result, }; @@ -98,7 +98,7 @@ pub fn chunk_range_inclusive( #[cfg(test)] mod test { use alloy::rpc::types::FeeHistory; - use services::state_committer::port::l1::{BlockFees, Fees}; + use services::fee_tracker::port::l1::{BlockFees, Fees}; use std::ops::RangeInclusive; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 2cb7fc31..c3a27c29 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - state_committer::port::l1::SequentialBlockFees, + fee_tracker::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -206,20 +206,11 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } } -impl services::state_committer::port::l1::Api for WebsocketClient { +impl services::fee_tracker::port::l1::Api for WebsocketClient { async fn current_height(&self) -> Result { self._get_block_number().await } - delegate! { - to (*self) { - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<(L1Tx, FragmentsSubmitted)>; - } - } async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -248,6 +239,22 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } } +impl services::state_committer::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + + delegate! { + to (*self) { + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<(L1Tx, FragmentsSubmitted)>; + } + } +} + #[cfg(test)] mod test { use alloy::eips::eip4844::DATA_GAS_PER_BLOB; diff --git a/packages/services/src/fee_tracker.rs b/packages/services/src/fee_tracker.rs new file mode 100644 index 00000000..a977c59c --- /dev/null +++ b/packages/services/src/fee_tracker.rs @@ -0,0 +1,4 @@ +pub mod port; +pub mod service; + +mod fee_analytics; diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs new file mode 100644 index 00000000..ce7a8648 --- /dev/null +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -0,0 +1,386 @@ +use std::ops::RangeInclusive; + +use crate::Error; + +use super::port::l1::{Api, Fees, SequentialBlockFees}; + +#[derive(Debug, Clone)] +pub struct FeeAnalytics

{ + fees_provider: P, +} + +impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } +} + +impl FeeAnalytics

{ + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + let fees = self.fees_provider.fees(block_range.clone()).await?; + + let received_height_range = fees.height_range(); + if received_height_range != block_range { + return Err(Error::from(format!( + "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" + ))); + } + + Ok(Self::mean(fees)) + } + + pub async fn fees_at_height(&self, height: u64) -> crate::Result { + let fee = self + .fees_provider + .fees(height..=height) + .await? + .into_iter() + .next() + .expect("sequential fees guaranteed not empty"); + + Ok(fee.fees) + } + + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } + } +} + +#[cfg(test)] +mod tests { + use itertools::Itertools; + use mockall::{predicate::eq, Sequence}; + + use crate::fee_tracker::port::l1::{testing, BlockFees}; + + use super::*; + + #[test] + fn can_create_valid_sequential_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn sequential_fees_cannot_be_empty() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn fees_must_be_sequential() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" + ); + } + + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested + #[test] + fn produced_iterator_gives_correct_values() { + // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order + let block_fees = vec![ + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); + assert_eq!( + iterated_fees, expectation, + "Expected iterator to yield the same block fees" + ); + } + use std::path::PathBuf; + + #[tokio::test] + async fn calculates_sma_correctly_for_last_1_block() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); + } + + #[tokio::test] + async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); + } + + #[tokio::test] + async fn errors_out_if_returned_fees_are_not_complete() { + // given + let mut fees = testing::incrementing_fees(5); + fees.remove(&4); + let fees_provider = testing::PreconfiguredFeeApi::new(fees); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let err = fee_analytics + .calculate_sma(0..=4) + .await + .expect_err("should have failed because returned fees are not complete"); + + // then + assert_eq!( + err.to_string(), + "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" + ); + } + + #[tokio::test] + async fn price_at_height_returns_correct_fee() { + // given + let fees_map = testing::incrementing_fees(5); + let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); + let fee_analytics = FeeAnalytics::new(fees_provider); + let height = 2; + + // when + let fee = fee_analytics.fees_at_height(height).await.unwrap(); + + // then + let expected_fee = Fees { + base_fee_per_gas: 3, + reward: 3, + base_fee_per_blob_gas: 3, + }; + assert_eq!( + fee, expected_fee, + "Fee at height {height} should be {expected_fee:?}" + ); + } + + // fn calculate_tx_fee(fees: &Fees) -> u128 { + // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + // } + // + // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + // let mut csv_writer = + // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + // .unwrap(); + // csv_writer + // .write_record(["height", "tx_fee"].iter()) + // .unwrap(); + // for (height, fee) in tx_fees { + // csv_writer + // .write_record([height.to_string(), fee.to_string()]) + // .unwrap(); + // } + // csv_writer.flush().unwrap(); + // } + + // #[tokio::test] + // async fn something() { + // let client = make_pub_eth_client().await; + // use services::fee_analytics::port::l1::FeesProvider; + // + // let current_block_height = 21408300; + // let starting_block_height = current_block_height - 48 * 3600 / 12; + // let data = client + // .fees(starting_block_height..=current_block_height) + // .await + // .into_iter() + // .collect::>(); + // + // let fee_lookup = data + // .iter() + // .map(|b| (b.height, b.fees)) + // .collect::>(); + // + // let short_sma = 25u64; + // let long_sma = 900; + // + // let current_tx_fees = data + // .iter() + // .map(|b| (b.height, calculate_tx_fee(&b.fees))) + // .collect::>(); + // + // save_tx_fees(¤t_tx_fees, "current_fees.csv"); + // + // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + // let fee_analytics = FeeAnalytics::new(local_client.clone()); + // + // let mut short_sma_tx_fees = vec![]; + // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - short_sma..=height) + // .await; + // + // let tx_fee = calculate_tx_fee(&fees); + // + // short_sma_tx_fees.push((height, tx_fee)); + // } + // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + // + // let decider = SendOrWaitDecider::new( + // FeeAnalytics::new(local_client.clone()), + // services::state_committer::fee_optimization::Config { + // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + // short: short_sma, + // long: long_sma, + // }, + // fee_thresholds: Feethresholds { + // max_l2_blocks_behind: 43200 * 3, + // start_discount_percentage: 0.2, + // end_premium_percentage: 0.2, + // always_acceptable_fee: 1000000000000000u128, + // }, + // }, + // ); + // + // let mut decisions = vec![]; + // let mut long_sma_tx_fees = vec![]; + // + // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - long_sma..=height) + // .await; + // let tx_fee = calculate_tx_fee(&fees); + // long_sma_tx_fees.push((height, tx_fee)); + // + // if decider + // .should_send_blob_tx( + // 6, + // Context { + // at_l1_height: height, + // num_l2_blocks_behind: (height - starting_block_height) * 12, + // }, + // ) + // .await + // { + // let current_fees = fee_lookup.get(&height).unwrap(); + // let current_tx_fee = calculate_tx_fee(current_fees); + // decisions.push((height, current_tx_fee)); + // } + // } + // + // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + // save_tx_fees(&decisions, "decisions.csv"); + // } +} diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs new file mode 100644 index 00000000..e9112e94 --- /dev/null +++ b/packages/services/src/fee_tracker/port.rs @@ -0,0 +1,463 @@ +pub mod l1 { + #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] + pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct BlockFees { + pub height: u64, + pub fees: Fees, + } + use std::ops::RangeInclusive; + + use itertools::Itertools; + + #[derive(Debug, PartialEq, Eq)] + pub struct SequentialBlockFees { + fees: Vec, + } + + #[derive(Debug)] + pub struct InvalidSequence(String); + + impl std::error::Error for InvalidSequence {} + + impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } + } + + impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } + } + + // Cannot be empty + #[allow(clippy::len_without_is_empty)] + impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } + + pub fn height_range(&self) -> RangeInclusive { + let start = self.fees.first().expect("not empty").height; + let end = self.fees.last().expect("not empty").height; + start..=end + } + } + impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + let heights = fees.iter().map(|f| f.height).collect::>(); + if !is_sequential { + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); + } + + Ok(Self { fees }) + } + } + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait Api { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + async fn current_height(&self) -> crate::Result; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use itertools::Itertools; + + use super::{Api, BlockFees, Fees, SequentialBlockFees}; + + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeeApi { + fees: Fees, + } + + impl ConstantFeeApi { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl Api for ConstantFeeApi { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = height_range + .into_iter() + .map(|height| BlockFees { + height, + fees: self.fees, + }) + .collect_vec(); + + Ok(fees.try_into().unwrap()) + } + + async fn current_height(&self) -> crate::Result { + Ok(0) + } + } + + #[derive(Debug, Clone)] + pub struct PreconfiguredFeeApi { + fees: BTreeMap, + } + + impl Api for PreconfiguredFeeApi { + async fn current_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) + } + + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = self + .fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_vec(); + + Ok(fees.try_into().expect("block fees not sequential")) + } + } + + impl PreconfiguredFeeApi { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, + }, + ) + }) + .collect() + } + } +} + +pub mod cache { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use tokio::sync::RwLock; + + use crate::Error; + + use super::l1::{Api, BlockFees, Fees, SequentialBlockFees}; + + #[derive(Debug)] + pub struct CachingApi

{ + fees_provider: P, + cache: RwLock>, + cache_limit: usize, + } + + impl

CachingApi

{ + pub fn new(fees_provider: P, cache_limit: usize) -> Self { + Self { + fees_provider, + cache: RwLock::new(BTreeMap::new()), + cache_limit, + } + } + } + + impl Api for CachingApi

{ + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + self.get_fees(height_range).await + } + + async fn current_height(&self) -> crate::Result { + self.fees_provider.current_height().await + } + } + + impl CachingApi

{ + pub async fn get_fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let mut missing_heights = vec![]; + + // Mind the scope to release the read lock + { + let cache = self.cache.read().await; + for height in height_range.clone() { + if !cache.contains_key(&height) { + missing_heights.push(height); + } + } + } + + if !missing_heights.is_empty() { + let fetched_fees = self + .fees_provider + .fees( + *missing_heights.first().expect("not empty") + ..=*missing_heights.last().expect("not empty"), + ) + .await?; + + let mut cache = self.cache.write().await; + for block_fee in fetched_fees { + cache.insert(block_fee.height, block_fee.fees); + } + } + + let fees: Vec<_> = { + let cache = self.cache.read().await; + height_range + .filter_map(|h| { + cache.get(&h).map(|f| BlockFees { + height: h, + fees: *f, + }) + }) + .collect() + }; + + self.shrink_cache().await; + + SequentialBlockFees::try_from(fees).map_err(|e| Error::Other(e.to_string())) + } + + async fn shrink_cache(&self) { + let mut cache = self.cache.write().await; + while cache.len() > self.cache_limit { + cache.pop_first(); + } + } + } + + #[cfg(test)] + mod tests { + use std::ops::RangeInclusive; + + use mockall::{predicate::eq, Sequence}; + + use crate::fee_tracker::port::{ + cache::CachingApi, + l1::{BlockFees, Fees, MockApi, SequentialBlockFees}, + }; + + #[tokio::test] + async fn caching_provider_avoids_duplicate_requests() { + // given + let mut mock_provider = MockApi::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // mock validates no extra calls made + } + + #[tokio::test] + async fn caching_provider_fetches_only_missing_blocks() { + // given + let mut mock_provider = MockApi::new(); + + let mut sequence = Sequence::new(); + mock_provider + .expect_fees() + .with(eq(0..=2)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + mock_provider + .expect_fees() + .with(eq(3..=5)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=2).await.unwrap(); + + // when + let _ = provider.get_fees(2..=5).await.unwrap(); + + // then + // not called for the overlapping area + } + + #[tokio::test] + async fn caching_provider_evicts_oldest_blocks() { + // given + let mut mock_provider = MockApi::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .times(2) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + mock_provider + .expect_fees() + .with(eq(5..=9)) + .times(1) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + let _ = provider.get_fees(5..=9).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // will refetch 0..=4 due to eviction + } + + #[tokio::test] + async fn caching_provider_handles_request_larger_than_cache() { + use mockall::predicate::*; + + // given + let mut mock_provider = MockApi::new(); + + let cache_limit = 5; + + mock_provider + .expect_fees() + .with(eq(0..=9)) + .times(1) + .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); + + let provider = CachingApi::new(mock_provider, cache_limit); + + // when + let result = provider.get_fees(0..=9).await.unwrap(); + + assert_eq!(result, generate_sequential_fees(0..=9)); + } + + fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { + SequentialBlockFees::try_from( + height_range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap() + } + } +} diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs new file mode 100644 index 00000000..8224c003 --- /dev/null +++ b/packages/services/src/fee_tracker/service.rs @@ -0,0 +1,370 @@ +use std::{cmp::min, num::NonZeroU32}; + +use tracing::info; + +use crate::{state_committer::service::SendOrWaitDecider, Error}; + +use super::{ + fee_analytics::FeeAnalytics, + port::l1::{Api, Fees}, +}; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub sma_periods: SmaPeriods, + pub fee_thresholds: FeeThresholds, +} + +#[cfg(feature = "test-helpers")] +impl Default for Config { + fn default() -> Self { + Config { + sma_periods: SmaPeriods { short: 1, long: 2 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + ..FeeThresholds::default() + }, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SmaPeriods { + pub short: u64, + pub long: u64, +} + +#[derive(Debug, Clone, Copy)] +pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU32, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, + pub always_acceptable_fee: u128, +} + +#[cfg(feature = "test-helpers")] +impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } +} + +impl SendOrWaitDecider for FeeTracker

{ + async fn should_send_blob_tx( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> crate::Result { + if self.too_far_behind(num_l2_blocks_behind) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); + return Ok(true); + } + + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs + let last_n_blocks = + |n: u64| at_l1_height.saturating_sub(n.saturating_sub(1))..=at_l1_height; + + let short_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) + .await?; + + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + + if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); + return Ok(true); + } + + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let max_upper_tx_fee = Self::calculate_max_upper_fee( + &self.config.fee_thresholds, + long_term_tx_fee, + num_l2_blocks_behind, + ); + + let should_send = short_term_tx_fee < max_upper_tx_fee; + + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } else { + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } + + Ok(should_send) + } +} + +#[derive(Default, Copy, Clone, Debug, PartialEq)] +pub struct Percentage(f64); + +impl TryFrom for Percentage { + type Error = Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } +} + +impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } +} + +impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } +} + +pub struct FeeTracker

{ + fee_analytics: FeeAnalytics

, + config: Config, +} + +impl FeeTracker

{ + fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { + num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + ) -> u128 { + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(num_l2_blocks_behind); + + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({})", + blocks_behind, + max_blocks_behind + ); + + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = Self::calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); + + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + Percentage::PPM + end_premium_ppm, + ); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) + } + + fn calculate_premium_increment( + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); + + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(Percentage::PPM) + .saturating_div(max_blocks_behind) + }; + + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) + } + + // TODO: Segfault maybe dont leak so much eth abstractions + fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; + let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; + + base_fee + blob_fee + fees.reward + } +} + +impl

FeeTracker

{ + pub fn new(fee_provider: P, config: Config) -> Self { + Self { + fee_analytics: FeeAnalytics::new(fee_provider), + config, + } + } +} + +#[cfg(test)] +mod tests { + use crate::fee_tracker::port::l1::testing::ConstantFeeApi; + + use super::*; + use test_case::test_case; + + struct Context { + num_l2_blocks_behind: u32, + at_l1_height: u64, + } + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + Context { + num_l2_blocks_behind: 0, + at_l1_height: 0, + }, + 1000; + "No blocks behind, multiplier should be 100%" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 2000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 2050; + "Half blocks behind with discount and premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 800, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 700; + "Start discount only, no premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + end_premium_percentage: 0.30.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 1150; + "End premium only, no discount" + )] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 10_000, + Context { + num_l2_blocks_behind: 99, + at_l1_height: 0, + }, + 11970; + "High fee with premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 1, + at_l1_height: 0, + }, + 12; + "Discount exceeds 100%, should be capped to 100%" +)] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + context: Context, + expected_max_upper_fee: u128, + ) { + let Context { + num_l2_blocks_behind, + at_l1_height, + } = context; + let max_upper_fee = FeeTracker::::calculate_max_upper_fee( + &fee_thresholds, + fee, + num_l2_blocks_behind, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } +} diff --git a/packages/services/src/fee_tracker/testing.rs b/packages/services/src/fee_tracker/testing.rs new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/packages/services/src/fee_tracker/testing.rs @@ -0,0 +1,2 @@ + + diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index c6ceb616..48123b0a 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,6 +2,7 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; +pub mod fee_tracker; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 22c6b436..35b4e5e6 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,6 +1,3 @@ -mod fee_algo; -mod fee_analytics; - pub mod service { use std::{ num::{NonZeroU32, NonZeroUsize}, @@ -8,77 +5,13 @@ pub mod service { }; use crate::{ - state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, + fee_tracker::service::FeeTracker, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Error, Result, Runner, }; use itertools::Itertools; use tracing::info; - use super::fee_algo::SendOrWaitDecider; - - #[derive(Debug, Clone, Copy)] - pub struct SmaPeriods { - pub short: u64, - pub long: u64, - } - - #[derive(Default, Copy, Clone, Debug, PartialEq)] - pub struct Percentage(f64); - - impl TryFrom for Percentage { - type Error = crate::Error; - - fn try_from(value: f64) -> std::result::Result { - if value < 0. { - return Err(Error::Other(format!("Invalid percentage value {value}"))); - } - - Ok(Self(value)) - } - } - - impl From for f64 { - fn from(value: Percentage) -> Self { - value.0 - } - } - - impl Percentage { - pub const ZERO: Self = Percentage(0.); - pub const PPM: u128 = 1_000_000; - - pub fn ppm(&self) -> u128 { - (self.0 * 1_000_000.) as u128 - } - } - - #[derive(Debug, Clone, Copy)] - pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: Percentage, - pub end_premium_percentage: Percentage, - pub always_acceptable_fee: u128, - } - - #[cfg(feature = "test-helpers")] - impl Default for FeeThresholds { - fn default() -> Self { - Self { - max_l2_blocks_behind: NonZeroU32::MAX, - start_discount_percentage: Percentage::ZERO, - end_premium_percentage: Percentage::ZERO, - always_acceptable_fee: u128::MAX, - } - } - } - - #[derive(Debug, Clone, Copy)] - pub struct FeeAlgoConfig { - pub sma_periods: SmaPeriods, - pub fee_thresholds: FeeThresholds, - } - // src/config.rs #[derive(Debug, Clone)] pub struct Config { @@ -87,7 +20,6 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub fee_algo: FeeAlgoConfig, } #[cfg(feature = "test-helpers")] @@ -98,31 +30,34 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - fee_algo: FeeAlgoConfig { - sma_periods: SmaPeriods { short: 1, long: 2 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - ..FeeThresholds::default() - }, - }, } } } + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + pub trait SendOrWaitDecider { + async fn should_send_blob_tx( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> Result; + } + /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider>, + decider: D, } - impl StateCommitter + impl StateCommitter where - L1: Clone + Send + Sync, Clock: crate::state_committer::port::Clock, { /// Creates a new `StateCommitter`. @@ -132,15 +67,10 @@ pub mod service { storage: Db, config: Config, clock: Clock, + decider: Decider, ) -> Self { let startup_time = clock.now(); - let price_algo = config.fee_algo; - // TODO: segfault, configure this cache - let decider = SendOrWaitDecider::new( - CachingFeesProvider::new(l1_adapter.clone(), 24 * 3600 / 12), - price_algo, - ); Self { l1_adapter, fuel_api, @@ -153,12 +83,13 @@ pub mod service { } } - impl StateCommitter + impl StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, + Decider: SendOrWaitDecider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -190,10 +121,8 @@ pub mod service { self.decider .should_send_blob_tx( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), - Context { - num_l2_blocks_behind, - at_l1_height: l1_height, - }, + num_l2_blocks_behind, + l1_height, ) .await } @@ -344,12 +273,13 @@ pub mod service { } } - impl Runner for StateCommitter + impl Runner for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, + Decider: SendOrWaitDecider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -376,7 +306,6 @@ pub mod port { use nonempty::NonEmpty; - pub use crate::state_committer::fee_analytics::{BlockFees, Fees, SequentialBlockFees}; use crate::{ types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, Result, @@ -398,80 +327,6 @@ pub mod port { fragments: NonEmpty, previous_tx: Option, ) -> Result<(L1Tx, FragmentsSubmitted)>; - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result; - } - - #[cfg(feature = "test-helpers")] - pub mod testing { - use std::{ops::RangeInclusive, sync::Arc}; - - use nonempty::NonEmpty; - - use crate::{ - state_committer::fee_analytics::{ - testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - FeesProvider, - }, - types::{FragmentsSubmitted, L1Tx}, - }; - - use super::{Api, Fees, MockApi, SequentialBlockFees}; - - #[derive(Clone)] - pub struct ApiMockWFees

{ - pub api: Arc, - fee_provider: P, - } - - impl ApiMockWFees { - pub fn new(api: MockApi) -> Self { - Self { - api: Arc::new(api), - fee_provider: ConstantFeesProvider::new(Fees::default()), - } - } - } - - impl

ApiMockWFees

{ - pub fn w_preconfigured_fees( - self, - fees: impl IntoIterator, - ) -> ApiMockWFees { - ApiMockWFees { - api: self.api, - fee_provider: PreconfiguredFeesProvider::new(fees), - } - } - } - - impl

Api for ApiMockWFees

- where - P: FeesProvider + Send + Sync, - { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - FeesProvider::fees(&self.fee_provider, height_range).await - } - - async fn current_height(&self) -> crate::Result { - self.api.current_height().await - } - - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> crate::Result<(L1Tx, FragmentsSubmitted)> { - self.api - .submit_state_fragments(fragments, previous_tx) - .await - } - } } } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs deleted file mode 100644 index 5e6cc4fa..00000000 --- a/packages/services/src/state_committer/fee_algo.rs +++ /dev/null @@ -1,674 +0,0 @@ -use std::cmp::min; - -use tracing::info; - -use crate::state_committer::service::Percentage; - -use super::{ - fee_analytics::{FeeAnalytics, FeesProvider}, - port::l1::Fees, - service::{FeeAlgoConfig, FeeThresholds}, -}; - -pub struct SendOrWaitDecider

{ - fee_analytics: FeeAnalytics

, - config: FeeAlgoConfig, -} - -impl

SendOrWaitDecider

{ - pub fn new(fee_provider: P, config: FeeAlgoConfig) -> Self { - Self { - fee_analytics: FeeAnalytics::new(fee_provider), - config, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct Context { - pub num_l2_blocks_behind: u32, - pub at_l1_height: u64, -} - -impl SendOrWaitDecider

{ - pub async fn should_send_blob_tx( - &self, - num_blobs: u32, - context: Context, - ) -> crate::Result { - if self.too_far_behind(context) { - info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", context.num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); - return Ok(true); - } - - // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller - // wants to send more than 6 blobs - let last_n_blocks = |n: u64| { - context.at_l1_height.saturating_sub(n.saturating_sub(1))..=context.at_l1_height - }; - - let short_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await?; - - let long_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await?; - - let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); - - if self.fee_always_acceptable(short_term_tx_fee) { - info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); - return Ok(true); - } - - let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - let max_upper_tx_fee = - Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); - - let should_send = short_term_tx_fee < max_upper_tx_fee; - - if should_send { - info!( - "Sending because short term price {} is lower than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } else { - info!( - "Not sending because short term price {} is higher than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } - - Ok(should_send) - } - - fn too_far_behind(&self, context: Context) -> bool { - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() - } - - fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee - } - - fn calculate_max_upper_fee( - fee_thresholds: &FeeThresholds, - fee: u128, - context: Context, - ) -> u128 { - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); - let blocks_behind = u128::from(context.num_l2_blocks_behind); - - debug_assert!( - blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({})", - blocks_behind, - max_blocks_behind - ); - - let start_discount_ppm = min( - fee_thresholds.start_discount_percentage.ppm(), - Percentage::PPM, - ); - let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); - - // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); - - // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = Self::calculate_premium_increment( - start_discount_ppm, - end_premium_ppm, - blocks_behind, - max_blocks_behind, - ); - - // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% - let multiplier_ppm = min( - base_multiplier.saturating_add(premium_increment), - Percentage::PPM + end_premium_ppm, - ); - - // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm) - .saturating_div(Percentage::PPM) - } - - fn calculate_premium_increment( - start_discount_ppm: u128, - end_premium_ppm: u128, - blocks_behind: u128, - max_blocks_behind: u128, - ) -> u128 { - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - - let proportion = if max_blocks_behind == 0 { - 0 - } else { - blocks_behind - .saturating_mul(Percentage::PPM) - .saturating_div(max_blocks_behind) - }; - - total_ppm - .saturating_mul(proportion) - .saturating_div(Percentage::PPM) - } - - // TODO: Segfault maybe dont leak so much eth abstractions - fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { - const DATA_GAS_PER_BLOB: u128 = 131_072u128; - const INTRINSIC_GAS: u128 = 21_000u128; - - let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; - let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; - - base_fee + blob_fee + fees.reward - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - state_committer::{ - fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - service::{FeeThresholds, Percentage, SmaPeriods}, - }, - types::NonNegative, - }; - - use test_case::test_case; - use tokio; - - fn generate_fees(config: FeeAlgoConfig, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { - let older_fees = std::iter::repeat_n( - old_fees, - (config.sma_periods.long - config.sma_periods.short) as usize, - ); - let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); - - older_fees - .chain(newer_fees) - .enumerate() - .map(|(i, f)| (i as u64, f)) - .collect() - } - - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, // not behind at all - true; - "Should send because all short-term fees are lower than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, - false; - "Should not send because all short-term fees are higher than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, - max_l2_blocks_behind: 100.try_into().unwrap(), - ..Default::default() - } - }, - 0, - true; - "Should send since short-term fee < always_acceptable_fee" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_blob_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_blob_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term reward is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term reward is higher" - )] - #[test_case( - // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because multiple short-term fees are lower" - )] - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because all fees are identical and no tolerance" - )] - #[test_case( - // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: short-term base_fee_per_gas and reward are lower, send" - )] - #[test_case( - // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Zero blobs: short-term reward is higher, don't send" - )] - #[test_case( - // Zero blobs don't care about higher short-term base_fee_per_blob_gas - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" - )] - // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. - #[test_case( - // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: Percentage::try_from(0.20).unwrap(), - end_premium_percentage: Percentage::try_from(0.20).unwrap(), - always_acceptable_fee: 0, - }, - }, - 0, - false; - "Early: short-term expensive, not send" - )] - #[test_case( - // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - } - }, - 100, - true; - "Later: after max wait, send regardless" - )] - #[test_case( - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - }, - 80, - true; - "Mid-wait: increased tolerance allows acceptance" - )] - #[test_case( - // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, - Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 2_700_000_000_000 - }, - }, - 0, - true; - "Always acceptable fee triggers immediate send" - )] - #[tokio::test] - async fn parameterized_send_or_wait_tests( - old_fees: Fees, - new_fees: Fees, - num_blobs: u32, - config: FeeAlgoConfig, - num_l2_blocks_behind: u32, - expected_decision: bool, - ) { - let fees = generate_fees(config, old_fees, new_fees); - let fees_provider = PreconfiguredFeesProvider::new(fees); - let current_block_height = fees_provider.current_block_height().await.unwrap(); - - let sut = SendOrWaitDecider::new(fees_provider, config); - - let should_send = sut - .should_send_blob_tx( - num_blobs, - Context { - at_l1_height: current_block_height, - num_l2_blocks_behind, - }, - ) - .await - .unwrap(); - - assert_eq!( - should_send, expected_decision, - "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", - ); - } - - #[test_case( - // Test Case 1: No blocks behind, no discount or premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - Context { - num_l2_blocks_behind: 0, - at_l1_height: 0, - }, - 1000; - "No blocks behind, multiplier should be 100%" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 2000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 2050; - "Half blocks behind with discount and premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 800, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 700; - "Start discount only, no premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - end_premium_percentage: 0.30.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 1150; - "End premium only, no discount" - )] - #[test_case( - // Test Case 8: High fee with premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 10_000, - Context { - num_l2_blocks_behind: 99, - at_l1_height: 0, - }, - 11970; - "High fee with premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 1.50.try_into().unwrap(), // 150% - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 1000, - Context { - num_l2_blocks_behind: 1, - at_l1_height: 0, - }, - 12; - "Discount exceeds 100%, should be capped to 100%" -)] - fn test_calculate_max_upper_fee( - fee_thresholds: FeeThresholds, - fee: u128, - context: Context, - expected_max_upper_fee: u128, - ) { - let max_upper_fee = SendOrWaitDecider::::calculate_max_upper_fee( - &fee_thresholds, - fee, - context, - ); - - assert_eq!( - max_upper_fee, expected_max_upper_fee, - "Expected max_upper_fee to be {}, but got {}", - expected_max_upper_fee, max_upper_fee - ); - } - #[tokio::test] - async fn test_send_when_too_far_behind_and_fee_provider_fails() { - // given - let config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 10.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }; - - // having no fees will make the validation in fee analytics fail - let fee_provider = PreconfiguredFeesProvider::new(vec![]); - let sut = SendOrWaitDecider::new(fee_provider, config); - - let context = Context { - num_l2_blocks_behind: 20, - at_l1_height: 100, - }; - - // when - let should_send = sut - .should_send_blob_tx(1, context) - .await - .expect("Should send despite fee provider failure"); - - // then - assert!( - should_send, - "Should send because too far behind, regardless of fee provider status" - ); - } -} diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs deleted file mode 100644 index 548223cb..00000000 --- a/packages/services/src/state_committer/fee_analytics.rs +++ /dev/null @@ -1,829 +0,0 @@ -use std::{collections::BTreeMap, ops::RangeInclusive}; - -use itertools::Itertools; -use tokio::sync::RwLock; - -use crate::state_committer::port::l1::Api; - -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] -pub struct Fees { - pub base_fee_per_gas: u128, - pub reward: u128, - pub base_fee_per_blob_gas: u128, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct BlockFees { - pub height: u64, - pub fees: Fees, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct SequentialBlockFees { - fees: Vec, -} - -#[allow(async_fn_in_trait)] -#[trait_variant::make(Send)] -#[cfg_attr(feature = "test-helpers", mockall::automock)] -pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> crate::Result; - async fn current_block_height(&self) -> crate::Result; -} - -#[derive(Debug)] -pub struct CachingFeesProvider

{ - fees_provider: P, - cache: RwLock>, - cache_limit: usize, -} - -impl

CachingFeesProvider

{ - pub fn new(fees_provider: P, cache_limit: usize) -> Self { - Self { - fees_provider, - cache: RwLock::new(BTreeMap::new()), - cache_limit, - } - } -} - -impl FeesProvider for CachingFeesProvider

{ - async fn fees(&self, height_range: RangeInclusive) -> crate::Result { - self.get_fees(height_range).await - } - - async fn current_block_height(&self) -> crate::Result { - self.fees_provider.current_block_height().await - } -} - -impl CachingFeesProvider

{ - pub async fn get_fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let mut missing_heights = vec![]; - - // Mind the scope to release the read lock - { - let cache = self.cache.read().await; - for height in height_range.clone() { - if !cache.contains_key(&height) { - missing_heights.push(height); - } - } - } - - if !missing_heights.is_empty() { - let fetched_fees = self - .fees_provider - .fees( - *missing_heights.first().expect("not empty") - ..=*missing_heights.last().expect("not empty"), - ) - .await?; - - let mut cache = self.cache.write().await; - for block_fee in fetched_fees { - cache.insert(block_fee.height, block_fee.fees); - } - } - - let fees: Vec<_> = { - let cache = self.cache.read().await; - height_range - .filter_map(|h| { - cache.get(&h).map(|f| BlockFees { - height: h, - fees: *f, - }) - }) - .collect() - }; - - self.shrink_cache().await; - - SequentialBlockFees::try_from(fees).map_err(|e| crate::Error::Other(e.to_string())) - } - - async fn shrink_cache(&self) { - let mut cache = self.cache.write().await; - while cache.len() > self.cache_limit { - cache.pop_first(); - } - } -} - -impl FeesProvider for T { - async fn fees(&self, height_range: RangeInclusive) -> crate::Result { - Api::fees(self, height_range).await - } - - async fn current_block_height(&self) -> crate::Result { - Api::current_height(self).await - } -} - -impl IntoIterator for SequentialBlockFees { - type Item = BlockFees; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.fees.into_iter() - } -} - -// Cannot be empty -#[allow(clippy::len_without_is_empty)] -impl SequentialBlockFees { - pub fn len(&self) -> usize { - self.fees.len() - } - - pub fn height_range(&self) -> RangeInclusive { - let start = self.fees.first().expect("not empty").height; - let end = self.fees.last().expect("not empty").height; - start..=end - } -} - -#[derive(Debug)] -pub struct InvalidSequence(String); - -impl std::error::Error for InvalidSequence {} - -impl std::fmt::Display for InvalidSequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } -} - -impl TryFrom> for SequentialBlockFees { - type Error = InvalidSequence; - fn try_from(mut fees: Vec) -> Result { - if fees.is_empty() { - return Err(InvalidSequence("Input cannot be empty".to_string())); - } - - fees.sort_by_key(|f| f.height); - - let is_sequential = fees - .iter() - .tuple_windows() - .all(|(l, r)| l.height + 1 == r.height); - - let heights = fees.iter().map(|f| f.height).collect::>(); - if !is_sequential { - return Err(InvalidSequence(format!( - "blocks are not sequential by height: {heights:?}" - ))); - } - - Ok(Self { fees }) - } -} - -#[cfg(feature = "test-helpers")] -pub mod testing { - use std::{collections::BTreeMap, ops::RangeInclusive}; - - use itertools::Itertools; - - use crate::state_committer::port::l1::{BlockFees, Fees}; - - use super::{FeesProvider, SequentialBlockFees}; - - #[derive(Debug, Clone, Copy)] - pub struct ConstantFeesProvider { - fees: Fees, - } - - impl ConstantFeesProvider { - pub fn new(fees: Fees) -> Self { - Self { fees } - } - } - - impl FeesProvider for ConstantFeesProvider { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = height_range - .into_iter() - .map(|height| BlockFees { - height, - fees: self.fees, - }) - .collect_vec(); - - Ok(fees.try_into().unwrap()) - } - - async fn current_block_height(&self) -> crate::Result { - Ok(0) - } - } - - #[derive(Debug, Clone)] - pub struct PreconfiguredFeesProvider { - fees: BTreeMap, - } - - impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> crate::Result { - Ok(*self - .fees - .keys() - .last() - .expect("no fees registered with PreconfiguredFeesProvider")) - } - - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = self - .fees - .iter() - .skip_while(|(height, _)| !height_range.contains(height)) - .take_while(|(height, _)| height_range.contains(height)) - .map(|(height, fees)| BlockFees { - height: *height, - fees: *fees, - }) - .collect_vec(); - - Ok(fees.try_into().expect("block fees not sequential")) - } - } - - impl PreconfiguredFeesProvider { - pub fn new(blocks: impl IntoIterator) -> Self { - Self { - fees: blocks.into_iter().collect(), - } - } - } - - pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { - (0..num_blocks) - .map(|i| { - ( - i, - Fees { - base_fee_per_gas: i as u128 + 1, - reward: i as u128 + 1, - base_fee_per_blob_gas: i as u128 + 1, - }, - ) - }) - .collect() - } -} - -#[derive(Debug, Clone)] -pub struct FeeAnalytics

{ - fees_provider: P, -} -impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } -} - -impl FeeAnalytics

{ - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - let fees = self.fees_provider.fees(block_range.clone()).await?; - - let received_height_range = fees.height_range(); - if received_height_range != block_range { - return Err(crate::Error::from(format!( - "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" - ))); - } - - Ok(Self::mean(fees)) - } - - pub async fn fees_at_height(&self, height: u64) -> crate::Result { - let fee = self - .fees_provider - .fees(height..=height) - .await? - .into_iter() - .next() - .expect("sequential fees guaranteed not empty"); - - Ok(fee.fees) - } - - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } - } -} - -#[cfg(test)] -mod tests { - use itertools::Itertools; - use mockall::{predicate::eq, Sequence}; - use testing::{incrementing_fees, PreconfiguredFeesProvider}; - - use super::*; - - #[test] - fn can_create_valid_sequential_fees() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees.clone()); - - // Then - assert!( - result.is_ok(), - "Expected SequentialBlockFees creation to succeed" - ); - let sequential_fees = result.unwrap(); - assert_eq!(sequential_fees.len(), block_fees.len()); - } - - #[test] - fn sequential_fees_cannot_be_empty() { - // Given - let block_fees: Vec = vec![]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for empty input" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"Input cannot be empty\")" - ); - } - - #[test] - fn fees_must_be_sequential() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 3, // Non-sequential height - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for non-sequential heights" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" - ); - } - - // TODO: segfault add more tests so that the in-order iteration invariant is properly tested - #[test] - fn produced_iterator_gives_correct_values() { - // Given - // notice the heights are out of order so that we validate that the returned sequence is in - // order - let block_fees = vec![ - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); - - // When - let iterated_fees: Vec = sequential_fees.into_iter().collect(); - - // Then - let expectation = block_fees - .into_iter() - .sorted_by_key(|b| b.height) - .collect_vec(); - assert_eq!( - iterated_fees, expectation, - "Expected iterator to yield the same block fees" - ); - } - use std::path::PathBuf; - - #[tokio::test] - async fn calculates_sma_correctly_for_last_1_block() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); - - // then - assert_eq!(sma.base_fee_per_gas, 5); - assert_eq!(sma.reward, 5); - assert_eq!(sma.base_fee_per_blob_gas, 5); - } - - #[tokio::test] - async fn calculates_sma_correctly_for_last_5_blocks() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); - - // then - let mean = (5 + 4 + 3 + 2 + 1) / 5; - assert_eq!(sma.base_fee_per_gas, mean); - assert_eq!(sma.reward, mean); - assert_eq!(sma.base_fee_per_blob_gas, mean); - } - - #[tokio::test] - async fn errors_out_if_returned_fees_are_not_complete() { - // given - let mut fees = testing::incrementing_fees(5); - fees.remove(&4); - let fees_provider = testing::PreconfiguredFeesProvider::new(fees); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let err = fee_analytics - .calculate_sma(0..=4) - .await - .expect_err("should have failed because returned fees are not complete"); - - // then - assert_eq!( - err.to_string(), - "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" - ); - } - - #[tokio::test] - async fn caching_provider_avoids_duplicate_requests() { - // given - let mut mock_provider = MockFeesProvider::new(); - - mock_provider - .expect_fees() - .with(eq(0..=4)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=4).await.unwrap(); - - // when - let _ = provider.get_fees(0..=4).await.unwrap(); - - // then - // mock validates no extra calls made - } - - #[tokio::test] - async fn caching_provider_fetches_only_missing_blocks() { - // Given: A mock FeesProvider - let mut mock_provider = MockFeesProvider::new(); - - // Expectation: The provider will fetch blocks 3..=5, since 0..=2 are cached - let mut sequence = Sequence::new(); - mock_provider - .expect_fees() - .with(eq(0..=2)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }) - .in_sequence(&mut sequence); - - mock_provider - .expect_fees() - .with(eq(3..=5)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }) - .in_sequence(&mut sequence); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=2).await.unwrap(); - - // when - let _ = provider.get_fees(2..=5).await.unwrap(); - - // then - // not called for the overlapping area - } - - fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { - SequentialBlockFees::try_from( - height_range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap() - } - - #[tokio::test] - async fn caching_provider_evicts_oldest_blocks() { - // given - let mut mock_provider = MockFeesProvider::new(); - - mock_provider - .expect_fees() - .with(eq(0..=4)) - .times(2) - .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); - - mock_provider - .expect_fees() - .with(eq(5..=9)) - .times(1) - .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=4).await.unwrap(); - let _ = provider.get_fees(5..=9).await.unwrap(); - - // when - let _ = provider.get_fees(0..=4).await.unwrap(); - - // then - // will refetch 0..=4 due to eviction - } - - #[tokio::test] - async fn caching_provider_handles_request_larger_than_cache() { - use mockall::predicate::*; - - // given - let mut mock_provider = MockFeesProvider::new(); - - let cache_limit = 5; - - mock_provider - .expect_fees() - .with(eq(0..=9)) - .times(1) - .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); - - let provider = CachingFeesProvider::new(mock_provider, cache_limit); - - // when - let result = provider.get_fees(0..=9).await.unwrap(); - - assert_eq!(result, generate_sequential_fees(0..=9)); - } - - #[tokio::test] - async fn price_at_height_returns_correct_fee() { - // given - let fees_map = incrementing_fees(5); - let fees_provider = PreconfiguredFeesProvider::new(fees_map.clone()); - let fee_analytics = FeeAnalytics::new(fees_provider); - let height = 2; - - // when - let fee = fee_analytics.fees_at_height(height).await.unwrap(); - - // then - let expected_fee = Fees { - base_fee_per_gas: 3, - reward: 3, - base_fee_per_blob_gas: 3, - }; - assert_eq!( - fee, expected_fee, - "Fee at height {height} should be {expected_fee:?}" - ); - } - - // fn calculate_tx_fee(fees: &Fees) -> u128 { - // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 - // } - // - // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - // let mut csv_writer = - // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) - // .unwrap(); - // csv_writer - // .write_record(["height", "tx_fee"].iter()) - // .unwrap(); - // for (height, fee) in tx_fees { - // csv_writer - // .write_record([height.to_string(), fee.to_string()]) - // .unwrap(); - // } - // csv_writer.flush().unwrap(); - // } - - // #[tokio::test] - // async fn something() { - // let client = make_pub_eth_client().await; - // use services::fee_analytics::port::l1::FeesProvider; - // - // let current_block_height = 21408300; - // let starting_block_height = current_block_height - 48 * 3600 / 12; - // let data = client - // .fees(starting_block_height..=current_block_height) - // .await - // .into_iter() - // .collect::>(); - // - // let fee_lookup = data - // .iter() - // .map(|b| (b.height, b.fees)) - // .collect::>(); - // - // let short_sma = 25u64; - // let long_sma = 900; - // - // let current_tx_fees = data - // .iter() - // .map(|b| (b.height, calculate_tx_fee(&b.fees))) - // .collect::>(); - // - // save_tx_fees(¤t_tx_fees, "current_fees.csv"); - // - // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - // let fee_analytics = FeeAnalytics::new(local_client.clone()); - // - // let mut short_sma_tx_fees = vec![]; - // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - short_sma..=height) - // .await; - // - // let tx_fee = calculate_tx_fee(&fees); - // - // short_sma_tx_fees.push((height, tx_fee)); - // } - // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - // - // let decider = SendOrWaitDecider::new( - // FeeAnalytics::new(local_client.clone()), - // services::state_committer::fee_optimization::Config { - // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { - // short: short_sma, - // long: long_sma, - // }, - // fee_thresholds: Feethresholds { - // max_l2_blocks_behind: 43200 * 3, - // start_discount_percentage: 0.2, - // end_premium_percentage: 0.2, - // always_acceptable_fee: 1000000000000000u128, - // }, - // }, - // ); - // - // let mut decisions = vec![]; - // let mut long_sma_tx_fees = vec![]; - // - // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - long_sma..=height) - // .await; - // let tx_fee = calculate_tx_fee(&fees); - // long_sma_tx_fees.push((height, tx_fee)); - // - // if decider - // .should_send_blob_tx( - // 6, - // Context { - // at_l1_height: height, - // num_l2_blocks_behind: (height - starting_block_height) * 12, - // }, - // ) - // .await - // { - // let current_fees = fee_lookup.get(&height).unwrap(); - // let current_tx_fee = calculate_tx_fee(current_fees); - // decisions.push((height, current_tx_fee)); - // } - // } - // - // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - // save_tx_fees(&decisions, "decisions.csv"); - // } -} diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs new file mode 100644 index 00000000..c8f5f0ed --- /dev/null +++ b/packages/services/tests/fee_tracker.rs @@ -0,0 +1,379 @@ +use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; +use services::fee_tracker::port::l1::Api; +use services::fee_tracker::service::FeeThresholds; +use services::fee_tracker::service::FeeTracker; +use services::fee_tracker::service::Percentage; +use services::fee_tracker::service::SmaPeriods; +use services::fee_tracker::{port::l1::Fees, service::Config}; +use services::state_committer::service::SendOrWaitDecider; +use test_case::test_case; + +fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (config.sma_periods.long - config.sma_periods.short) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() +} + +#[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + 6, + Config { + sma_periods: services::fee_tracker::service::SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, // not behind at all + true; + "Should send because all short-term fees are lower than long-term" + )] +#[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, + false; + "Should not send because all short-term fees are higher than long-term" + )] +#[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + max_l2_blocks_behind: 100.try_into().unwrap(), + ..Default::default() + } + }, + 0, + true; + "Should send since short-term fee < always_acceptable_fee" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_gas is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_gas is higher" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_blob_gas is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_blob_gas is higher" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term reward is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term reward is higher" + )] +#[test_case( + // Multiple short-term fees are lower + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because multiple short-term fees are lower" + )] +#[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because all fees are identical and no tolerance" + )] +#[test_case( + // Zero blobs scenario: blob fee differences don't matter + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" + )] +#[test_case( + // Zero blobs but short-term reward is higher + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Zero blobs: short-term reward is higher, don't send" + )] +#[test_case( + // Zero blobs don't care about higher short-term base_fee_per_blob_gas + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" + )] +// Initially not send, but as num_l2_blocks_behind increases, acceptance grows. +#[test_case( + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), + always_acceptable_fee: 0, + }, + }, + 0, + false; + "Early: short-term expensive, not send" + )] +#[test_case( + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + } + }, + 100, + true; + "Later: after max wait, send regardless" + )] +#[test_case( + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + }, + 80, + true; + "Mid-wait: increased tolerance allows acceptance" + )] +#[test_case( + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, + Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 2_700_000_000_000 + }, + }, + 0, + true; + "Always acceptable fee triggers immediate send" + )] +#[tokio::test] +async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + num_blobs: u32, + config: Config, + num_l2_blocks_behind: u32, + expected_decision: bool, +) { + let fees = generate_fees(config, old_fees, new_fees); + let fees_provider = PreconfiguredFeeApi::new(fees); + let current_block_height = fees_provider.current_height().await.unwrap(); + + let sut = FeeTracker::new(fees_provider, config); + + let should_send = sut + .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) + .await + .unwrap(); + + assert_eq!( + should_send, expected_decision, + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", + ); +} + +#[tokio::test] +async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let fee_provider = PreconfiguredFeeApi::new(vec![]); + let sut = FeeTracker::new(fee_provider, config); + + // when + let should_send = sut + .should_send_blob_tx(1, 20, 100) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); +} diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 4ada71e7..b02946a5 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,13 @@ use services::{ - state_committer::{ - port::l1::{testing::ApiMockWFees, Fees}, - service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, + fee_tracker::{ + port::l1::{testing::PreconfiguredFeeApi, Fees}, + service::{Config as FeeTrackerConfig, FeeThresholds, FeeTracker, SmaPeriods}, }, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; use std::time::Duration; +use test_helpers::noop_fee_tracker; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -30,7 +31,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -40,6 +41,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), + noop_fee_tracker(), ); // when @@ -73,7 +75,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -83,6 +85,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time beyond the timeout @@ -108,7 +111,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -118,6 +121,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time less than the timeout @@ -153,7 +157,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -163,6 +167,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), + noop_fee_tracker(), ); // when @@ -199,7 +204,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -209,6 +214,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time to exceed the timeout since last finalized fragment @@ -245,7 +251,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> .expect_current_height() .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -255,6 +261,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time beyond the timeout from startup @@ -303,7 +310,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -314,6 +321,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Submit the initial fragments @@ -387,7 +395,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -415,17 +423,17 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // When @@ -493,7 +501,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -512,17 +520,17 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock).w_preconfigured_fees(fee_sequence), + l1_mock, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // when @@ -590,7 +598,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), @@ -618,17 +626,17 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // when @@ -695,7 +703,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_tracker_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -724,17 +732,17 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_tracker_config), ); // when diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 20ac4b38..5777b8da 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,13 +3,15 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - state_committer::port::l1::testing::ApiMockWFees, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, }; use test_case::test_case; -use test_helpers::mocks::{self, l1::TxStatus}; +use test_helpers::{ + mocks::{self, l1::TxStatus}, + noop_fee_tracker, +}; #[tokio::test] async fn successful_finalized_tx() -> Result<()> { @@ -446,7 +448,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -454,6 +456,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Orig tx @@ -549,7 +552,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -557,6 +560,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 3f8c7cdc..f7f35683 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,9 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::state_committer::port::l1::testing::ApiMockWFees; +use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::Fees; +use services::fee_tracker::service::FeeTracker; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -485,6 +487,10 @@ pub mod mocks { } } +pub fn noop_fee_tracker() -> FeeTracker { + FeeTracker::new(ConstantFeeApi::new(Fees::default()), Default::default()) +} + pub struct Setup { db: DbWithProcess, test_clock: TestClock, @@ -545,7 +551,7 @@ impl Setup { .return_once(move || Box::pin(async { Ok(0) })); StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -556,6 +562,7 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), + noop_fee_tracker(), ) .run() .await @@ -583,7 +590,7 @@ impl Setup { let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, fuel_mock, self.db(), services::StateCommitterConfig { @@ -591,9 +598,9 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - ..Default::default() }, self.test_clock.clone(), + noop_fee_tracker(), ); committer.run().await.unwrap(); From bfc5eb01571c25d1c07176a0d895152ee15bbb22 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:43:47 +0100 Subject: [PATCH 038/136] add helper to lessen test boilerplate --- packages/services/tests/state_committer.rs | 21 ++++++++++----------- packages/test-helpers/src/lib.rs | 9 ++++++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index b02946a5..ab903e1c 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,13 +1,13 @@ use services::{ fee_tracker::{ - port::l1::{testing::PreconfiguredFeeApi, Fees}, - service::{Config as FeeTrackerConfig, FeeThresholds, FeeTracker, SmaPeriods}, + port::l1::Fees, + service::{Config as FeeTrackerConfig, FeeThresholds, SmaPeriods}, }, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; use std::time::Duration; -use test_helpers::noop_fee_tracker; +use test_helpers::{noop_fee_tracker, preconfigured_fee_tracker}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -318,7 +318,6 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 5.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(60), - ..Default::default() }, test_clock.clone(), noop_fee_tracker(), @@ -395,7 +394,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_algo_config = services::fee_tracker::service::Config { + let fee_tracker_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -433,7 +432,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // When @@ -501,7 +500,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fee_algo_config = FeeTrackerConfig { + let fee_tracker_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -530,7 +529,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when @@ -598,7 +597,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fee_algo_config = FeeTrackerConfig { + let fee_tracker_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), @@ -636,7 +635,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when @@ -742,7 +741,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_tracker_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index f7f35683..2e0f6a72 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::testing::{ConstantFeeApi, PreconfiguredFeeApi}; use services::fee_tracker::port::l1::Fees; use services::fee_tracker::service::FeeTracker; use services::types::{ @@ -491,6 +491,13 @@ pub fn noop_fee_tracker() -> FeeTracker { FeeTracker::new(ConstantFeeApi::new(Fees::default()), Default::default()) } +pub fn preconfigured_fee_tracker( + fee_sequence: impl IntoIterator, + config: services::fee_tracker::service::Config, +) -> FeeTracker { + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), config) +} + pub struct Setup { db: DbWithProcess, test_clock: TestClock, From d39838e9dfbdbfe84c06d61bb1c417545d6079d9 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:45:36 +0100 Subject: [PATCH 039/136] remove context --- packages/services/src/fee_tracker/service.rs | 41 ++++---------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 8224c003..b99b7a4b 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -249,11 +249,6 @@ mod tests { use super::*; use test_case::test_case; - struct Context { - num_l2_blocks_behind: u32, - at_l1_height: u64, - } - #[test_case( // Test Case 1: No blocks behind, no discount or premium FeeThresholds { @@ -262,10 +257,7 @@ mod tests { ..Default::default() }, 1000, - Context { - num_l2_blocks_behind: 0, - at_l1_height: 0, - }, + 0, 1000; "No blocks behind, multiplier should be 100%" )] @@ -277,10 +269,7 @@ mod tests { always_acceptable_fee: 0, }, 2000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 2050; "Half blocks behind with discount and premium" )] @@ -292,10 +281,7 @@ mod tests { ..Default::default() }, 800, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 700; "Start discount only, no premium" )] @@ -307,10 +293,7 @@ mod tests { ..Default::default() }, 1000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 1150; "End premium only, no discount" )] @@ -323,10 +306,7 @@ mod tests { always_acceptable_fee: 0, }, 10_000, - Context { - num_l2_blocks_behind: 99, - at_l1_height: 0, - }, + 99, 11970; "High fee with premium" )] @@ -338,23 +318,16 @@ mod tests { always_acceptable_fee: 0, }, 1000, - Context { - num_l2_blocks_behind: 1, - at_l1_height: 0, - }, + 1, 12; "Discount exceeds 100%, should be capped to 100%" )] fn test_calculate_max_upper_fee( fee_thresholds: FeeThresholds, fee: u128, - context: Context, + num_l2_blocks_behind: u32, expected_max_upper_fee: u128, ) { - let Context { - num_l2_blocks_behind, - at_l1_height, - } = context; let max_upper_fee = FeeTracker::::calculate_max_upper_fee( &fee_thresholds, fee, From f908f3237bdf1218dbbabf936d80d34c8dbf2859 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:47:54 +0100 Subject: [PATCH 040/136] cargo fix --- packages/services/src/fee_tracker/fee_analytics.rs | 2 -- packages/services/src/state_committer.rs | 9 ++------- packages/services/tests/fee_tracker.rs | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index ce7a8648..9052a578 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -65,7 +65,6 @@ impl FeeAnalytics

{ #[cfg(test)] mod tests { use itertools::Itertools; - use mockall::{predicate::eq, Sequence}; use crate::fee_tracker::port::l1::{testing, BlockFees}; @@ -199,7 +198,6 @@ mod tests { "Expected iterator to yield the same block fees" ); } - use std::path::PathBuf; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 35b4e5e6..f66e4d5a 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,13 +1,9 @@ pub mod service { - use std::{ - num::{NonZeroU32, NonZeroUsize}, - time::Duration, - }; + use std::{num::NonZeroUsize, time::Duration}; use crate::{ - fee_tracker::service::FeeTracker, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, + Result, Runner, }; use itertools::Itertools; use tracing::info; @@ -302,7 +298,6 @@ pub mod port { }; pub mod l1 { - use std::ops::RangeInclusive; use nonempty::NonEmpty; diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index c8f5f0ed..1ee2c820 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,4 +1,3 @@ -use services::fee_tracker::port::l1::testing::ConstantFeeApi; use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; use services::fee_tracker::port::l1::Api; use services::fee_tracker::service::FeeThresholds; From d87d5cb4f0a34f03b2b4c46495bef392ac02d6a1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 15:59:42 +0100 Subject: [PATCH 041/136] add fee tracker to be run periodically --- committer/src/config.rs | 9 +- committer/src/main.rs | 10 ++ committer/src/setup.rs | 62 +++++--- e2e/src/committer.rs | 1 + .../services/src/fee_tracker/fee_analytics.rs | 25 ++-- packages/services/src/fee_tracker/service.rs | 132 ++++++++++++++++-- packages/services/src/fee_tracker/testing.rs | 2 - packages/services/src/state_committer.rs | 33 +++++ packages/services/tests/fee_tracker.rs | 45 +++--- packages/services/tests/state_committer.rs | 20 ++- 10 files changed, 266 insertions(+), 73 deletions(-) delete mode 100644 packages/services/src/fee_tracker/testing.rs diff --git a/committer/src/config.rs b/committer/src/config.rs index 5b3b6b03..eca2d76e 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, str::FromStr, time::Duration, }; @@ -93,6 +93,9 @@ pub struct App { /// How often to check for finalized l1 txs #[serde(deserialize_with = "human_readable_duration")] pub tx_finalization_check_interval: Duration, + /// How often to check for l1 prices + #[serde(deserialize_with = "human_readable_duration")] + pub l1_prices_check_interval: Duration, /// Number of L1 blocks that need to pass to accept the tx as finalized pub num_blocks_to_finalize_tx: u64, /// Interval after which to bump a pending tx @@ -119,10 +122,10 @@ pub struct App { #[derive(Debug, Clone, Deserialize)] pub struct FeeAlgoConfig { /// Short-term period for Simple Moving Average (SMA) in block numbers - pub short_sma_blocks: u64, + pub short_sma_blocks: NonZeroU64, /// Long-term period for Simple Moving Average (SMA) in block numbers - pub long_sma_blocks: u64, + pub long_sma_blocks: NonZeroU64, /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees pub max_l2_blocks_behind: NonZeroU32, diff --git a/committer/src/main.rs b/committer/src/main.rs index 5810df75..c2662738 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -72,12 +72,21 @@ async fn main() -> Result<()> { &metrics_registry, ); + let (fee_tracker, fee_tracker_handle) = setup::fee_tracker( + ethereum_rpc.clone(), + cancel_token.clone(), + &config, + &metrics_registry, + )?; + let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, + &metrics_registry, + fee_tracker, )?; let state_importer_handle = @@ -104,6 +113,7 @@ async fn main() -> Result<()> { handles.push(block_bundler); handles.push(state_listener_handle); handles.push(state_pruner_handle); + handles.push(fee_tracker_handle); } launch_api_server( diff --git a/committer/src/setup.rs b/committer/src/setup.rs index fac00a21..0cfe9c43 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -118,27 +118,9 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, + registry: &Registry, + fee_tracker: FeeTracker, ) -> Result> { - let fee_tracker = FeeTracker::new( - l1.clone(), - services::fee_tracker::service::Config { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config - .app - .fee_algo - .start_discount_percentage - .try_into()?, - end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, - }, - ); - let state_committer = services::StateCommitter::new( l1, fuel, @@ -153,6 +135,8 @@ pub fn state_committer( fee_tracker, ); + state_committer.register_metrics(registry); + Ok(schedule_polling( config.app.tx_finalization_check_interval, state_committer, @@ -337,3 +321,41 @@ pub async fn shut_down( storage.close().await; Ok(()) } + +pub fn fee_tracker( + l1: L1, + cancel_token: CancellationToken, + config: &config::Config, + registry: &Registry, +) -> Result<(FeeTracker, tokio::task::JoinHandle<()>)> { + let fee_tracker = FeeTracker::new( + l1, + services::fee_tracker::service::Config { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, + ); + + fee_tracker.register_metrics(registry); + + let handle = schedule_polling( + config.app.tx_finalization_check_interval, + fee_tracker.clone(), + "Fee Tracker", + cancel_token, + ); + + Ok((fee_tracker, handle)) +} diff --git a/e2e/src/committer.rs b/e2e/src/committer.rs index 4cdc21e2..e6a6f171 100644 --- a/e2e/src/committer.rs +++ b/e2e/src/committer.rs @@ -73,6 +73,7 @@ impl Committer { .env("COMMITTER__APP__HOST", "127.0.0.1") .env("COMMITTER__APP__BLOCK_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__TX_FINALIZATION_CHECK_INTERVAL", "5s") + .env("COMMITTER__APP__L1_PRICES_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__NUM_BLOCKS_TO_FINALIZE_TX", "3") .env("COMMITTER__APP__GAS_BUMP_TIMEOUT", "300s") .env("COMMITTER__APP__TX_MAX_FEE", "4000000000000000") diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index 9052a578..1f0d8144 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -2,7 +2,7 @@ use std::ops::RangeInclusive; use crate::Error; -use super::port::l1::{Api, Fees, SequentialBlockFees}; +use super::port::l1::{Api, BlockFees, Fees, SequentialBlockFees}; #[derive(Debug, Clone)] pub struct FeeAnalytics

{ @@ -29,7 +29,9 @@ impl FeeAnalytics

{ Ok(Self::mean(fees)) } - pub async fn fees_at_height(&self, height: u64) -> crate::Result { + pub async fn latest_fees(&self) -> crate::Result { + let height = self.fees_provider.current_height().await?; + let fee = self .fees_provider .fees(height..=height) @@ -38,7 +40,7 @@ impl FeeAnalytics

{ .next() .expect("sequential fees guaranteed not empty"); - Ok(fee.fees) + Ok(fee) } fn mean(fees: SequentialBlockFees) -> Fees { @@ -252,21 +254,24 @@ mod tests { } #[tokio::test] - async fn price_at_height_returns_correct_fee() { + async fn latest_fees_on_fee_analytics() { // given let fees_map = testing::incrementing_fees(5); let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); let fee_analytics = FeeAnalytics::new(fees_provider); - let height = 2; + let height = 4; // when - let fee = fee_analytics.fees_at_height(height).await.unwrap(); + let fee = fee_analytics.latest_fees().await.unwrap(); // then - let expected_fee = Fees { - base_fee_per_gas: 3, - reward: 3, - base_fee_per_blob_gas: 3, + let expected_fee = BlockFees { + height, + fees: Fees { + base_fee_per_gas: 5, + reward: 5, + base_fee_per_blob_gas: 5, + }, }; assert_eq!( fee, expected_fee, diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index b99b7a4b..8cd176a0 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -1,12 +1,20 @@ -use std::{cmp::min, num::NonZeroU32}; +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64}, + ops::RangeInclusive, +}; +use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, +}; use tracing::info; -use crate::{state_committer::service::SendOrWaitDecider, Error}; +use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; use super::{ fee_analytics::FeeAnalytics, - port::l1::{Api, Fees}, + port::l1::{Api, BlockFees, Fees}, }; #[derive(Debug, Clone, Copy)] @@ -19,7 +27,10 @@ pub struct Config { impl Default for Config { fn default() -> Self { Config { - sma_periods: SmaPeriods { short: 1, long: 2 }, + sma_periods: SmaPeriods { + short: 1.try_into().expect("not zero"), + long: 2.try_into().expect("not zero"), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), ..FeeThresholds::default() @@ -30,8 +41,8 @@ impl Default for Config { #[derive(Debug, Clone, Copy)] pub struct SmaPeriods { - pub short: u64, - pub long: u64, + pub short: NonZeroU64, + pub long: NonZeroU64, } #[derive(Debug, Clone, Copy)] @@ -60,7 +71,7 @@ impl SendOrWaitDecider for FeeTracker

{ num_blobs: u32, num_l2_blocks_behind: u32, at_l1_height: u64, - ) -> crate::Result { + ) -> Result { if self.too_far_behind(num_l2_blocks_behind) { info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); return Ok(true); @@ -68,8 +79,7 @@ impl SendOrWaitDecider for FeeTracker

{ // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs - let last_n_blocks = - |n: u64| at_l1_height.saturating_sub(n.saturating_sub(1))..=at_l1_height; + let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); let short_term_sma = self .fee_analytics @@ -81,14 +91,14 @@ impl SendOrWaitDecider for FeeTracker

{ .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; - let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &short_term_sma); if self.fee_always_acceptable(short_term_tx_fee) { info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); return Ok(true); } - let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &long_term_sma); let max_upper_tx_fee = Self::calculate_max_upper_fee( &self.config.fee_thresholds, long_term_tx_fee, @@ -143,9 +153,56 @@ impl Percentage { } } +#[derive(Debug, Clone)] +struct Metrics { + current_blob_tx_fee: IntGauge, + short_term_blob_tx_fee: IntGauge, + long_term_blob_tx_fee: IntGauge, +} + +impl Default for Metrics { + fn default() -> Self { + let current_blob_tx_fee = IntGauge::with_opts(Opts::new( + "current_blob_tx_fee", + "The current fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let short_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "short_term_blob_tx_fee", + "The short term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let long_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "long_term_blob_tx_fee", + "The long term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + Self { + current_blob_tx_fee, + short_term_blob_tx_fee, + long_term_blob_tx_fee, + } + } +} + +impl

RegistersMetrics for FeeTracker

{ + fn metrics(&self) -> Vec> { + vec![ + Box::new(self.metrics.current_blob_tx_fee.clone()), + Box::new(self.metrics.short_term_blob_tx_fee.clone()), + Box::new(self.metrics.long_term_blob_tx_fee.clone()), + ] + } +} + +#[derive(Clone)] pub struct FeeTracker

{ fee_analytics: FeeAnalytics

, config: Config, + metrics: Metrics, } impl FeeTracker

{ @@ -167,7 +224,7 @@ impl FeeTracker

{ debug_assert!( blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({})", + "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", blocks_behind, max_blocks_behind ); @@ -222,7 +279,7 @@ impl FeeTracker

{ } // TODO: Segfault maybe dont leak so much eth abstractions - fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; @@ -231,6 +288,44 @@ impl FeeTracker

{ base_fee + blob_fee + fees.reward } + + fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + } + + pub async fn update_metrics(&self) -> Result<()> { + let latest_fees = self.fee_analytics.latest_fees().await?; + let short_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.config.sma_periods.short, + )) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.config.sma_periods.long, + )) + .await?; + + let calc_fee = + |fees: &Fees| i64::try_from(Self::calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX); + + self.metrics + .current_blob_tx_fee + .set(calc_fee(&latest_fees.fees)); + self.metrics + .short_term_blob_tx_fee + .set(calc_fee(&short_term_sma)); + self.metrics + .long_term_blob_tx_fee + .set(calc_fee(&long_term_sma)); + + Ok(()) + } } impl

FeeTracker

{ @@ -238,10 +333,21 @@ impl

FeeTracker

{ Self { fee_analytics: FeeAnalytics::new(fee_provider), config, + metrics: Metrics::default(), } } } +impl

Runner for FeeTracker

+where + P: crate::fee_tracker::port::l1::Api + Send + Sync, +{ + async fn run(&mut self) -> Result<()> { + self.update_metrics().await?; + Ok(()) + } +} + #[cfg(test)] mod tests { use crate::fee_tracker::port::l1::testing::ConstantFeeApi; diff --git a/packages/services/src/fee_tracker/testing.rs b/packages/services/src/fee_tracker/testing.rs deleted file mode 100644 index 139597f9..00000000 --- a/packages/services/src/fee_tracker/testing.rs +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index f66e4d5a..caa47636 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -6,6 +6,10 @@ pub mod service { Result, Runner, }; use itertools::Itertools; + use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, + }; use tracing::info; // src/config.rs @@ -41,6 +45,29 @@ pub mod service { ) -> Result; } + struct Metrics { + num_l2_blocks_behind: IntGauge, + } + + impl Default for Metrics { + fn default() -> Self { + let num_l2_blocks_behind = IntGauge::with_opts(Opts::new( + "num_l2_blocks_behind", + "How many L2 blocks have been produced since the starting height of the oldest bundle we're committing", + )).expect("metric config to be correct"); + + Self { + num_l2_blocks_behind, + } + } + } + + impl RegistersMetrics for StateCommitter { + fn metrics(&self) -> Vec> { + vec![Box::new(self.metrics.num_l2_blocks_behind.clone())] + } + } + /// The `StateCommitter` is responsible for committing state fragments to L1. pub struct StateCommitter { l1_adapter: L1, @@ -50,6 +77,7 @@ pub mod service { clock: Clock, startup_time: DateTime, decider: D, + metrics: Metrics, } impl StateCommitter @@ -75,6 +103,7 @@ pub mod service { clock, startup_time, decider, + metrics: Default::default(), } } } @@ -114,6 +143,10 @@ pub mod service { let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); + self.metrics + .num_l2_blocks_behind + .set(num_l2_blocks_behind as i64); + self.decider .should_send_blob_tx( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 1ee2c820..7b92c0c1 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -11,9 +11,9 @@ use test_case::test_case; fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, - (config.sma_periods.long - config.sma_periods.short) as usize, + (config.sma_periods.long.get() - config.sma_periods.short.get()) as usize, ); - let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short.get() as usize); older_fees .chain(newer_fees) @@ -27,7 +27,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_periods: services::fee_tracker::service::SmaPeriods { short: 2, long: 6 }, + sma_periods: services::fee_tracker::service::SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -43,7 +43,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -59,7 +59,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), @@ -75,7 +75,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -91,7 +91,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -107,7 +107,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -123,7 +123,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -139,7 +139,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -155,7 +155,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -172,7 +172,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -188,7 +188,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -205,7 +205,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -222,7 +222,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -239,7 +239,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -257,7 +257,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: Percentage::try_from(0.20).unwrap(), @@ -275,7 +275,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -292,7 +292,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -310,7 +310,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -352,7 +352,10 @@ async fn parameterized_send_or_wait_tests( async fn test_send_when_too_far_behind_and_fee_provider_fails() { // given let config = Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 10.try_into().unwrap(), always_acceptable_fee: 0, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index ab903e1c..345fe363 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -395,7 +395,10 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ]; let fee_tracker_config = services::fee_tracker::service::Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -501,7 +504,10 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ]; let fee_tracker_config = FeeTrackerConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -598,7 +604,10 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_tracker_config = FeeTrackerConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), always_acceptable_fee: 0, @@ -703,7 +712,10 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ]; let fee_tracker_config = services::fee_tracker::service::Config { - sma_periods: SmaPeriods { short: 2, long: 5 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 5.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), From ebb846467a4f60616dff60b625fde4fc554fa375 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 18:35:56 +0100 Subject: [PATCH 042/136] revert debug logs --- e2e/src/whole_stack.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/src/whole_stack.rs b/e2e/src/whole_stack.rs index 1a3e257d..59e8e779 100644 --- a/e2e/src/whole_stack.rs +++ b/e2e/src/whole_stack.rs @@ -60,7 +60,7 @@ impl WholeStack { let db = start_db().await?; let committer = start_committer( - true, + logs, blob_support, db.clone(), ð_node, From c8a7ca7d8195b38c3296ed9413173aca6928563d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 19 Dec 2024 01:48:22 +0100 Subject: [PATCH 043/136] enable cache --- committer/src/main.rs | 1 + committer/src/setup.rs | 11 +++++++---- packages/services/src/fee_tracker/port.rs | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 9a654eea..202ecece 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,6 +7,7 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; +use services::fee_tracker::port::cache::CachingApi; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 0cfe9c43..8e2fcd18 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,10 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_tracker::service::{FeeThresholds, FeeTracker, SmaPeriods}, + fee_tracker::{ + port::cache::CachingApi, + service::{FeeThresholds, FeeTracker, SmaPeriods}, + }, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -119,7 +122,7 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, - fee_tracker: FeeTracker, + fee_tracker: FeeTracker>, ) -> Result> { let state_committer = services::StateCommitter::new( l1, @@ -327,9 +330,9 @@ pub fn fee_tracker( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, -) -> Result<(FeeTracker, tokio::task::JoinHandle<()>)> { +) -> Result<(FeeTracker>, tokio::task::JoinHandle<()>)> { let fee_tracker = FeeTracker::new( - l1, + CachingApi::new(l1, 24 * 3600 / 12), services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: config.app.fee_algo.short_sma_blocks, diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs index e9112e94..e7ba4f55 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_tracker/port.rs @@ -188,7 +188,7 @@ pub mod l1 { } pub mod cache { - use std::{collections::BTreeMap, ops::RangeInclusive}; + use std::{collections::BTreeMap, ops::RangeInclusive, sync::Arc}; use tokio::sync::RwLock; @@ -196,10 +196,10 @@ pub mod cache { use super::l1::{Api, BlockFees, Fees, SequentialBlockFees}; - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct CachingApi

{ fees_provider: P, - cache: RwLock>, + cache: Arc>>, cache_limit: usize, } @@ -207,7 +207,7 @@ pub mod cache { pub fn new(fees_provider: P, cache_limit: usize) -> Self { Self { fees_provider, - cache: RwLock::new(BTreeMap::new()), + cache: Arc::new(RwLock::new(BTreeMap::new())), cache_limit, } } From c1a9ea33c974206f02d892e8e56a30d2a49d295a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 19 Dec 2024 14:57:26 +0100 Subject: [PATCH 044/136] add more logging to debug --- packages/services/src/fee_tracker/service.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 8cd176a0..d5a9a4de 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -105,6 +105,8 @@ impl SendOrWaitDecider for FeeTracker

{ num_l2_blocks_behind, ); + info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); + let should_send = short_term_tx_fee < max_upper_tx_fee; if should_send { @@ -252,6 +254,8 @@ impl FeeTracker

{ Percentage::PPM + end_premium_ppm, ); + info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); + // 3. Final fee: eg. 105% of the base fee fee.saturating_mul(multiplier_ppm) .saturating_div(Percentage::PPM) From 173a26934c1f698b08371c9bbcf6f24b9ba1ff2a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 23 Dec 2024 10:34:55 +0100 Subject: [PATCH 045/136] fix bug, minimum not maximum starting height when calculating l2 blocks behind --- packages/services/src/state_committer.rs | 42 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index caa47636..de728ad8 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -137,15 +137,8 @@ pub mod service { let l1_height = self.l1_adapter.current_height().await?; let l2_height = self.fuel_api.latest_height().await?; - let oldest_l2_block_in_fragments = fragments - .maximum_by_key(|b| b.oldest_block_in_bundle) - .oldest_block_in_bundle; - - let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - - self.metrics - .num_l2_blocks_behind - .set(num_l2_blocks_behind as i64); + let num_l2_blocks_behind = self.num_l2_blocks_behind(fragments, l2_height); + self.update_l2_blocks_behind_metric(num_l2_blocks_behind); self.decider .should_send_blob_tx( @@ -156,6 +149,18 @@ pub mod service { .await } + fn num_l2_blocks_behind( + &self, + fragments: &NonEmpty, + l2_height: u32, + ) -> u32 { + let oldest_l2_block_in_fragments = fragments + .minimum_by_key(|b| b.oldest_block_in_bundle) + .oldest_block_in_bundle; + + l2_height.saturating_sub(oldest_l2_block_in_fragments) + } + async fn submit_fragments( &self, fragments: NonEmpty, @@ -224,7 +229,23 @@ pub mod service { .oldest_nonfinalized_fragments(starting_height, 6) .await?; - Ok(NonEmpty::collect(existing_fragments)) + let fragments = NonEmpty::collect(existing_fragments); + + if let Some(fragments) = fragments.as_ref() { + // Tracking the metric here as well to get updates more often -- because + // submit_fragments might not be called + self.update_l2_blocks_behind_metric( + self.num_l2_blocks_behind(fragments, latest_height), + ); + } + + Ok(fragments) + } + + fn update_l2_blocks_behind_metric(&self, l2_blocks_behind: u32) { + self.metrics + .num_l2_blocks_behind + .set(l2_blocks_behind as i64); } async fn should_submit_fragments(&self, fragment_count: NonZeroUsize) -> Result { @@ -256,6 +277,7 @@ pub mod service { self.submit_fragments(fragments, None).await?; } } + Ok(()) } From 3a554e400e7c35baf31173a19ea242f59e88cb1b Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 23 Dec 2024 11:57:13 +0100 Subject: [PATCH 046/136] mult reward by INTRINSIC_GAS --- packages/services/src/fee_tracker/service.rs | 12 ++++++++---- packages/services/tests/fee_tracker.rs | 7 +++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index d5a9a4de..680afa13 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -287,10 +287,14 @@ impl FeeTracker

{ const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; - let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; - let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; - - base_fee + blob_fee + fees.reward + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas); + let blob_fee = fees + .base_fee_per_blob_gas + .saturating_mul(num_blobs as u128) + .saturating_mul(DATA_GAS_PER_BLOB); + let reward_fee = fees.reward.saturating_mul(INTRINSIC_GAS); + + base_fee.saturating_add(blob_fee).saturating_add(reward_fee) } fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 7b92c0c1..6936791b 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -61,14 +61,14 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, max_l2_blocks_behind: 100.try_into().unwrap(), ..Default::default() } }, 0, true; - "Should send since short-term fee < always_acceptable_fee" + "Should send since short-term fee less than always_acceptable_fee" )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, @@ -234,9 +234,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Zero blobs: short-term reward is higher, don't send" )] #[test_case( - // Zero blobs don't care about higher short-term base_fee_per_blob_gas Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, + Fees { base_fee_per_gas: 2000, reward: 6000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, From 1131d93cf9c940e55b67ef2db9e6a78b82d0acfe Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 23 Dec 2024 15:12:40 +0100 Subject: [PATCH 047/136] incorporate fee check into should submit_fragments --- packages/services/src/state_committer.rs | 59 +++++++++++++----------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index de728ad8..19af7aac 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -133,7 +133,7 @@ pub mod service { Ok(std_elapsed >= self.config.fragment_accumulation_timeout) } - async fn should_send_tx(&self, fragments: &NonEmpty) -> Result { + async fn fees_acceptable(&self, fragments: &NonEmpty) -> Result { let l1_height = self.l1_adapter.current_height().await?; let l2_height = self.fuel_api.latest_height().await?; @@ -166,10 +166,6 @@ pub mod service { fragments: NonEmpty, previous_tx: Option, ) -> Result<()> { - if !self.should_send_tx(&fragments).await? { - info!("decided against sending fragments due to high fees"); - return Ok(()); - } info!("about to send at most {} fragments", fragments.len()); let data = fragments.clone().map(|f| f.fragment); @@ -248,32 +244,41 @@ pub mod service { .set(l2_blocks_behind as i64); } - async fn should_submit_fragments(&self, fragment_count: NonZeroUsize) -> Result { - if fragment_count >= self.config.fragments_to_accumulate { - return Ok(true); - } - info!( - "have only {} out of the target {} fragments per tx", - fragment_count, self.config.fragments_to_accumulate - ); + async fn should_submit_fragments( + &self, + fragments: &NonEmpty, + ) -> Result { + let fragment_count = fragments.len_nonzero(); + + let expired = || async { + let expired = self.is_timeout_expired().await?; + if expired { + info!( + "fragment accumulation timeout expired, available {}/{} fragments", + fragment_count, self.config.fragments_to_accumulate + ); + } + Result::Ok(expired) + }; - let expired = self.is_timeout_expired().await?; - if expired { - info!( - "fragment accumulation timeout expired, proceeding with {} fragments", - fragment_count - ); - } + let enough_fragments = || { + let enough_fragments = fragment_count >= self.config.fragments_to_accumulate; + if !enough_fragments { + info!( + "not enough fragments {}/{}", + fragment_count, self.config.fragments_to_accumulate + ); + }; + enough_fragments + }; - Ok(expired) + // wrapped in closures so that we short-circuit *and* reduce redundant logs + Ok((enough_fragments() || expired().await?) && self.fees_acceptable(fragments).await?) } async fn submit_fragments_if_ready(&self) -> Result<()> { if let Some(fragments) = self.next_fragments_to_submit().await? { - if self - .should_submit_fragments(fragments.len_nonzero()) - .await? - { + if self.should_submit_fragments(&fragments).await? { self.submit_fragments(fragments, None).await?; } } @@ -317,7 +322,9 @@ pub mod service { ); let fragments = self.fragments_submitted_by_tx(previous_tx.hash).await?; - self.submit_fragments(fragments, Some(previous_tx)).await?; + if self.fees_acceptable(&fragments).await? { + self.submit_fragments(fragments, Some(previous_tx)).await?; + } } Ok(()) From eeb87ae8813b1da6b93d7d9cf30a064a7202a60a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 24 Dec 2024 18:46:01 +0100 Subject: [PATCH 048/136] WIP, saving progress against powerloss --- ...fb8eb4358479355cc4fe0a85ad1ea8d6bd014.json | 20 ++++++++ packages/adapters/storage/src/lib.rs | 4 ++ packages/adapters/storage/src/postgres.rs | 15 ++++++ .../adapters/storage/src/test_instance.rs | 4 ++ packages/services/src/state_committer.rs | 49 +++++++++---------- 5 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 .sqlx/query-ca516d5a6f16e7877c129ec68edfb8eb4358479355cc4fe0a85ad1ea8d6bd014.json diff --git a/.sqlx/query-ca516d5a6f16e7877c129ec68edfb8eb4358479355cc4fe0a85ad1ea8d6bd014.json b/.sqlx/query-ca516d5a6f16e7877c129ec68edfb8eb4358479355cc4fe0a85ad1ea8d6bd014.json new file mode 100644 index 00000000..9ca9b604 --- /dev/null +++ b/.sqlx/query-ca516d5a6f16e7877c129ec68edfb8eb4358479355cc4fe0a85ad1ea8d6bd014.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(end_height) AS latest_bundled_height FROM bundles", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "latest_bundled_height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "ca516d5a6f16e7877c129ec68edfb8eb4358479355cc4fe0a85ad1ea8d6bd014" +} diff --git a/packages/adapters/storage/src/lib.rs b/packages/adapters/storage/src/lib.rs index 8ea34bbc..356c760c 100644 --- a/packages/adapters/storage/src/lib.rs +++ b/packages/adapters/storage/src/lib.rs @@ -168,6 +168,10 @@ impl services::state_committer::port::Storage for Postgres { async fn get_latest_pending_txs(&self) -> Result> { self._get_latest_pending_txs().await.map_err(Into::into) } + + async fn latest_bundled_height(&self) -> Result> { + self._latest_bundled_height().await.map_err(Into::into) + } } impl services::state_pruner::port::Storage for Postgres { diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index fb19bdc7..a239bf44 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -595,6 +595,21 @@ impl Postgres { .transpose() } + pub(crate) async fn _latest_bundled_height(&self) -> Result> { + sqlx::query!("SELECT MAX(end_height) AS latest_bundled_height FROM bundles") + .fetch_optional(&self.connection_pool) + .await? + .map(|height| { + let height = height + .latest_bundled_height + .expect("end height is not NULLable"); + u32::try_from(height).map_err(|_| { + crate::error::Error::Conversion(format!("invalid block height: {height}")) + }) + }) + .transpose() + } + pub(crate) async fn _update_tx_state( &self, hash: [u8; 32], diff --git a/packages/adapters/storage/src/test_instance.rs b/packages/adapters/storage/src/test_instance.rs index b4baa4ea..4dbb70e6 100644 --- a/packages/adapters/storage/src/test_instance.rs +++ b/packages/adapters/storage/src/test_instance.rs @@ -329,6 +329,10 @@ impl services::state_committer::port::Storage for DbWithProcess { async fn get_latest_pending_txs(&self) -> services::Result> { self.db._get_latest_pending_txs().await.map_err(Into::into) } + + async fn latest_bundled_height(&self) -> services::Result> { + self.db._latest_bundled_height().await.map_err(Into::into) + } } impl services::status_reporter::port::Storage for DbWithProcess { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 19af7aac..68459fd4 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -46,25 +46,26 @@ pub mod service { } struct Metrics { - num_l2_blocks_behind: IntGauge, + current_height_to_commit: IntGauge, } impl Default for Metrics { fn default() -> Self { - let num_l2_blocks_behind = IntGauge::with_opts(Opts::new( - "num_l2_blocks_behind", - "How many L2 blocks have been produced since the starting height of the oldest bundle we're committing", - )).expect("metric config to be correct"); + let current_height_to_commit = IntGauge::with_opts(Opts::new( + "current_height_to_commit", + "The starting l2 height of the bundle we're committing/will commit next", + )) + .expect("metric config to be correct"); Self { - num_l2_blocks_behind, + current_height_to_commit, } } } impl RegistersMetrics for StateCommitter { fn metrics(&self) -> Vec> { - vec![Box::new(self.metrics.num_l2_blocks_behind.clone())] + vec![Box::new(self.metrics.current_height_to_commit.clone())] } } @@ -137,8 +138,10 @@ pub mod service { let l1_height = self.l1_adapter.current_height().await?; let l2_height = self.fuel_api.latest_height().await?; - let num_l2_blocks_behind = self.num_l2_blocks_behind(fragments, l2_height); - self.update_l2_blocks_behind_metric(num_l2_blocks_behind); + let oldest_l2_block = self.oldest_l2_block_in_fragments(fragments); + self.update_oldest_block_metric(oldest_l2_block); + + let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block); self.decider .should_send_blob_tx( @@ -149,16 +152,10 @@ pub mod service { .await } - fn num_l2_blocks_behind( - &self, - fragments: &NonEmpty, - l2_height: u32, - ) -> u32 { - let oldest_l2_block_in_fragments = fragments + fn oldest_l2_block_in_fragments(&self, fragments: &NonEmpty) -> u32 { + fragments .minimum_by_key(|b| b.oldest_block_in_bundle) - .oldest_block_in_bundle; - - l2_height.saturating_sub(oldest_l2_block_in_fragments) + .oldest_block_in_bundle } async fn submit_fragments( @@ -230,18 +227,16 @@ pub mod service { if let Some(fragments) = fragments.as_ref() { // Tracking the metric here as well to get updates more often -- because // submit_fragments might not be called - self.update_l2_blocks_behind_metric( - self.num_l2_blocks_behind(fragments, latest_height), - ); + self.update_oldest_block_metric(self.oldest_l2_block_in_fragments(fragments)); } Ok(fragments) } - fn update_l2_blocks_behind_metric(&self, l2_blocks_behind: u32) { + fn update_oldest_block_metric(&self, oldest_height: u32) { self.metrics - .num_l2_blocks_behind - .set(l2_blocks_behind as i64); + .current_height_to_commit + .set(oldest_height as i64); } async fn should_submit_fragments( @@ -281,6 +276,10 @@ pub mod service { if self.should_submit_fragments(&fragments).await? { self.submit_fragments(fragments, None).await?; } + } else { + let oldest_bundled_height = self.storage.latest_bundled_height().await?; + + // TODO: segfault } Ok(()) @@ -415,7 +414,7 @@ pub mod port { starting_height: u32, limit: usize, ) -> Result>; - + async fn latest_bundled_height(&self) -> Result>; async fn fragments_submitted_by_tx(&self, tx_hash: [u8; 32]) -> Result>; async fn get_latest_pending_txs(&self) -> Result>; From 3484efccedbe69d545849efc27cae90af743f20a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 24 Dec 2024 19:59:27 +0100 Subject: [PATCH 049/136] have metrics even when fragments should not be submitted --- packages/services/src/state_committer.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 68459fd4..071619fb 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -277,9 +277,22 @@ pub mod service { self.submit_fragments(fragments, None).await?; } } else { - let oldest_bundled_height = self.storage.latest_bundled_height().await?; - - // TODO: segfault + // TODO: segfault test this + // if we have no fragments to submit, that means that we're up to date and new + // blocks haven't been bundled yet + let current_height_to_commit = + if let Some(height) = self.storage.latest_bundled_height().await? { + height.saturating_add(1) + } else { + self.fuel_api + .latest_height() + .await? + .saturating_sub(self.config.lookback_window) + }; + + self.metrics + .current_height_to_commit + .set(current_height_to_commit as i64); } Ok(()) From 77959867aaae43e8d66c6a53628b0cc2f40c56e3 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 24 Dec 2024 20:18:31 +0100 Subject: [PATCH 050/136] add todo --- packages/services/src/fee_tracker/service.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 680afa13..2e5a3201 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -148,6 +148,7 @@ impl From for f64 { impl Percentage { pub const ZERO: Self = Percentage(0.); + // TODO: segfault, does PPM really make a difference? pub const PPM: u128 = 1_000_000; pub fn ppm(&self) -> u128 { From 0b0bea7dd7442f1bcd79a710716b40d5756b1271 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 24 Dec 2024 20:59:35 +0100 Subject: [PATCH 051/136] fix typo --- packages/adapters/storage/src/postgres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index 51b46f72..c3d32404 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -602,7 +602,7 @@ impl Postgres { .map(|height| { let height = height .latest_bundled_height - .expect("end height is not NULLable"); + .expect("end height is not NULL-able"); u32::try_from(height).map_err(|_| { crate::error::Error::Conversion(format!("invalid block height: {height}")) }) From 466db65bb17d049fd54f8644259ca1ca5c2b1c83 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 25 Dec 2024 15:57:05 +0100 Subject: [PATCH 052/136] fees cannot be zero --- committer/src/config.rs | 4 +- committer/src/setup.rs | 2 +- packages/adapters/eth/src/fee_conversion.rs | 77 +++++----- .../services/src/fee_tracker/fee_analytics.rs | 86 ++++++----- packages/services/src/fee_tracker/port.rs | 86 ++++------- packages/services/src/fee_tracker/service.rs | 9 +- packages/services/tests/fee_tracker.rs | 72 ++++----- packages/services/tests/state_committer.rs | 144 +++++++++--------- 8 files changed, 240 insertions(+), 240 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index eca2d76e..6715f82f 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -93,9 +93,9 @@ pub struct App { /// How often to check for finalized l1 txs #[serde(deserialize_with = "human_readable_duration")] pub tx_finalization_check_interval: Duration, - /// How often to check for l1 prices + /// How often to check for l1 fees #[serde(deserialize_with = "human_readable_duration")] - pub l1_prices_check_interval: Duration, + pub l1_fee_check_interval: Duration, /// Number of L1 blocks that need to pass to accept the tx as finalized pub num_blocks_to_finalize_tx: u64, /// Interval after which to bump a pending tx diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 8e2fcd18..9c606916 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -354,7 +354,7 @@ pub fn fee_tracker( fee_tracker.register_metrics(registry); let handle = schedule_polling( - config.app.tx_finalization_check_interval, + config.app.l1_fee_check_interval, fee_tracker.clone(), "Fee Tracker", cancel_token, diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 0d34073c..76df3ba2 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -55,13 +55,16 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { ) .take(number_of_blocks) .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { - height, - fees: Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - }, + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| { + // TODO: segfault add tests for detecting invalid 0s + BlockFees { + height, + fees: Fees { + base_fee_per_gas: base_fee_per_gas.try_into().unwrap(), + reward: reward.try_into().unwrap(), + base_fee_per_blob_gas: base_fee_per_blob_gas.try_into().unwrap(), + }, + } }, ) .collect(); @@ -373,9 +376,9 @@ mod test { let expected = vec![BlockFees { height: 600, fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, + base_fee_per_gas: 100.try_into().unwrap(), + reward: 10.try_into().unwrap(), + base_fee_per_blob_gas: 150.try_into().unwrap(), }, }]; assert_eq!( @@ -404,25 +407,25 @@ mod test { BlockFees { height: 700, fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, + base_fee_per_gas: 100.try_into().unwrap(), + reward: 10.try_into().unwrap(), + base_fee_per_blob_gas: 150.try_into().unwrap(), }, }, BlockFees { height: 701, fees: Fees { - base_fee_per_gas: 200, - reward: 20, - base_fee_per_blob_gas: 250, + base_fee_per_gas: 200.try_into().unwrap(), + reward: 20.try_into().unwrap(), + base_fee_per_blob_gas: 250.try_into().unwrap(), }, }, BlockFees { height: 702, fees: Fees { - base_fee_per_gas: 300, - reward: 30, - base_fee_per_blob_gas: 350, + base_fee_per_gas: 300.try_into().unwrap(), + reward: 30.try_into().unwrap(), + base_fee_per_blob_gas: 350.try_into().unwrap(), }, }, ]; @@ -452,17 +455,17 @@ mod test { BlockFees { height: u64::MAX - 2, fees: Fees { - base_fee_per_gas: u128::MAX - 2, - reward: u128::MAX - 4, - base_fee_per_blob_gas: u128::MAX - 3, + base_fee_per_gas: (u128::MAX - 2).try_into().unwrap(), + reward: (u128::MAX - 4).try_into().unwrap(), + base_fee_per_blob_gas: (u128::MAX - 3).try_into().unwrap(), }, }, BlockFees { height: u64::MAX - 1, fees: Fees { - base_fee_per_gas: u128::MAX - 1, - reward: u128::MAX - 3, - base_fee_per_blob_gas: u128::MAX - 2, + base_fee_per_gas: (u128::MAX - 1).try_into().unwrap(), + reward: (u128::MAX - 3).try_into().unwrap(), + base_fee_per_blob_gas: (u128::MAX - 2).try_into().unwrap(), }, }, ]; @@ -492,33 +495,33 @@ mod test { BlockFees { height: 800, fees: Fees { - base_fee_per_gas: 500, - reward: 50, - base_fee_per_blob_gas: 550, + base_fee_per_gas: 500.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 550.try_into().unwrap(), }, }, BlockFees { height: 801, fees: Fees { - base_fee_per_gas: 600, - reward: 60, - base_fee_per_blob_gas: 650, + base_fee_per_gas: 600.try_into().unwrap(), + reward: 60.try_into().unwrap(), + base_fee_per_blob_gas: 650.try_into().unwrap(), }, }, BlockFees { height: 802, fees: Fees { - base_fee_per_gas: 700, - reward: 70, - base_fee_per_blob_gas: 750, + base_fee_per_gas: 700.try_into().unwrap(), + reward: 70.try_into().unwrap(), + base_fee_per_blob_gas: 750.try_into().unwrap(), }, }, BlockFees { height: 803, fees: Fees { - base_fee_per_gas: 800, - reward: 80, - base_fee_per_blob_gas: 850, + base_fee_per_gas: 800.try_into().unwrap(), + reward: 80.try_into().unwrap(), + base_fee_per_blob_gas: 850.try_into().unwrap(), }, }, ]; diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index 1f0d8144..a3430566 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -1,4 +1,4 @@ -use std::ops::RangeInclusive; +use std::{num::NonZeroU128, ops::RangeInclusive}; use crate::Error; @@ -49,17 +49,35 @@ impl FeeAnalytics

{ let total = fees .into_iter() .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + .fold(Fees::default(), |acc, f| { + let base_fee_per_gas = acc + .base_fee_per_gas + .saturating_add(f.base_fee_per_gas.get()); + let reward = acc.reward.saturating_add(f.reward.get()); + let base_fee_per_blob_gas = acc + .base_fee_per_blob_gas + .saturating_add(f.base_fee_per_blob_gas.get()); + + Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + } }); - // TODO: segfault should we round to nearest here? + let divide_by_count = |value: NonZeroU128| { + let minimum_fee = NonZeroU128::try_from(1).unwrap(); + value + .get() + .saturating_div(count) + .try_into() + .unwrap_or(minimum_fee) + }; + Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + base_fee_per_gas: divide_by_count(total.base_fee_per_gas), + reward: divide_by_count(total.reward), + base_fee_per_blob_gas: divide_by_count(total.base_fee_per_blob_gas), } } } @@ -79,17 +97,17 @@ mod tests { BlockFees { height: 1, fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), }, }, BlockFees { height: 2, fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), }, }, ]; @@ -132,17 +150,17 @@ mod tests { BlockFees { height: 1, fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), }, }, BlockFees { height: 3, // Non-sequential height fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), }, }, ]; @@ -171,17 +189,17 @@ mod tests { BlockFees { height: 2, fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), }, }, BlockFees { height: 1, fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), }, }, ]; @@ -211,9 +229,9 @@ mod tests { let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); // then - assert_eq!(sma.base_fee_per_gas, 5); - assert_eq!(sma.reward, 5); - assert_eq!(sma.base_fee_per_blob_gas, 5); + assert_eq!(sma.base_fee_per_gas, 6.try_into().unwrap()); + assert_eq!(sma.reward, 6.try_into().unwrap()); + assert_eq!(sma.base_fee_per_blob_gas, 6.try_into().unwrap()); } #[tokio::test] @@ -226,7 +244,7 @@ mod tests { let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); // then - let mean = (5 + 4 + 3 + 2 + 1) / 5; + let mean = ((5 + 4 + 3 + 2 + 1) / 5).try_into().unwrap(); assert_eq!(sma.base_fee_per_gas, mean); assert_eq!(sma.reward, mean); assert_eq!(sma.base_fee_per_blob_gas, mean); @@ -268,9 +286,9 @@ mod tests { let expected_fee = BlockFees { height, fees: Fees { - base_fee_per_gas: 5, - reward: 5, - base_fee_per_blob_gas: 5, + base_fee_per_gas: 5.try_into().unwrap(), + reward: 5.try_into().unwrap(), + base_fee_per_blob_gas: 5.try_into().unwrap(), }, }; assert_eq!( diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs index e7ba4f55..58526beb 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_tracker/port.rs @@ -1,9 +1,19 @@ pub mod l1 { - #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Fees { - pub base_fee_per_gas: u128, - pub reward: u128, - pub base_fee_per_blob_gas: u128, + pub base_fee_per_gas: NonZeroU128, + pub reward: NonZeroU128, + pub base_fee_per_blob_gas: NonZeroU128, + } + + impl Default for Fees { + fn default() -> Self { + Fees { + base_fee_per_gas: 1.try_into().unwrap(), + reward: 1.try_into().unwrap(), + base_fee_per_blob_gas: 1.try_into().unwrap(), + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -11,7 +21,7 @@ pub mod l1 { pub height: u64, pub fees: Fees, } - use std::ops::RangeInclusive; + use std::{num::NonZeroU128, ops::RangeInclusive}; use itertools::Itertools; @@ -173,12 +183,13 @@ pub mod l1 { pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { (0..num_blocks) .map(|i| { + let fee = (u128::from(i) + 1).try_into().unwrap(); ( i, Fees { - base_fee_per_gas: i as u128 + 1, - reward: i as u128 + 1, - base_fee_per_blob_gas: i as u128 + 1, + base_fee_per_gas: fee, + reward: fee, + base_fee_per_blob_gas: fee, }, ) }) @@ -305,19 +316,7 @@ pub mod cache { .once() .return_once(|range| { Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) + Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) }) }); @@ -343,19 +342,7 @@ pub mod cache { .once() .return_once(|range| { Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) + Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) }) }) .in_sequence(&mut sequence); @@ -366,19 +353,7 @@ pub mod cache { .once() .return_once(|range| { Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) + Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) }) }) .in_sequence(&mut sequence); @@ -447,13 +422,16 @@ pub mod cache { fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { SequentialBlockFees::try_from( height_range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, + .map(|h| { + let fee = u128::from(h + 1).try_into().unwrap(); + BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: fee, + reward: fee, + base_fee_per_blob_gas: fee, + }, + } }) .collect::>(), ) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 2e5a3201..53ec7964 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -14,7 +14,7 @@ use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; use super::{ fee_analytics::FeeAnalytics, - port::l1::{Api, BlockFees, Fees}, + port::l1::{Api, Fees}, }; #[derive(Debug, Clone, Copy)] @@ -288,12 +288,13 @@ impl FeeTracker

{ const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; - let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas); + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); let blob_fee = fees .base_fee_per_blob_gas - .saturating_mul(num_blobs as u128) + .get() + .saturating_mul(u128::from(num_blobs)) .saturating_mul(DATA_GAS_PER_BLOB); - let reward_fee = fees.reward.saturating_mul(INTRINSIC_GAS); + let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); base_fee.saturating_add(blob_fee).saturating_add(reward_fee) } diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 6936791b..18c27040 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -23,8 +23,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe } #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, 6, Config { sma_periods: services::fee_tracker::service::SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -39,8 +39,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send because all short-term fees are lower than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -55,8 +55,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should not send because all short-term fees are higher than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -71,8 +71,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send since short-term fee less than always_acceptable_fee" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -87,8 +87,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send because short-term base_fee_per_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -103,8 +103,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -119,8 +119,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -135,8 +135,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -151,8 +151,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send because short-term reward is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -168,8 +168,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe )] #[test_case( // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, + Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -184,8 +184,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Should send because multiple short-term fees are lower" )] #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -201,8 +201,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe )] #[test_case( // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -218,8 +218,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe )] #[test_case( // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -234,8 +234,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Zero blobs: short-term reward is higher, don't send" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2000, reward: 6000, base_fee_per_blob_gas: 50_000_000 }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -252,8 +252,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. #[test_case( // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -270,8 +270,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe )] #[test_case( // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -287,8 +287,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Later: after max wait, send regardless" )] #[test_case( - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -305,8 +305,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe )] #[test_case( // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, - Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, + Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 345fe363..907e45b5 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -347,49 +347,49 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ( 0, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 1, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 2, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ( 3, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ( 4, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ( 5, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ]; @@ -456,49 +456,49 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ( 0, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ( 1, Fees { - base_fee_per_gas: 3000, - reward: 3000, - base_fee_per_blob_gas: 3000, + base_fee_per_gas: 3000.try_into().unwrap(), + reward: 3000.try_into().unwrap(), + base_fee_per_blob_gas: 3000.try_into().unwrap(), }, ), ( 2, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 3, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 4, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 5, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ]; @@ -556,49 +556,49 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ( 0, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ( 1, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ( 2, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ( 3, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ( 4, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ( 5, Fees { - base_fee_per_gas: 7000, - reward: 7000, - base_fee_per_blob_gas: 7000, + base_fee_per_gas: 7000.try_into().unwrap(), + reward: 7000.try_into().unwrap(), + base_fee_per_blob_gas: 7000.try_into().unwrap(), }, ), ]; @@ -664,49 +664,49 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ( 95, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 96, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 97, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 98, Fees { - base_fee_per_gas: 5000, - reward: 5000, - base_fee_per_blob_gas: 5000, + base_fee_per_gas: 5000.try_into().unwrap(), + reward: 5000.try_into().unwrap(), + base_fee_per_blob_gas: 5000.try_into().unwrap(), }, ), ( 99, Fees { - base_fee_per_gas: 5800, - reward: 5800, - base_fee_per_blob_gas: 5800, + base_fee_per_gas: 5800.try_into().unwrap(), + reward: 5800.try_into().unwrap(), + base_fee_per_blob_gas: 5800.try_into().unwrap(), }, ), ( 100, Fees { - base_fee_per_gas: 5800, - reward: 5800, - base_fee_per_blob_gas: 5800, + base_fee_per_gas: 5800.try_into().unwrap(), + reward: 5800.try_into().unwrap(), + base_fee_per_blob_gas: 5800.try_into().unwrap(), }, ), ]; From 6df8eff3af34afb07655a49da61a93814ffdab7a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 25 Dec 2024 16:05:53 +0100 Subject: [PATCH 053/136] add test for when mean equals 0 to be rounded up to 1 --- packages/adapters/eth/src/fee_conversion.rs | 71 ++++++++++++++++--- .../services/src/fee_tracker/fee_analytics.rs | 42 +++++++++++ packages/services/src/fee_tracker/port.rs | 2 +- packages/services/tests/fee_tracker.rs | 3 + 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 76df3ba2..7134af3d 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -1,4 +1,4 @@ -use std::ops::RangeInclusive; +use std::{num::NonZeroU128, ops::RangeInclusive}; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; @@ -47,7 +47,7 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { }) .try_collect()?; - let values = izip!( + izip!( (fees.oldest_block..), fees.base_fee_per_gas.into_iter(), fees.base_fee_per_blob_gas.into_iter(), @@ -57,19 +57,25 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { .map( |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| { // TODO: segfault add tests for detecting invalid 0s - BlockFees { + let convert_to_nonzero = |value: u128| { + NonZeroU128::try_from(value).map_err(|_| { + crate::error::Error::Other( + "historical fee response returned a 0 fee, which we deem as invalid" + .to_string(), + ) + }) + }; + Ok(BlockFees { height, fees: Fees { - base_fee_per_gas: base_fee_per_gas.try_into().unwrap(), - reward: reward.try_into().unwrap(), - base_fee_per_blob_gas: base_fee_per_blob_gas.try_into().unwrap(), + base_fee_per_gas: convert_to_nonzero(base_fee_per_gas)?, + reward: convert_to_nonzero(reward)?, + base_fee_per_blob_gas: convert_to_nonzero(base_fee_per_blob_gas)?, }, - } + }) }, ) - .collect(); - - Ok(values) + .try_collect() } pub fn chunk_range_inclusive( @@ -531,4 +537,49 @@ mod test { "Expected BlockFees entries matching the full range chunk" ); } + + #[test] + fn unpack_fee_history_invalid_zero_fees_or_rewards() { + // Test case where base_fee_per_gas contains a zero + let fees_with_zero_base_fee = FeeHistory { + oldest_block: 1000, + base_fee_per_gas: vec![100, 0, 200], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + let result = fee_conversion::unpack_fee_history(fees_with_zero_base_fee); + assert!( + result.is_err(), + "Expected error due to zero base_fee_per_gas" + ); + + // Test case where base_fee_per_blob_gas contains a zero + let fees_with_zero_blob_fee = FeeHistory { + oldest_block: 1001, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 0, 350], + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + let result = fee_conversion::unpack_fee_history(fees_with_zero_blob_fee); + assert!( + result.is_err(), + "Expected error due to zero base_fee_per_blob_gas" + ); + + // Test case where reward is zero + let fees_with_zero_reward = FeeHistory { + oldest_block: 1002, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![0], vec![20]]), + ..Default::default() + }; + + let result = fee_conversion::unpack_fee_history(fees_with_zero_reward); + assert!(result.is_err(), "Expected error due to zero reward"); + } } diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index a3430566..ac1980f9 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -297,6 +297,48 @@ mod tests { ); } + #[tokio::test] + async fn mean_is_at_least_one_when_totals_are_zero() { + // given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 1.try_into().unwrap(), + reward: 1.try_into().unwrap(), + base_fee_per_blob_gas: 1.try_into().unwrap(), + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 1.try_into().unwrap(), + reward: 1.try_into().unwrap(), + base_fee_per_blob_gas: 1.try_into().unwrap(), + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees).unwrap(); + let mean = FeeAnalytics::::mean(sequential_fees.clone()); + + // then + assert_eq!( + mean.base_fee_per_gas, + 1.try_into().unwrap(), + "base_fee_per_gas should be set to 1 when total is 0" + ); + assert_eq!( + mean.reward, + 1.try_into().unwrap(), + "reward should be set to 1 when total is 0" + ); + assert_eq!( + mean.base_fee_per_blob_gas, + 1.try_into().unwrap(), + "base_fee_per_blob_gas should be set to 1 when total is 0" + ); + } + // fn calculate_tx_fee(fees: &Fees) -> u128 { // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 // } diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs index 58526beb..47dbb283 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_tracker/port.rs @@ -25,7 +25,7 @@ pub mod l1 { use itertools::Itertools; - #[derive(Debug, PartialEq, Eq)] + #[derive(Debug, PartialEq, Eq, Clone)] pub struct SequentialBlockFees { fees: Vec, } diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 18c27040..67afd54b 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,5 +1,8 @@ +use services::fee_tracker::port::l1::testing; use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; use services::fee_tracker::port::l1::Api; +use services::fee_tracker::port::l1::BlockFees; +use services::fee_tracker::port::l1::SequentialBlockFees; use services::fee_tracker::service::FeeThresholds; use services::fee_tracker::service::FeeTracker; use services::fee_tracker::service::Percentage; From 5b2b35dbe4b27a56764783ede2f0a662bde83463 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 25 Dec 2024 16:38:20 +0100 Subject: [PATCH 054/136] cleanup --- committer/src/main.rs | 1 - packages/adapters/eth/src/fee_conversion.rs | 1 - packages/services/src/fee_tracker/fee_analytics.rs | 1 - packages/services/src/fee_tracker/service.rs | 2 -- packages/services/tests/fee_tracker.rs | 3 --- 5 files changed, 8 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 202ecece..9a654eea 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,7 +7,6 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; -use services::fee_tracker::port::cache::CachingApi; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 7134af3d..507a90cc 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -56,7 +56,6 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { .take(number_of_blocks) .map( |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| { - // TODO: segfault add tests for detecting invalid 0s let convert_to_nonzero = |value: u128| { NonZeroU128::try_from(value).map_err(|_| { crate::error::Error::Other( diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index ac1980f9..99580f03 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -179,7 +179,6 @@ mod tests { ); } - // TODO: segfault add more tests so that the in-order iteration invariant is properly tested #[test] fn produced_iterator_gives_correct_values() { // Given diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 53ec7964..6d4803bb 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -148,7 +148,6 @@ impl From for f64 { impl Percentage { pub const ZERO: Self = Percentage(0.); - // TODO: segfault, does PPM really make a difference? pub const PPM: u128 = 1_000_000; pub fn ppm(&self) -> u128 { @@ -283,7 +282,6 @@ impl FeeTracker

{ .saturating_div(Percentage::PPM) } - // TODO: Segfault maybe dont leak so much eth abstractions fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 67afd54b..18c27040 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,8 +1,5 @@ -use services::fee_tracker::port::l1::testing; use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; use services::fee_tracker::port::l1::Api; -use services::fee_tracker::port::l1::BlockFees; -use services::fee_tracker::port::l1::SequentialBlockFees; use services::fee_tracker::service::FeeThresholds; use services::fee_tracker::service::FeeTracker; use services::fee_tracker::service::Percentage; From eeda63c3a05fdc2bbd0d99e527f8b5903bad1b0c Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 25 Dec 2024 16:39:01 +0100 Subject: [PATCH 055/136] merge imports --- committer/src/main.rs | 4 ++-- e2e/src/eth_node/state_contract.rs | 2 +- e2e/src/lib.rs | 2 +- packages/adapters/eth/src/fee_conversion.rs | 4 ++-- packages/adapters/fuel/src/client.rs | 6 +++-- packages/adapters/fuel/src/lib.rs | 1 - packages/adapters/storage/src/lib.rs | 12 ++++++---- packages/adapters/storage/src/postgres.rs | 21 ++++++++-------- .../adapters/storage/src/test_instance.rs | 11 +++++---- packages/services/src/block_bundler.rs | 13 +++++----- .../services/src/block_bundler/bundler.rs | 9 +++---- packages/services/src/block_committer.rs | 10 ++++---- packages/services/src/block_importer.rs | 5 ++-- .../services/src/fee_tracker/fee_analytics.rs | 6 ++--- packages/services/src/fee_tracker/port.rs | 3 +-- packages/services/src/fee_tracker/service.rs | 7 +++--- packages/services/src/lib.rs | 4 ++-- packages/services/src/state_committer.rs | 12 ++++++---- packages/services/src/state_listener.rs | 9 +++---- packages/services/src/state_pruner.rs | 6 ++--- .../services/src/wallet_balance_tracker.rs | 11 ++++----- packages/services/tests/block_bundler.rs | 10 ++++---- packages/services/tests/block_committer.rs | 8 +++---- packages/services/tests/fee_tracker.rs | 15 ++++++------ packages/services/tests/state_committer.rs | 3 ++- packages/services/tests/status_reporter.rs | 6 +++-- packages/test-helpers/src/lib.rs | 24 ++++++++++--------- 27 files changed, 118 insertions(+), 106 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 9a654eea..e2704cf6 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -102,7 +102,7 @@ async fn main() -> Result<()> { ); // Enable pruner once the issue is resolved - //TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 + // TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 // let state_pruner_handle = setup::state_pruner( // storage.clone(), // cancel_token.clone(), @@ -116,7 +116,7 @@ async fn main() -> Result<()> { handles.push(state_listener_handle); handles.push(fee_tracker_handle); // Enable pruner once the issue is resolved - //TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 + // TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 // handles.push(state_pruner_handle); } diff --git a/e2e/src/eth_node/state_contract.rs b/e2e/src/eth_node/state_contract.rs index 2d514e21..ca005865 100644 --- a/e2e/src/eth_node/state_contract.rs +++ b/e2e/src/eth_node/state_contract.rs @@ -10,7 +10,7 @@ use alloy::{ use eth::{AwsClient, AwsConfig, Signer, Signers, WebsocketClient}; use fs_extra::dir::{copy, CopyOptions}; use serde::Deserialize; -use services::{types::fuel::FuelBlock, types::Address}; +use services::types::{fuel::FuelBlock, Address}; use tokio::process::Command; use url::Url; diff --git a/e2e/src/lib.rs b/e2e/src/lib.rs index 94ee8bd9..4459a594 100644 --- a/e2e/src/lib.rs +++ b/e2e/src/lib.rs @@ -85,7 +85,7 @@ mod tests { } // Enable test once the issue is resolved - //TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 + // TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 #[ignore] #[tokio::test(flavor = "multi_thread")] async fn old_state_will_be_pruned() -> Result<()> { diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 507a90cc..7edfdbf8 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -105,11 +105,11 @@ pub fn chunk_range_inclusive( #[cfg(test)] mod test { + use std::ops::RangeInclusive; + use alloy::rpc::types::FeeHistory; use services::fee_tracker::port::l1::{BlockFees, Fees}; - use std::ops::RangeInclusive; - use crate::fee_conversion::{self}; #[test] diff --git a/packages/adapters/fuel/src/client.rs b/packages/adapters/fuel/src/client.rs index 8bc10c57..09ccb9d6 100644 --- a/packages/adapters/fuel/src/client.rs +++ b/packages/adapters/fuel/src/client.rs @@ -12,11 +12,13 @@ use futures::{stream, Stream, StreamExt}; use metrics::{ prometheus::core::Collector, ConnectionHealthTracker, HealthChecker, RegistersMetrics, }; -use services::types::{CompressedFuelBlock, NonEmpty}; +use services::{ + types::{CompressedFuelBlock, NonEmpty}, + Error, Result, +}; use url::Url; use crate::metrics::Metrics; -use services::{Error, Result}; #[derive(Clone)] pub struct HttpClient { diff --git a/packages/adapters/fuel/src/lib.rs b/packages/adapters/fuel/src/lib.rs index 82189de9..2c54a593 100644 --- a/packages/adapters/fuel/src/lib.rs +++ b/packages/adapters/fuel/src/lib.rs @@ -9,7 +9,6 @@ mod metrics; pub use client::*; use delegate::delegate; - use services::Result; impl services::block_importer::port::fuel::Api for client::HttpClient { diff --git a/packages/adapters/storage/src/lib.rs b/packages/adapters/storage/src/lib.rs index fdecf3f2..68eadfc6 100644 --- a/packages/adapters/storage/src/lib.rs +++ b/packages/adapters/storage/src/lib.rs @@ -455,8 +455,9 @@ mod tests { #[tokio::test] async fn can_get_last_time_a_fragment_was_finalized() { - use services::state_committer::port::Storage; - use services::state_listener::port::Storage as ListenerStorage; + use services::{ + state_committer::port::Storage, state_listener::port::Storage as ListenerStorage, + }; // given let storage = start_db().await; @@ -919,9 +920,10 @@ mod tests { #[tokio::test] async fn can_update_costs() -> Result<()> { - use services::cost_reporter::port::Storage; - use services::state_committer::port::Storage as StateStorage; - use services::state_listener::port::Storage as ListenerStorage; + use services::{ + cost_reporter::port::Storage, state_committer::port::Storage as StateStorage, + state_listener::port::Storage as ListenerStorage, + }; // given let storage = start_db().await; diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index c3d32404..f3168916 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -1,6 +1,5 @@ use std::{collections::HashMap, ops::RangeInclusive}; -use crate::postgres::tables::u128_to_bigdecimal; use itertools::Itertools; use metrics::{prometheus::IntGauge, RegistersMetrics}; use services::types::{ @@ -14,7 +13,10 @@ use sqlx::{ }; use super::error::{Error, Result}; -use crate::mappings::tables::{self, L1TxState}; +use crate::{ + mappings::tables::{self, L1TxState}, + postgres::tables::u128_to_bigdecimal, +}; #[derive(Debug, Clone)] struct Metrics { @@ -1167,15 +1169,13 @@ fn create_ranges(heights: Vec) -> Vec> { mod tests { use std::{env, fs, path::Path}; + use rand::Rng; + use services::types::{CollectNonEmpty, Fragment, L1Tx, TransactionState}; use sqlx::{Executor, PgPool, Row}; use tokio::time::Instant; - use crate::test_instance; - use super::*; - - use rand::Rng; - use services::types::{CollectNonEmpty, Fragment, L1Tx, TransactionState}; + use crate::test_instance; #[tokio::test] async fn test_second_migration_applies_successfully() { @@ -1438,9 +1438,10 @@ mod tests { #[tokio::test] async fn stress_test_update_costs() -> Result<()> { - use services::block_bundler::port::Storage; - use services::state_committer::port::Storage as CommitterStorage; - use services::state_listener::port::Storage as ListenerStorage; + use services::{ + block_bundler::port::Storage, state_committer::port::Storage as CommitterStorage, + state_listener::port::Storage as ListenerStorage, + }; let mut rng = rand::thread_rng(); diff --git a/packages/adapters/storage/src/test_instance.rs b/packages/adapters/storage/src/test_instance.rs index c9139bb6..e1f12fbe 100644 --- a/packages/adapters/storage/src/test_instance.rs +++ b/packages/adapters/storage/src/test_instance.rs @@ -1,3 +1,9 @@ +use std::{ + borrow::Cow, + ops::RangeInclusive, + sync::{Arc, Weak}, +}; + use delegate::delegate; use services::{ block_bundler, block_committer, block_importer, @@ -8,11 +14,6 @@ use services::{ }, }; use sqlx::Executor; -use std::{ - borrow::Cow, - ops::RangeInclusive, - sync::{Arc, Weak}, -}; use testcontainers::{ core::{ContainerPort, WaitFor}, runners::AsyncRunner, diff --git a/packages/services/src/block_bundler.rs b/packages/services/src/block_bundler.rs index b475f37d..4a182504 100644 --- a/packages/services/src/block_bundler.rs +++ b/packages/services/src/block_bundler.rs @@ -3,11 +3,6 @@ pub mod bundler; pub mod service { use std::{num::NonZeroUsize, time::Duration}; - use super::bundler::{Bundle, BundleProposal, BundlerFactory, Metadata}; - use crate::{ - types::{DateTime, Utc}, - Error, Result, Runner, - }; use metrics::{ custom_exponential_buckets, prometheus::{histogram_opts, linear_buckets, Histogram, IntGauge}, @@ -15,6 +10,12 @@ pub mod service { }; use tracing::info; + use super::bundler::{Bundle, BundleProposal, BundlerFactory, Metadata}; + use crate::{ + types::{DateTime, Utc}, + Error, Result, Runner, + }; + #[derive(Debug, Clone, Copy)] pub struct Config { pub optimization_time_limit: Duration, @@ -332,13 +333,13 @@ pub mod port { pub mod test_helpers { use std::num::NonZeroUsize; - use crate::types::{storage::SequentialFuelBlocks, NonNegative}; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, Mutex, }; use super::bundler::{Bundle, BundleProposal, BundlerFactory}; + use crate::types::{storage::SequentialFuelBlocks, NonNegative}; pub struct ControllableBundler { can_advance: UnboundedReceiver<()>, diff --git a/packages/services/src/block_bundler/bundler.rs b/packages/services/src/block_bundler/bundler.rs index 5a30c84b..614a6357 100644 --- a/packages/services/src/block_bundler/bundler.rs +++ b/packages/services/src/block_bundler/bundler.rs @@ -1,5 +1,10 @@ use std::{cmp::min, collections::VecDeque, fmt::Display, num::NonZeroUsize, ops::RangeInclusive}; +use bytesize::ByteSize; +use fuel_block_committer_encoding::bundle::{self, BundleV1}; +use itertools::Itertools; +use rayon::prelude::*; + use crate::{ types::{ storage::SequentialFuelBlocks, CollectNonEmpty, CompressedFuelBlock, Fragment, NonEmpty, @@ -7,10 +12,6 @@ use crate::{ }, Result, }; -use bytesize::ByteSize; -use fuel_block_committer_encoding::bundle::{self, BundleV1}; -use itertools::Itertools; -use rayon::prelude::*; #[derive(Debug, Clone, PartialEq)] pub struct Metadata { diff --git a/packages/services/src/block_committer.rs b/packages/services/src/block_committer.rs index 92c4d19b..7c765dc2 100644 --- a/packages/services/src/block_committer.rs +++ b/packages/services/src/block_committer.rs @@ -1,13 +1,12 @@ pub mod service { use std::num::NonZeroU32; + use tracing::info; + use crate::{ types::{fuel::FuelBlock, BlockSubmission, NonNegative, TransactionState}, - Error, Result, + Error, Result, Runner, }; - use tracing::info; - - use crate::Runner; pub struct BlockCommitter { l1_adapter: L1, @@ -240,9 +239,10 @@ pub mod port { } pub mod fuel { - use crate::Result; pub use fuel_core_client::client::types::block::Block as FuelBlock; + use crate::Result; + #[allow(async_fn_in_trait)] #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] diff --git a/packages/services/src/block_importer.rs b/packages/services/src/block_importer.rs index a95fb2d1..76870f29 100644 --- a/packages/services/src/block_importer.rs +++ b/packages/services/src/block_importer.rs @@ -1,10 +1,11 @@ pub mod service { + use futures::TryStreamExt; + use tracing::info; + use crate::{ types::{nonempty, CompressedFuelBlock, NonEmpty}, Result, Runner, }; - use futures::TryStreamExt; - use tracing::info; /// The `BlockImporter` is responsible for importing blocks from the Fuel blockchain /// into local storage. It fetches blocks from the Fuel API diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index 99580f03..89625710 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -1,8 +1,7 @@ use std::{num::NonZeroU128, ops::RangeInclusive}; -use crate::Error; - use super::port::l1::{Api, BlockFees, Fees, SequentialBlockFees}; +use crate::Error; #[derive(Debug, Clone)] pub struct FeeAnalytics

{ @@ -86,9 +85,8 @@ impl FeeAnalytics

{ mod tests { use itertools::Itertools; - use crate::fee_tracker::port::l1::{testing, BlockFees}; - use super::*; + use crate::fee_tracker::port::l1::{testing, BlockFees}; #[test] fn can_create_valid_sequential_fees() { diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs index 47dbb283..4d9fa17d 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_tracker/port.rs @@ -203,9 +203,8 @@ pub mod cache { use tokio::sync::RwLock; - use crate::Error; - use super::l1::{Api, BlockFees, Fees, SequentialBlockFees}; + use crate::Error; #[derive(Debug, Clone)] pub struct CachingApi

{ diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 6d4803bb..a191624a 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -10,12 +10,11 @@ use metrics::{ }; use tracing::info; -use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; - use super::{ fee_analytics::FeeAnalytics, port::l1::{Api, Fees}, }; +use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; #[derive(Debug, Clone, Copy)] pub struct Config { @@ -358,10 +357,10 @@ where #[cfg(test)] mod tests { - use crate::fee_tracker::port::l1::testing::ConstantFeeApi; + use test_case::test_case; use super::*; - use test_case::test_case; + use crate::fee_tracker::port::l1::testing::ConstantFeeApi; #[test_case( // Test Case 1: No blocks behind, no discount or premium diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 48123b0a..6c7195cd 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -12,8 +12,8 @@ pub mod types; pub mod wallet_balance_tracker; pub use block_bundler::{ - bundler::Factory as BundlerFactory, service::BlockBundler, - service::Config as BlockBundlerConfig, + bundler::Factory as BundlerFactory, + service::{BlockBundler, Config as BlockBundlerConfig}, }; #[cfg(feature = "test-helpers")] pub use block_bundler::{ diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 071619fb..c2b29887 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,10 +1,6 @@ pub mod service { use std::{num::NonZeroUsize, time::Duration}; - use crate::{ - types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, - }; use itertools::Itertools; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, @@ -12,6 +8,11 @@ pub mod service { }; use tracing::info; + use crate::{ + types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, + Result, Runner, + }; + // src/config.rs #[derive(Debug, Clone)] pub struct Config { @@ -400,9 +401,10 @@ pub mod port { } pub mod fuel { - use crate::Result; pub use fuel_core_client::client::types::block::Block as FuelBlock; + use crate::Result; + #[allow(async_fn_in_trait)] #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] diff --git a/packages/services/src/state_listener.rs b/packages/services/src/state_listener.rs index 86a16cbf..61a28370 100644 --- a/packages/services/src/state_listener.rs +++ b/packages/services/src/state_listener.rs @@ -1,16 +1,17 @@ pub mod service { use std::collections::HashSet; - use crate::{ - types::{L1Tx, TransactionCostUpdate, TransactionState}, - Runner, - }; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, RegistersMetrics, }; use tracing::info; + use crate::{ + types::{L1Tx, TransactionCostUpdate, TransactionState}, + Runner, + }; + pub struct StateListener { l1_adapter: L1, storage: Db, diff --git a/packages/services/src/state_pruner.rs b/packages/services/src/state_pruner.rs index b3ed4eef..327af849 100644 --- a/packages/services/src/state_pruner.rs +++ b/packages/services/src/state_pruner.rs @@ -1,13 +1,13 @@ pub mod service { + use std::time::Duration; + use metrics::{ prometheus::{core::Collector, IntGauge}, RegistersMetrics, }; - use crate::{Result, Runner}; - use std::time::Duration; - use super::create_int_gauge; + use crate::{Result, Runner}; pub struct StatePruner { storage: Db, diff --git a/packages/services/src/wallet_balance_tracker.rs b/packages/services/src/wallet_balance_tracker.rs index dd5f5526..a779f0bf 100644 --- a/packages/services/src/wallet_balance_tracker.rs +++ b/packages/services/src/wallet_balance_tracker.rs @@ -1,16 +1,15 @@ pub mod service { use std::collections::HashMap; - use crate::{ - types::{Address, U256}, - Result, - }; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, RegistersMetrics, }; - use crate::Runner; + use crate::{ + types::{Address, U256}, + Result, Runner, + }; struct Balance { gauge: IntGauge, @@ -99,7 +98,6 @@ mod tests { use std::str::FromStr; - use crate::types::Address; use alloy::primitives::U256; use metrics::{ prometheus::{proto::Metric, Registry}, @@ -109,6 +107,7 @@ mod tests { use service::WalletBalanceTracker; use super::*; + use crate::types::Address; #[tokio::test] async fn updates_metrics() { diff --git a/packages/services/tests/block_bundler.rs b/packages/services/tests/block_bundler.rs index 802e9195..4ae2218b 100644 --- a/packages/services/tests/block_bundler.rs +++ b/packages/services/tests/block_bundler.rs @@ -223,8 +223,9 @@ async fn does_nothing_if_not_enough_blocks() -> Result<()> { #[tokio::test] async fn stops_accumulating_blocks_if_time_runs_out_measured_from_component_creation() -> Result<()> { - use services::block_bundler::port::Storage as BundlerStorage; - use services::state_committer::port::Storage; + use services::{ + block_bundler::port::Storage as BundlerStorage, state_committer::port::Storage, + }; // given let setup = test_helpers::Setup::init().await; @@ -561,8 +562,9 @@ async fn doesnt_stop_advancing_if_there_is_still_time_to_optimize() -> Result<() #[tokio::test] async fn skips_blocks_outside_lookback_window() -> Result<()> { - use services::block_bundler::port::Storage as BundlerStorage; - use services::state_committer::port::Storage; + use services::{ + block_bundler::port::Storage as BundlerStorage, state_committer::port::Storage, + }; // given let setup = test_helpers::Setup::init().await; diff --git a/packages/services/tests/block_committer.rs b/packages/services/tests/block_committer.rs index 0002df32..708113a8 100644 --- a/packages/services/tests/block_committer.rs +++ b/packages/services/tests/block_committer.rs @@ -1,11 +1,11 @@ -use services::types::{TransactionResponse, TransactionState, Utc}; use services::{ block_committer::{port::Storage, service::BlockCommitter}, + types::{TransactionResponse, TransactionState, Utc}, Runner, }; -use test_helpers::{ - mocks::fuel::{given_a_block, given_fetcher, given_secret_key}, - mocks::l1::{expects_contract_submission, expects_transaction_response, FullL1Mock}, +use test_helpers::mocks::{ + fuel::{given_a_block, given_fetcher, given_secret_key}, + l1::{expects_contract_submission, expects_transaction_response, FullL1Mock}, }; #[tokio::test] diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 18c27040..0bd9cf77 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,11 +1,10 @@ -use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; -use services::fee_tracker::port::l1::Api; -use services::fee_tracker::service::FeeThresholds; -use services::fee_tracker::service::FeeTracker; -use services::fee_tracker::service::Percentage; -use services::fee_tracker::service::SmaPeriods; -use services::fee_tracker::{port::l1::Fees, service::Config}; -use services::state_committer::service::SendOrWaitDecider; +use services::{ + fee_tracker::{ + port::l1::{testing::PreconfiguredFeeApi, Api, Fees}, + service::{Config, FeeThresholds, FeeTracker, Percentage, SmaPeriods}, + }, + state_committer::service::SendOrWaitDecider, +}; use test_case::test_case; fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 907e45b5..ac77fc55 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use services::{ fee_tracker::{ port::l1::Fees, @@ -6,7 +8,6 @@ use services::{ types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use std::time::Duration; use test_helpers::{noop_fee_tracker, preconfigured_fee_tracker}; #[tokio::test] diff --git a/packages/services/tests/status_reporter.rs b/packages/services/tests/status_reporter.rs index faebed55..f75366bd 100644 --- a/packages/services/tests/status_reporter.rs +++ b/packages/services/tests/status_reporter.rs @@ -2,8 +2,10 @@ use std::sync::Arc; use clock::TestClock; use rand::Rng; -use services::status_reporter::service::{Status, StatusReport, StatusReporter}; -use services::types::{BlockSubmission, BlockSubmissionTx}; +use services::{ + status_reporter::service::{Status, StatusReport, StatusReporter}, + types::{BlockSubmission, BlockSubmissionTx}, +}; use storage::PostgresProcess; #[tokio::test] diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 2e0f6a72..1e4609a8 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,19 +8,21 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_tracker::port::l1::testing::{ConstantFeeApi, PreconfiguredFeeApi}; -use services::fee_tracker::port::l1::Fees; -use services::fee_tracker::service::FeeTracker; -use services::types::{ - BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, -}; -use storage::{DbWithProcess, PostgresProcess}; - -use services::{block_committer::service::BlockCommitter, Runner}; use services::{ - block_importer::service::BlockImporter, state_listener::service::StateListener, BlockBundler, - BlockBundlerConfig, BundlerFactory, StateCommitter, + block_committer::service::BlockCommitter, + block_importer::service::BlockImporter, + fee_tracker::{ + port::l1::{ + testing::{ConstantFeeApi, PreconfiguredFeeApi}, + Fees, + }, + service::FeeTracker, + }, + state_listener::service::StateListener, + types::{BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty}, + BlockBundler, BlockBundlerConfig, BundlerFactory, Runner, StateCommitter, }; +use storage::{DbWithProcess, PostgresProcess}; pub fn random_data(size: impl Into) -> NonEmpty { let size = size.into(); From dde123bcbae0dc887ae6dad85b9a506d52c13fed Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 12:51:34 +0100 Subject: [PATCH 056/136] separate algo from fee tracker --- Cargo.lock | 12 + Cargo.toml | 2 +- committer/src/main.rs | 6 +- committer/src/setup.rs | 69 +- packages/adapters/eth/src/fee_conversion.rs | 4 +- packages/adapters/eth/src/lib.rs | 4 +- ...{fee_tracker.rs => fee_metrics_updater.rs} | 2 +- .../fee_analytics.rs | 19 +- .../port.rs | 7 +- .../src/fee_metrics_updater/service.rs | 143 ++++ packages/services/src/fee_tracker/service.rs | 456 ----------- packages/services/src/lib.rs | 2 +- packages/services/src/state_committer.rs | 740 +++++++++++++++++- packages/services/tests/fee_tracker.rs | 377 --------- packages/services/tests/state_committer.rs | 43 +- packages/services/tests/state_listener.rs | 6 +- packages/test-helpers/src/lib.rs | 22 +- 17 files changed, 990 insertions(+), 924 deletions(-) rename packages/services/src/{fee_tracker.rs => fee_metrics_updater.rs} (58%) rename packages/services/src/{fee_tracker => fee_metrics_updater}/fee_analytics.rs (95%) rename packages/services/src/{fee_tracker => fee_metrics_updater}/port.rs (98%) create mode 100644 packages/services/src/fee_metrics_updater/service.rs delete mode 100644 packages/services/src/fee_tracker/service.rs diff --git a/Cargo.lock b/Cargo.lock index 84196586..e117ebf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2747,6 +2747,18 @@ dependencies = [ "bytes", ] +[[package]] +name = "fee_algo_simulation" +version = "0.10.5" +dependencies = [ + "alloy", + "anyhow", + "serde", + "serde_json", + "services", + "tokio", +] + [[package]] name = "ff" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index edb584d5..ed0face1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ resolver = "2" members = [ "committer", - "e2e", + "e2e", "fee_algo_simulation", "packages/adapters/clock", "packages/adapters/eth", "packages/adapters/fuel", diff --git a/committer/src/main.rs b/committer/src/main.rs index e2704cf6..aab86f3e 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -72,7 +72,7 @@ async fn main() -> Result<()> { &metrics_registry, ); - let (fee_tracker, fee_tracker_handle) = setup::fee_tracker( + let (fee_analytics, fee_metrics_updater_handle) = setup::fee_metrics_updater( ethereum_rpc.clone(), cancel_token.clone(), &config, @@ -86,7 +86,7 @@ async fn main() -> Result<()> { cancel_token.clone(), &config, &metrics_registry, - fee_tracker, + fee_analytics, )?; let state_importer_handle = @@ -114,7 +114,7 @@ async fn main() -> Result<()> { handles.push(state_importer_handle); handles.push(block_bundler); handles.push(state_listener_handle); - handles.push(fee_tracker_handle); + handles.push(fee_metrics_updater_handle); // Enable pruner once the issue is resolved // TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 // handles.push(state_pruner_handle); diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 62d0e74e..eda99b3b 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,11 +9,12 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_tracker::{ + fee_metrics_updater::{ + fee_analytics::{self, FeeAnalytics}, port::cache::CachingApi, - service::{FeeThresholds, FeeTracker, SmaPeriods}, + service::{FeeMetricsUpdater, SmaPeriods}, }, - state_committer::port::Storage, + state_committer::{port::Storage, service::FeeThresholds}, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -122,8 +123,21 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, - fee_tracker: FeeTracker>, + fee_metrics_updater: FeeAnalytics>, ) -> Result> { + let algo_config = services::state_committer::service::AlgoConfig { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config.app.fee_algo.start_discount_percentage.try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }; + let state_committer = services::StateCommitter::new( l1, fuel, @@ -133,9 +147,10 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, + fee_algo: algo_config, }, SystemClock, - fee_tracker, + fee_metrics_updater, ); state_committer.register_metrics(registry); @@ -325,40 +340,38 @@ pub async fn shut_down( Ok(()) } -pub fn fee_tracker( +pub fn fee_metrics_updater( l1: L1, cancel_token: CancellationToken, config: &config::Config, registry: &Registry, -) -> Result<(FeeTracker>, tokio::task::JoinHandle<()>)> { - let fee_tracker = FeeTracker::new( - CachingApi::new(l1, 24 * 3600 / 12), - services::fee_tracker::service::Config { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config - .app - .fee_algo - .start_discount_percentage - .try_into()?, - end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, +) -> Result<(FeeAnalytics>, tokio::task::JoinHandle<()>)> { + let algo_config = services::state_committer::service::AlgoConfig { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, }, - ); + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config.app.fee_algo.start_discount_percentage.try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }; + + let api = CachingApi::new(l1, 24 * 3600 / 12); + let fee_analytics = FeeAnalytics::new(api); + + let fee_metrics_updater = FeeMetricsUpdater::new(fee_analytics.clone(), algo_config.sma_periods); - fee_tracker.register_metrics(registry); + fee_metrics_updater.register_metrics(registry); let handle = schedule_polling( config.app.l1_fee_check_interval, - fee_tracker.clone(), + fee_metrics_updater, "Fee Tracker", cancel_token, ); - Ok((fee_tracker, handle)) + Ok((fee_analytics, handle)) } diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 7edfdbf8..45eeeaa7 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -3,7 +3,7 @@ use std::{num::NonZeroU128, ops::RangeInclusive}; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; use services::{ - fee_tracker::port::l1::{BlockFees, Fees}, + fee_metrics_updater::port::l1::{BlockFees, Fees}, Result, }; @@ -108,7 +108,7 @@ mod test { use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; - use services::fee_tracker::port::l1::{BlockFees, Fees}; + use services::fee_metrics_updater::port::l1::{BlockFees, Fees}; use crate::fee_conversion::{self}; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index c3a27c29..a5490ddd 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_tracker::port::l1::SequentialBlockFees, + fee_metrics_updater::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -206,7 +206,7 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } } -impl services::fee_tracker::port::l1::Api for WebsocketClient { +impl services::fee_metrics_updater::port::l1::Api for WebsocketClient { async fn current_height(&self) -> Result { self._get_block_number().await } diff --git a/packages/services/src/fee_tracker.rs b/packages/services/src/fee_metrics_updater.rs similarity index 58% rename from packages/services/src/fee_tracker.rs rename to packages/services/src/fee_metrics_updater.rs index a977c59c..f4e90cb9 100644 --- a/packages/services/src/fee_tracker.rs +++ b/packages/services/src/fee_metrics_updater.rs @@ -1,4 +1,4 @@ pub mod port; pub mod service; -mod fee_analytics; +pub mod fee_analytics; diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_metrics_updater/fee_analytics.rs similarity index 95% rename from packages/services/src/fee_tracker/fee_analytics.rs rename to packages/services/src/fee_metrics_updater/fee_analytics.rs index 89625710..8d2de2a0 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_metrics_updater/fee_analytics.rs @@ -3,6 +3,8 @@ use std::{num::NonZeroU128, ops::RangeInclusive}; use super::port::l1::{Api, BlockFees, Fees, SequentialBlockFees}; use crate::Error; +// TODO: segfault, move this higher because it is used by both the state committer and the fee +// tracker #[derive(Debug, Clone)] pub struct FeeAnalytics

{ fees_provider: P, @@ -81,12 +83,27 @@ impl FeeAnalytics

{ } } +pub fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); + let blob_fee = fees + .base_fee_per_blob_gas + .get() + .saturating_mul(u128::from(num_blobs)) + .saturating_mul(DATA_GAS_PER_BLOB); + let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); + + base_fee.saturating_add(blob_fee).saturating_add(reward_fee) +} + #[cfg(test)] mod tests { use itertools::Itertools; use super::*; - use crate::fee_tracker::port::l1::{testing, BlockFees}; + use crate::fee_metrics_updater::port::l1::{testing, BlockFees}; #[test] fn can_create_valid_sequential_fees() { diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_metrics_updater/port.rs similarity index 98% rename from packages/services/src/fee_tracker/port.rs rename to packages/services/src/fee_metrics_updater/port.rs index 4d9fa17d..2383e4f4 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_metrics_updater/port.rs @@ -1,5 +1,5 @@ pub mod l1 { - #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct Fees { pub base_fee_per_gas: NonZeroU128, pub reward: NonZeroU128, @@ -16,7 +16,7 @@ pub mod l1 { } } - #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct BlockFees { pub height: u64, pub fees: Fees, @@ -24,6 +24,7 @@ pub mod l1 { use std::{num::NonZeroU128, ops::RangeInclusive}; use itertools::Itertools; + use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Clone)] pub struct SequentialBlockFees { @@ -299,7 +300,7 @@ pub mod cache { use mockall::{predicate::eq, Sequence}; - use crate::fee_tracker::port::{ + use crate::fee_metrics_updater::port::{ cache::CachingApi, l1::{BlockFees, Fees, MockApi, SequentialBlockFees}, }; diff --git a/packages/services/src/fee_metrics_updater/service.rs b/packages/services/src/fee_metrics_updater/service.rs new file mode 100644 index 00000000..7f595d90 --- /dev/null +++ b/packages/services/src/fee_metrics_updater/service.rs @@ -0,0 +1,143 @@ +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64}, + ops::RangeInclusive, +}; + +use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, +}; +use tracing::info; + +use super::{ + fee_analytics::{self, FeeAnalytics}, + port::l1::{Api, Fees}, +}; +use crate::{Error, Result, Runner}; + +#[derive(Debug, Clone)] +struct Metrics { + current_blob_tx_fee: IntGauge, + short_term_blob_tx_fee: IntGauge, + long_term_blob_tx_fee: IntGauge, +} + +impl Default for Metrics { + fn default() -> Self { + let current_blob_tx_fee = IntGauge::with_opts(Opts::new( + "current_blob_tx_fee", + "The current fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let short_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "short_term_blob_tx_fee", + "The short term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let long_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "long_term_blob_tx_fee", + "The long term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + Self { + current_blob_tx_fee, + short_term_blob_tx_fee, + long_term_blob_tx_fee, + } + } +} + +impl

RegistersMetrics for FeeMetricsUpdater

{ + fn metrics(&self) -> Vec> { + vec![ + Box::new(self.metrics.current_blob_tx_fee.clone()), + Box::new(self.metrics.short_term_blob_tx_fee.clone()), + Box::new(self.metrics.long_term_blob_tx_fee.clone()), + ] + } +} + +#[derive(Clone)] +pub struct FeeMetricsUpdater

{ + fee_analytics: FeeAnalytics

, + metrics: Metrics, + metrics_sma: SmaPeriods, +} + +#[derive(Debug, Clone, Copy)] +pub struct SmaPeriods { + pub short: NonZeroU64, + pub long: NonZeroU64, +} +impl FeeMetricsUpdater

{ + fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + } + + pub async fn update_metrics(&self) -> Result<()> { + let latest_fees = self.fee_analytics.latest_fees().await?; + let short_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.metrics_sma.short, + )) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.metrics_sma.long, + )) + .await?; + + let calc_fee = |fees: &Fees| { + i64::try_from(fee_analytics::calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX) + }; + + self.metrics + .current_blob_tx_fee + .set(calc_fee(&latest_fees.fees)); + self.metrics + .short_term_blob_tx_fee + .set(calc_fee(&short_term_sma)); + self.metrics + .long_term_blob_tx_fee + .set(calc_fee(&long_term_sma)); + + Ok(()) + } +} + +impl

FeeMetricsUpdater

{ + pub fn new(fee_analytics: FeeAnalytics

, metrics_sma: SmaPeriods) -> Self { + Self { + fee_analytics, + metrics_sma, + metrics: Metrics::default(), + } + } +} + +impl

Runner for FeeMetricsUpdater

+where + P: crate::fee_metrics_updater::port::l1::Api + Send + Sync, +{ + async fn run(&mut self) -> Result<()> { + self.update_metrics().await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use test_case::test_case; + + use super::*; + use crate::fee_metrics_updater::port::l1::testing::ConstantFeeApi; +} diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs deleted file mode 100644 index a191624a..00000000 --- a/packages/services/src/fee_tracker/service.rs +++ /dev/null @@ -1,456 +0,0 @@ -use std::{ - cmp::min, - num::{NonZeroU32, NonZeroU64}, - ops::RangeInclusive, -}; - -use metrics::{ - prometheus::{core::Collector, IntGauge, Opts}, - RegistersMetrics, -}; -use tracing::info; - -use super::{ - fee_analytics::FeeAnalytics, - port::l1::{Api, Fees}, -}; -use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; - -#[derive(Debug, Clone, Copy)] -pub struct Config { - pub sma_periods: SmaPeriods, - pub fee_thresholds: FeeThresholds, -} - -#[cfg(feature = "test-helpers")] -impl Default for Config { - fn default() -> Self { - Config { - sma_periods: SmaPeriods { - short: 1.try_into().expect("not zero"), - long: 2.try_into().expect("not zero"), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - ..FeeThresholds::default() - }, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct SmaPeriods { - pub short: NonZeroU64, - pub long: NonZeroU64, -} - -#[derive(Debug, Clone, Copy)] -pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: Percentage, - pub end_premium_percentage: Percentage, - pub always_acceptable_fee: u128, -} - -#[cfg(feature = "test-helpers")] -impl Default for FeeThresholds { - fn default() -> Self { - Self { - max_l2_blocks_behind: NonZeroU32::MAX, - start_discount_percentage: Percentage::ZERO, - end_premium_percentage: Percentage::ZERO, - always_acceptable_fee: u128::MAX, - } - } -} - -impl SendOrWaitDecider for FeeTracker

{ - async fn should_send_blob_tx( - &self, - num_blobs: u32, - num_l2_blocks_behind: u32, - at_l1_height: u64, - ) -> Result { - if self.too_far_behind(num_l2_blocks_behind) { - info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); - return Ok(true); - } - - // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller - // wants to send more than 6 blobs - let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); - - let short_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await?; - - let long_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await?; - - let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &short_term_sma); - - if self.fee_always_acceptable(short_term_tx_fee) { - info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); - return Ok(true); - } - - let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &long_term_sma); - let max_upper_tx_fee = Self::calculate_max_upper_fee( - &self.config.fee_thresholds, - long_term_tx_fee, - num_l2_blocks_behind, - ); - - info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); - - let should_send = short_term_tx_fee < max_upper_tx_fee; - - if should_send { - info!( - "Sending because short term price {} is lower than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } else { - info!( - "Not sending because short term price {} is higher than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } - - Ok(should_send) - } -} - -#[derive(Default, Copy, Clone, Debug, PartialEq)] -pub struct Percentage(f64); - -impl TryFrom for Percentage { - type Error = Error; - - fn try_from(value: f64) -> std::result::Result { - if value < 0. { - return Err(Error::Other(format!("Invalid percentage value {value}"))); - } - - Ok(Self(value)) - } -} - -impl From for f64 { - fn from(value: Percentage) -> Self { - value.0 - } -} - -impl Percentage { - pub const ZERO: Self = Percentage(0.); - pub const PPM: u128 = 1_000_000; - - pub fn ppm(&self) -> u128 { - (self.0 * 1_000_000.) as u128 - } -} - -#[derive(Debug, Clone)] -struct Metrics { - current_blob_tx_fee: IntGauge, - short_term_blob_tx_fee: IntGauge, - long_term_blob_tx_fee: IntGauge, -} - -impl Default for Metrics { - fn default() -> Self { - let current_blob_tx_fee = IntGauge::with_opts(Opts::new( - "current_blob_tx_fee", - "The current fee for a transaction with 6 blobs", - )) - .expect("metric config to be correct"); - - let short_term_blob_tx_fee = IntGauge::with_opts(Opts::new( - "short_term_blob_tx_fee", - "The short term fee for a transaction with 6 blobs", - )) - .expect("metric config to be correct"); - - let long_term_blob_tx_fee = IntGauge::with_opts(Opts::new( - "long_term_blob_tx_fee", - "The long term fee for a transaction with 6 blobs", - )) - .expect("metric config to be correct"); - - Self { - current_blob_tx_fee, - short_term_blob_tx_fee, - long_term_blob_tx_fee, - } - } -} - -impl

RegistersMetrics for FeeTracker

{ - fn metrics(&self) -> Vec> { - vec![ - Box::new(self.metrics.current_blob_tx_fee.clone()), - Box::new(self.metrics.short_term_blob_tx_fee.clone()), - Box::new(self.metrics.long_term_blob_tx_fee.clone()), - ] - } -} - -#[derive(Clone)] -pub struct FeeTracker

{ - fee_analytics: FeeAnalytics

, - config: Config, - metrics: Metrics, -} - -impl FeeTracker

{ - fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { - num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() - } - - fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee - } - - fn calculate_max_upper_fee( - fee_thresholds: &FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - ) -> u128 { - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); - let blocks_behind = u128::from(num_l2_blocks_behind); - - debug_assert!( - blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", - blocks_behind, - max_blocks_behind - ); - - let start_discount_ppm = min( - fee_thresholds.start_discount_percentage.ppm(), - Percentage::PPM, - ); - let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); - - // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); - - // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = Self::calculate_premium_increment( - start_discount_ppm, - end_premium_ppm, - blocks_behind, - max_blocks_behind, - ); - - // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% - let multiplier_ppm = min( - base_multiplier.saturating_add(premium_increment), - Percentage::PPM + end_premium_ppm, - ); - - info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); - - // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm) - .saturating_div(Percentage::PPM) - } - - fn calculate_premium_increment( - start_discount_ppm: u128, - end_premium_ppm: u128, - blocks_behind: u128, - max_blocks_behind: u128, - ) -> u128 { - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - - let proportion = if max_blocks_behind == 0 { - 0 - } else { - blocks_behind - .saturating_mul(Percentage::PPM) - .saturating_div(max_blocks_behind) - }; - - total_ppm - .saturating_mul(proportion) - .saturating_div(Percentage::PPM) - } - - fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { - const DATA_GAS_PER_BLOB: u128 = 131_072u128; - const INTRINSIC_GAS: u128 = 21_000u128; - - let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); - let blob_fee = fees - .base_fee_per_blob_gas - .get() - .saturating_mul(u128::from(num_blobs)) - .saturating_mul(DATA_GAS_PER_BLOB); - let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); - - base_fee.saturating_add(blob_fee).saturating_add(reward_fee) - } - - fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { - current_block.saturating_sub(n.get().saturating_sub(1))..=current_block - } - - pub async fn update_metrics(&self) -> Result<()> { - let latest_fees = self.fee_analytics.latest_fees().await?; - let short_term_sma = self - .fee_analytics - .calculate_sma(Self::last_n_blocks( - latest_fees.height, - self.config.sma_periods.short, - )) - .await?; - - let long_term_sma = self - .fee_analytics - .calculate_sma(Self::last_n_blocks( - latest_fees.height, - self.config.sma_periods.long, - )) - .await?; - - let calc_fee = - |fees: &Fees| i64::try_from(Self::calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX); - - self.metrics - .current_blob_tx_fee - .set(calc_fee(&latest_fees.fees)); - self.metrics - .short_term_blob_tx_fee - .set(calc_fee(&short_term_sma)); - self.metrics - .long_term_blob_tx_fee - .set(calc_fee(&long_term_sma)); - - Ok(()) - } -} - -impl

FeeTracker

{ - pub fn new(fee_provider: P, config: Config) -> Self { - Self { - fee_analytics: FeeAnalytics::new(fee_provider), - config, - metrics: Metrics::default(), - } - } -} - -impl

Runner for FeeTracker

-where - P: crate::fee_tracker::port::l1::Api + Send + Sync, -{ - async fn run(&mut self) -> Result<()> { - self.update_metrics().await?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use test_case::test_case; - - use super::*; - use crate::fee_tracker::port::l1::testing::ConstantFeeApi; - - #[test_case( - // Test Case 1: No blocks behind, no discount or premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 0, - 1000; - "No blocks behind, multiplier should be 100%" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 2000, - 50, - 2050; - "Half blocks behind with discount and premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 800, - 50, - 700; - "Start discount only, no premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - end_premium_percentage: 0.30.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 50, - 1150; - "End premium only, no discount" - )] - #[test_case( - // Test Case 8: High fee with premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 10_000, - 99, - 11970; - "High fee with premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 1.50.try_into().unwrap(), // 150% - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 1000, - 1, - 12; - "Discount exceeds 100%, should be capped to 100%" -)] - fn test_calculate_max_upper_fee( - fee_thresholds: FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - expected_max_upper_fee: u128, - ) { - let max_upper_fee = FeeTracker::::calculate_max_upper_fee( - &fee_thresholds, - fee, - num_l2_blocks_behind, - ); - - assert_eq!( - max_upper_fee, expected_max_upper_fee, - "Expected max_upper_fee to be {}, but got {}", - expected_max_upper_fee, max_upper_fee - ); - } -} diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 6c7195cd..479bcbfd 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,7 +2,7 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; -pub mod fee_tracker; +pub mod fee_metrics_updater; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index c2b29887..9c5021f4 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,5 +1,10 @@ pub mod service { - use std::{num::NonZeroUsize, time::Duration}; + use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + ops::RangeInclusive, + time::Duration, + }; use itertools::Itertools; use metrics::{ @@ -9,8 +14,12 @@ pub mod service { use tracing::info; use crate::{ + fee_metrics_updater::{ + fee_analytics::{self, FeeAnalytics}, + service::SmaPeriods, + }, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, + Error, Result, Runner, }; // src/config.rs @@ -21,6 +30,7 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, + pub fee_algo: AlgoConfig, } #[cfg(feature = "test-helpers")] @@ -31,19 +41,237 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), + fee_algo: Default::default(), } } } - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - pub trait SendOrWaitDecider { + #[derive(Debug, Clone, Copy)] + pub struct AlgoConfig { + pub sma_periods: SmaPeriods, + pub fee_thresholds: FeeThresholds, + } + + #[cfg(feature = "test-helpers")] + impl Default for AlgoConfig { + fn default() -> Self { + Self { + sma_periods: SmaPeriods { + short: 1.try_into().expect("not zero"), + long: 2.try_into().expect("not zero"), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + ..FeeThresholds::default() + }, + } + } + } + + #[derive(Debug, Clone, Copy)] + pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU32, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, + pub always_acceptable_fee: u128, + } + + #[cfg(feature = "test-helpers")] + impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } + } + + #[derive(Default, Copy, Clone, Debug, PartialEq)] + pub struct Percentage(f64); + + impl TryFrom for Percentage { + type Error = Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } + } + + impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } + } + + impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } + } + + pub struct SmaFeeAlgo

{ + fee_analytics: FeeAnalytics

, + config: AlgoConfig, + } + + impl

SmaFeeAlgo

{ + pub fn new(fee_analytics: FeeAnalytics

, config: AlgoConfig) -> Self { + Self { + fee_analytics, + config, + } + } + + fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { + num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + + fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + } + + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + ) -> u128 { + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(num_l2_blocks_behind); + + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", + blocks_behind, + max_blocks_behind + ); + + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = Self::calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); + + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + Percentage::PPM + end_premium_ppm, + ); + + info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) + } + + fn calculate_premium_increment( + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); + + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(Percentage::PPM) + .saturating_div(max_blocks_behind) + }; + + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) + } + } + + impl

SmaFeeAlgo

+ where + P: crate::fee_metrics_updater::port::l1::Api + Send + Sync, + { async fn should_send_blob_tx( &self, num_blobs: u32, num_l2_blocks_behind: u32, at_l1_height: u64, - ) -> Result; + ) -> Result { + if self.too_far_behind(num_l2_blocks_behind) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); + return Ok(true); + } + + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs + let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); + + let short_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) + .await?; + + let short_term_tx_fee = + fee_analytics::calculate_blob_tx_fee(num_blobs, &short_term_sma); + + if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); + return Ok(true); + } + + let long_term_tx_fee = fee_analytics::calculate_blob_tx_fee(num_blobs, &long_term_sma); + let max_upper_tx_fee = Self::calculate_max_upper_fee( + &self.config.fee_thresholds, + long_term_tx_fee, + num_l2_blocks_behind, + ); + + info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); + + let should_send = short_term_tx_fee < max_upper_tx_fee; + + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } else { + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } + + Ok(should_send) + } } struct Metrics { @@ -71,18 +299,18 @@ pub mod service { } /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, - decider: D, metrics: Metrics, + fee_algo: SmaFeeAlgo, } - impl StateCommitter + impl StateCommitter where Clock: crate::state_committer::port::Clock, { @@ -93,30 +321,30 @@ pub mod service { storage: Db, config: Config, clock: Clock, - decider: Decider, + fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); Self { + fee_algo: SmaFeeAlgo::new(fee_analytics, config.fee_algo), l1_adapter, fuel_api, storage, config, clock, startup_time, - decider, metrics: Default::default(), } } } - impl StateCommitter + impl StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, - Decider: SendOrWaitDecider, + FeeProvider: crate::fee_metrics_updater::port::l1::Api + Sync, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -144,7 +372,7 @@ pub mod service { let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block); - self.decider + self.fee_algo .should_send_blob_tx( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), num_l2_blocks_behind, @@ -344,13 +572,14 @@ pub mod service { } } - impl Runner for StateCommitter + impl Runner + for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, - Decider: SendOrWaitDecider + Send + Sync, + FeeProvider: crate::fee_metrics_updater::port::l1::Api + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -362,6 +591,485 @@ pub mod service { Ok(()) } } + + #[cfg(test)] + mod tests { + use crate::fee_metrics_updater::port::l1::testing::PreconfiguredFeeApi; + use crate::fee_metrics_updater::port::l1::{Api, Fees}; + use test_case::test_case; + + use super::*; + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + 0, + 1000; + "No blocks behind, multiplier should be 100%" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 2000, + 50, + 2050; + "Half blocks behind with discount and premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 800, + 50, + 700; + "Start discount only, no premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + end_premium_percentage: 0.30.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + 50, + 1150; + "End premium only, no discount" + )] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 10_000, + 99, + 11970; + "High fee with premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 1000, + 1, + 12; + "Discount exceeds 100%, should be capped to 100%" +)] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + expected_max_upper_fee: u128, + ) { + use crate::fee_metrics_updater::port::l1::testing::ConstantFeeApi; + + let max_upper_fee = SmaFeeAlgo::::calculate_max_upper_fee( + &fee_thresholds, + fee, + num_l2_blocks_behind, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } + + fn generate_fees( + sma_periods: SmaPeriods, + old_fees: Fees, + new_fees: Fees, + ) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (sma_periods.long.get() - sma_periods.short.get()) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, sma_periods.short.get() as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() + } + + #[test_case( + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, // not behind at all + true; + "Should send because all short-term fees are lower than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, + false; + "Should not send because all short-term fees are higher than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, + max_l2_blocks_behind: 100.try_into().unwrap(), + ..Default::default() + } + }, + 0, + true; + "Should send since short-term fee less than always_acceptable_fee" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_gas is lower" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_gas is higher" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_blob_gas is lower" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_blob_gas is higher" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term reward is lower" + )] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term reward is higher" + )] + #[test_case( + // Multiple short-term fees are lower + Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because multiple short-term fees are lower" + )] + #[test_case( + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because all fees are identical and no tolerance" + )] + #[test_case( + // Zero blobs scenario: blob fee differences don't matter + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" + )] + #[test_case( + // Zero blobs but short-term reward is higher + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Zero blobs: short-term reward is higher, don't send" + )] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" + )] + // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. + #[test_case( + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), + always_acceptable_fee: 0, + }, + }, + 0, + false; + "Early: short-term expensive, not send" + )] + #[test_case( + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + } + }, + 100, + true; + "Later: after max wait, send regardless" + )] + #[test_case( + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + }, + 80, + true; + "Mid-wait: increased tolerance allows acceptance" + )] + #[test_case( + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 2_700_000_000_000 + }, + }, + 0, + true; + "Always acceptable fee triggers immediate send" + )] + #[tokio::test] + async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + num_blobs: u32, + config: AlgoConfig, + num_l2_blocks_behind: u32, + expected_decision: bool, + ) { + let fees = generate_fees(config.sma_periods, old_fees, new_fees); + let api = PreconfiguredFeeApi::new(fees); + let current_block_height = api.current_height().await.unwrap(); + let fees_analytics = FeeAnalytics::new(api); + + let sut = SmaFeeAlgo::new(fees_analytics, config); + + let should_send = sut + .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) + .await + .unwrap(); + + assert_eq!( + should_send, expected_decision, + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", + ); + } + + #[tokio::test] + async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = AlgoConfig { + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let fee_analytics = FeeAnalytics::new(PreconfiguredFeeApi::new(vec![])); + let sut = SmaFeeAlgo::new(fee_analytics, config); + + // when + let should_send = sut + .should_send_blob_tx(1, 20, 100) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); + } + } } pub mod port { diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 0bd9cf77..139597f9 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,379 +1,2 @@ -use services::{ - fee_tracker::{ - port::l1::{testing::PreconfiguredFeeApi, Api, Fees}, - service::{Config, FeeThresholds, FeeTracker, Percentage, SmaPeriods}, - }, - state_committer::service::SendOrWaitDecider, -}; -use test_case::test_case; -fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { - let older_fees = std::iter::repeat_n( - old_fees, - (config.sma_periods.long.get() - config.sma_periods.short.get()) as usize, - ); - let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short.get() as usize); - older_fees - .chain(newer_fees) - .enumerate() - .map(|(i, f)| (i as u64, f)) - .collect() -} - -#[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - 6, - Config { - sma_periods: services::fee_tracker::service::SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, // not behind at all - true; - "Should send because all short-term fees are lower than long-term" - )] -#[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, - false; - "Should not send because all short-term fees are higher than long-term" - )] -#[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - 6, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, - max_l2_blocks_behind: 100.try_into().unwrap(), - ..Default::default() - } - }, - 0, - true; - "Should send since short-term fee less than always_acceptable_fee" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_gas is lower" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_gas is higher" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_blob_gas is lower" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_blob_gas is higher" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term reward is lower" - )] -#[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - 5, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term reward is higher" - )] -#[test_case( - // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, - 6, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because multiple short-term fees are lower" - )] -#[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because all fees are identical and no tolerance" - )] -#[test_case( - // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: short-term base_fee_per_gas and reward are lower, send" - )] -#[test_case( - // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Zero blobs: short-term reward is higher, don't send" - )] -#[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, - 0, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" - )] -// Initially not send, but as num_l2_blocks_behind increases, acceptance grows. -#[test_case( - // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: Percentage::try_from(0.20).unwrap(), - end_premium_percentage: Percentage::try_from(0.20).unwrap(), - always_acceptable_fee: 0, - }, - }, - 0, - false; - "Early: short-term expensive, not send" - )] -#[test_case( - // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - } - }, - 100, - true; - "Later: after max wait, send regardless" - )] -#[test_case( - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - }, - 80, - true; - "Mid-wait: increased tolerance allows acceptance" - )] -#[test_case( - // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, - 1, - Config { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 2_700_000_000_000 - }, - }, - 0, - true; - "Always acceptable fee triggers immediate send" - )] -#[tokio::test] -async fn parameterized_send_or_wait_tests( - old_fees: Fees, - new_fees: Fees, - num_blobs: u32, - config: Config, - num_l2_blocks_behind: u32, - expected_decision: bool, -) { - let fees = generate_fees(config, old_fees, new_fees); - let fees_provider = PreconfiguredFeeApi::new(fees); - let current_block_height = fees_provider.current_height().await.unwrap(); - - let sut = FeeTracker::new(fees_provider, config); - - let should_send = sut - .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) - .await - .unwrap(); - - assert_eq!( - should_send, expected_decision, - "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", - ); -} - -#[tokio::test] -async fn test_send_when_too_far_behind_and_fee_provider_fails() { - // given - let config = Config { - sma_periods: SmaPeriods { - short: 2.try_into().unwrap(), - long: 6.try_into().unwrap(), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 10.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }; - - // having no fees will make the validation in fee analytics fail - let fee_provider = PreconfiguredFeeApi::new(vec![]); - let sut = FeeTracker::new(fee_provider, config); - - // when - let should_send = sut - .should_send_blob_tx(1, 20, 100) - .await - .expect("Should send despite fee provider failure"); - - // then - assert!( - should_send, - "Should send because too far behind, regardless of fee provider status" - ); -} diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index ac77fc55..ca1e6b56 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,14 +1,12 @@ use std::time::Duration; use services::{ - fee_tracker::{ - port::l1::Fees, - service::{Config as FeeTrackerConfig, FeeThresholds, SmaPeriods}, - }, + fee_metrics_updater::{port::l1::Fees, service::SmaPeriods}, + state_committer::service::{AlgoConfig, FeeThresholds}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use test_helpers::{noop_fee_tracker, preconfigured_fee_tracker}; +use test_helpers::{noop_fee_analytics, preconfigured_fee_analytics}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -42,7 +40,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - noop_fee_tracker(), + noop_fee_analytics(), ); // when @@ -86,7 +84,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Advance time beyond the timeout @@ -122,7 +120,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Advance time less than the timeout @@ -168,7 +166,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - noop_fee_tracker(), + noop_fee_analytics(), ); // when @@ -215,7 +213,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Advance time to exceed the timeout since last finalized fragment @@ -262,7 +260,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Advance time beyond the timeout from startup @@ -319,9 +317,10 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 5.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(60), + ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Submit the initial fragments @@ -395,7 +394,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_tracker_config = services::fee_tracker::service::Config { + let fee_algo = AlgoConfig { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap(), @@ -433,10 +432,11 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo, ..Default::default() }, setup.test_clock(), - preconfigured_fee_tracker(fee_sequence, fee_tracker_config), + preconfigured_fee_analytics(fee_sequence), ); // When @@ -504,7 +504,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fee_tracker_config = FeeTrackerConfig { + let fee_algo = AlgoConfig { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap(), @@ -533,10 +533,11 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo, ..Default::default() }, setup.test_clock(), - preconfigured_fee_tracker(fee_sequence, fee_tracker_config), + preconfigured_fee_analytics(fee_sequence), ); // when @@ -604,7 +605,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fee_tracker_config = FeeTrackerConfig { + let fee_algo = AlgoConfig { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap(), @@ -642,10 +643,11 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo, ..Default::default() }, setup.test_clock(), - preconfigured_fee_tracker(fee_sequence, fee_tracker_config), + preconfigured_fee_analytics(fee_sequence), ); // when @@ -712,7 +714,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ), ]; - let fee_tracker_config = services::fee_tracker::service::Config { + let fee_algo = AlgoConfig { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 5.try_into().unwrap(), @@ -751,10 +753,11 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo, ..Default::default() }, setup.test_clock(), - preconfigured_fee_tracker(fee_sequence, fee_tracker_config), + preconfigured_fee_analytics(fee_sequence), ); // when diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 5777b8da..a54e7042 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -10,7 +10,7 @@ use services::{ use test_case::test_case; use test_helpers::{ mocks::{self, l1::TxStatus}, - noop_fee_tracker, + noop_fee_analytics, }; #[tokio::test] @@ -456,7 +456,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Orig tx @@ -560,7 +560,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 1e4609a8..cbfec200 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -11,13 +11,15 @@ use rand::{Rng, RngCore}; use services::{ block_committer::service::BlockCommitter, block_importer::service::BlockImporter, - fee_tracker::{ + fee_metrics_updater::{ + fee_analytics::FeeAnalytics, port::l1::{ testing::{ConstantFeeApi, PreconfiguredFeeApi}, Fees, }, - service::FeeTracker, + service::{FeeMetricsUpdater, SmaPeriods}, }, + state_committer::service::SmaFeeAlgo, state_listener::service::StateListener, types::{BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty}, BlockBundler, BlockBundlerConfig, BundlerFactory, Runner, StateCommitter, @@ -489,15 +491,14 @@ pub mod mocks { } } -pub fn noop_fee_tracker() -> FeeTracker { - FeeTracker::new(ConstantFeeApi::new(Fees::default()), Default::default()) +pub fn noop_fee_analytics() -> FeeAnalytics { + FeeAnalytics::new(ConstantFeeApi::new(Fees::default())) } -pub fn preconfigured_fee_tracker( +pub fn preconfigured_fee_analytics( fee_sequence: impl IntoIterator, - config: services::fee_tracker::service::Config, -) -> FeeTracker { - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), config) +) -> FeeAnalytics { + FeeAnalytics::new(PreconfiguredFeeApi::new(fee_sequence)) } pub struct Setup { @@ -571,7 +572,7 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ) .run() .await @@ -607,9 +608,10 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), + ..Default::default() }, self.test_clock.clone(), - noop_fee_tracker(), + noop_fee_analytics(), ); committer.run().await.unwrap(); From dbfe64d37ffa9a947e55334bffe9deb69796609e Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 12:54:12 +0100 Subject: [PATCH 057/136] rename and cleanup imports --- committer/src/main.rs | 4 ++-- committer/src/setup.rs | 16 ++++++++-------- packages/adapters/eth/src/fee_conversion.rs | 4 ++-- packages/adapters/eth/src/lib.rs | 4 ++-- ...ee_metrics_updater.rs => historical_fees.rs} | 0 .../fee_analytics.rs | 2 +- .../port.rs | 2 +- .../service.rs | 17 ++++++----------- packages/services/src/lib.rs | 2 +- packages/services/src/state_committer.rs | 14 +++++++------- packages/services/tests/state_committer.rs | 2 +- packages/test-helpers/src/lib.rs | 4 +--- 12 files changed, 32 insertions(+), 39 deletions(-) rename packages/services/src/{fee_metrics_updater.rs => historical_fees.rs} (100%) rename packages/services/src/{fee_metrics_updater => historical_fees}/fee_analytics.rs (99%) rename packages/services/src/{fee_metrics_updater => historical_fees}/port.rs (99%) rename packages/services/src/{fee_metrics_updater => historical_fees}/service.rs (91%) diff --git a/committer/src/main.rs b/committer/src/main.rs index aab86f3e..0a05c988 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -72,7 +72,7 @@ async fn main() -> Result<()> { &metrics_registry, ); - let (fee_analytics, fee_metrics_updater_handle) = setup::fee_metrics_updater( + let (fee_analytics, historical_fees_handle) = setup::historical_fees( ethereum_rpc.clone(), cancel_token.clone(), &config, @@ -114,7 +114,7 @@ async fn main() -> Result<()> { handles.push(state_importer_handle); handles.push(block_bundler); handles.push(state_listener_handle); - handles.push(fee_metrics_updater_handle); + handles.push(historical_fees_handle); // Enable pruner once the issue is resolved // TODO: https://github.com/FuelLabs/fuel-block-committer/issues/173 // handles.push(state_pruner_handle); diff --git a/committer/src/setup.rs b/committer/src/setup.rs index eda99b3b..b547cd75 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,8 +9,8 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_metrics_updater::{ - fee_analytics::{self, FeeAnalytics}, + historical_fees::{ + fee_analytics::{FeeAnalytics}, port::cache::CachingApi, service::{FeeMetricsUpdater, SmaPeriods}, }, @@ -123,7 +123,7 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, - fee_metrics_updater: FeeAnalytics>, + historical_fees: FeeAnalytics>, ) -> Result> { let algo_config = services::state_committer::service::AlgoConfig { sma_periods: SmaPeriods { @@ -150,7 +150,7 @@ pub fn state_committer( fee_algo: algo_config, }, SystemClock, - fee_metrics_updater, + historical_fees, ); state_committer.register_metrics(registry); @@ -340,7 +340,7 @@ pub async fn shut_down( Ok(()) } -pub fn fee_metrics_updater( +pub fn historical_fees( l1: L1, cancel_token: CancellationToken, config: &config::Config, @@ -362,13 +362,13 @@ pub fn fee_metrics_updater( let api = CachingApi::new(l1, 24 * 3600 / 12); let fee_analytics = FeeAnalytics::new(api); - let fee_metrics_updater = FeeMetricsUpdater::new(fee_analytics.clone(), algo_config.sma_periods); + let historical_fees = FeeMetricsUpdater::new(fee_analytics.clone(), algo_config.sma_periods); - fee_metrics_updater.register_metrics(registry); + historical_fees.register_metrics(registry); let handle = schedule_polling( config.app.l1_fee_check_interval, - fee_metrics_updater, + historical_fees, "Fee Tracker", cancel_token, ); diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 45eeeaa7..501414de 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -3,7 +3,7 @@ use std::{num::NonZeroU128, ops::RangeInclusive}; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; use services::{ - fee_metrics_updater::port::l1::{BlockFees, Fees}, + historical_fees::port::l1::{BlockFees, Fees}, Result, }; @@ -108,7 +108,7 @@ mod test { use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; - use services::fee_metrics_updater::port::l1::{BlockFees, Fees}; + use services::historical_fees::port::l1::{BlockFees, Fees}; use crate::fee_conversion::{self}; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index a5490ddd..5fb3e481 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_metrics_updater::port::l1::SequentialBlockFees, + historical_fees::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -206,7 +206,7 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } } -impl services::fee_metrics_updater::port::l1::Api for WebsocketClient { +impl services::historical_fees::port::l1::Api for WebsocketClient { async fn current_height(&self) -> Result { self._get_block_number().await } diff --git a/packages/services/src/fee_metrics_updater.rs b/packages/services/src/historical_fees.rs similarity index 100% rename from packages/services/src/fee_metrics_updater.rs rename to packages/services/src/historical_fees.rs diff --git a/packages/services/src/fee_metrics_updater/fee_analytics.rs b/packages/services/src/historical_fees/fee_analytics.rs similarity index 99% rename from packages/services/src/fee_metrics_updater/fee_analytics.rs rename to packages/services/src/historical_fees/fee_analytics.rs index 8d2de2a0..03cfa7c4 100644 --- a/packages/services/src/fee_metrics_updater/fee_analytics.rs +++ b/packages/services/src/historical_fees/fee_analytics.rs @@ -103,7 +103,7 @@ mod tests { use itertools::Itertools; use super::*; - use crate::fee_metrics_updater::port::l1::{testing, BlockFees}; + use crate::historical_fees::port::l1::{testing, BlockFees}; #[test] fn can_create_valid_sequential_fees() { diff --git a/packages/services/src/fee_metrics_updater/port.rs b/packages/services/src/historical_fees/port.rs similarity index 99% rename from packages/services/src/fee_metrics_updater/port.rs rename to packages/services/src/historical_fees/port.rs index 2383e4f4..f1cfaf6e 100644 --- a/packages/services/src/fee_metrics_updater/port.rs +++ b/packages/services/src/historical_fees/port.rs @@ -300,7 +300,7 @@ pub mod cache { use mockall::{predicate::eq, Sequence}; - use crate::fee_metrics_updater::port::{ + use crate::historical_fees::port::{ cache::CachingApi, l1::{BlockFees, Fees, MockApi, SequentialBlockFees}, }; diff --git a/packages/services/src/fee_metrics_updater/service.rs b/packages/services/src/historical_fees/service.rs similarity index 91% rename from packages/services/src/fee_metrics_updater/service.rs rename to packages/services/src/historical_fees/service.rs index 7f595d90..d778c4e7 100644 --- a/packages/services/src/fee_metrics_updater/service.rs +++ b/packages/services/src/historical_fees/service.rs @@ -1,20 +1,15 @@ -use std::{ - cmp::min, - num::{NonZeroU32, NonZeroU64}, - ops::RangeInclusive, -}; +use std::{num::NonZeroU64, ops::RangeInclusive}; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, RegistersMetrics, }; -use tracing::info; use super::{ fee_analytics::{self, FeeAnalytics}, port::l1::{Api, Fees}, }; -use crate::{Error, Result, Runner}; +use crate::{Result, Runner}; #[derive(Debug, Clone)] struct Metrics { @@ -126,7 +121,7 @@ impl

FeeMetricsUpdater

{ impl

Runner for FeeMetricsUpdater

where - P: crate::fee_metrics_updater::port::l1::Api + Send + Sync, + P: crate::historical_fees::port::l1::Api + Send + Sync, { async fn run(&mut self) -> Result<()> { self.update_metrics().await?; @@ -136,8 +131,8 @@ where #[cfg(test)] mod tests { - use test_case::test_case; + - use super::*; - use crate::fee_metrics_updater::port::l1::testing::ConstantFeeApi; + + } diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 479bcbfd..bd4668c6 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,7 +2,7 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; -pub mod fee_metrics_updater; +pub mod historical_fees; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 9c5021f4..81dba403 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -14,7 +14,7 @@ pub mod service { use tracing::info; use crate::{ - fee_metrics_updater::{ + historical_fees::{ fee_analytics::{self, FeeAnalytics}, service::SmaPeriods, }, @@ -212,7 +212,7 @@ pub mod service { impl

SmaFeeAlgo

where - P: crate::fee_metrics_updater::port::l1::Api + Send + Sync, + P: crate::historical_fees::port::l1::Api + Send + Sync, { async fn should_send_blob_tx( &self, @@ -344,7 +344,7 @@ pub mod service { FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, - FeeProvider: crate::fee_metrics_updater::port::l1::Api + Sync, + FeeProvider: crate::historical_fees::port::l1::Api + Sync, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -579,7 +579,7 @@ pub mod service { FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, - FeeProvider: crate::fee_metrics_updater::port::l1::Api + Send + Sync, + FeeProvider: crate::historical_fees::port::l1::Api + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -594,8 +594,8 @@ pub mod service { #[cfg(test)] mod tests { - use crate::fee_metrics_updater::port::l1::testing::PreconfiguredFeeApi; - use crate::fee_metrics_updater::port::l1::{Api, Fees}; + use crate::historical_fees::port::l1::testing::PreconfiguredFeeApi; + use crate::historical_fees::port::l1::{Api, Fees}; use test_case::test_case; use super::*; @@ -679,7 +679,7 @@ pub mod service { num_l2_blocks_behind: u32, expected_max_upper_fee: u128, ) { - use crate::fee_metrics_updater::port::l1::testing::ConstantFeeApi; + use crate::historical_fees::port::l1::testing::ConstantFeeApi; let max_upper_fee = SmaFeeAlgo::::calculate_max_upper_fee( &fee_thresholds, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index ca1e6b56..a0e4d1fd 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,7 +1,7 @@ use std::time::Duration; use services::{ - fee_metrics_updater::{port::l1::Fees, service::SmaPeriods}, + historical_fees::{port::l1::Fees, service::SmaPeriods}, state_committer::service::{AlgoConfig, FeeThresholds}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index cbfec200..98e9acc0 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -11,15 +11,13 @@ use rand::{Rng, RngCore}; use services::{ block_committer::service::BlockCommitter, block_importer::service::BlockImporter, - fee_metrics_updater::{ + historical_fees::{ fee_analytics::FeeAnalytics, port::l1::{ testing::{ConstantFeeApi, PreconfiguredFeeApi}, Fees, }, - service::{FeeMetricsUpdater, SmaPeriods}, }, - state_committer::service::SmaFeeAlgo, state_listener::service::StateListener, types::{BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty}, BlockBundler, BlockBundlerConfig, BundlerFactory, Runner, StateCommitter, From d92f8811b73fbff8b6086c81b0bd2f9ebaf66fb1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 13:22:46 +0100 Subject: [PATCH 058/136] removed fee analytics, merged with historical_fees and added a separate metrics runner --- committer/src/setup.rs | 28 +- packages/services/src/historical_fees.rs | 2 - .../src/historical_fees/fee_analytics.rs | 463 ------------------ packages/services/src/historical_fees/port.rs | 213 ++++++++ .../services/src/historical_fees/service.rs | 336 +++++++++++-- packages/services/src/state_committer.rs | 37 +- packages/services/tests/state_committer.rs | 16 +- packages/services/tests/state_listener.rs | 6 +- packages/test-helpers/src/lib.rs | 14 +- 9 files changed, 552 insertions(+), 563 deletions(-) delete mode 100644 packages/services/src/historical_fees/fee_analytics.rs diff --git a/committer/src/setup.rs b/committer/src/setup.rs index b547cd75..17ee2b84 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -10,9 +10,8 @@ use metrics::{ use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, historical_fees::{ - fee_analytics::{FeeAnalytics}, port::cache::CachingApi, - service::{FeeMetricsUpdater, SmaPeriods}, + service::{HistoricalFees, SmaPeriods}, }, state_committer::{port::Storage, service::FeeThresholds}, state_listener::service::StateListener, @@ -123,7 +122,7 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, - historical_fees: FeeAnalytics>, + historical_fees: HistoricalFees>, ) -> Result> { let algo_config = services::state_committer::service::AlgoConfig { sma_periods: SmaPeriods { @@ -345,33 +344,24 @@ pub fn historical_fees( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, -) -> Result<(FeeAnalytics>, tokio::task::JoinHandle<()>)> { - let algo_config = services::state_committer::service::AlgoConfig { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config.app.fee_algo.start_discount_percentage.try_into()?, - end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, +) -> Result<(HistoricalFees>, tokio::task::JoinHandle<()>)> { + let sma_periods = SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, }; let api = CachingApi::new(l1, 24 * 3600 / 12); - let fee_analytics = FeeAnalytics::new(api); - let historical_fees = FeeMetricsUpdater::new(fee_analytics.clone(), algo_config.sma_periods); + let historical_fees = HistoricalFees::new(api); historical_fees.register_metrics(registry); let handle = schedule_polling( config.app.l1_fee_check_interval, - historical_fees, + historical_fees.fee_metrics_updater(sma_periods), "Fee Tracker", cancel_token, ); - Ok((fee_analytics, handle)) + Ok((historical_fees, handle)) } diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs index f4e90cb9..7691d7e1 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/historical_fees.rs @@ -1,4 +1,2 @@ pub mod port; pub mod service; - -pub mod fee_analytics; diff --git a/packages/services/src/historical_fees/fee_analytics.rs b/packages/services/src/historical_fees/fee_analytics.rs deleted file mode 100644 index 03cfa7c4..00000000 --- a/packages/services/src/historical_fees/fee_analytics.rs +++ /dev/null @@ -1,463 +0,0 @@ -use std::{num::NonZeroU128, ops::RangeInclusive}; - -use super::port::l1::{Api, BlockFees, Fees, SequentialBlockFees}; -use crate::Error; - -// TODO: segfault, move this higher because it is used by both the state committer and the fee -// tracker -#[derive(Debug, Clone)] -pub struct FeeAnalytics

{ - fees_provider: P, -} - -impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } -} - -impl FeeAnalytics

{ - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - let fees = self.fees_provider.fees(block_range.clone()).await?; - - let received_height_range = fees.height_range(); - if received_height_range != block_range { - return Err(Error::from(format!( - "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" - ))); - } - - Ok(Self::mean(fees)) - } - - pub async fn latest_fees(&self) -> crate::Result { - let height = self.fees_provider.current_height().await?; - - let fee = self - .fees_provider - .fees(height..=height) - .await? - .into_iter() - .next() - .expect("sequential fees guaranteed not empty"); - - Ok(fee) - } - - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| { - let base_fee_per_gas = acc - .base_fee_per_gas - .saturating_add(f.base_fee_per_gas.get()); - let reward = acc.reward.saturating_add(f.reward.get()); - let base_fee_per_blob_gas = acc - .base_fee_per_blob_gas - .saturating_add(f.base_fee_per_blob_gas.get()); - - Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - } - }); - - let divide_by_count = |value: NonZeroU128| { - let minimum_fee = NonZeroU128::try_from(1).unwrap(); - value - .get() - .saturating_div(count) - .try_into() - .unwrap_or(minimum_fee) - }; - - Fees { - base_fee_per_gas: divide_by_count(total.base_fee_per_gas), - reward: divide_by_count(total.reward), - base_fee_per_blob_gas: divide_by_count(total.base_fee_per_blob_gas), - } - } -} - -pub fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { - const DATA_GAS_PER_BLOB: u128 = 131_072u128; - const INTRINSIC_GAS: u128 = 21_000u128; - - let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); - let blob_fee = fees - .base_fee_per_blob_gas - .get() - .saturating_mul(u128::from(num_blobs)) - .saturating_mul(DATA_GAS_PER_BLOB); - let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); - - base_fee.saturating_add(blob_fee).saturating_add(reward_fee) -} - -#[cfg(test)] -mod tests { - use itertools::Itertools; - - use super::*; - use crate::historical_fees::port::l1::{testing, BlockFees}; - - #[test] - fn can_create_valid_sequential_fees() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100.try_into().unwrap(), - reward: 50.try_into().unwrap(), - base_fee_per_blob_gas: 10.try_into().unwrap(), - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110.try_into().unwrap(), - reward: 55.try_into().unwrap(), - base_fee_per_blob_gas: 15.try_into().unwrap(), - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees.clone()); - - // Then - assert!( - result.is_ok(), - "Expected SequentialBlockFees creation to succeed" - ); - let sequential_fees = result.unwrap(); - assert_eq!(sequential_fees.len(), block_fees.len()); - } - - #[test] - fn sequential_fees_cannot_be_empty() { - // Given - let block_fees: Vec = vec![]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for empty input" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"Input cannot be empty\")" - ); - } - - #[test] - fn fees_must_be_sequential() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100.try_into().unwrap(), - reward: 50.try_into().unwrap(), - base_fee_per_blob_gas: 10.try_into().unwrap(), - }, - }, - BlockFees { - height: 3, // Non-sequential height - fees: Fees { - base_fee_per_gas: 110.try_into().unwrap(), - reward: 55.try_into().unwrap(), - base_fee_per_blob_gas: 15.try_into().unwrap(), - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for non-sequential heights" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" - ); - } - - #[test] - fn produced_iterator_gives_correct_values() { - // Given - // notice the heights are out of order so that we validate that the returned sequence is in - // order - let block_fees = vec![ - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110.try_into().unwrap(), - reward: 55.try_into().unwrap(), - base_fee_per_blob_gas: 15.try_into().unwrap(), - }, - }, - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100.try_into().unwrap(), - reward: 50.try_into().unwrap(), - base_fee_per_blob_gas: 10.try_into().unwrap(), - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); - - // When - let iterated_fees: Vec = sequential_fees.into_iter().collect(); - - // Then - let expectation = block_fees - .into_iter() - .sorted_by_key(|b| b.height) - .collect_vec(); - assert_eq!( - iterated_fees, expectation, - "Expected iterator to yield the same block fees" - ); - } - - #[tokio::test] - async fn calculates_sma_correctly_for_last_1_block() { - // given - let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); - - // then - assert_eq!(sma.base_fee_per_gas, 6.try_into().unwrap()); - assert_eq!(sma.reward, 6.try_into().unwrap()); - assert_eq!(sma.base_fee_per_blob_gas, 6.try_into().unwrap()); - } - - #[tokio::test] - async fn calculates_sma_correctly_for_last_5_blocks() { - // given - let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); - - // then - let mean = ((5 + 4 + 3 + 2 + 1) / 5).try_into().unwrap(); - assert_eq!(sma.base_fee_per_gas, mean); - assert_eq!(sma.reward, mean); - assert_eq!(sma.base_fee_per_blob_gas, mean); - } - - #[tokio::test] - async fn errors_out_if_returned_fees_are_not_complete() { - // given - let mut fees = testing::incrementing_fees(5); - fees.remove(&4); - let fees_provider = testing::PreconfiguredFeeApi::new(fees); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let err = fee_analytics - .calculate_sma(0..=4) - .await - .expect_err("should have failed because returned fees are not complete"); - - // then - assert_eq!( - err.to_string(), - "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" - ); - } - - #[tokio::test] - async fn latest_fees_on_fee_analytics() { - // given - let fees_map = testing::incrementing_fees(5); - let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); - let fee_analytics = FeeAnalytics::new(fees_provider); - let height = 4; - - // when - let fee = fee_analytics.latest_fees().await.unwrap(); - - // then - let expected_fee = BlockFees { - height, - fees: Fees { - base_fee_per_gas: 5.try_into().unwrap(), - reward: 5.try_into().unwrap(), - base_fee_per_blob_gas: 5.try_into().unwrap(), - }, - }; - assert_eq!( - fee, expected_fee, - "Fee at height {height} should be {expected_fee:?}" - ); - } - - #[tokio::test] - async fn mean_is_at_least_one_when_totals_are_zero() { - // given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 1.try_into().unwrap(), - reward: 1.try_into().unwrap(), - base_fee_per_blob_gas: 1.try_into().unwrap(), - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 1.try_into().unwrap(), - reward: 1.try_into().unwrap(), - base_fee_per_blob_gas: 1.try_into().unwrap(), - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees).unwrap(); - let mean = FeeAnalytics::::mean(sequential_fees.clone()); - - // then - assert_eq!( - mean.base_fee_per_gas, - 1.try_into().unwrap(), - "base_fee_per_gas should be set to 1 when total is 0" - ); - assert_eq!( - mean.reward, - 1.try_into().unwrap(), - "reward should be set to 1 when total is 0" - ); - assert_eq!( - mean.base_fee_per_blob_gas, - 1.try_into().unwrap(), - "base_fee_per_blob_gas should be set to 1 when total is 0" - ); - } - - // fn calculate_tx_fee(fees: &Fees) -> u128 { - // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 - // } - // - // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - // let mut csv_writer = - // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) - // .unwrap(); - // csv_writer - // .write_record(["height", "tx_fee"].iter()) - // .unwrap(); - // for (height, fee) in tx_fees { - // csv_writer - // .write_record([height.to_string(), fee.to_string()]) - // .unwrap(); - // } - // csv_writer.flush().unwrap(); - // } - - // #[tokio::test] - // async fn something() { - // let client = make_pub_eth_client().await; - // use services::fee_analytics::port::l1::FeesProvider; - // - // let current_block_height = 21408300; - // let starting_block_height = current_block_height - 48 * 3600 / 12; - // let data = client - // .fees(starting_block_height..=current_block_height) - // .await - // .into_iter() - // .collect::>(); - // - // let fee_lookup = data - // .iter() - // .map(|b| (b.height, b.fees)) - // .collect::>(); - // - // let short_sma = 25u64; - // let long_sma = 900; - // - // let current_tx_fees = data - // .iter() - // .map(|b| (b.height, calculate_tx_fee(&b.fees))) - // .collect::>(); - // - // save_tx_fees(¤t_tx_fees, "current_fees.csv"); - // - // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - // let fee_analytics = FeeAnalytics::new(local_client.clone()); - // - // let mut short_sma_tx_fees = vec![]; - // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - short_sma..=height) - // .await; - // - // let tx_fee = calculate_tx_fee(&fees); - // - // short_sma_tx_fees.push((height, tx_fee)); - // } - // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - // - // let decider = SendOrWaitDecider::new( - // FeeAnalytics::new(local_client.clone()), - // services::state_committer::fee_optimization::Config { - // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { - // short: short_sma, - // long: long_sma, - // }, - // fee_thresholds: Feethresholds { - // max_l2_blocks_behind: 43200 * 3, - // start_discount_percentage: 0.2, - // end_premium_percentage: 0.2, - // always_acceptable_fee: 1000000000000000u128, - // }, - // }, - // ); - // - // let mut decisions = vec![]; - // let mut long_sma_tx_fees = vec![]; - // - // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - long_sma..=height) - // .await; - // let tx_fee = calculate_tx_fee(&fees); - // long_sma_tx_fees.push((height, tx_fee)); - // - // if decider - // .should_send_blob_tx( - // 6, - // Context { - // at_l1_height: height, - // num_l2_blocks_behind: (height - starting_block_height) * 12, - // }, - // ) - // .await - // { - // let current_fees = fee_lookup.get(&height).unwrap(); - // let current_tx_fee = calculate_tx_fee(current_fees); - // decisions.push((height, current_tx_fee)); - // } - // } - // - // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - // save_tx_fees(&decisions, "decisions.csv"); - // } -} diff --git a/packages/services/src/historical_fees/port.rs b/packages/services/src/historical_fees/port.rs index f1cfaf6e..0a9006d2 100644 --- a/packages/services/src/historical_fees/port.rs +++ b/packages/services/src/historical_fees/port.rs @@ -53,6 +53,44 @@ pub mod l1 { // Cannot be empty #[allow(clippy::len_without_is_empty)] impl SequentialBlockFees { + pub fn mean(&self) -> Fees { + let count = self.len() as u128; + + let total = self + .fees + .iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| { + let base_fee_per_gas = acc + .base_fee_per_gas + .saturating_add(f.base_fee_per_gas.get()); + let reward = acc.reward.saturating_add(f.reward.get()); + let base_fee_per_blob_gas = acc + .base_fee_per_blob_gas + .saturating_add(f.base_fee_per_blob_gas.get()); + + Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + } + }); + + let divide_by_count = |value: NonZeroU128| { + let minimum_fee = NonZeroU128::try_from(1).unwrap(); + value + .get() + .saturating_div(count) + .try_into() + .unwrap_or(minimum_fee) + }; + + Fees { + base_fee_per_gas: divide_by_count(total.base_fee_per_gas), + reward: divide_by_count(total.reward), + base_fee_per_blob_gas: divide_by_count(total.base_fee_per_blob_gas), + } + } pub fn len(&self) -> usize { self.fees.len() } @@ -197,6 +235,181 @@ pub mod l1 { .collect() } } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn can_create_valid_sequential_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn sequential_fees_cannot_be_empty() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn fees_must_be_sequential() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" + ); + } + + #[test] + fn produced_iterator_gives_correct_values() { + // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order + let block_fees = vec![ + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110.try_into().unwrap(), + reward: 55.try_into().unwrap(), + base_fee_per_blob_gas: 15.try_into().unwrap(), + }, + }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100.try_into().unwrap(), + reward: 50.try_into().unwrap(), + base_fee_per_blob_gas: 10.try_into().unwrap(), + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); + assert_eq!( + iterated_fees, expectation, + "Expected iterator to yield the same block fees" + ); + } + + #[tokio::test] + async fn mean_is_at_least_one_when_totals_are_zero() { + // given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 1.try_into().unwrap(), + reward: 1.try_into().unwrap(), + base_fee_per_blob_gas: 1.try_into().unwrap(), + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 1.try_into().unwrap(), + reward: 1.try_into().unwrap(), + base_fee_per_blob_gas: 1.try_into().unwrap(), + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees).unwrap(); + let mean = sequential_fees.mean(); + + // then + assert_eq!( + mean.base_fee_per_gas, + 1.try_into().unwrap(), + "base_fee_per_gas should be set to 1 when total is 0" + ); + assert_eq!( + mean.reward, + 1.try_into().unwrap(), + "reward should be set to 1 when total is 0" + ); + assert_eq!( + mean.base_fee_per_blob_gas, + 1.try_into().unwrap(), + "base_fee_per_blob_gas should be set to 1 when total is 0" + ); + } + } } pub mod cache { diff --git a/packages/services/src/historical_fees/service.rs b/packages/services/src/historical_fees/service.rs index d778c4e7..272e4b12 100644 --- a/packages/services/src/historical_fees/service.rs +++ b/packages/services/src/historical_fees/service.rs @@ -1,15 +1,15 @@ -use std::{num::NonZeroU64, ops::RangeInclusive}; +use std::{ + num::NonZeroU64, + ops::RangeInclusive, +}; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, RegistersMetrics, }; -use super::{ - fee_analytics::{self, FeeAnalytics}, - port::l1::{Api, Fees}, -}; -use crate::{Result, Runner}; +use super::port::l1::{Api, BlockFees, Fees}; +use crate::{Error, Result, Runner}; #[derive(Debug, Clone)] struct Metrics { @@ -46,7 +46,7 @@ impl Default for Metrics { } } -impl

RegistersMetrics for FeeMetricsUpdater

{ +impl

RegistersMetrics for HistoricalFees

{ fn metrics(&self) -> Vec> { vec![ Box::new(self.metrics.current_blob_tx_fee.clone()), @@ -57,10 +57,9 @@ impl

RegistersMetrics for FeeMetricsUpdater

{ } #[derive(Clone)] -pub struct FeeMetricsUpdater

{ - fee_analytics: FeeAnalytics

, +pub struct HistoricalFees

{ + fee_provider: P, metrics: Metrics, - metrics_sma: SmaPeriods, } #[derive(Debug, Clone, Copy)] @@ -68,40 +67,115 @@ pub struct SmaPeriods { pub short: NonZeroU64, pub long: NonZeroU64, } -impl FeeMetricsUpdater

{ - fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { - current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + +pub fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); + let blob_fee = fees + .base_fee_per_blob_gas + .get() + .saturating_mul(u128::from(num_blobs)) + .saturating_mul(DATA_GAS_PER_BLOB); + let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); + + base_fee.saturating_add(blob_fee).saturating_add(reward_fee) +} + +impl HistoricalFees

{ + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + let fees = self.fee_provider.fees(block_range.clone()).await?; + + let received_height_range = fees.height_range(); + if received_height_range != block_range { + return Err(Error::from(format!( + "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" + ))); + } + + Ok(fees.mean()) + } + + pub async fn latest_fees(&self) -> crate::Result { + let height = self.fee_provider.current_height().await?; + + let fee = self + .fee_provider + .fees(height..=height) + .await? + .into_iter() + .next() + .expect("sequential fees guaranteed not empty"); + + Ok(fee) + } +} + +fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block +} + +impl

HistoricalFees

{ + pub fn new(fee_provider: P) -> Self { + Self { + fee_provider, + metrics: Metrics::default(), + } } + pub fn fee_metrics_updater(&self, periods: SmaPeriods) -> FeeMetricsUpdater

+ where + Self: Clone, + { + FeeMetricsUpdater { + historical_fees: self.clone(), + sma_periods: periods, + } + } +} + +pub struct FeeMetricsUpdater

{ + historical_fees: HistoricalFees

, + sma_periods: SmaPeriods, +} + +impl

FeeMetricsUpdater

{ + pub fn new(historical_fees: HistoricalFees

, sma_periods: SmaPeriods) -> Self { + Self { + historical_fees, + sma_periods, + } + } +} + +impl FeeMetricsUpdater

{ pub async fn update_metrics(&self) -> Result<()> { - let latest_fees = self.fee_analytics.latest_fees().await?; + let metrics_sma = self.sma_periods; + let latest_fees = self.historical_fees.latest_fees().await?; let short_term_sma = self - .fee_analytics - .calculate_sma(Self::last_n_blocks( - latest_fees.height, - self.metrics_sma.short, - )) + .historical_fees + .calculate_sma(last_n_blocks(latest_fees.height, metrics_sma.short)) .await?; let long_term_sma = self - .fee_analytics - .calculate_sma(Self::last_n_blocks( - latest_fees.height, - self.metrics_sma.long, - )) + .historical_fees + .calculate_sma(last_n_blocks(latest_fees.height, metrics_sma.long)) .await?; - let calc_fee = |fees: &Fees| { - i64::try_from(fee_analytics::calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX) - }; + let calc_fee = + |fees: &Fees| i64::try_from(calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX); - self.metrics + self.historical_fees + .metrics .current_blob_tx_fee .set(calc_fee(&latest_fees.fees)); - self.metrics + self.historical_fees + .metrics .short_term_blob_tx_fee .set(calc_fee(&short_term_sma)); - self.metrics + self.historical_fees + .metrics .long_term_blob_tx_fee .set(calc_fee(&long_term_sma)); @@ -109,19 +183,9 @@ impl FeeMetricsUpdater

{ } } -impl

FeeMetricsUpdater

{ - pub fn new(fee_analytics: FeeAnalytics

, metrics_sma: SmaPeriods) -> Self { - Self { - fee_analytics, - metrics_sma, - metrics: Metrics::default(), - } - } -} - impl

Runner for FeeMetricsUpdater

where - P: crate::historical_fees::port::l1::Api + Send + Sync, + P: Api + Send + Sync, { async fn run(&mut self) -> Result<()> { self.update_metrics().await?; @@ -133,6 +197,192 @@ where mod tests { - - + use super::*; + use crate::historical_fees::port::l1::{testing, BlockFees}; + + #[tokio::test] + async fn calculates_sma_correctly_for_last_1_block() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let sut = HistoricalFees::new(fees_provider); + + // when + let sma = sut.calculate_sma(4..=4).await.unwrap(); + + // then + assert_eq!(sma.base_fee_per_gas, 6.try_into().unwrap()); + assert_eq!(sma.reward, 6.try_into().unwrap()); + assert_eq!(sma.base_fee_per_blob_gas, 6.try_into().unwrap()); + } + + #[tokio::test] + async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let sut = HistoricalFees::new(fees_provider); + + // when + let sma = sut.calculate_sma(0..=4).await.unwrap(); + + // then + let mean = ((5 + 4 + 3 + 2 + 1) / 5).try_into().unwrap(); + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); + } + + #[tokio::test] + async fn errors_out_if_returned_fees_are_not_complete() { + // given + let mut fees = testing::incrementing_fees(5); + fees.remove(&4); + let fees_provider = testing::PreconfiguredFeeApi::new(fees); + let sut = HistoricalFees::new(fees_provider); + + // when + let err = sut + .calculate_sma(0..=4) + .await + .expect_err("should have failed because returned fees are not complete"); + + // then + assert_eq!( + err.to_string(), + "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" + ); + } + + #[tokio::test] + async fn latest_fees_on_fee_analytics() { + // given + let fees_map = testing::incrementing_fees(5); + let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); + let sut = HistoricalFees::new(fees_provider); + let height = 4; + + // when + let fee = sut.latest_fees().await.unwrap(); + + // then + let expected_fee = BlockFees { + height, + fees: Fees { + base_fee_per_gas: 5.try_into().unwrap(), + reward: 5.try_into().unwrap(), + base_fee_per_blob_gas: 5.try_into().unwrap(), + }, + }; + assert_eq!( + fee, expected_fee, + "Fee at height {height} should be {expected_fee:?}" + ); + } + + // fn calculate_tx_fee(fees: &Fees) -> u128 { + // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + // } + // + // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + // let mut csv_writer = + // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + // .unwrap(); + // csv_writer + // .write_record(["height", "tx_fee"].iter()) + // .unwrap(); + // for (height, fee) in tx_fees { + // csv_writer + // .write_record([height.to_string(), fee.to_string()]) + // .unwrap(); + // } + // csv_writer.flush().unwrap(); + // } + + // #[tokio::test] + // async fn something() { + // let client = make_pub_eth_client().await; + // use services::fee_analytics::port::l1::FeesProvider; + // + // let current_block_height = 21408300; + // let starting_block_height = current_block_height - 48 * 3600 / 12; + // let data = client + // .fees(starting_block_height..=current_block_height) + // .await + // .into_iter() + // .collect::>(); + // + // let fee_lookup = data + // .iter() + // .map(|b| (b.height, b.fees)) + // .collect::>(); + // + // let short_sma = 25u64; + // let long_sma = 900; + // + // let current_tx_fees = data + // .iter() + // .map(|b| (b.height, calculate_tx_fee(&b.fees))) + // .collect::>(); + // + // save_tx_fees(¤t_tx_fees, "current_fees.csv"); + // + // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + // let fee_analytics = FeeAnalytics::new(local_client.clone()); + // + // let mut short_sma_tx_fees = vec![]; + // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - short_sma..=height) + // .await; + // + // let tx_fee = calculate_tx_fee(&fees); + // + // short_sma_tx_fees.push((height, tx_fee)); + // } + // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + // + // let decider = SendOrWaitDecider::new( + // FeeAnalytics::new(local_client.clone()), + // services::state_committer::fee_optimization::Config { + // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + // short: short_sma, + // long: long_sma, + // }, + // fee_thresholds: Feethresholds { + // max_l2_blocks_behind: 43200 * 3, + // start_discount_percentage: 0.2, + // end_premium_percentage: 0.2, + // always_acceptable_fee: 1000000000000000u128, + // }, + // }, + // ); + // + // let mut decisions = vec![]; + // let mut long_sma_tx_fees = vec![]; + // + // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - long_sma..=height) + // .await; + // let tx_fee = calculate_tx_fee(&fees); + // long_sma_tx_fees.push((height, tx_fee)); + // + // if decider + // .should_send_blob_tx( + // 6, + // Context { + // at_l1_height: height, + // num_l2_blocks_behind: (height - starting_block_height) * 12, + // }, + // ) + // .await + // { + // let current_fees = fee_lookup.get(&height).unwrap(); + // let current_tx_fee = calculate_tx_fee(current_fees); + // decisions.push((height, current_tx_fee)); + // } + // } + // + // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + // save_tx_fees(&decisions, "decisions.csv"); + // } } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 81dba403..953a4ae9 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -15,8 +15,8 @@ pub mod service { use crate::{ historical_fees::{ - fee_analytics::{self, FeeAnalytics}, - service::SmaPeriods, + self, + service::{HistoricalFees, SmaPeriods}, }, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Error, Result, Runner, @@ -119,14 +119,14 @@ pub mod service { } pub struct SmaFeeAlgo

{ - fee_analytics: FeeAnalytics

, + historical_fees: HistoricalFees

, config: AlgoConfig, } impl

SmaFeeAlgo

{ - pub fn new(fee_analytics: FeeAnalytics

, config: AlgoConfig) -> Self { + pub fn new(historical_fees: HistoricalFees

, config: AlgoConfig) -> Self { Self { - fee_analytics, + historical_fees, config, } } @@ -212,7 +212,7 @@ pub mod service { impl

SmaFeeAlgo

where - P: crate::historical_fees::port::l1::Api + Send + Sync, + P: historical_fees::port::l1::Api + Send + Sync, { async fn should_send_blob_tx( &self, @@ -230,24 +230,25 @@ pub mod service { let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); let short_term_sma = self - .fee_analytics + .historical_fees .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await?; let long_term_sma = self - .fee_analytics + .historical_fees .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; let short_term_tx_fee = - fee_analytics::calculate_blob_tx_fee(num_blobs, &short_term_sma); + historical_fees::service::calculate_blob_tx_fee(num_blobs, &short_term_sma); if self.fee_always_acceptable(short_term_tx_fee) { info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); return Ok(true); } - let long_term_tx_fee = fee_analytics::calculate_blob_tx_fee(num_blobs, &long_term_sma); + let long_term_tx_fee = + historical_fees::service::calculate_blob_tx_fee(num_blobs, &long_term_sma); let max_upper_tx_fee = Self::calculate_max_upper_fee( &self.config.fee_thresholds, long_term_tx_fee, @@ -321,12 +322,12 @@ pub mod service { storage: Db, config: Config, clock: Clock, - fee_analytics: FeeAnalytics, + historical_fees: HistoricalFees, ) -> Self { let startup_time = clock.now(); Self { - fee_algo: SmaFeeAlgo::new(fee_analytics, config.fee_algo), + fee_algo: SmaFeeAlgo::new(historical_fees, config.fee_algo), l1_adapter, fuel_api, storage, @@ -344,7 +345,7 @@ pub mod service { FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, - FeeProvider: crate::historical_fees::port::l1::Api + Sync, + FeeProvider: historical_fees::port::l1::Api + Sync, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -579,7 +580,7 @@ pub mod service { FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, - FeeProvider: crate::historical_fees::port::l1::Api + Send + Sync, + FeeProvider: historical_fees::port::l1::Api + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -1023,9 +1024,9 @@ pub mod service { let fees = generate_fees(config.sma_periods, old_fees, new_fees); let api = PreconfiguredFeeApi::new(fees); let current_block_height = api.current_height().await.unwrap(); - let fees_analytics = FeeAnalytics::new(api); + let historical_fees = HistoricalFees::new(api); - let sut = SmaFeeAlgo::new(fees_analytics, config); + let sut = SmaFeeAlgo::new(historical_fees, config); let should_send = sut .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) @@ -1054,8 +1055,8 @@ pub mod service { }; // having no fees will make the validation in fee analytics fail - let fee_analytics = FeeAnalytics::new(PreconfiguredFeeApi::new(vec![])); - let sut = SmaFeeAlgo::new(fee_analytics, config); + let historical_fees = HistoricalFees::new(PreconfiguredFeeApi::new(vec![])); + let sut = SmaFeeAlgo::new(historical_fees, config); // when let should_send = sut diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index a0e4d1fd..0f642f89 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -6,7 +6,7 @@ use services::{ types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use test_helpers::{noop_fee_analytics, preconfigured_fee_analytics}; +use test_helpers::{noop_historical_fees, preconfigured_fee_analytics}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -40,7 +40,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - noop_fee_analytics(), + noop_historical_fees(), ); // when @@ -84,7 +84,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Advance time beyond the timeout @@ -120,7 +120,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Advance time less than the timeout @@ -166,7 +166,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - noop_fee_analytics(), + noop_historical_fees(), ); // when @@ -213,7 +213,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Advance time to exceed the timeout since last finalized fragment @@ -260,7 +260,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Advance time beyond the timeout from startup @@ -320,7 +320,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Submit the initial fragments diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index a54e7042..3408d793 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -10,7 +10,7 @@ use services::{ use test_case::test_case; use test_helpers::{ mocks::{self, l1::TxStatus}, - noop_fee_analytics, + noop_historical_fees, }; #[tokio::test] @@ -456,7 +456,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Orig tx @@ -560,7 +560,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 98e9acc0..0543b8f9 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -12,11 +12,11 @@ use services::{ block_committer::service::BlockCommitter, block_importer::service::BlockImporter, historical_fees::{ - fee_analytics::FeeAnalytics, port::l1::{ testing::{ConstantFeeApi, PreconfiguredFeeApi}, Fees, }, + service::HistoricalFees, }, state_listener::service::StateListener, types::{BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty}, @@ -489,14 +489,14 @@ pub mod mocks { } } -pub fn noop_fee_analytics() -> FeeAnalytics { - FeeAnalytics::new(ConstantFeeApi::new(Fees::default())) +pub fn noop_historical_fees() -> HistoricalFees { + HistoricalFees::new(ConstantFeeApi::new(Fees::default())) } pub fn preconfigured_fee_analytics( fee_sequence: impl IntoIterator, -) -> FeeAnalytics { - FeeAnalytics::new(PreconfiguredFeeApi::new(fee_sequence)) +) -> HistoricalFees { + HistoricalFees::new(PreconfiguredFeeApi::new(fee_sequence)) } pub struct Setup { @@ -570,7 +570,7 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ) .run() .await @@ -609,7 +609,7 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - noop_fee_analytics(), + noop_historical_fees(), ); committer.run().await.unwrap(); From 503cd0ef47ede7c0484b87a129ec4e46bf7e6a87 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 13:26:01 +0100 Subject: [PATCH 059/136] move state committer port and service to separate file --- .../services/src/historical_fees/service.rs | 7 +- packages/services/src/state_committer.rs | 1158 +---------------- packages/services/src/state_committer/port.rs | 77 ++ .../services/src/state_committer/service.rs | 1065 +++++++++++++++ 4 files changed, 1145 insertions(+), 1162 deletions(-) create mode 100644 packages/services/src/state_committer/port.rs create mode 100644 packages/services/src/state_committer/service.rs diff --git a/packages/services/src/historical_fees/service.rs b/packages/services/src/historical_fees/service.rs index 272e4b12..72819dfb 100644 --- a/packages/services/src/historical_fees/service.rs +++ b/packages/services/src/historical_fees/service.rs @@ -1,7 +1,4 @@ -use std::{ - num::NonZeroU64, - ops::RangeInclusive, -}; +use std::{num::NonZeroU64, ops::RangeInclusive}; use metrics::{ prometheus::{core::Collector, IntGauge, Opts}, @@ -195,8 +192,6 @@ where #[cfg(test)] mod tests { - - use super::*; use crate::historical_fees::port::l1::{testing, BlockFees}; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 953a4ae9..7691d7e1 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,1156 +1,2 @@ -pub mod service { - use std::{ - cmp::min, - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, - ops::RangeInclusive, - time::Duration, - }; - - use itertools::Itertools; - use metrics::{ - prometheus::{core::Collector, IntGauge, Opts}, - RegistersMetrics, - }; - use tracing::info; - - use crate::{ - historical_fees::{ - self, - service::{HistoricalFees, SmaPeriods}, - }, - types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, - }; - - // src/config.rs - #[derive(Debug, Clone)] - pub struct Config { - /// The lookback window in blocks to determine the starting height. - pub lookback_window: u32, - pub fragment_accumulation_timeout: Duration, - pub fragments_to_accumulate: NonZeroUsize, - pub gas_bump_timeout: Duration, - pub fee_algo: AlgoConfig, - } - - #[cfg(feature = "test-helpers")] - impl Default for Config { - fn default() -> Self { - Self { - lookback_window: 1000, - fragment_accumulation_timeout: Duration::from_secs(0), - fragments_to_accumulate: 1.try_into().unwrap(), - gas_bump_timeout: Duration::from_secs(300), - fee_algo: Default::default(), - } - } - } - - #[derive(Debug, Clone, Copy)] - pub struct AlgoConfig { - pub sma_periods: SmaPeriods, - pub fee_thresholds: FeeThresholds, - } - - #[cfg(feature = "test-helpers")] - impl Default for AlgoConfig { - fn default() -> Self { - Self { - sma_periods: SmaPeriods { - short: 1.try_into().expect("not zero"), - long: 2.try_into().expect("not zero"), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - ..FeeThresholds::default() - }, - } - } - } - - #[derive(Debug, Clone, Copy)] - pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: Percentage, - pub end_premium_percentage: Percentage, - pub always_acceptable_fee: u128, - } - - #[cfg(feature = "test-helpers")] - impl Default for FeeThresholds { - fn default() -> Self { - Self { - max_l2_blocks_behind: NonZeroU32::MAX, - start_discount_percentage: Percentage::ZERO, - end_premium_percentage: Percentage::ZERO, - always_acceptable_fee: u128::MAX, - } - } - } - - #[derive(Default, Copy, Clone, Debug, PartialEq)] - pub struct Percentage(f64); - - impl TryFrom for Percentage { - type Error = Error; - - fn try_from(value: f64) -> std::result::Result { - if value < 0. { - return Err(Error::Other(format!("Invalid percentage value {value}"))); - } - - Ok(Self(value)) - } - } - - impl From for f64 { - fn from(value: Percentage) -> Self { - value.0 - } - } - - impl Percentage { - pub const ZERO: Self = Percentage(0.); - pub const PPM: u128 = 1_000_000; - - pub fn ppm(&self) -> u128 { - (self.0 * 1_000_000.) as u128 - } - } - - pub struct SmaFeeAlgo

{ - historical_fees: HistoricalFees

, - config: AlgoConfig, - } - - impl

SmaFeeAlgo

{ - pub fn new(historical_fees: HistoricalFees

, config: AlgoConfig) -> Self { - Self { - historical_fees, - config, - } - } - - fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { - num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() - } - - fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee - } - - fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { - current_block.saturating_sub(n.get().saturating_sub(1))..=current_block - } - - fn calculate_max_upper_fee( - fee_thresholds: &FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - ) -> u128 { - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); - let blocks_behind = u128::from(num_l2_blocks_behind); - - debug_assert!( - blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", - blocks_behind, - max_blocks_behind - ); - - let start_discount_ppm = min( - fee_thresholds.start_discount_percentage.ppm(), - Percentage::PPM, - ); - let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); - - // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); - - // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = Self::calculate_premium_increment( - start_discount_ppm, - end_premium_ppm, - blocks_behind, - max_blocks_behind, - ); - - // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% - let multiplier_ppm = min( - base_multiplier.saturating_add(premium_increment), - Percentage::PPM + end_premium_ppm, - ); - - info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); - - // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm) - .saturating_div(Percentage::PPM) - } - - fn calculate_premium_increment( - start_discount_ppm: u128, - end_premium_ppm: u128, - blocks_behind: u128, - max_blocks_behind: u128, - ) -> u128 { - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - - let proportion = if max_blocks_behind == 0 { - 0 - } else { - blocks_behind - .saturating_mul(Percentage::PPM) - .saturating_div(max_blocks_behind) - }; - - total_ppm - .saturating_mul(proportion) - .saturating_div(Percentage::PPM) - } - } - - impl

SmaFeeAlgo

- where - P: historical_fees::port::l1::Api + Send + Sync, - { - async fn should_send_blob_tx( - &self, - num_blobs: u32, - num_l2_blocks_behind: u32, - at_l1_height: u64, - ) -> Result { - if self.too_far_behind(num_l2_blocks_behind) { - info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); - return Ok(true); - } - - // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller - // wants to send more than 6 blobs - let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); - - let short_term_sma = self - .historical_fees - .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await?; - - let long_term_sma = self - .historical_fees - .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await?; - - let short_term_tx_fee = - historical_fees::service::calculate_blob_tx_fee(num_blobs, &short_term_sma); - - if self.fee_always_acceptable(short_term_tx_fee) { - info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); - return Ok(true); - } - - let long_term_tx_fee = - historical_fees::service::calculate_blob_tx_fee(num_blobs, &long_term_sma); - let max_upper_tx_fee = Self::calculate_max_upper_fee( - &self.config.fee_thresholds, - long_term_tx_fee, - num_l2_blocks_behind, - ); - - info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); - - let should_send = short_term_tx_fee < max_upper_tx_fee; - - if should_send { - info!( - "Sending because short term price {} is lower than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } else { - info!( - "Not sending because short term price {} is higher than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } - - Ok(should_send) - } - } - - struct Metrics { - current_height_to_commit: IntGauge, - } - - impl Default for Metrics { - fn default() -> Self { - let current_height_to_commit = IntGauge::with_opts(Opts::new( - "current_height_to_commit", - "The starting l2 height of the bundle we're committing/will commit next", - )) - .expect("metric config to be correct"); - - Self { - current_height_to_commit, - } - } - } - - impl RegistersMetrics for StateCommitter { - fn metrics(&self) -> Vec> { - vec![Box::new(self.metrics.current_height_to_commit.clone())] - } - } - - /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { - l1_adapter: L1, - fuel_api: FuelApi, - storage: Db, - config: Config, - clock: Clock, - startup_time: DateTime, - metrics: Metrics, - fee_algo: SmaFeeAlgo, - } - - impl StateCommitter - where - Clock: crate::state_committer::port::Clock, - { - /// Creates a new `StateCommitter`. - pub fn new( - l1_adapter: L1, - fuel_api: FuelApi, - storage: Db, - config: Config, - clock: Clock, - historical_fees: HistoricalFees, - ) -> Self { - let startup_time = clock.now(); - - Self { - fee_algo: SmaFeeAlgo::new(historical_fees, config.fee_algo), - l1_adapter, - fuel_api, - storage, - config, - clock, - startup_time, - metrics: Default::default(), - } - } - } - - impl StateCommitter - where - L1: crate::state_committer::port::l1::Api + Send + Sync, - FuelApi: crate::state_committer::port::fuel::Api, - Db: crate::state_committer::port::Storage, - Clock: crate::state_committer::port::Clock, - FeeProvider: historical_fees::port::l1::Api + Sync, - { - async fn get_reference_time(&self) -> Result> { - Ok(self - .storage - .last_time_a_fragment_was_finalized() - .await? - .unwrap_or(self.startup_time)) - } - - async fn is_timeout_expired(&self) -> Result { - let reference_time = self.get_reference_time().await?; - let elapsed = self.clock.now() - reference_time; - let std_elapsed = elapsed - .to_std() - .map_err(|e| crate::Error::Other(format!("Failed to convert time: {}", e)))?; - Ok(std_elapsed >= self.config.fragment_accumulation_timeout) - } - - async fn fees_acceptable(&self, fragments: &NonEmpty) -> Result { - let l1_height = self.l1_adapter.current_height().await?; - let l2_height = self.fuel_api.latest_height().await?; - - let oldest_l2_block = self.oldest_l2_block_in_fragments(fragments); - self.update_oldest_block_metric(oldest_l2_block); - - let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block); - - self.fee_algo - .should_send_blob_tx( - u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), - num_l2_blocks_behind, - l1_height, - ) - .await - } - - fn oldest_l2_block_in_fragments(&self, fragments: &NonEmpty) -> u32 { - fragments - .minimum_by_key(|b| b.oldest_block_in_bundle) - .oldest_block_in_bundle - } - - async fn submit_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<()> { - info!("about to send at most {} fragments", fragments.len()); - - let data = fragments.clone().map(|f| f.fragment); - - match self - .l1_adapter - .submit_state_fragments(data, previous_tx) - .await - { - Ok((submitted_tx, submitted_fragments)) => { - let fragment_ids = fragments - .iter() - .map(|f| f.id) - .take(submitted_fragments.num_fragments.get()) - .collect_nonempty() - .expect("non-empty vec"); - - let ids = fragment_ids - .iter() - .map(|id| id.as_u32().to_string()) - .join(", "); - - let tx_hash = submitted_tx.hash; - self.storage - .record_pending_tx(submitted_tx, fragment_ids, self.clock.now()) - .await?; - - tracing::info!("Submitted fragments {ids} with tx {}", hex::encode(tx_hash)); - Ok(()) - } - Err(e) => { - let ids = fragments - .iter() - .map(|f| f.id.as_u32().to_string()) - .join(", "); - - tracing::error!("Failed to submit fragments {ids}: {e}"); - - Err(e) - } - } - } - - async fn latest_pending_transaction(&self) -> Result> { - let tx = self.storage.get_latest_pending_txs().await?; - Ok(tx) - } - - async fn next_fragments_to_submit(&self) -> Result>> { - let latest_height = self.fuel_api.latest_height().await?; - let starting_height = latest_height.saturating_sub(self.config.lookback_window); - - // although we shouldn't know at this layer how many fragments the L1 can accept, we ignore - // this for now and put the eth value of max blobs per block (6). - let existing_fragments = self - .storage - .oldest_nonfinalized_fragments(starting_height, 6) - .await?; - - let fragments = NonEmpty::collect(existing_fragments); - - if let Some(fragments) = fragments.as_ref() { - // Tracking the metric here as well to get updates more often -- because - // submit_fragments might not be called - self.update_oldest_block_metric(self.oldest_l2_block_in_fragments(fragments)); - } - - Ok(fragments) - } - - fn update_oldest_block_metric(&self, oldest_height: u32) { - self.metrics - .current_height_to_commit - .set(oldest_height as i64); - } - - async fn should_submit_fragments( - &self, - fragments: &NonEmpty, - ) -> Result { - let fragment_count = fragments.len_nonzero(); - - let expired = || async { - let expired = self.is_timeout_expired().await?; - if expired { - info!( - "fragment accumulation timeout expired, available {}/{} fragments", - fragment_count, self.config.fragments_to_accumulate - ); - } - Result::Ok(expired) - }; - - let enough_fragments = || { - let enough_fragments = fragment_count >= self.config.fragments_to_accumulate; - if !enough_fragments { - info!( - "not enough fragments {}/{}", - fragment_count, self.config.fragments_to_accumulate - ); - }; - enough_fragments - }; - - // wrapped in closures so that we short-circuit *and* reduce redundant logs - Ok((enough_fragments() || expired().await?) && self.fees_acceptable(fragments).await?) - } - - async fn submit_fragments_if_ready(&self) -> Result<()> { - if let Some(fragments) = self.next_fragments_to_submit().await? { - if self.should_submit_fragments(&fragments).await? { - self.submit_fragments(fragments, None).await?; - } - } else { - // TODO: segfault test this - // if we have no fragments to submit, that means that we're up to date and new - // blocks haven't been bundled yet - let current_height_to_commit = - if let Some(height) = self.storage.latest_bundled_height().await? { - height.saturating_add(1) - } else { - self.fuel_api - .latest_height() - .await? - .saturating_sub(self.config.lookback_window) - }; - - self.metrics - .current_height_to_commit - .set(current_height_to_commit as i64); - } - - Ok(()) - } - - fn elapsed_since_tx_submitted(&self, tx: &L1Tx) -> Result { - let created_at = tx.created_at.expect("tx to have timestamp"); - - self.clock.elapsed(created_at) - } - - async fn fragments_submitted_by_tx( - &self, - tx_hash: [u8; 32], - ) -> Result> { - let fragments = self.storage.fragments_submitted_by_tx(tx_hash).await?; - - match NonEmpty::collect(fragments) { - Some(fragments) => Ok(fragments), - None => Err(crate::Error::Other(format!( - "no fragments found for previously submitted tx {}", - hex::encode(tx_hash) - ))), - } - } - - async fn resubmit_fragments_if_stalled(&self) -> Result<()> { - let Some(previous_tx) = self.latest_pending_transaction().await? else { - return Ok(()); - }; - - let elapsed = self.elapsed_since_tx_submitted(&previous_tx)?; - - if elapsed >= self.config.gas_bump_timeout { - info!( - "replacing tx {} because it was pending for {}s", - hex::encode(previous_tx.hash), - elapsed.as_secs() - ); - - let fragments = self.fragments_submitted_by_tx(previous_tx.hash).await?; - if self.fees_acceptable(&fragments).await? { - self.submit_fragments(fragments, Some(previous_tx)).await?; - } - } - - Ok(()) - } - } - - impl Runner - for StateCommitter - where - L1: crate::state_committer::port::l1::Api + Send + Sync, - FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, - Db: crate::state_committer::port::Storage + Clone + Send + Sync, - Clock: crate::state_committer::port::Clock + Send + Sync, - FeeProvider: historical_fees::port::l1::Api + Send + Sync, - { - async fn run(&mut self) -> Result<()> { - if self.storage.has_nonfinalized_txs().await? { - self.resubmit_fragments_if_stalled().await? - } else { - self.submit_fragments_if_ready().await? - }; - - Ok(()) - } - } - - #[cfg(test)] - mod tests { - use crate::historical_fees::port::l1::testing::PreconfiguredFeeApi; - use crate::historical_fees::port::l1::{Api, Fees}; - use test_case::test_case; - - use super::*; - - #[test_case( - // Test Case 1: No blocks behind, no discount or premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 0, - 1000; - "No blocks behind, multiplier should be 100%" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 2000, - 50, - 2050; - "Half blocks behind with discount and premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 800, - 50, - 700; - "Start discount only, no premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - end_premium_percentage: 0.30.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 50, - 1150; - "End premium only, no discount" - )] - #[test_case( - // Test Case 8: High fee with premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 10_000, - 99, - 11970; - "High fee with premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 1.50.try_into().unwrap(), // 150% - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 1000, - 1, - 12; - "Discount exceeds 100%, should be capped to 100%" -)] - fn test_calculate_max_upper_fee( - fee_thresholds: FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - expected_max_upper_fee: u128, - ) { - use crate::historical_fees::port::l1::testing::ConstantFeeApi; - - let max_upper_fee = SmaFeeAlgo::::calculate_max_upper_fee( - &fee_thresholds, - fee, - num_l2_blocks_behind, - ); - - assert_eq!( - max_upper_fee, expected_max_upper_fee, - "Expected max_upper_fee to be {}, but got {}", - expected_max_upper_fee, max_upper_fee - ); - } - - fn generate_fees( - sma_periods: SmaPeriods, - old_fees: Fees, - new_fees: Fees, - ) -> Vec<(u64, Fees)> { - let older_fees = std::iter::repeat_n( - old_fees, - (sma_periods.long.get() - sma_periods.short.get()) as usize, - ); - let newer_fees = std::iter::repeat_n(new_fees, sma_periods.short.get() as usize); - - older_fees - .chain(newer_fees) - .enumerate() - .map(|(i, f)| (i as u64, f)) - .collect() - } - - #[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, // not behind at all - true; - "Should send because all short-term fees are lower than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, - false; - "Should not send because all short-term fees are higher than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, - max_l2_blocks_behind: 100.try_into().unwrap(), - ..Default::default() - } - }, - 0, - true; - "Should send since short-term fee less than always_acceptable_fee" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_blob_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_blob_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term reward is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term reward is higher" - )] - #[test_case( - // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because multiple short-term fees are lower" - )] - #[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because all fees are identical and no tolerance" - )] - #[test_case( - // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: short-term base_fee_per_gas and reward are lower, send" - )] - #[test_case( - // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Zero blobs: short-term reward is higher, don't send" - )] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" - )] - // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. - #[test_case( - // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: Percentage::try_from(0.20).unwrap(), - end_premium_percentage: Percentage::try_from(0.20).unwrap(), - always_acceptable_fee: 0, - }, - }, - 0, - false; - "Early: short-term expensive, not send" - )] - #[test_case( - // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - } - }, - 100, - true; - "Later: after max wait, send regardless" - )] - #[test_case( - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - }, - 80, - true; - "Mid-wait: increased tolerance allows acceptance" - )] - #[test_case( - // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 2_700_000_000_000 - }, - }, - 0, - true; - "Always acceptable fee triggers immediate send" - )] - #[tokio::test] - async fn parameterized_send_or_wait_tests( - old_fees: Fees, - new_fees: Fees, - num_blobs: u32, - config: AlgoConfig, - num_l2_blocks_behind: u32, - expected_decision: bool, - ) { - let fees = generate_fees(config.sma_periods, old_fees, new_fees); - let api = PreconfiguredFeeApi::new(fees); - let current_block_height = api.current_height().await.unwrap(); - let historical_fees = HistoricalFees::new(api); - - let sut = SmaFeeAlgo::new(historical_fees, config); - - let should_send = sut - .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) - .await - .unwrap(); - - assert_eq!( - should_send, expected_decision, - "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", - ); - } - - #[tokio::test] - async fn test_send_when_too_far_behind_and_fee_provider_fails() { - // given - let config = AlgoConfig { - sma_periods: SmaPeriods { - short: 2.try_into().unwrap(), - long: 6.try_into().unwrap(), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 10.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }; - - // having no fees will make the validation in fee analytics fail - let historical_fees = HistoricalFees::new(PreconfiguredFeeApi::new(vec![])); - let sut = SmaFeeAlgo::new(historical_fees, config); - - // when - let should_send = sut - .should_send_blob_tx(1, 20, 100) - .await - .expect("Should send despite fee provider failure"); - - // then - assert!( - should_send, - "Should send because too far behind, regardless of fee provider status" - ); - } - } -} - -pub mod port { - use nonempty::NonEmpty; - - use crate::{ - types::{storage::BundleFragment, DateTime, L1Tx, NonNegative, Utc}, - Error, Result, - }; - - pub mod l1 { - - use nonempty::NonEmpty; - - use crate::{ - types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, - Result, - }; - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - #[cfg_attr(feature = "test-helpers", mockall::automock)] - pub trait Contract: Send + Sync { - async fn submit(&self, hash: [u8; 32], height: u32) -> Result; - } - - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - #[cfg_attr(feature = "test-helpers", mockall::automock)] - pub trait Api { - async fn current_height(&self) -> Result; - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<(L1Tx, FragmentsSubmitted)>; - } - } - - pub mod fuel { - pub use fuel_core_client::client::types::block::Block as FuelBlock; - - use crate::Result; - - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - #[cfg_attr(feature = "test-helpers", mockall::automock)] - pub trait Api: Send + Sync { - async fn latest_height(&self) -> Result; - } - } - - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - pub trait Storage: Send + Sync { - async fn has_nonfinalized_txs(&self) -> Result; - async fn last_time_a_fragment_was_finalized(&self) -> Result>>; - async fn record_pending_tx( - &self, - tx: L1Tx, - fragment_id: NonEmpty>, - created_at: DateTime, - ) -> Result<()>; - async fn oldest_nonfinalized_fragments( - &self, - starting_height: u32, - limit: usize, - ) -> Result>; - async fn latest_bundled_height(&self) -> Result>; - async fn fragments_submitted_by_tx(&self, tx_hash: [u8; 32]) - -> Result>; - async fn get_latest_pending_txs(&self) -> Result>; - } - - pub trait Clock { - fn now(&self) -> DateTime; - fn elapsed(&self, since: DateTime) -> Result { - self.now() - .signed_duration_since(since) - .to_std() - .map_err(|e| Error::Other(format!("failed to convert time: {}", e))) - } - } -} +pub mod port; +pub mod service; diff --git a/packages/services/src/state_committer/port.rs b/packages/services/src/state_committer/port.rs new file mode 100644 index 00000000..0e77e0df --- /dev/null +++ b/packages/services/src/state_committer/port.rs @@ -0,0 +1,77 @@ +use nonempty::NonEmpty; + +use crate::{ + types::{storage::BundleFragment, DateTime, L1Tx, NonNegative, Utc}, + Error, Result, +}; + +pub mod l1 { + use nonempty::NonEmpty; + + use crate::{ + types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, + Result, + }; + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait Contract: Send + Sync { + async fn submit(&self, hash: [u8; 32], height: u32) -> Result; + } + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait Api { + async fn current_height(&self) -> Result; + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<(L1Tx, FragmentsSubmitted)>; + } +} + +pub mod fuel { + pub use fuel_core_client::client::types::block::Block as FuelBlock; + + use crate::Result; + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait Api: Send + Sync { + async fn latest_height(&self) -> Result; + } +} + +#[allow(async_fn_in_trait)] +#[trait_variant::make(Send)] +pub trait Storage: Send + Sync { + async fn has_nonfinalized_txs(&self) -> Result; + async fn last_time_a_fragment_was_finalized(&self) -> Result>>; + async fn record_pending_tx( + &self, + tx: L1Tx, + fragment_id: NonEmpty>, + created_at: DateTime, + ) -> Result<()>; + async fn oldest_nonfinalized_fragments( + &self, + starting_height: u32, + limit: usize, + ) -> Result>; + async fn latest_bundled_height(&self) -> Result>; + async fn fragments_submitted_by_tx(&self, tx_hash: [u8; 32]) -> Result>; + async fn get_latest_pending_txs(&self) -> Result>; +} + +pub trait Clock { + fn now(&self) -> DateTime; + fn elapsed(&self, since: DateTime) -> Result { + self.now() + .signed_duration_since(since) + .to_std() + .map_err(|e| Error::Other(format!("failed to convert time: {}", e))) + } +} diff --git a/packages/services/src/state_committer/service.rs b/packages/services/src/state_committer/service.rs new file mode 100644 index 00000000..83a46fc8 --- /dev/null +++ b/packages/services/src/state_committer/service.rs @@ -0,0 +1,1065 @@ +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + ops::RangeInclusive, + time::Duration, +}; + +use itertools::Itertools; +use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, +}; +use tracing::info; + +use crate::{ + historical_fees::{ + self, + service::{HistoricalFees, SmaPeriods}, + }, + types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, + Error, Result, Runner, +}; + +// src/config.rs +#[derive(Debug, Clone)] +pub struct Config { + /// The lookback window in blocks to determine the starting height. + pub lookback_window: u32, + pub fragment_accumulation_timeout: Duration, + pub fragments_to_accumulate: NonZeroUsize, + pub gas_bump_timeout: Duration, + pub fee_algo: AlgoConfig, +} + +#[cfg(feature = "test-helpers")] +impl Default for Config { + fn default() -> Self { + Self { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(0), + fragments_to_accumulate: 1.try_into().unwrap(), + gas_bump_timeout: Duration::from_secs(300), + fee_algo: Default::default(), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct AlgoConfig { + pub sma_periods: SmaPeriods, + pub fee_thresholds: FeeThresholds, +} + +#[cfg(feature = "test-helpers")] +impl Default for AlgoConfig { + fn default() -> Self { + Self { + sma_periods: SmaPeriods { + short: 1.try_into().expect("not zero"), + long: 2.try_into().expect("not zero"), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + ..FeeThresholds::default() + }, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU32, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, + pub always_acceptable_fee: u128, +} + +#[cfg(feature = "test-helpers")] +impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } +} + +#[derive(Default, Copy, Clone, Debug, PartialEq)] +pub struct Percentage(f64); + +impl TryFrom for Percentage { + type Error = Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } +} + +impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } +} + +impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } +} + +pub struct SmaFeeAlgo

{ + historical_fees: HistoricalFees

, + config: AlgoConfig, +} + +impl

SmaFeeAlgo

{ + pub fn new(historical_fees: HistoricalFees

, config: AlgoConfig) -> Self { + Self { + historical_fees, + config, + } + } + + fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { + num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + + fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + } + + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + ) -> u128 { + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(num_l2_blocks_behind); + + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", + blocks_behind, + max_blocks_behind + ); + + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = Self::calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); + + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + Percentage::PPM + end_premium_ppm, + ); + + info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) + } + + fn calculate_premium_increment( + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); + + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(Percentage::PPM) + .saturating_div(max_blocks_behind) + }; + + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) + } +} + +impl

SmaFeeAlgo

+where + P: historical_fees::port::l1::Api + Send + Sync, +{ + async fn should_send_blob_tx( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> Result { + if self.too_far_behind(num_l2_blocks_behind) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); + return Ok(true); + } + + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs + let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); + + let short_term_sma = self + .historical_fees + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) + .await?; + + let long_term_sma = self + .historical_fees + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) + .await?; + + let short_term_tx_fee = + historical_fees::service::calculate_blob_tx_fee(num_blobs, &short_term_sma); + + if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); + return Ok(true); + } + + let long_term_tx_fee = + historical_fees::service::calculate_blob_tx_fee(num_blobs, &long_term_sma); + let max_upper_tx_fee = Self::calculate_max_upper_fee( + &self.config.fee_thresholds, + long_term_tx_fee, + num_l2_blocks_behind, + ); + + info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); + + let should_send = short_term_tx_fee < max_upper_tx_fee; + + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } else { + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } + + Ok(should_send) + } +} + +struct Metrics { + current_height_to_commit: IntGauge, +} + +impl Default for Metrics { + fn default() -> Self { + let current_height_to_commit = IntGauge::with_opts(Opts::new( + "current_height_to_commit", + "The starting l2 height of the bundle we're committing/will commit next", + )) + .expect("metric config to be correct"); + + Self { + current_height_to_commit, + } + } +} + +impl RegistersMetrics for StateCommitter { + fn metrics(&self) -> Vec> { + vec![Box::new(self.metrics.current_height_to_commit.clone())] + } +} + +/// The `StateCommitter` is responsible for committing state fragments to L1. +pub struct StateCommitter { + l1_adapter: L1, + fuel_api: FuelApi, + storage: Db, + config: Config, + clock: Clock, + startup_time: DateTime, + metrics: Metrics, + fee_algo: SmaFeeAlgo, +} + +impl StateCommitter +where + Clock: crate::state_committer::port::Clock, +{ + /// Creates a new `StateCommitter`. + pub fn new( + l1_adapter: L1, + fuel_api: FuelApi, + storage: Db, + config: Config, + clock: Clock, + historical_fees: HistoricalFees, + ) -> Self { + let startup_time = clock.now(); + + Self { + fee_algo: SmaFeeAlgo::new(historical_fees, config.fee_algo), + l1_adapter, + fuel_api, + storage, + config, + clock, + startup_time, + metrics: Default::default(), + } + } +} + +impl StateCommitter +where + L1: crate::state_committer::port::l1::Api + Send + Sync, + FuelApi: crate::state_committer::port::fuel::Api, + Db: crate::state_committer::port::Storage, + Clock: crate::state_committer::port::Clock, + FeeProvider: historical_fees::port::l1::Api + Sync, +{ + async fn get_reference_time(&self) -> Result> { + Ok(self + .storage + .last_time_a_fragment_was_finalized() + .await? + .unwrap_or(self.startup_time)) + } + + async fn is_timeout_expired(&self) -> Result { + let reference_time = self.get_reference_time().await?; + let elapsed = self.clock.now() - reference_time; + let std_elapsed = elapsed + .to_std() + .map_err(|e| crate::Error::Other(format!("Failed to convert time: {}", e)))?; + Ok(std_elapsed >= self.config.fragment_accumulation_timeout) + } + + async fn fees_acceptable(&self, fragments: &NonEmpty) -> Result { + let l1_height = self.l1_adapter.current_height().await?; + let l2_height = self.fuel_api.latest_height().await?; + + let oldest_l2_block = self.oldest_l2_block_in_fragments(fragments); + self.update_oldest_block_metric(oldest_l2_block); + + let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block); + + self.fee_algo + .should_send_blob_tx( + u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), + num_l2_blocks_behind, + l1_height, + ) + .await + } + + fn oldest_l2_block_in_fragments(&self, fragments: &NonEmpty) -> u32 { + fragments + .minimum_by_key(|b| b.oldest_block_in_bundle) + .oldest_block_in_bundle + } + + async fn submit_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<()> { + info!("about to send at most {} fragments", fragments.len()); + + let data = fragments.clone().map(|f| f.fragment); + + match self + .l1_adapter + .submit_state_fragments(data, previous_tx) + .await + { + Ok((submitted_tx, submitted_fragments)) => { + let fragment_ids = fragments + .iter() + .map(|f| f.id) + .take(submitted_fragments.num_fragments.get()) + .collect_nonempty() + .expect("non-empty vec"); + + let ids = fragment_ids + .iter() + .map(|id| id.as_u32().to_string()) + .join(", "); + + let tx_hash = submitted_tx.hash; + self.storage + .record_pending_tx(submitted_tx, fragment_ids, self.clock.now()) + .await?; + + tracing::info!("Submitted fragments {ids} with tx {}", hex::encode(tx_hash)); + Ok(()) + } + Err(e) => { + let ids = fragments + .iter() + .map(|f| f.id.as_u32().to_string()) + .join(", "); + + tracing::error!("Failed to submit fragments {ids}: {e}"); + + Err(e) + } + } + } + + async fn latest_pending_transaction(&self) -> Result> { + let tx = self.storage.get_latest_pending_txs().await?; + Ok(tx) + } + + async fn next_fragments_to_submit(&self) -> Result>> { + let latest_height = self.fuel_api.latest_height().await?; + let starting_height = latest_height.saturating_sub(self.config.lookback_window); + + // although we shouldn't know at this layer how many fragments the L1 can accept, we ignore + // this for now and put the eth value of max blobs per block (6). + let existing_fragments = self + .storage + .oldest_nonfinalized_fragments(starting_height, 6) + .await?; + + let fragments = NonEmpty::collect(existing_fragments); + + if let Some(fragments) = fragments.as_ref() { + // Tracking the metric here as well to get updates more often -- because + // submit_fragments might not be called + self.update_oldest_block_metric(self.oldest_l2_block_in_fragments(fragments)); + } + + Ok(fragments) + } + + fn update_oldest_block_metric(&self, oldest_height: u32) { + self.metrics + .current_height_to_commit + .set(oldest_height as i64); + } + + async fn should_submit_fragments(&self, fragments: &NonEmpty) -> Result { + let fragment_count = fragments.len_nonzero(); + + let expired = || async { + let expired = self.is_timeout_expired().await?; + if expired { + info!( + "fragment accumulation timeout expired, available {}/{} fragments", + fragment_count, self.config.fragments_to_accumulate + ); + } + Result::Ok(expired) + }; + + let enough_fragments = || { + let enough_fragments = fragment_count >= self.config.fragments_to_accumulate; + if !enough_fragments { + info!( + "not enough fragments {}/{}", + fragment_count, self.config.fragments_to_accumulate + ); + }; + enough_fragments + }; + + // wrapped in closures so that we short-circuit *and* reduce redundant logs + Ok((enough_fragments() || expired().await?) && self.fees_acceptable(fragments).await?) + } + + async fn submit_fragments_if_ready(&self) -> Result<()> { + if let Some(fragments) = self.next_fragments_to_submit().await? { + if self.should_submit_fragments(&fragments).await? { + self.submit_fragments(fragments, None).await?; + } + } else { + // TODO: segfault test this + // if we have no fragments to submit, that means that we're up to date and new + // blocks haven't been bundled yet + let current_height_to_commit = + if let Some(height) = self.storage.latest_bundled_height().await? { + height.saturating_add(1) + } else { + self.fuel_api + .latest_height() + .await? + .saturating_sub(self.config.lookback_window) + }; + + self.metrics + .current_height_to_commit + .set(current_height_to_commit as i64); + } + + Ok(()) + } + + fn elapsed_since_tx_submitted(&self, tx: &L1Tx) -> Result { + let created_at = tx.created_at.expect("tx to have timestamp"); + + self.clock.elapsed(created_at) + } + + async fn fragments_submitted_by_tx( + &self, + tx_hash: [u8; 32], + ) -> Result> { + let fragments = self.storage.fragments_submitted_by_tx(tx_hash).await?; + + match NonEmpty::collect(fragments) { + Some(fragments) => Ok(fragments), + None => Err(crate::Error::Other(format!( + "no fragments found for previously submitted tx {}", + hex::encode(tx_hash) + ))), + } + } + + async fn resubmit_fragments_if_stalled(&self) -> Result<()> { + let Some(previous_tx) = self.latest_pending_transaction().await? else { + return Ok(()); + }; + + let elapsed = self.elapsed_since_tx_submitted(&previous_tx)?; + + if elapsed >= self.config.gas_bump_timeout { + info!( + "replacing tx {} because it was pending for {}s", + hex::encode(previous_tx.hash), + elapsed.as_secs() + ); + + let fragments = self.fragments_submitted_by_tx(previous_tx.hash).await?; + if self.fees_acceptable(&fragments).await? { + self.submit_fragments(fragments, Some(previous_tx)).await?; + } + } + + Ok(()) + } +} + +impl Runner + for StateCommitter +where + L1: crate::state_committer::port::l1::Api + Send + Sync, + FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, + Db: crate::state_committer::port::Storage + Clone + Send + Sync, + Clock: crate::state_committer::port::Clock + Send + Sync, + FeeProvider: historical_fees::port::l1::Api + Send + Sync, +{ + async fn run(&mut self) -> Result<()> { + if self.storage.has_nonfinalized_txs().await? { + self.resubmit_fragments_if_stalled().await? + } else { + self.submit_fragments_if_ready().await? + }; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::historical_fees::port::l1::testing::PreconfiguredFeeApi; + use crate::historical_fees::port::l1::{Api, Fees}; + use test_case::test_case; + + use super::*; + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + 0, + 1000; + "No blocks behind, multiplier should be 100%" +)] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 2000, + 50, + 2050; + "Half blocks behind with discount and premium" +)] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 800, + 50, + 700; + "Start discount only, no premium" +)] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + end_premium_percentage: 0.30.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + 50, + 1150; + "End premium only, no discount" +)] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 10_000, + 99, + 11970; + "High fee with premium" +)] + #[test_case( +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, +}, +1000, +1, +12; +"Discount exceeds 100%, should be capped to 100%" +)] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + expected_max_upper_fee: u128, + ) { + use crate::historical_fees::port::l1::testing::ConstantFeeApi; + + let max_upper_fee = SmaFeeAlgo::::calculate_max_upper_fee( + &fee_thresholds, + fee, + num_l2_blocks_behind, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } + + fn generate_fees(sma_periods: SmaPeriods, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (sma_periods.long.get() - sma_periods.short.get()) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, sma_periods.short.get() as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() + } + + #[test_case( + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, // not behind at all + true; + "Should send because all short-term fees are lower than long-term" +)] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, + false; + "Should not send because all short-term fees are higher than long-term" +)] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, + max_l2_blocks_behind: 100.try_into().unwrap(), + ..Default::default() + } + }, + 0, + true; + "Should send since short-term fee less than always_acceptable_fee" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_gas is lower" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_gas is higher" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_blob_gas is lower" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_blob_gas is higher" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term reward is lower" +)] + #[test_case( + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, + 5, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term reward is higher" +)] + #[test_case( + // Multiple short-term fees are lower + Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because multiple short-term fees are lower" +)] + #[test_case( + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 6, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because all fees are identical and no tolerance" +)] + #[test_case( + // Zero blobs scenario: blob fee differences don't matter + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" +)] + #[test_case( + // Zero blobs but short-term reward is higher + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Zero blobs: short-term reward is higher, don't send" +)] + #[test_case( + Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, + 0, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" +)] + // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. + #[test_case( + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 +Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, +Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), + always_acceptable_fee: 0, + }, + }, + 0, + false; + "Early: short-term expensive, not send" +)] + #[test_case( + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + } + }, + 100, + true; + "Later: after max wait, send regardless" +)] + #[test_case( + Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, + Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + }, + 80, + true; + "Mid-wait: increased tolerance allows acceptance" +)] + #[test_case( + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, + Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, + 1, + AlgoConfig { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 2_700_000_000_000 + }, + }, + 0, + true; + "Always acceptable fee triggers immediate send" +)] + #[tokio::test] + async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + num_blobs: u32, + config: AlgoConfig, + num_l2_blocks_behind: u32, + expected_decision: bool, + ) { + let fees = generate_fees(config.sma_periods, old_fees, new_fees); + let api = PreconfiguredFeeApi::new(fees); + let current_block_height = api.current_height().await.unwrap(); + let historical_fees = HistoricalFees::new(api); + + let sut = SmaFeeAlgo::new(historical_fees, config); + + let should_send = sut + .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) + .await + .unwrap(); + + assert_eq!( + should_send, expected_decision, + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", + ); + } + + #[tokio::test] + async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = AlgoConfig { + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let historical_fees = HistoricalFees::new(PreconfiguredFeeApi::new(vec![])); + let sut = SmaFeeAlgo::new(historical_fees, config); + + // when + let should_send = sut + .should_send_blob_tx(1, 20, 100) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); + } +} From 20787e999d12fae8936db8978b946c8edc26dbc3 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 13:41:35 +0100 Subject: [PATCH 060/136] fee algo separated into module --- committer/src/setup.rs | 4 +- packages/services/src/lib.rs | 2 +- packages/services/src/state_committer.rs | 2 + .../services/src/state_committer/fee_algo.rs | 687 +++++++++++++++++ .../services/src/state_committer/service.rs | 689 +----------------- packages/services/tests/fee_tracker.rs | 1 - packages/services/tests/state_committer.rs | 6 +- 7 files changed, 705 insertions(+), 686 deletions(-) create mode 100644 packages/services/src/state_committer/fee_algo.rs diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 17ee2b84..41637945 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -13,7 +13,7 @@ use services::{ port::cache::CachingApi, service::{HistoricalFees, SmaPeriods}, }, - state_committer::{port::Storage, service::FeeThresholds}, + state_committer::{port::Storage, FeeThresholds}, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -124,7 +124,7 @@ pub fn state_committer( registry: &Registry, historical_fees: HistoricalFees>, ) -> Result> { - let algo_config = services::state_committer::service::AlgoConfig { + let algo_config = services::state_committer::AlgoConfig { sma_periods: SmaPeriods { short: config.app.fee_algo.short_sma_blocks, long: config.app.fee_algo.long_sma_blocks, diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index bd4668c6..8d91f439 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,8 +2,8 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; -pub mod historical_fees; pub mod health_reporter; +pub mod historical_fees; pub mod state_committer; pub mod state_listener; pub mod state_pruner; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 7691d7e1..73c924ed 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,2 +1,4 @@ +mod fee_algo; +pub use fee_algo::{Config as AlgoConfig, FeeThresholds, Percentage}; pub mod port; pub mod service; diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs new file mode 100644 index 00000000..381cabb3 --- /dev/null +++ b/packages/services/src/state_committer/fee_algo.rs @@ -0,0 +1,687 @@ +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64}, + ops::RangeInclusive, +}; + +use tracing::info; + +use crate::{ + historical_fees::{ + self, + service::{HistoricalFees, SmaPeriods}, + }, + Error, Result, +}; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub sma_periods: SmaPeriods, + pub fee_thresholds: FeeThresholds, +} + +#[cfg(feature = "test-helpers")] +impl Default for Config { + fn default() -> Self { + Self { + sma_periods: SmaPeriods { + short: 1.try_into().expect("not zero"), + long: 2.try_into().expect("not zero"), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + ..FeeThresholds::default() + }, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU32, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, + pub always_acceptable_fee: u128, +} + +#[cfg(feature = "test-helpers")] +impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } +} + +#[derive(Default, Copy, Clone, Debug, PartialEq)] +pub struct Percentage(f64); + +impl TryFrom for Percentage { + type Error = Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } +} + +impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } +} + +impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } +} + +pub struct SmaFeeAlgo

{ + historical_fees: HistoricalFees

, + config: Config, +} + +impl

SmaFeeAlgo

{ + pub fn new(historical_fees: HistoricalFees

, config: Config) -> Self { + Self { + historical_fees, + config, + } + } + + fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { + num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } +} + +fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block +} + +fn calculate_premium_increment( + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, +) -> u128 { + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); + + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(Percentage::PPM) + .saturating_div(max_blocks_behind) + }; + + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) +} +fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, +) -> u128 { + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(num_l2_blocks_behind); + + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", + blocks_behind, + max_blocks_behind +); + + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); + + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + Percentage::PPM + end_premium_ppm, + ); + + info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) +} + +impl

SmaFeeAlgo

+where + P: historical_fees::port::l1::Api + Send + Sync, +{ + pub async fn fees_acceptable( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> Result { + if self.too_far_behind(num_l2_blocks_behind) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); + return Ok(true); + } + + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs + let last_n_blocks = |n| last_n_blocks(at_l1_height, n); + + let short_term_sma = self + .historical_fees + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) + .await?; + + let long_term_sma = self + .historical_fees + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) + .await?; + + let short_term_tx_fee = + historical_fees::service::calculate_blob_tx_fee(num_blobs, &short_term_sma); + + if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); + return Ok(true); + } + + let long_term_tx_fee = + historical_fees::service::calculate_blob_tx_fee(num_blobs, &long_term_sma); + let max_upper_tx_fee = calculate_max_upper_fee( + &self.config.fee_thresholds, + long_term_tx_fee, + num_l2_blocks_behind, + ); + + info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); + + let should_send = short_term_tx_fee < max_upper_tx_fee; + + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } else { + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } + + Ok(should_send) + } +} +#[cfg(test)] +mod test { + pub use test_case::test_case; + + use super::Config; + use crate::{ + historical_fees::{ + port::l1::{testing::PreconfiguredFeeApi, Api, Fees}, + service::{HistoricalFees, SmaPeriods}, + }, + state_committer::{ + fee_algo::{calculate_max_upper_fee, SmaFeeAlgo}, + FeeThresholds, Percentage, + }, + }; + + #[test_case( +// Test Case 1: No blocks behind, no discount or premium +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() +}, +1000, +0, +1000; +"No blocks behind, multiplier should be 100%" +)] + #[test_case( +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, +}, +2000, +50, +2050; +"Half blocks behind with discount and premium" +)] + #[test_case( +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() +}, +800, +50, +700; +"Start discount only, no premium" +)] + #[test_case( +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + end_premium_percentage: 0.30.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() +}, +1000, +50, +1150; +"End premium only, no discount" +)] + #[test_case( +// Test Case 8: High fee with premium +FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, +}, +10_000, +99, +11970; +"High fee with premium" +)] + #[test_case( +FeeThresholds { +max_l2_blocks_behind: 100.try_into().unwrap(), +start_discount_percentage: 1.50.try_into().unwrap(), // 150% +end_premium_percentage: 0.20.try_into().unwrap(), +always_acceptable_fee: 0, +}, +1000, +1, +12; +"Discount exceeds 100%, should be capped to 100%" +)] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + expected_max_upper_fee: u128, + ) { + let max_upper_fee = calculate_max_upper_fee(&fee_thresholds, fee, num_l2_blocks_behind); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } + + fn generate_fees(sma_periods: SmaPeriods, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (sma_periods.long.get() - sma_periods.short.get()) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, sma_periods.short.get() as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() + } + + #[test_case( +Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, +6, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, +}, +0, // not behind at all +true; +"Should send because all short-term fees are lower than long-term" +)] + #[test_case( +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, +Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +6, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, +}, +0, +false; +"Should not send because all short-term fees are higher than long-term" +)] + #[test_case( +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, +Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, +6, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, + max_l2_blocks_behind: 100.try_into().unwrap(), + ..Default::default() + } +}, +0, +true; +"Should send since short-term fee less than always_acceptable_fee" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Should send because short-term base_fee_per_gas is lower" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +false; +"Should not send because short-term base_fee_per_gas is higher" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Should send because short-term base_fee_per_blob_gas is lower" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +false; +"Should not send because short-term base_fee_per_blob_gas is higher" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Should send because short-term reward is lower" +)] + #[test_case( +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +5, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +false; +"Should not send because short-term reward is higher" +)] + #[test_case( +// Multiple short-term fees are lower +Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, +6, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Should send because multiple short-term fees are lower" +)] + #[test_case( +Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +6, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +false; +"Should not send because all fees are identical and no tolerance" +)] + #[test_case( +// Zero blobs scenario: blob fee differences don't matter +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +0, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Zero blobs: short-term base_fee_per_gas and reward are lower, send" +)] + #[test_case( +// Zero blobs but short-term reward is higher +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +0, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +false; +"Zero blobs: short-term reward is higher, don't send" +)] + #[test_case( +Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, +0, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } +}, +0, +true; +"Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" +)] + // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. + #[test_case( +// Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 +Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, +Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +1, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), + always_acceptable_fee: 0, + }, +}, +0, +false; +"Early: short-term expensive, not send" +)] + #[test_case( +// At max_l2_blocks_behind, send regardless +Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, +Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +1, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + } +}, +100, +true; +"Later: after max wait, send regardless" +)] + #[test_case( +Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, +Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +1, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, +}, +80, +true; +"Mid-wait: increased tolerance allows acceptance" +)] + #[test_case( +// Short-term fee is huge, but always_acceptable_fee is large, so send immediately +Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, +1, +Config { + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 2_700_000_000_000 + }, +}, +0, +true; +"Always acceptable fee triggers immediate send" +)] + #[tokio::test] + async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + num_blobs: u32, + config: Config, + num_l2_blocks_behind: u32, + expected_decision: bool, + ) { + let fees = generate_fees(config.sma_periods, old_fees, new_fees); + let api = PreconfiguredFeeApi::new(fees); + let current_block_height = api.current_height().await.unwrap(); + let historical_fees = HistoricalFees::new(api); + + let sut = SmaFeeAlgo::new(historical_fees, config); + + let should_send = sut + .fees_acceptable(num_blobs, num_l2_blocks_behind, current_block_height) + .await + .unwrap(); + + assert_eq!( + should_send, expected_decision, + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", +); + } +} diff --git a/packages/services/src/state_committer/service.rs b/packages/services/src/state_committer/service.rs index 83a46fc8..65315380 100644 --- a/packages/services/src/state_committer/service.rs +++ b/packages/services/src/state_committer/service.rs @@ -1,9 +1,4 @@ -use std::{ - cmp::min, - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, - ops::RangeInclusive, - time::Duration, -}; +use std::{num::NonZeroUsize, time::Duration}; use itertools::Itertools; use metrics::{ @@ -12,13 +7,11 @@ use metrics::{ }; use tracing::info; +use super::{fee_algo::SmaFeeAlgo, AlgoConfig}; use crate::{ - historical_fees::{ - self, - service::{HistoricalFees, SmaPeriods}, - }, + historical_fees::{self, service::HistoricalFees}, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, + Result, Runner, }; // src/config.rs @@ -45,235 +38,6 @@ impl Default for Config { } } -#[derive(Debug, Clone, Copy)] -pub struct AlgoConfig { - pub sma_periods: SmaPeriods, - pub fee_thresholds: FeeThresholds, -} - -#[cfg(feature = "test-helpers")] -impl Default for AlgoConfig { - fn default() -> Self { - Self { - sma_periods: SmaPeriods { - short: 1.try_into().expect("not zero"), - long: 2.try_into().expect("not zero"), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - ..FeeThresholds::default() - }, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: Percentage, - pub end_premium_percentage: Percentage, - pub always_acceptable_fee: u128, -} - -#[cfg(feature = "test-helpers")] -impl Default for FeeThresholds { - fn default() -> Self { - Self { - max_l2_blocks_behind: NonZeroU32::MAX, - start_discount_percentage: Percentage::ZERO, - end_premium_percentage: Percentage::ZERO, - always_acceptable_fee: u128::MAX, - } - } -} - -#[derive(Default, Copy, Clone, Debug, PartialEq)] -pub struct Percentage(f64); - -impl TryFrom for Percentage { - type Error = Error; - - fn try_from(value: f64) -> std::result::Result { - if value < 0. { - return Err(Error::Other(format!("Invalid percentage value {value}"))); - } - - Ok(Self(value)) - } -} - -impl From for f64 { - fn from(value: Percentage) -> Self { - value.0 - } -} - -impl Percentage { - pub const ZERO: Self = Percentage(0.); - pub const PPM: u128 = 1_000_000; - - pub fn ppm(&self) -> u128 { - (self.0 * 1_000_000.) as u128 - } -} - -pub struct SmaFeeAlgo

{ - historical_fees: HistoricalFees

, - config: AlgoConfig, -} - -impl

SmaFeeAlgo

{ - pub fn new(historical_fees: HistoricalFees

, config: AlgoConfig) -> Self { - Self { - historical_fees, - config, - } - } - - fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { - num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() - } - - fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee - } - - fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { - current_block.saturating_sub(n.get().saturating_sub(1))..=current_block - } - - fn calculate_max_upper_fee( - fee_thresholds: &FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - ) -> u128 { - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); - let blocks_behind = u128::from(num_l2_blocks_behind); - - debug_assert!( - blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", - blocks_behind, - max_blocks_behind - ); - - let start_discount_ppm = min( - fee_thresholds.start_discount_percentage.ppm(), - Percentage::PPM, - ); - let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); - - // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); - - // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = Self::calculate_premium_increment( - start_discount_ppm, - end_premium_ppm, - blocks_behind, - max_blocks_behind, - ); - - // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% - let multiplier_ppm = min( - base_multiplier.saturating_add(premium_increment), - Percentage::PPM + end_premium_ppm, - ); - - info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); - - // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm) - .saturating_div(Percentage::PPM) - } - - fn calculate_premium_increment( - start_discount_ppm: u128, - end_premium_ppm: u128, - blocks_behind: u128, - max_blocks_behind: u128, - ) -> u128 { - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - - let proportion = if max_blocks_behind == 0 { - 0 - } else { - blocks_behind - .saturating_mul(Percentage::PPM) - .saturating_div(max_blocks_behind) - }; - - total_ppm - .saturating_mul(proportion) - .saturating_div(Percentage::PPM) - } -} - -impl

SmaFeeAlgo

-where - P: historical_fees::port::l1::Api + Send + Sync, -{ - async fn should_send_blob_tx( - &self, - num_blobs: u32, - num_l2_blocks_behind: u32, - at_l1_height: u64, - ) -> Result { - if self.too_far_behind(num_l2_blocks_behind) { - info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); - return Ok(true); - } - - // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller - // wants to send more than 6 blobs - let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); - - let short_term_sma = self - .historical_fees - .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await?; - - let long_term_sma = self - .historical_fees - .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await?; - - let short_term_tx_fee = - historical_fees::service::calculate_blob_tx_fee(num_blobs, &short_term_sma); - - if self.fee_always_acceptable(short_term_tx_fee) { - info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); - return Ok(true); - } - - let long_term_tx_fee = - historical_fees::service::calculate_blob_tx_fee(num_blobs, &long_term_sma); - let max_upper_tx_fee = Self::calculate_max_upper_fee( - &self.config.fee_thresholds, - long_term_tx_fee, - num_l2_blocks_behind, - ); - - info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); - - let should_send = short_term_tx_fee < max_upper_tx_fee; - - if should_send { - info!( - "Sending because short term price {} is lower than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } else { - info!( - "Not sending because short term price {} is higher than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } - - Ok(should_send) - } -} - struct Metrics { current_height_to_commit: IntGauge, } @@ -373,7 +137,7 @@ where let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block); self.fee_algo - .should_send_blob_tx( + .fees_acceptable( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), num_l2_blocks_behind, l1_height, @@ -591,445 +355,12 @@ where #[cfg(test)] mod tests { - use crate::historical_fees::port::l1::testing::PreconfiguredFeeApi; - use crate::historical_fees::port::l1::{Api, Fees}; - use test_case::test_case; + use historical_fees::service::SmaPeriods; use super::*; - - #[test_case( - // Test Case 1: No blocks behind, no discount or premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 0, - 1000; - "No blocks behind, multiplier should be 100%" -)] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 2000, - 50, - 2050; - "Half blocks behind with discount and premium" -)] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 800, - 50, - 700; - "Start discount only, no premium" -)] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - end_premium_percentage: 0.30.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - 50, - 1150; - "End premium only, no discount" -)] - #[test_case( - // Test Case 8: High fee with premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 10_000, - 99, - 11970; - "High fee with premium" -)] - #[test_case( -FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 1.50.try_into().unwrap(), // 150% - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, -}, -1000, -1, -12; -"Discount exceeds 100%, should be capped to 100%" -)] - fn test_calculate_max_upper_fee( - fee_thresholds: FeeThresholds, - fee: u128, - num_l2_blocks_behind: u32, - expected_max_upper_fee: u128, - ) { - use crate::historical_fees::port::l1::testing::ConstantFeeApi; - - let max_upper_fee = SmaFeeAlgo::::calculate_max_upper_fee( - &fee_thresholds, - fee, - num_l2_blocks_behind, - ); - - assert_eq!( - max_upper_fee, expected_max_upper_fee, - "Expected max_upper_fee to be {}, but got {}", - expected_max_upper_fee, max_upper_fee - ); - } - - fn generate_fees(sma_periods: SmaPeriods, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { - let older_fees = std::iter::repeat_n( - old_fees, - (sma_periods.long.get() - sma_periods.short.get()) as usize, - ); - let newer_fees = std::iter::repeat_n(new_fees, sma_periods.short.get() as usize); - - older_fees - .chain(newer_fees) - .enumerate() - .map(|(i, f)| (i as u64, f)) - .collect() - } - - #[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, // not behind at all - true; - "Should send because all short-term fees are lower than long-term" -)] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, - false; - "Should not send because all short-term fees are higher than long-term" -)] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, - max_l2_blocks_behind: 100.try_into().unwrap(), - ..Default::default() - } - }, - 0, - true; - "Should send since short-term fee less than always_acceptable_fee" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_gas is lower" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_gas is higher" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_blob_gas is lower" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_blob_gas is higher" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term reward is lower" -)] - #[test_case( - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, - 5, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term reward is higher" -)] - #[test_case( - // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because multiple short-term fees are lower" -)] - #[test_case( - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 6, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because all fees are identical and no tolerance" -)] - #[test_case( - // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: short-term base_fee_per_gas and reward are lower, send" -)] - #[test_case( - // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Zero blobs: short-term reward is higher, don't send" -)] - #[test_case( - Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, - 0, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" -)] - // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. - #[test_case( - // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 -Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, -Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: Percentage::try_from(0.20).unwrap(), - end_premium_percentage: Percentage::try_from(0.20).unwrap(), - always_acceptable_fee: 0, - }, - }, - 0, - false; - "Early: short-term expensive, not send" -)] - #[test_case( - // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - } - }, - 100, - true; - "Later: after max wait, send regardless" -)] - #[test_case( - Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, - Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - }, - 80, - true; - "Mid-wait: increased tolerance allows acceptance" -)] - #[test_case( - // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, - Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, - 1, - AlgoConfig { - sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 2_700_000_000_000 - }, - }, - 0, - true; - "Always acceptable fee triggers immediate send" -)] - #[tokio::test] - async fn parameterized_send_or_wait_tests( - old_fees: Fees, - new_fees: Fees, - num_blobs: u32, - config: AlgoConfig, - num_l2_blocks_behind: u32, - expected_decision: bool, - ) { - let fees = generate_fees(config.sma_periods, old_fees, new_fees); - let api = PreconfiguredFeeApi::new(fees); - let current_block_height = api.current_height().await.unwrap(); - let historical_fees = HistoricalFees::new(api); - - let sut = SmaFeeAlgo::new(historical_fees, config); - - let should_send = sut - .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) - .await - .unwrap(); - - assert_eq!( - should_send, expected_decision, - "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", - ); - } + use crate::{ + historical_fees::port::l1::testing::PreconfiguredFeeApi, state_committer::FeeThresholds, + }; #[tokio::test] async fn test_send_when_too_far_behind_and_fee_provider_fails() { @@ -1052,7 +383,7 @@ Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap() // when let should_send = sut - .should_send_blob_tx(1, 20, 100) + .fees_acceptable(1, 20, 100) .await .expect("Should send despite fee provider failure"); diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 139597f9..8b137891 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,2 +1 @@ - diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 0f642f89..1efcd748 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -2,7 +2,7 @@ use std::time::Duration; use services::{ historical_fees::{port::l1::Fees, service::SmaPeriods}, - state_committer::service::{AlgoConfig, FeeThresholds}, + state_committer::{AlgoConfig, FeeThresholds}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -394,7 +394,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_algo = AlgoConfig { + let config = AlgoConfig { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap(), @@ -432,7 +432,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo, + fee_algo: config, ..Default::default() }, setup.test_clock(), From 0639c02f357e3caedb430e303afaabd7cd769f32 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 26 Dec 2024 18:22:28 +0100 Subject: [PATCH 061/136] finish removing NonZeroU128 fees --- e2e/src/committer.rs | 2 +- e2e/src/whole_stack.rs | 2 +- packages/adapters/eth/src/fee_conversion.rs | 89 ++++--------------- packages/services/src/historical_fees/port.rs | 67 +++++--------- .../services/src/historical_fees/service.rs | 19 ++-- .../services/src/state_committer/fee_algo.rs | 72 +++++++-------- 6 files changed, 85 insertions(+), 166 deletions(-) diff --git a/e2e/src/committer.rs b/e2e/src/committer.rs index f92a117e..5fe783ff 100644 --- a/e2e/src/committer.rs +++ b/e2e/src/committer.rs @@ -73,7 +73,7 @@ impl Committer { .env("COMMITTER__APP__HOST", "127.0.0.1") .env("COMMITTER__APP__BLOCK_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__TX_FINALIZATION_CHECK_INTERVAL", "5s") - .env("COMMITTER__APP__L1_PRICES_CHECK_INTERVAL", "5s") + .env("COMMITTER__APP__L1_FEE_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__NUM_BLOCKS_TO_FINALIZE_TX", "3") .env("COMMITTER__APP__GAS_BUMP_TIMEOUT", "300s") .env("COMMITTER__APP__TX_MAX_FEE", "4000000000000000") diff --git a/e2e/src/whole_stack.rs b/e2e/src/whole_stack.rs index 59e8e779..1a3e257d 100644 --- a/e2e/src/whole_stack.rs +++ b/e2e/src/whole_stack.rs @@ -60,7 +60,7 @@ impl WholeStack { let db = start_db().await?; let committer = start_committer( - logs, + true, blob_support, db.clone(), ð_node, diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 501414de..fb6dc410 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -1,4 +1,4 @@ -use std::{num::NonZeroU128, ops::RangeInclusive}; +use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; @@ -47,7 +47,7 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { }) .try_collect()?; - izip!( + let fees = izip!( (fees.oldest_block..), fees.base_fee_per_gas.into_iter(), fees.base_fee_per_blob_gas.into_iter(), @@ -55,26 +55,18 @@ pub fn unpack_fee_history(fees: FeeHistory) -> Result> { ) .take(number_of_blocks) .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| { - let convert_to_nonzero = |value: u128| { - NonZeroU128::try_from(value).map_err(|_| { - crate::error::Error::Other( - "historical fee response returned a 0 fee, which we deem as invalid" - .to_string(), - ) - }) - }; - Ok(BlockFees { - height, - fees: Fees { - base_fee_per_gas: convert_to_nonzero(base_fee_per_gas)?, - reward: convert_to_nonzero(reward)?, - base_fee_per_blob_gas: convert_to_nonzero(base_fee_per_blob_gas)?, - }, - }) + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, }, ) - .try_collect() + .collect(); + + Ok(fees) } pub fn chunk_range_inclusive( @@ -460,17 +452,17 @@ mod test { BlockFees { height: u64::MAX - 2, fees: Fees { - base_fee_per_gas: (u128::MAX - 2).try_into().unwrap(), - reward: (u128::MAX - 4).try_into().unwrap(), - base_fee_per_blob_gas: (u128::MAX - 3).try_into().unwrap(), + base_fee_per_gas: u128::MAX - 2, + reward: u128::MAX - 4, + base_fee_per_blob_gas: u128::MAX - 3, }, }, BlockFees { height: u64::MAX - 1, fees: Fees { - base_fee_per_gas: (u128::MAX - 1).try_into().unwrap(), - reward: (u128::MAX - 3).try_into().unwrap(), - base_fee_per_blob_gas: (u128::MAX - 2).try_into().unwrap(), + base_fee_per_gas: u128::MAX - 1, + reward: u128::MAX - 3, + base_fee_per_blob_gas: u128::MAX - 2, }, }, ]; @@ -536,49 +528,4 @@ mod test { "Expected BlockFees entries matching the full range chunk" ); } - - #[test] - fn unpack_fee_history_invalid_zero_fees_or_rewards() { - // Test case where base_fee_per_gas contains a zero - let fees_with_zero_base_fee = FeeHistory { - oldest_block: 1000, - base_fee_per_gas: vec![100, 0, 200], - base_fee_per_blob_gas: vec![150, 250, 350], - reward: Some(vec![vec![10], vec![20]]), - ..Default::default() - }; - - let result = fee_conversion::unpack_fee_history(fees_with_zero_base_fee); - assert!( - result.is_err(), - "Expected error due to zero base_fee_per_gas" - ); - - // Test case where base_fee_per_blob_gas contains a zero - let fees_with_zero_blob_fee = FeeHistory { - oldest_block: 1001, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 0, 350], - reward: Some(vec![vec![10], vec![20]]), - ..Default::default() - }; - - let result = fee_conversion::unpack_fee_history(fees_with_zero_blob_fee); - assert!( - result.is_err(), - "Expected error due to zero base_fee_per_blob_gas" - ); - - // Test case where reward is zero - let fees_with_zero_reward = FeeHistory { - oldest_block: 1002, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 250, 350], - reward: Some(vec![vec![0], vec![20]]), - ..Default::default() - }; - - let result = fee_conversion::unpack_fee_history(fees_with_zero_reward); - assert!(result.is_err(), "Expected error due to zero reward"); - } } diff --git a/packages/services/src/historical_fees/port.rs b/packages/services/src/historical_fees/port.rs index 0a9006d2..7754bd5b 100644 --- a/packages/services/src/historical_fees/port.rs +++ b/packages/services/src/historical_fees/port.rs @@ -1,9 +1,9 @@ pub mod l1 { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct Fees { - pub base_fee_per_gas: NonZeroU128, - pub reward: NonZeroU128, - pub base_fee_per_blob_gas: NonZeroU128, + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, } impl Default for Fees { @@ -21,7 +21,7 @@ pub mod l1 { pub height: u64, pub fees: Fees, } - use std::{num::NonZeroU128, ops::RangeInclusive}; + use std::ops::RangeInclusive; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -31,6 +31,8 @@ pub mod l1 { fees: Vec, } + // Doesn't detect that we use the contents in the Display impl + #[allow(dead_code)] #[derive(Debug)] pub struct InvalidSequence(String); @@ -61,13 +63,11 @@ pub mod l1 { .iter() .map(|bf| bf.fees) .fold(Fees::default(), |acc, f| { - let base_fee_per_gas = acc - .base_fee_per_gas - .saturating_add(f.base_fee_per_gas.get()); - let reward = acc.reward.saturating_add(f.reward.get()); + let base_fee_per_gas = acc.base_fee_per_gas.saturating_add(f.base_fee_per_gas); + let reward = acc.reward.saturating_add(f.reward); let base_fee_per_blob_gas = acc .base_fee_per_blob_gas - .saturating_add(f.base_fee_per_blob_gas.get()); + .saturating_add(f.base_fee_per_blob_gas); Fees { base_fee_per_gas, @@ -76,19 +76,10 @@ pub mod l1 { } }); - let divide_by_count = |value: NonZeroU128| { - let minimum_fee = NonZeroU128::try_from(1).unwrap(); - value - .get() - .saturating_div(count) - .try_into() - .unwrap_or(minimum_fee) - }; - Fees { - base_fee_per_gas: divide_by_count(total.base_fee_per_gas), - reward: divide_by_count(total.reward), - base_fee_per_blob_gas: divide_by_count(total.base_fee_per_blob_gas), + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), } } pub fn len(&self) -> usize { @@ -222,7 +213,7 @@ pub mod l1 { pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { (0..num_blocks) .map(|i| { - let fee = (u128::from(i) + 1).try_into().unwrap(); + let fee = u128::from(i) + 1; ( i, Fees { @@ -394,18 +385,12 @@ pub mod l1 { // then assert_eq!( - mean.base_fee_per_gas, - 1.try_into().unwrap(), + mean.base_fee_per_gas, 1, "base_fee_per_gas should be set to 1 when total is 0" ); + assert_eq!(mean.reward, 1, "reward should be set to 1 when total is 0"); assert_eq!( - mean.reward, - 1.try_into().unwrap(), - "reward should be set to 1 when total is 0" - ); - assert_eq!( - mean.base_fee_per_blob_gas, - 1.try_into().unwrap(), + mean.base_fee_per_blob_gas, 1, "base_fee_per_blob_gas should be set to 1 when total is 0" ); } @@ -527,11 +512,7 @@ pub mod cache { .expect_fees() .with(eq(0..=4)) .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) - }) - }); + .return_once(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); let provider = CachingApi::new(mock_provider, 5); let _ = provider.get_fees(0..=4).await.unwrap(); @@ -553,22 +534,14 @@ pub mod cache { .expect_fees() .with(eq(0..=2)) .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) - }) - }) + .return_once(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })) .in_sequence(&mut sequence); mock_provider .expect_fees() .with(eq(3..=5)) .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from(generate_sequential_fees(range)).unwrap()) - }) - }) + .return_once(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })) .in_sequence(&mut sequence); let provider = CachingApi::new(mock_provider, 5); @@ -636,7 +609,7 @@ pub mod cache { SequentialBlockFees::try_from( height_range .map(|h| { - let fee = u128::from(h + 1).try_into().unwrap(); + let fee = u128::from(h + 1); BlockFees { height: h, fees: Fees { diff --git a/packages/services/src/historical_fees/service.rs b/packages/services/src/historical_fees/service.rs index 72819dfb..8b85f25e 100644 --- a/packages/services/src/historical_fees/service.rs +++ b/packages/services/src/historical_fees/service.rs @@ -69,13 +69,12 @@ pub fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; - let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas.get()); + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas); let blob_fee = fees .base_fee_per_blob_gas - .get() .saturating_mul(u128::from(num_blobs)) .saturating_mul(DATA_GAS_PER_BLOB); - let reward_fee = fees.reward.get().saturating_mul(INTRINSIC_GAS); + let reward_fee = fees.reward.saturating_mul(INTRINSIC_GAS); base_fee.saturating_add(blob_fee).saturating_add(reward_fee) } @@ -205,9 +204,9 @@ mod tests { let sma = sut.calculate_sma(4..=4).await.unwrap(); // then - assert_eq!(sma.base_fee_per_gas, 6.try_into().unwrap()); - assert_eq!(sma.reward, 6.try_into().unwrap()); - assert_eq!(sma.base_fee_per_blob_gas, 6.try_into().unwrap()); + assert_eq!(sma.base_fee_per_gas, 6); + assert_eq!(sma.reward, 6); + assert_eq!(sma.base_fee_per_blob_gas, 6); } #[tokio::test] @@ -220,7 +219,7 @@ mod tests { let sma = sut.calculate_sma(0..=4).await.unwrap(); // then - let mean = ((5 + 4 + 3 + 2 + 1) / 5).try_into().unwrap(); + let mean = (5 + 4 + 3 + 2 + 1) / 5; assert_eq!(sma.base_fee_per_gas, mean); assert_eq!(sma.reward, mean); assert_eq!(sma.base_fee_per_blob_gas, mean); @@ -262,9 +261,9 @@ mod tests { let expected_fee = BlockFees { height, fees: Fees { - base_fee_per_gas: 5.try_into().unwrap(), - reward: 5.try_into().unwrap(), - base_fee_per_blob_gas: 5.try_into().unwrap(), + base_fee_per_gas: 5, + reward: 5, + base_fee_per_blob_gas: 5, }, }; assert_eq!( diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 381cabb3..138db4bf 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -360,8 +360,8 @@ always_acceptable_fee: 0, } #[test_case( -Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, +Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000}, +Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -376,8 +376,8 @@ true; "Should send because all short-term fees are lower than long-term" )] #[test_case( -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, -Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000}, +Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -392,8 +392,8 @@ false; "Should not send because all short-term fees are higher than long-term" )] #[test_case( -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 3000.try_into().unwrap()}, -Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap()}, +Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000}, +Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -408,8 +408,8 @@ true; "Should send since short-term fee less than always_acceptable_fee" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, -Fees { base_fee_per_gas: 1500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -424,8 +424,8 @@ true; "Should send because short-term base_fee_per_gas is lower" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -440,8 +440,8 @@ false; "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 900.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -456,8 +456,8 @@ true; "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 3000.try_into().unwrap(), base_fee_per_blob_gas: 1100.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -472,8 +472,8 @@ false; "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 9000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap()}, +Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -488,8 +488,8 @@ true; "Should send because short-term reward is lower" )] #[test_case( -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 10000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 11000.try_into().unwrap(), base_fee_per_blob_gas: 1000.try_into().unwrap() }, +Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000}, +Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000}, 5, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -505,8 +505,8 @@ false; )] #[test_case( // Multiple short-term fees are lower -Fees { base_fee_per_gas: 4000.try_into().unwrap(), reward: 8000.try_into().unwrap(), base_fee_per_blob_gas: 4000.try_into().unwrap() }, -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 3500.try_into().unwrap() }, +Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000}, +Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -521,8 +521,8 @@ true; "Should send because multiple short-term fees are lower" )] #[test_case( -Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, -Fees { base_fee_per_gas: 5000.try_into().unwrap(), reward: 5000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000}, +Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000}, 6, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -538,8 +538,8 @@ false; )] #[test_case( // Zero blobs scenario: blob fee differences don't matter -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2500.try_into().unwrap(), reward: 5500.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000}, +Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000}, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -555,8 +555,8 @@ true; )] #[test_case( // Zero blobs but short-term reward is higher -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 7000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000}, +Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000}, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -571,8 +571,8 @@ false; "Zero blobs: short-term reward is higher, don't send" )] #[test_case( -Fees { base_fee_per_gas: 3000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 5000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2000.try_into().unwrap(), reward: 6000.try_into().unwrap(), base_fee_per_blob_gas: 50_000_000.try_into().unwrap() }, +Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000}, +Fees { base_fee_per_gas: 2000, reward: 6000, base_fee_per_blob_gas: 50_000_000}, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -589,8 +589,8 @@ true; // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. #[test_case( // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 -Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, -Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +Fees { base_fee_per_gas: 6000, reward: 1, base_fee_per_blob_gas: 6000}, +Fees { base_fee_per_gas: 7000, reward: 1, base_fee_per_blob_gas: 7000}, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -607,8 +607,8 @@ false; )] #[test_case( // At max_l2_blocks_behind, send regardless -Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, -Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +Fees { base_fee_per_gas: 6000, reward: 1, base_fee_per_blob_gas: 6000}, +Fees { base_fee_per_gas: 7000, reward: 1, base_fee_per_blob_gas: 7000}, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, @@ -624,8 +624,8 @@ true; "Later: after max wait, send regardless" )] #[test_case( -Fees { base_fee_per_gas: 6000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 6000.try_into().unwrap() }, -Fees { base_fee_per_gas: 7000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 7000.try_into().unwrap() }, +Fees { base_fee_per_gas: 6000, reward: 1, base_fee_per_blob_gas: 6000}, +Fees { base_fee_per_gas: 7000, reward: 1, base_fee_per_blob_gas: 7000}, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, @@ -642,8 +642,8 @@ true; )] #[test_case( // Short-term fee is huge, but always_acceptable_fee is large, so send immediately -Fees { base_fee_per_gas: 100_000.try_into().unwrap(), reward: 1.try_into().unwrap(), base_fee_per_blob_gas: 100_000.try_into().unwrap() }, -Fees { base_fee_per_gas: 2_000_000.try_into().unwrap(), reward: 1_000_000.try_into().unwrap(), base_fee_per_blob_gas: 20_000_000.try_into().unwrap() }, +Fees { base_fee_per_gas: 100_000, reward: 1, base_fee_per_blob_gas: 100_000}, +Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000}, 1, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, From 5ce370dae68e47548062ee9866b1742a1e9deaf8 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 30 Dec 2024 13:14:22 +0100 Subject: [PATCH 062/136] simulation progressing --- Cargo.lock | 315 +++++++++++++++++- Cargo.toml | 5 +- fee_algo_simulation/Cargo.toml | 28 ++ fee_algo_simulation/src/main.rs | 200 +++++++++++ packages/adapters/eth/src/blob_encoder.rs | 137 ++++++++ .../{fee_conversion.rs => fee_api_helpers.rs} | 76 +++-- packages/adapters/eth/src/http.rs | 57 ++++ packages/adapters/eth/src/lib.rs | 258 +------------- packages/adapters/eth/src/websocket.rs | 89 ++++- .../adapters/eth/src/websocket/connection.rs | 14 +- packages/services/src/historical_fees/port.rs | 8 + .../services/src/state_committer/fee_algo.rs | 1 + 12 files changed, 907 insertions(+), 281 deletions(-) create mode 100644 fee_algo_simulation/Cargo.toml create mode 100644 fee_algo_simulation/src/main.rs create mode 100644 packages/adapters/eth/src/blob_encoder.rs rename packages/adapters/eth/src/{fee_conversion.rs => fee_api_helpers.rs} (85%) create mode 100644 packages/adapters/eth/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index e117ebf0..f5f570ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1667,6 +1667,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" +[[package]] +name = "bytemuck" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" + [[package]] name = "byteorder" version = "1.5.0" @@ -1751,8 +1757,10 @@ checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1914,6 +1922,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.3" @@ -2036,6 +2050,42 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types", + "foreign-types 0.5.0", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + +[[package]] +name = "core-text" +version = "20.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" +dependencies = [ + "core-foundation 0.9.4", + "core-graphics", + "foreign-types 0.5.0", + "libc", +] + [[package]] name = "counter" version = "0.5.7" @@ -2470,6 +2520,15 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "dlib" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" +dependencies = [ + "libloading", +] + [[package]] name = "docker_credential" version = "1.3.1" @@ -2499,6 +2558,18 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dwrote" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70182709525a3632b2ba96b6569225467b18ecb4a77f46d255f713a6bebf05fd" +dependencies = [ + "lazy_static", + "libc", + "winapi", + "wio", +] + [[package]] name = "e2e" version = "0.10.5" @@ -2747,16 +2818,28 @@ dependencies = [ "bytes", ] +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fee_algo_simulation" version = "0.10.5" dependencies = [ - "alloy", "anyhow", + "eth", + "itertools 0.13.0", + "plotters", "serde", "serde_json", "services", "tokio", + "xdg", ] [[package]] @@ -2797,6 +2880,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + [[package]] name = "flume" version = "0.11.1" @@ -2820,13 +2909,59 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +[[package]] +name = "font-kit" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64b34f4efd515f905952d91bc185039863705592c0c53ae6d979805dd154520" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "core-foundation 0.9.4", + "core-graphics", + "core-text", + "dirs", + "dwrote", + "float-ord", + "freetype-sys", + "lazy_static", + "libc", + "log", + "pathfinder_geometry", + "pathfinder_simd", + "walkdir", + "winapi", + "yeslogic-fontconfig-sys", +] + [[package]] name = "foreign-types" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -2835,6 +2970,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2850,6 +2991,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" +[[package]] +name = "freetype-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7edc5b9669349acfda99533e9e0bcf26a51862ab43b08ee7745c55d28eb134" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3286,6 +3438,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.1" @@ -3984,6 +4146,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", + "png", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -4113,6 +4289,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.74" @@ -4177,6 +4359,16 @@ version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -4327,6 +4519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -4518,7 +4711,7 @@ checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -4664,6 +4857,25 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pathfinder_geometry" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b7e7b4ea703700ce73ebf128e1450eb69c3a8329199ffbfb9b2a0418e5ad3" +dependencies = [ + "log", + "pathfinder_simd", +] + +[[package]] +name = "pathfinder_simd" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf07ef4804cfa9aea3b04a7bbdd5a40031dbb6b4f2cbaf2b011666c80c5b4f2" +dependencies = [ + "rustc_version 0.4.1", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -4769,6 +4981,65 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "chrono", + "font-kit", + "image", + "lazy_static", + "num-traits", + "pathfinder_geometry", + "plotters-backend", + "plotters-bitmap", + "plotters-svg", + "ttf-parser", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-bitmap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ce181e3f6bf82d6c1dc569103ca7b1bd964c60ba03d7e6cdfbb3e3eb7f7405" +dependencies = [ + "gif", + "image", + "plotters-backend", +] + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portpicker" version = "0.1.1" @@ -6887,6 +7158,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + [[package]] name = "tungstenite" version = "0.23.0" @@ -7203,6 +7480,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "whoami" version = "1.5.2" @@ -7456,6 +7739,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d129932f4644ac2396cb456385cbf9e63b5b30c6e8dc4820bdca4eb082037a5" +dependencies = [ + "winapi", +] + [[package]] name = "write16" version = "1.0.0" @@ -7496,6 +7788,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xdg" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213b7324336b53d2414b2db8537e56544d981803139155afa84f76eeebb7a546" + [[package]] name = "xmlparser" version = "0.13.6" @@ -7508,6 +7806,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yeslogic-fontconfig-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503a066b4c037c440169d995b869046827dbc71263f6e8f3be6d77d4f3229dbd" +dependencies = [ + "dlib", + "once_cell", + "pkg-config", +] + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index ed0face1..0ac3e8b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,8 @@ resolver = "2" members = [ "committer", - "e2e", "fee_algo_simulation", + "e2e", + "fee_algo_simulation", "packages/adapters/clock", "packages/adapters/eth", "packages/adapters/fuel", @@ -38,6 +39,8 @@ actix-web = { version = "4", default-features = false } bitvec = { version = "1.0", default-features = false } bytesize = { version = "1.3", default-features = false } alloy = { version = "0.3.6", default-features = false } +xdg = { version = "2.5", default-features = false } +plotters = { version = "0.3", default-features = false } proptest = { version = "1.0", default-features = false } rayon = { version = "1.10", default-features = false } num_cpus = { version = "1.16", default-features = false } diff --git a/fee_algo_simulation/Cargo.toml b/fee_algo_simulation/Cargo.toml new file mode 100644 index 00000000..d1b51081 --- /dev/null +++ b/fee_algo_simulation/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "fee_algo_simulation" +version.workspace = true +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish.workspace = true + +[dependencies] +services = { workspace = true } +anyhow = { workspace = true } +xdg = { workspace = true } +plotters = { version = "0.3", features = ["default"] } +itertools = { workspace = true, features = ["use_std"] } +eth = { workspace = true } +# TODO: segfault features +tokio = { workspace = true, features = [ + "macros", + "rt-multi-thread", + "process", + "fs", +] } + +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } diff --git a/fee_algo_simulation/src/main.rs b/fee_algo_simulation/src/main.rs new file mode 100644 index 00000000..2ffa3bc6 --- /dev/null +++ b/fee_algo_simulation/src/main.rs @@ -0,0 +1,200 @@ +use std::{ops::RangeInclusive, path::PathBuf}; + +use anyhow::Result; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use services::{ + block_importer::port::fuel::__mock_MockApi_Api::__compressed_blocks_in_height_range, + historical_fees::{ + self, + port::{ + cache::CachingApi, + l1::{Api, BlockFees, Fees, SequentialBlockFees}, + }, + service::{calculate_blob_tx_fee, HistoricalFees, SmaPeriods}, + }, + state_committer::{AlgoConfig, FeeThresholds}, +}; +use xdg::BaseDirectories; + +#[derive(Debug, Serialize, Deserialize, Default)] +struct SavedFees { + fees: Vec, +} + +const URL: &str = "https://eth.llamarpc.com"; + +pub struct PersistentApi

{ + provider: P, +} + +fn fee_file() -> PathBuf { + let xdg = BaseDirectories::with_prefix("fee_simulation").unwrap(); + if let Some(cache) = xdg.find_cache_file("fee_cache.json") { + cache + } else { + xdg.place_data_file("fee_cache.json").unwrap() + } +} + +fn load_cache() -> Vec<(u64, Fees)> { + let Ok(contents) = std::fs::read_to_string(fee_file()) else { + return vec![]; + }; + + let fees: SavedFees = serde_json::from_str(&contents).unwrap_or_default(); + + fees.fees.into_iter().map(|f| (f.height, f.fees)).collect() +} + +fn save_cache(cache: impl IntoIterator) -> anyhow::Result<()> { + let fees = SavedFees { + fees: cache + .into_iter() + .map(|(height, fees)| BlockFees { height, fees }) + .collect(), + }; + + std::fs::write(fee_file(), serde_json::to_string(&fees)?)?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let client = eth::HttpClient::new(URL).unwrap(); + let num_blocks_per_month = 30 * 24 * 3600 / 12; + + let caching_api = CachingApi::new(client, num_blocks_per_month * 2); + caching_api.import(load_cache()).await; + + let historical_fees = HistoricalFees::new(caching_api.clone()); + + let ending_height = 21514918u64; + let amount_of_blocks = num_blocks_per_month; + + let config = AlgoConfig { + sma_periods: SmaPeriods { + short: 25.try_into().unwrap(), + long: 300.try_into().unwrap(), + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: (8 * 3600).try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 1000000000000000, + }, + }; + + let fees = caching_api + .fees(ending_height - amount_of_blocks as u64..=ending_height) + .await?; + + plot_fees(fees)?; + + save_cache(caching_api.export().await)?; + + Ok(()) +} + +use plotters::prelude::*; + +const OUT_FILE_NAME: &str = "sample.png"; +fn plot_fees(fees: SequentialBlockFees) -> Result<()> { + let root_area = BitMapBackend::new(OUT_FILE_NAME, (1024, 768)).into_drawing_area(); + + root_area.fill(&WHITE)?; + + let root_area = root_area.titled("Fees", ("sans-serif", 60))?; + + let fees: Vec<_> = fees + .into_iter() + .map(|block_fees| { + ( + block_fees.height, + calculate_blob_tx_fee(6, &block_fees.fees), + ) + }) + .collect_vec(); + + let min_height = fees.first().unwrap().0; + let max_height = fees.last().unwrap().0; + + let max_fee = fees.iter().map(|(_, fees)| fees).max().unwrap(); + + let mut cc = ChartBuilder::on(&root_area) + .margin(50) + .set_left_and_bottom_label_area_size(50) + // .set_all_label_area_size(50) + .build_cartesian_2d(min_height..max_height + 1, 0..max_fee.saturating_mul(2))?; + + cc.configure_mesh() + .x_labels(20) + .y_labels(10) + .disable_mesh() + .x_label_formatter(&|v| format!("{:.1}", v)) + .y_label_formatter(&|v| format!("{:.3}", (*v as f64 / 10f64.powi(18)))) + .draw()?; + + cc.draw_series(LineSeries::new(fees, &RED))? + .label("Current fees") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], RED)); + + // cc.draw_series(LineSeries::new( + // x_axis.values().map(|x| (x, x.cos())), + // &BLUE, + // ))? + // .label("Cosine") + // .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], BLUE)); + + cc.configure_series_labels().border_style(BLACK).draw()?; + + /* + // It's possible to use a existing pointing element + cc.draw_series(PointSeries::<_, _, Circle<_>>::new( + (-3.0f32..2.1f32).step(1.0).values().map(|x| (x, x.sin())), + 5, + Into::::into(&RGBColor(255,0,0)).filled(), + ))?;*/ + + // Otherwise you can use a function to construct your pointing element yourself + // cc.draw_series(PointSeries::of_element( + // (-3.0f32..2.1f32).step(1.0).values().map(|x| (x, x.sin())), + // 5, + // ShapeStyle::from(&RED).filled(), + // &|coord, size, style| { + // EmptyElement::at(coord) + // + Circle::new((0, 0), size, style) + // + Text::new(format!("{:?}", coord), (0, 15), ("sans-serif", 15)) + // }, + // ))?; + + // let drawing_areas = lower.split_evenly((1, 2)); + + // for (drawing_area, idx) in drawing_areas.iter().zip(1..) { + // let mut cc = ChartBuilder::on(drawing_area) + // .x_label_area_size(30) + // .y_label_area_size(30) + // .margin_right(20) + // .caption(format!("y = x^{}", 1 + 2 * idx), ("sans-serif", 40)) + // .build_cartesian_2d(-1f32..1f32, -1f32..1f32)?; + // cc.configure_mesh() + // .x_labels(5) + // .y_labels(3) + // .max_light_lines(4) + // .draw()?; + // + // cc.draw_series(LineSeries::new( + // (-1f32..1f32) + // .step(0.01) + // .values() + // .map(|x| (x, x.powf(idx as f32 * 2.0 + 1.0))), + // &BLUE, + // ))?; + // } + + // To avoid the IO failure being ignored silently, we manually call the present function + root_area.present().expect("Unable to write result to file, please make sure 'plotters-doc-data' dir exists under current dir"); + println!("Result has been saved to {}", OUT_FILE_NAME); + Ok(()) +} diff --git a/packages/adapters/eth/src/blob_encoder.rs b/packages/adapters/eth/src/blob_encoder.rs new file mode 100644 index 00000000..e65bd947 --- /dev/null +++ b/packages/adapters/eth/src/blob_encoder.rs @@ -0,0 +1,137 @@ +use std::num::NonZeroUsize; + +use alloy::{ + consensus::BlobTransactionSidecar, + eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, +}; +use itertools::{izip, Itertools}; + +use fuel_block_committer_encoding::blob; +use services::{ + types::{Fragment, NonEmpty, NonNegative}, + Result, +}; + +#[derive(Debug, Copy, Clone)] +pub struct BlobEncoder; + +impl BlobEncoder { + #[cfg(feature = "test-helpers")] + pub const FRAGMENT_SIZE: usize = BYTES_PER_BLOB; + + pub(crate) fn sidecar_from_fragments( + fragments: impl IntoIterator, + ) -> crate::error::Result { + let mut sidecar = BlobTransactionSidecar::default(); + + for fragment in fragments { + let data = Vec::from(fragment.data); + + sidecar.blobs.push(Default::default()); + let current_blob = sidecar.blobs.last_mut().expect("just added it"); + + sidecar.commitments.push(Default::default()); + let current_commitment = sidecar.commitments.last_mut().expect("just added it"); + + sidecar.proofs.push(Default::default()); + let current_proof = sidecar.proofs.last_mut().expect("just added it"); + + let read_location = data.as_slice(); + + current_blob.copy_from_slice(&read_location[..BYTES_PER_BLOB]); + let read_location = &read_location[BYTES_PER_BLOB..]; + + current_commitment.copy_from_slice(&read_location[..48]); + let read_location = &read_location[48..]; + + current_proof.copy_from_slice(&read_location[..48]); + } + + Ok(sidecar) + } +} + +impl services::block_bundler::port::l1::FragmentEncoder for BlobEncoder { + fn encode(&self, data: NonEmpty, id: NonNegative) -> Result> { + let data = Vec::from(data); + let encoder = blob::Encoder::default(); + let decoder = blob::Decoder::default(); + + let blobs = encoder.encode(&data, id.as_u32()).map_err(|e| { + crate::error::Error::Other(format!("failed to encode data as blobs: {e}")) + })?; + + let bits_usage: Vec<_> = blobs + .iter() + .map(|blob| { + let blob::Header::V1(header) = decoder.read_header(blob).map_err(|e| { + crate::error::Error::Other(format!("failed to read blob header: {e}")) + })?; + Result::Ok(header.num_bits) + }) + .try_collect()?; + + let sidecar = blob::generate_sidecar(blobs) + .map_err(|e| crate::error::Error::Other(format!("failed to generate sidecar: {e}")))?; + + let fragments = izip!( + &sidecar.blobs, + &sidecar.commitments, + &sidecar.proofs, + bits_usage + ) + .map(|(blob, commitment, proof, used_bits)| { + let mut data_commitment_and_proof = vec![0; blob.len() + 48 * 2]; + let write_location = &mut data_commitment_and_proof[..]; + + write_location[..blob.len()].copy_from_slice(blob.as_slice()); + let write_location = &mut write_location[blob.len()..]; + + write_location[..48].copy_from_slice(&(**commitment)); + let write_location = &mut write_location[48..]; + + write_location[..48].copy_from_slice(&(**proof)); + + let bits_per_blob = BYTES_PER_BLOB as u32 * 8; + + Fragment { + data: NonEmpty::from_vec(data_commitment_and_proof).expect("known to be non-empty"), + unused_bytes: bits_per_blob.saturating_sub(used_bits).saturating_div(8), + total_bytes: bits_per_blob + .saturating_div(8) + .try_into() + .expect("known to be non-zero"), + } + }) + .collect(); + + Ok(NonEmpty::from_vec(fragments).expect("known to be non-empty")) + } + + fn gas_usage(&self, num_bytes: NonZeroUsize) -> u64 { + blob::Encoder::default().blobs_needed_to_encode(num_bytes.get()) as u64 * DATA_GAS_PER_BLOB + } +} + +#[cfg(test)] +mod test { + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; + use fuel_block_committer_encoding::blob; + use services::block_bundler::port::l1::FragmentEncoder; + + use crate::blob_encoder::{self}; + + #[test] + fn gas_usage_correctly_calculated() { + // given + let num_bytes = 400_000; + let encoder = blob::Encoder::default(); + assert_eq!(encoder.blobs_needed_to_encode(num_bytes), 4); + + // when + let gas_usage = blob_encoder::BlobEncoder.gas_usage(num_bytes.try_into().unwrap()); + + // then + assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); + } +} diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_api_helpers.rs similarity index 85% rename from packages/adapters/eth/src/fee_conversion.rs rename to packages/adapters/eth/src/fee_api_helpers.rs index fb6dc410..aee679f3 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_api_helpers.rs @@ -1,13 +1,47 @@ -use std::ops::RangeInclusive; +use std::{future::Future, ops::RangeInclusive}; use alloy::rpc::types::FeeHistory; +use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - historical_fees::port::l1::{BlockFees, Fees}, + historical_fees::port::l1::{BlockFees, Fees, SequentialBlockFees}, Result, }; +use static_assertions::const_assert; + +pub async fn batch_requests<'a, 'b, Fut, F>( + height_range: RangeInclusive, + get_fees: F, +) -> Result +where + 'a: 'b, + F: Fn(RangeInclusive, &'a [f64]) -> Fut, + Fut: Future> + 'b, +{ + const REWARD_PERCENTILE: f64 = + alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; + // so that a alloy version bump doesn't surprise us + const_assert!(REWARD_PERCENTILE == 20.0,); + + // There is a comment in alloy about not doing more than 1024 blocks at a time + const RPC_LIMIT: u64 = 1024; + + let fees: Vec = stream::iter(chunk_range_inclusive(height_range, RPC_LIMIT)) + .then(|range| get_fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) + .try_collect() + .await?; + + let mut unpacked_fees = vec![]; + for fee in fees { + unpacked_fees.extend(unpack_fee_history(fee)?); + } + + unpacked_fees + .try_into() + .map_err(|e| services::Error::Other(format!("{e}"))) +} -pub fn unpack_fee_history(fees: FeeHistory) -> Result> { +fn unpack_fee_history(fees: FeeHistory) -> Result> { let number_of_blocks = if fees.base_fee_per_gas.is_empty() { 0 } else { @@ -102,7 +136,7 @@ mod test { use alloy::rpc::types::FeeHistory; use services::historical_fees::port::l1::{BlockFees, Fees}; - use crate::fee_conversion::{self}; + use crate::fee_api_helpers::{chunk_range_inclusive, unpack_fee_history}; #[test] fn test_chunk_size_zero() { @@ -111,7 +145,7 @@ mod test { let chunk_size = 0; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected: Vec> = vec![]; @@ -128,7 +162,7 @@ mod test { let chunk_size = 10; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![1..=5]; @@ -145,7 +179,7 @@ mod test { let chunk_size = 2; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; @@ -159,7 +193,7 @@ mod test { let chunk_size = 3; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; @@ -176,7 +210,7 @@ mod test { let chunk_size = 1; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![5..=5]; @@ -193,7 +227,7 @@ mod test { let chunk_size = 50; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![100..=100]; @@ -210,7 +244,7 @@ mod test { let chunk_size = 1; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; @@ -227,7 +261,7 @@ mod test { let chunk_size = 11; // when - let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + let result = chunk_range_inclusive(initial_range, chunk_size); // then let expected = vec![20..=30]; @@ -249,7 +283,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees); + let result = unpack_fee_history(fees); // then let expected: Vec = vec![]; @@ -272,7 +306,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees.clone()); + let result = unpack_fee_history(fees.clone()); // then let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); @@ -295,7 +329,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees.clone()); + let result = unpack_fee_history(fees.clone()); // then let expected_error = @@ -319,7 +353,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees.clone()); + let result = unpack_fee_history(fees.clone()); // then let expected_error = @@ -343,7 +377,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees.clone()); + let result = unpack_fee_history(fees.clone()); // then let expected_error = @@ -367,7 +401,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees); + let result = unpack_fee_history(fees); // then let expected = vec![BlockFees { @@ -397,7 +431,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees); + let result = unpack_fee_history(fees); // then let expected = vec![ @@ -445,7 +479,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees.clone()); + let result = unpack_fee_history(fees.clone()); // then let expected = vec![ @@ -485,7 +519,7 @@ mod test { }; // when - let result = fee_conversion::unpack_fee_history(fees); + let result = unpack_fee_history(fees); // then let expected = vec![ diff --git a/packages/adapters/eth/src/http.rs b/packages/adapters/eth/src/http.rs new file mode 100644 index 00000000..02516a33 --- /dev/null +++ b/packages/adapters/eth/src/http.rs @@ -0,0 +1,57 @@ +use std::ops::RangeInclusive; + +use alloy::providers::Provider as AlloyProvider; +use alloy::providers::ProviderBuilder; + +use services::historical_fees::port::l1::SequentialBlockFees; + +use alloy::transports::http::Client; + +use alloy::transports::http::Http; + +use alloy::providers::RootProvider; + +use crate::fee_api_helpers::batch_requests; + +#[derive(Debug, Clone)] +pub struct Provider { + pub(crate) provider: RootProvider>, +} + +impl Provider { + pub fn new(url: &str) -> crate::Result { + let url = url + .parse() + .map_err(|e| crate::error::Error::Other(format!("invalid url: {url}: {e}")))?; + let provider = ProviderBuilder::new().on_http(url); + + Ok(Self { provider }) + } +} + +impl services::historical_fees::port::l1::Api for Provider { + async fn fees(&self, height_range: RangeInclusive) -> crate::Result { + batch_requests(height_range, |sub_range, percentiles| async move { + let last_block = *sub_range.end(); + let block_count = sub_range.count() as u64; + let fees = self + .provider + .get_fee_history( + block_count, + alloy::eips::BlockNumberOrTag::Number(last_block), + percentiles, + ) + .await + .map_err(|e| services::Error::Network(format!("failed to get fee history: {e}")))?; + + Ok(fees) + }) + .await + } + async fn current_height(&self) -> crate::Result { + self.provider + .get_block_number() + .await + .map_err(|e| services::Error::Network(format!("failed to get block number: {e}"))) + } +} diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 5fb3e481..4313f409 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -8,13 +8,14 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, + providers::{}, rpc::types::FeeHistory, + transports::http::{}, }; -use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - historical_fees::port::l1::SequentialBlockFees, + historical_fees::port::l1::{ SequentialBlockFees}, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -24,256 +25,15 @@ use services::{ mod aws; mod error; -mod fee_conversion; +mod fee_api_helpers; mod metrics; mod websocket; +mod blob_encoder; +mod http; + pub use alloy::primitives::Address; +pub use blob_encoder::BlobEncoder; pub use aws::*; -use fuel_block_committer_encoding::blob::{self, generate_sidecar}; -use static_assertions::const_assert; pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; - -#[derive(Debug, Copy, Clone)] -pub struct BlobEncoder; - -pub async fn make_pub_eth_client() -> WebsocketClient { - let signers = Signers::for_keys(crate::L1Keys { - main: crate::L1Key::Private( - "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), - ), - blob: None, - }) - .await - .unwrap(); - - crate::WebsocketClient::connect( - "wss://ethereum-rpc.publicnode.com".parse().unwrap(), - Default::default(), - signers, - 10, - crate::TxConfig { - tx_max_fee: u128::MAX, - send_tx_request_timeout: Duration::MAX, - }, - ) - .await - .unwrap() -} -impl BlobEncoder { - #[cfg(feature = "test-helpers")] - pub const FRAGMENT_SIZE: usize = BYTES_PER_BLOB; - - pub(crate) fn sidecar_from_fragments( - fragments: impl IntoIterator, - ) -> crate::error::Result { - let mut sidecar = BlobTransactionSidecar::default(); - - for fragment in fragments { - let data = Vec::from(fragment.data); - - sidecar.blobs.push(Default::default()); - let current_blob = sidecar.blobs.last_mut().expect("just added it"); - - sidecar.commitments.push(Default::default()); - let current_commitment = sidecar.commitments.last_mut().expect("just added it"); - - sidecar.proofs.push(Default::default()); - let current_proof = sidecar.proofs.last_mut().expect("just added it"); - - let read_location = data.as_slice(); - - current_blob.copy_from_slice(&read_location[..BYTES_PER_BLOB]); - let read_location = &read_location[BYTES_PER_BLOB..]; - - current_commitment.copy_from_slice(&read_location[..48]); - let read_location = &read_location[48..]; - - current_proof.copy_from_slice(&read_location[..48]); - } - - Ok(sidecar) - } -} - -impl services::block_bundler::port::l1::FragmentEncoder for BlobEncoder { - fn encode(&self, data: NonEmpty, id: NonNegative) -> Result> { - let data = Vec::from(data); - let encoder = blob::Encoder::default(); - let decoder = blob::Decoder::default(); - - let blobs = encoder.encode(&data, id.as_u32()).map_err(|e| { - crate::error::Error::Other(format!("failed to encode data as blobs: {e}")) - })?; - - let bits_usage: Vec<_> = blobs - .iter() - .map(|blob| { - let blob::Header::V1(header) = decoder.read_header(blob).map_err(|e| { - crate::error::Error::Other(format!("failed to read blob header: {e}")) - })?; - Result::Ok(header.num_bits) - }) - .try_collect()?; - - let sidecar = generate_sidecar(blobs) - .map_err(|e| crate::error::Error::Other(format!("failed to generate sidecar: {e}")))?; - - let fragments = izip!( - &sidecar.blobs, - &sidecar.commitments, - &sidecar.proofs, - bits_usage - ) - .map(|(blob, commitment, proof, used_bits)| { - let mut data_commitment_and_proof = vec![0; blob.len() + 48 * 2]; - let write_location = &mut data_commitment_and_proof[..]; - - write_location[..blob.len()].copy_from_slice(blob.as_slice()); - let write_location = &mut write_location[blob.len()..]; - - write_location[..48].copy_from_slice(&(**commitment)); - let write_location = &mut write_location[48..]; - - write_location[..48].copy_from_slice(&(**proof)); - - let bits_per_blob = BYTES_PER_BLOB as u32 * 8; - - Fragment { - data: NonEmpty::from_vec(data_commitment_and_proof).expect("known to be non-empty"), - unused_bytes: bits_per_blob.saturating_sub(used_bits).saturating_div(8), - total_bytes: bits_per_blob - .saturating_div(8) - .try_into() - .expect("known to be non-zero"), - } - }) - .collect(); - - Ok(NonEmpty::from_vec(fragments).expect("known to be non-empty")) - } - - fn gas_usage(&self, num_bytes: NonZeroUsize) -> u64 { - blob::Encoder::default().blobs_needed_to_encode(num_bytes.get()) as u64 * DATA_GAS_PER_BLOB - } -} - -impl services::block_committer::port::l1::Contract for WebsocketClient { - delegate! { - to self { - async fn submit(&self, hash: [u8; 32], height: u32) -> Result; - fn commit_interval(&self) -> NonZeroU32; - } - } -} - -impl services::state_listener::port::l1::Api for WebsocketClient { - delegate! { - to (*self) { - async fn get_transaction_response(&self, tx_hash: [u8; 32],) -> Result>; - async fn is_squeezed_out(&self, tx_hash: [u8; 32],) -> Result; - } - } - - async fn get_block_number(&self) -> Result { - let block_num = self._get_block_number().await?; - let height = L1Height::try_from(block_num)?; - - Ok(height) - } -} - -impl services::wallet_balance_tracker::port::l1::Api for WebsocketClient { - delegate! { - to (*self) { - async fn balance(&self, address: Address) -> Result; - } - } -} - -impl services::block_committer::port::l1::Api for WebsocketClient { - delegate! { - to (*self) { - async fn get_transaction_response(&self, tx_hash: [u8; 32],) -> Result>; - } - } - - async fn get_block_number(&self) -> Result { - let block_num = self._get_block_number().await?; - let height = L1Height::try_from(block_num)?; - - Ok(height) - } -} - -impl services::historical_fees::port::l1::Api for WebsocketClient { - async fn current_height(&self) -> Result { - self._get_block_number().await - } - - async fn fees(&self, height_range: RangeInclusive) -> Result { - const REWARD_PERCENTILE: f64 = - alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; - // so that a alloy version bump doesn't surprise us - const_assert!(REWARD_PERCENTILE == 20.0,); - - // There is a comment in alloy about not doing more than 1024 blocks at a time - const RPC_LIMIT: u64 = 1024; - - let fees: Vec = stream::iter(fee_conversion::chunk_range_inclusive( - height_range, - RPC_LIMIT, - )) - .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) - .try_collect() - .await?; - - let mut unpacked_fees = vec![]; - for fee in fees { - unpacked_fees.extend(fee_conversion::unpack_fee_history(fee)?); - } - - unpacked_fees - .try_into() - .map_err(|e| services::Error::Other(format!("{e}"))) - } -} - -impl services::state_committer::port::l1::Api for WebsocketClient { - async fn current_height(&self) -> Result { - self._get_block_number().await - } - - delegate! { - to (*self) { - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<(L1Tx, FragmentsSubmitted)>; - } - } -} - -#[cfg(test)] -mod test { - use alloy::eips::eip4844::DATA_GAS_PER_BLOB; - use fuel_block_committer_encoding::blob; - use services::block_bundler::port::l1::FragmentEncoder; - - use crate::BlobEncoder; - - #[test] - fn gas_usage_correctly_calculated() { - // given - let num_bytes = 400_000; - let encoder = blob::Encoder::default(); - assert_eq!(encoder.blobs_needed_to_encode(num_bytes), 4); - - // when - let gas_usage = BlobEncoder.gas_usage(num_bytes.try_into().unwrap()); - - // then - assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); - } -} +pub use http::Provider as HttpClient; diff --git a/packages/adapters/eth/src/websocket.rs b/packages/adapters/eth/src/websocket.rs index 8b64038b..0bcd9821 100644 --- a/packages/adapters/eth/src/websocket.rs +++ b/packages/adapters/eth/src/websocket.rs @@ -8,20 +8,28 @@ use alloy::{ rpc::types::FeeHistory, signers::{local::PrivateKeySigner, Signature}, }; +use delegate::delegate; +use futures::{stream, StreamExt, TryStreamExt}; use serde::Deserialize; use services::{ + historical_fees::port::l1::SequentialBlockFees, types::{ - BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx, NonEmpty, TransactionResponse, U256, + BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, + TransactionResponse, U256, }, Result, }; +use static_assertions::const_assert; use url::Url; use self::{ connection::WsConnection, health_tracking_middleware::{EthApi, HealthTrackingMiddleware}, }; -use crate::{AwsClient, AwsConfig}; +use crate::{ + fee_api_helpers::{self, batch_requests}, + http, AwsClient, AwsConfig, +}; mod connection; mod health_tracking_middleware; @@ -33,6 +41,83 @@ pub struct WebsocketClient { contract_caller_address: Address, } +impl services::block_committer::port::l1::Contract for WebsocketClient { + delegate! { + to self { + async fn submit(&self, hash: [u8; 32], height: u32) -> Result; + fn commit_interval(&self) -> NonZeroU32; + } + } +} + +impl services::state_listener::port::l1::Api for WebsocketClient { + delegate! { + to (*self) { + async fn get_transaction_response(&self, tx_hash: [u8; 32],) -> Result>; + async fn is_squeezed_out(&self, tx_hash: [u8; 32],) -> Result; + } + } + + async fn get_block_number(&self) -> Result { + let block_num = self._get_block_number().await?; + let height = L1Height::try_from(block_num)?; + + Ok(height) + } +} + +impl services::wallet_balance_tracker::port::l1::Api for WebsocketClient { + delegate! { + to (*self) { + async fn balance(&self, address: Address) -> Result; + } + } +} + +impl services::block_committer::port::l1::Api for WebsocketClient { + delegate! { + to (*self) { + async fn get_transaction_response(&self, tx_hash: [u8; 32],) -> Result>; + } + } + + async fn get_block_number(&self) -> Result { + let block_num = self._get_block_number().await?; + let height = L1Height::try_from(block_num)?; + + Ok(height) + } +} + +impl services::historical_fees::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + + async fn fees(&self, height_range: RangeInclusive) -> Result { + batch_requests(height_range, move |sub_range, percentiles| async move { + self.fees(sub_range, percentiles).await + }) + .await + } +} + +impl services::state_committer::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + + delegate! { + to (*self) { + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<(L1Tx, FragmentsSubmitted)>; + } + } +} + #[derive(Debug, Clone, PartialEq)] pub enum L1Key { Kms(String), diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index f6729d21..e653e59c 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -31,8 +31,8 @@ use url::Url; use super::{health_tracking_middleware::EthApi, Signers}; use crate::{ + blob_encoder::{self, BlobEncoder}, error::{Error, Result}, - BlobEncoder, }; pub type WsProvider = alloy::providers::fillers::FillProvider< @@ -276,7 +276,7 @@ impl EthApi for WsConnection { let num_fragments = min(fragments.len(), 6); let limited_fragments = fragments.into_iter().take(num_fragments); - let sidecar = BlobEncoder::sidecar_from_fragments(limited_fragments)?; + let sidecar = blob_encoder::BlobEncoder::sidecar_from_fragments(limited_fragments)?; let blob_tx = match previous_tx { Some(previous_tx) => { @@ -485,6 +485,8 @@ mod tests { use alloy::{node_bindings::Anvil, signers::local::PrivateKeySigner}; use services::{block_bundler::port::l1::FragmentEncoder, types::nonempty}; + use crate::blob_encoder; + use super::*; #[test] @@ -539,8 +541,8 @@ mod tests { }; let data = nonempty![1, 2, 3]; - let fragments = BlobEncoder.encode(data, 1.into()).unwrap(); - let sidecar = BlobEncoder::sidecar_from_fragments(fragments.clone()).unwrap(); + let fragments = blob_encoder::BlobEncoder.encode(data, 1.into()).unwrap(); + let sidecar = blob_encoder::BlobEncoder::sidecar_from_fragments(fragments.clone()).unwrap(); // create a tx with the help of the provider to get gas fields, hash etc let tx = TransactionRequest::default() @@ -617,7 +619,9 @@ mod tests { }; let data = nonempty![1, 2, 3]; - let fragment = BlobEncoder.encode(data, 1.try_into().unwrap()).unwrap(); + let fragment = blob_encoder::BlobEncoder + .encode(data, 1.try_into().unwrap()) + .unwrap(); // when let result = connection.submit_state_fragments(fragment, None).await; diff --git a/packages/services/src/historical_fees/port.rs b/packages/services/src/historical_fees/port.rs index 7754bd5b..13c1bb7c 100644 --- a/packages/services/src/historical_fees/port.rs +++ b/packages/services/src/historical_fees/port.rs @@ -420,6 +420,14 @@ pub mod cache { cache_limit, } } + + pub async fn import(&self, fees: impl IntoIterator) { + self.cache.write().await.extend(fees); + } + + pub async fn export(&self) -> impl IntoIterator { + self.cache.read().await.clone() + } } impl Api for CachingApi

{ diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 138db4bf..bcb7123b 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -39,6 +39,7 @@ impl Default for Config { #[derive(Debug, Clone, Copy)] pub struct FeeThresholds { pub max_l2_blocks_behind: NonZeroU32, + // TODO: segfault a better way to express this pub start_discount_percentage: Percentage, pub end_premium_percentage: Percentage, pub always_acceptable_fee: u128, From 6529d415bd3fed77dc2aa005d775097fa980dee0 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 30 Dec 2024 13:55:09 +0100 Subject: [PATCH 063/136] can plot current price --- Cargo.lock | 75 ++++++++ fee_algo_simulation/Cargo.toml | 2 + fee_algo_simulation/src/main.rs | 292 +++++++++++++++++++------------- 3 files changed, 253 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5f570ef..c199b736 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,6 +1459,61 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -2832,6 +2887,7 @@ name = "fee_algo_simulation" version = "0.10.5" dependencies = [ "anyhow", + "axum", "eth", "itertools 0.13.0", "plotters", @@ -3819,6 +3875,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -4477,6 +4534,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -6005,6 +6068,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_repr" version = "0.1.19" @@ -7068,8 +7141,10 @@ dependencies = [ "futures-util", "pin-project-lite", "sync_wrapper 0.1.2", + "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/fee_algo_simulation/Cargo.toml b/fee_algo_simulation/Cargo.toml index d1b51081..969f37d9 100644 --- a/fee_algo_simulation/Cargo.toml +++ b/fee_algo_simulation/Cargo.toml @@ -15,6 +15,8 @@ anyhow = { workspace = true } xdg = { workspace = true } plotters = { version = "0.3", features = ["default"] } itertools = { workspace = true, features = ["use_std"] } +# TODO: move this to workspace level +axum = { version = "0.7" } eth = { workspace = true } # TODO: segfault features tokio = { workspace = true, features = [ diff --git a/fee_algo_simulation/src/main.rs b/fee_algo_simulation/src/main.rs index 2ffa3bc6..d92971cf 100644 --- a/fee_algo_simulation/src/main.rs +++ b/fee_algo_simulation/src/main.rs @@ -1,8 +1,17 @@ -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{net::SocketAddr, ops::RangeInclusive, path::PathBuf}; use anyhow::Result; +use axum::{ + extract::{Query, State}, + http::StatusCode, + response::{Html, IntoResponse}, + routing::get, + Json, Router, +}; use itertools::Itertools; use serde::{Deserialize, Serialize}; +use xdg::BaseDirectories; + use services::{ block_importer::port::fuel::__mock_MockApi_Api::__compressed_blocks_in_height_range, historical_fees::{ @@ -15,7 +24,9 @@ use services::{ }, state_committer::{AlgoConfig, FeeThresholds}, }; -use xdg::BaseDirectories; + +// If you're using ethers-based client: +use services::historical_fees::port::l1 as eth; #[derive(Debug, Serialize, Deserialize, Default)] struct SavedFees { @@ -28,6 +39,7 @@ pub struct PersistentApi

{ provider: P, } +/// Same fee_cache.json location logic fn fee_file() -> PathBuf { let xdg = BaseDirectories::with_prefix("fee_simulation").unwrap(); if let Some(cache) = xdg.find_cache_file("fee_cache.json") { @@ -37,16 +49,16 @@ fn fee_file() -> PathBuf { } } +/// Load from disk fn load_cache() -> Vec<(u64, Fees)> { let Ok(contents) = std::fs::read_to_string(fee_file()) else { return vec![]; }; - let fees: SavedFees = serde_json::from_str(&contents).unwrap_or_default(); - fees.fees.into_iter().map(|f| (f.height, f.fees)).collect() } +/// Save to disk fn save_cache(cache: impl IntoIterator) -> anyhow::Result<()> { let fees = SavedFees { fees: cache @@ -54,26 +66,159 @@ fn save_cache(cache: impl IntoIterator) -> anyhow::Result<() .map(|(height, fees)| BlockFees { height, fees }) .collect(), }; - std::fs::write(fee_file(), serde_json::to_string(&fees)?)?; - Ok(()) } +/// Shared state across routes +#[derive(Clone)] +struct AppState { + caching_api: CachingApi<::eth::HttpClient>, + historical_fees: HistoricalFees>, + default_config: AlgoConfig, + num_blocks_per_month: u64, +} + +/// Query params for /fees +#[derive(Debug, Deserialize)] +struct FeeParams { + ending_height: Option, + amount_of_blocks: Option, +} + +/// GET /fees +/// Returns an array of `(height, blobFee)`. +async fn get_fees( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + let ending_height = params.ending_height.unwrap_or(21_514_918); + let amount_of_blocks = params + .amount_of_blocks + .unwrap_or(state.num_blocks_per_month); + + let start_height = ending_height.saturating_sub(amount_of_blocks); + let range = start_height..=ending_height; + + // Actually fetch from the caching API + let fees_res = state.caching_api.fees(range).await; + let Ok(seq_fees) = fees_res else { + return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to fetch fees").into_response(); + }; + + // Convert to (blockHeight, blobFee) + let data: Vec<(u64, u128)> = seq_fees + .into_iter() + .map(|block_fees| { + let blob_fee = calculate_blob_tx_fee(6, &block_fees.fees); + (block_fees.height, blob_fee) + }) + .collect(); + + Json(data).into_response() +} + +/// The HTML page at GET / +async fn index_html() -> Html<&'static str> { + Html( + r#" + + + + Fee Simulator + + + + +

Fee Simulator - Plot Blob Tx Fee Only

+
+ + + + + + + +
+ +
+ + + + +
+ +
+ + + + +"#, + ) +} + #[tokio::main] async fn main() -> Result<()> { - let client = eth::HttpClient::new(URL).unwrap(); + // 1) Create your ETH HTTP client + let client = ::eth::HttpClient::new(URL).unwrap(); + + // 2) ~1 month = 2160 blocks (approx) if 12s per block let num_blocks_per_month = 30 * 24 * 3600 / 12; + // 3) Build your CachingApi & import any existing cache let caching_api = CachingApi::new(client, num_blocks_per_month * 2); caching_api.import(load_cache()).await; + // 4) Build your HistoricalFees let historical_fees = HistoricalFees::new(caching_api.clone()); - let ending_height = 21514918u64; - let amount_of_blocks = num_blocks_per_month; - - let config = AlgoConfig { + // 5) Same default config you had + let default_config = AlgoConfig { sma_periods: SmaPeriods { short: 25.try_into().unwrap(), long: 300.try_into().unwrap(), @@ -86,115 +231,30 @@ async fn main() -> Result<()> { }, }; - let fees = caching_api - .fees(ending_height - amount_of_blocks as u64..=ending_height) - .await?; - - plot_fees(fees)?; - - save_cache(caching_api.export().await)?; - - Ok(()) -} + // 6) Bundle everything into state + let state = AppState { + caching_api, + historical_fees, + default_config, + num_blocks_per_month: num_blocks_per_month as u64, + }; -use plotters::prelude::*; + // 7) Axum router: serve front-end + fees endpoint + let app = Router::new() + .route("/", get(index_html)) + .route("/fees", get(get_fees)) + .with_state(state); -const OUT_FILE_NAME: &str = "sample.png"; -fn plot_fees(fees: SequentialBlockFees) -> Result<()> { - let root_area = BitMapBackend::new(OUT_FILE_NAME, (1024, 768)).into_drawing_area(); + // 8) Run server + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + println!("Server listening on http://{}", addr); - root_area.fill(&WHITE)?; + // run our app with hyper + let listener = tokio::net::TcpListener::bind("127.0.0.1:3000") + .await + .unwrap(); - let root_area = root_area.titled("Fees", ("sans-serif", 60))?; + axum::serve(listener, app).await.unwrap(); - let fees: Vec<_> = fees - .into_iter() - .map(|block_fees| { - ( - block_fees.height, - calculate_blob_tx_fee(6, &block_fees.fees), - ) - }) - .collect_vec(); - - let min_height = fees.first().unwrap().0; - let max_height = fees.last().unwrap().0; - - let max_fee = fees.iter().map(|(_, fees)| fees).max().unwrap(); - - let mut cc = ChartBuilder::on(&root_area) - .margin(50) - .set_left_and_bottom_label_area_size(50) - // .set_all_label_area_size(50) - .build_cartesian_2d(min_height..max_height + 1, 0..max_fee.saturating_mul(2))?; - - cc.configure_mesh() - .x_labels(20) - .y_labels(10) - .disable_mesh() - .x_label_formatter(&|v| format!("{:.1}", v)) - .y_label_formatter(&|v| format!("{:.3}", (*v as f64 / 10f64.powi(18)))) - .draw()?; - - cc.draw_series(LineSeries::new(fees, &RED))? - .label("Current fees") - .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], RED)); - - // cc.draw_series(LineSeries::new( - // x_axis.values().map(|x| (x, x.cos())), - // &BLUE, - // ))? - // .label("Cosine") - // .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], BLUE)); - - cc.configure_series_labels().border_style(BLACK).draw()?; - - /* - // It's possible to use a existing pointing element - cc.draw_series(PointSeries::<_, _, Circle<_>>::new( - (-3.0f32..2.1f32).step(1.0).values().map(|x| (x, x.sin())), - 5, - Into::::into(&RGBColor(255,0,0)).filled(), - ))?;*/ - - // Otherwise you can use a function to construct your pointing element yourself - // cc.draw_series(PointSeries::of_element( - // (-3.0f32..2.1f32).step(1.0).values().map(|x| (x, x.sin())), - // 5, - // ShapeStyle::from(&RED).filled(), - // &|coord, size, style| { - // EmptyElement::at(coord) - // + Circle::new((0, 0), size, style) - // + Text::new(format!("{:?}", coord), (0, 15), ("sans-serif", 15)) - // }, - // ))?; - - // let drawing_areas = lower.split_evenly((1, 2)); - - // for (drawing_area, idx) in drawing_areas.iter().zip(1..) { - // let mut cc = ChartBuilder::on(drawing_area) - // .x_label_area_size(30) - // .y_label_area_size(30) - // .margin_right(20) - // .caption(format!("y = x^{}", 1 + 2 * idx), ("sans-serif", 40)) - // .build_cartesian_2d(-1f32..1f32, -1f32..1f32)?; - // cc.configure_mesh() - // .x_labels(5) - // .y_labels(3) - // .max_light_lines(4) - // .draw()?; - // - // cc.draw_series(LineSeries::new( - // (-1f32..1f32) - // .step(0.01) - // .values() - // .map(|x| (x, x.powf(idx as f32 * 2.0 + 1.0))), - // &BLUE, - // ))?; - // } - - // To avoid the IO failure being ignored silently, we manually call the present function - root_area.present().expect("Unable to write result to file, please make sure 'plotters-doc-data' dir exists under current dir"); - println!("Result has been saved to {}", OUT_FILE_NAME); Ok(()) } From 23d9ef0e54c0fd3e9b4cc12af33a3c0f3236e77c Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 30 Dec 2024 14:43:52 +0100 Subject: [PATCH 064/136] can simulate algo params --- fee_algo_simulation/src/main.rs | 356 ++++++++++++++---- packages/adapters/eth/src/blob_encoder.rs | 3 +- packages/adapters/eth/src/http.rs | 13 +- packages/adapters/eth/src/lib.rs | 34 +- packages/adapters/eth/src/websocket.rs | 8 +- .../adapters/eth/src/websocket/connection.rs | 5 +- packages/services/src/state_committer.rs | 2 +- 7 files changed, 292 insertions(+), 129 deletions(-) diff --git a/fee_algo_simulation/src/main.rs b/fee_algo_simulation/src/main.rs index d92971cf..a3a2e39a 100644 --- a/fee_algo_simulation/src/main.rs +++ b/fee_algo_simulation/src/main.rs @@ -1,4 +1,9 @@ -use std::{net::SocketAddr, ops::RangeInclusive, path::PathBuf}; +use std::{ + net::SocketAddr, + num::{NonZeroU32, NonZeroU64}, + ops::RangeInclusive, + path::PathBuf, +}; use anyhow::Result; use axum::{ @@ -8,25 +13,18 @@ use axum::{ routing::get, Json, Router, }; -use itertools::Itertools; use serde::{Deserialize, Serialize}; -use xdg::BaseDirectories; - use services::{ - block_importer::port::fuel::__mock_MockApi_Api::__compressed_blocks_in_height_range, historical_fees::{ - self, port::{ cache::CachingApi, - l1::{Api, BlockFees, Fees, SequentialBlockFees}, + l1::{Api, BlockFees, Fees}, }, service::{calculate_blob_tx_fee, HistoricalFees, SmaPeriods}, }, - state_committer::{AlgoConfig, FeeThresholds}, + state_committer::{AlgoConfig, FeeThresholds, Percentage, SmaFeeAlgo}, }; - -// If you're using ethers-based client: -use services::historical_fees::port::l1 as eth; +use xdg::BaseDirectories; #[derive(Debug, Serialize, Deserialize, Default)] struct SavedFees { @@ -35,10 +33,6 @@ struct SavedFees { const URL: &str = "https://eth.llamarpc.com"; -pub struct PersistentApi

{ - provider: P, -} - /// Same fee_cache.json location logic fn fee_file() -> PathBuf { let xdg = BaseDirectories::with_prefix("fee_simulation").unwrap(); @@ -73,9 +67,8 @@ fn save_cache(cache: impl IntoIterator) -> anyhow::Result<() /// Shared state across routes #[derive(Clone)] struct AppState { - caching_api: CachingApi<::eth::HttpClient>, - historical_fees: HistoricalFees>, - default_config: AlgoConfig, + caching_api: CachingApi, + historical_fees: HistoricalFees>, num_blocks_per_month: u64, } @@ -84,40 +77,146 @@ struct AppState { struct FeeParams { ending_height: Option, amount_of_blocks: Option, + + // Fee Algo settings + short: Option, + long: Option, + max_l2_blocks_behind: Option, + start_discount_percentage: Option, + end_premium_percentage: Option, + always_acceptable_fee: Option, + + // How many L2 blocks behind are we? If none is given, default 0 + num_l2_blocks_behind: Option, +} + +/// Response struct for each fee data point +#[derive(Debug, Serialize)] +struct FeeDataPoint { + #[serde(rename = "blockHeight")] + block_height: u64, + #[serde(rename = "currentFee")] + current_fee: String, // Serialize u128 as String + #[serde(rename = "shortFee")] + short_fee: String, // Serialize u128 as String + #[serde(rename = "longFee")] + long_fee: String, // Serialize u128 as String + acceptable: bool, } /// GET /fees -/// Returns an array of `(height, blobFee)`. +/// +/// Returns JSON: each item is { blockHeight, currentFee, shortFee, longFee, acceptable } async fn get_fees( State(state): State, Query(params): Query, ) -> impl IntoResponse { + // 1) Resolve user inputs or use defaults let ending_height = params.ending_height.unwrap_or(21_514_918); let amount_of_blocks = params .amount_of_blocks .unwrap_or(state.num_blocks_per_month); + let short = params.short.unwrap_or(25); // default short + let long = params.long.unwrap_or(300); // default long + + let max_l2 = params.max_l2_blocks_behind.unwrap_or(8 * 3600); + let start_discount = params.start_discount_percentage.unwrap_or(0.10); + let end_premium = params.end_premium_percentage.unwrap_or(0.20); + let always_acceptable_fee = params + .always_acceptable_fee + .map(|v| v.parse().unwrap()) + .unwrap_or(1_000_000_000_000_000); + let num_l2_blocks_behind = params.num_l2_blocks_behind.unwrap_or(0); + + // 2) Build an SmaFeeAlgo config from user’s inputs + // Notice we reuse your “fee_algo::Config” (renamed FeeAlgoConfig in this file). + let config = AlgoConfig { + sma_periods: SmaPeriods { + short: match NonZeroU64::new(short) { + Some(nz) => nz, + None => NonZeroU64::new(1).unwrap(), + }, + long: match NonZeroU64::new(long) { + Some(nz) => nz, + None => NonZeroU64::new(1).unwrap(), + }, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: match NonZeroU32::new(max_l2) { + Some(nz) => nz, + None => NonZeroU32::new(1).unwrap(), + }, + start_discount_percentage: match Percentage::try_from(start_discount) { + Ok(p) => p, + Err(_) => Percentage::ZERO, + }, + end_premium_percentage: match Percentage::try_from(end_premium) { + Ok(p) => p, + Err(_) => Percentage::ZERO, + }, + always_acceptable_fee, + }, + }; + + let sma_algo = SmaFeeAlgo::new(state.historical_fees.clone(), config); + + // 3) Determine which blocks to fetch let start_height = ending_height.saturating_sub(amount_of_blocks); let range = start_height..=ending_height; - // Actually fetch from the caching API + // 4) Actually fetch from the caching API let fees_res = state.caching_api.fees(range).await; let Ok(seq_fees) = fees_res else { return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to fetch fees").into_response(); }; - // Convert to (blockHeight, blobFee) - let data: Vec<(u64, u128)> = seq_fees - .into_iter() - .map(|block_fees| { - let blob_fee = calculate_blob_tx_fee(6, &block_fees.fees); - (block_fees.height, blob_fee) - }) - .collect(); + // 5) Prepare data points + let mut data = Vec::with_capacity(seq_fees.len()); + + for block_fees in seq_fees.into_iter() { + let block_height = block_fees.height; + let current_fee = calculate_blob_tx_fee(6, &block_fees.fees); + + // fetch the shortTerm + longTerm SMA at exactly this height + let short_term_sma = state + .historical_fees + .calculate_sma(last_n_blocks(block_height, config.sma_periods.short)) + .await + .unwrap(); + + let long_term_sma = state + .historical_fees + .calculate_sma(last_n_blocks(block_height, config.sma_periods.long)) + .await + .unwrap(); + + let short_fee = calculate_blob_tx_fee(6, &short_term_sma); + let long_fee = calculate_blob_tx_fee(6, &long_term_sma); + + let acceptable = sma_algo + .fees_acceptable(6, num_l2_blocks_behind, block_height) + .await + .unwrap(); + + data.push(FeeDataPoint { + block_height, + current_fee: current_fee.to_string(), + short_fee: short_fee.to_string(), + long_fee: long_fee.to_string(), + acceptable, + }); + } + // Return as JSON Json(data).into_response() } +/// Helper: compute [current_block - (n-1) .. current_block] +fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block +} + /// The HTML page at GET / async fn index_html() -> Html<&'static str> { Html( @@ -125,30 +224,54 @@ async fn index_html() -> Html<&'static str> { - Fee Simulator - + Fee Simulator - Multiple Series + Shading -

Fee Simulator - Plot Blob Tx Fee Only

+

Fee Simulator - Multiple Series + Shading

+
- + - -
+
+ + + + + +
-
- - - - + + + + + + + + +
+ + + + + + + +
+ + + + + + +
-
+
@@ -205,10 +417,10 @@ async fn index_html() -> Html<&'static str> { #[tokio::main] async fn main() -> Result<()> { // 1) Create your ETH HTTP client - let client = ::eth::HttpClient::new(URL).unwrap(); + let client = eth::HttpClient::new(URL).unwrap(); - // 2) ~1 month = 2160 blocks (approx) if 12s per block - let num_blocks_per_month = 30 * 24 * 3600 / 12; + // 2) ~1 month = 216000 blocks (approx) if 12s per block + let num_blocks_per_month = 30 * 24 * 3600 / 12; // 259200 blocks // 3) Build your CachingApi & import any existing cache let caching_api = CachingApi::new(client, num_blocks_per_month * 2); @@ -217,25 +429,10 @@ async fn main() -> Result<()> { // 4) Build your HistoricalFees let historical_fees = HistoricalFees::new(caching_api.clone()); - // 5) Same default config you had - let default_config = AlgoConfig { - sma_periods: SmaPeriods { - short: 25.try_into().unwrap(), - long: 300.try_into().unwrap(), - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: (8 * 3600).try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 1000000000000000, - }, - }; - // 6) Bundle everything into state let state = AppState { - caching_api, + caching_api: caching_api.clone(), historical_fees, - default_config, num_blocks_per_month: num_blocks_per_month as u64, }; @@ -255,6 +452,7 @@ async fn main() -> Result<()> { .unwrap(); axum::serve(listener, app).await.unwrap(); + save_cache(caching_api.export().await)?; Ok(()) } diff --git a/packages/adapters/eth/src/blob_encoder.rs b/packages/adapters/eth/src/blob_encoder.rs index e65bd947..ca1c8c5f 100644 --- a/packages/adapters/eth/src/blob_encoder.rs +++ b/packages/adapters/eth/src/blob_encoder.rs @@ -4,9 +4,8 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, }; -use itertools::{izip, Itertools}; - use fuel_block_committer_encoding::blob; +use itertools::{izip, Itertools}; use services::{ types::{Fragment, NonEmpty, NonNegative}, Result, diff --git a/packages/adapters/eth/src/http.rs b/packages/adapters/eth/src/http.rs index 02516a33..e47dba69 100644 --- a/packages/adapters/eth/src/http.rs +++ b/packages/adapters/eth/src/http.rs @@ -1,16 +1,11 @@ use std::ops::RangeInclusive; -use alloy::providers::Provider as AlloyProvider; -use alloy::providers::ProviderBuilder; - +use alloy::{ + providers::{Provider as AlloyProvider, ProviderBuilder, RootProvider}, + transports::http::{Client, Http}, +}; use services::historical_fees::port::l1::SequentialBlockFees; -use alloy::transports::http::Client; - -use alloy::transports::http::Http; - -use alloy::providers::RootProvider; - use crate::fee_api_helpers::batch_requests; #[derive(Debug, Clone)] diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 4313f409..b4ef06a1 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -1,39 +1,15 @@ -use std::{ - num::{NonZeroU32, NonZeroUsize}, - ops::RangeInclusive, - time::Duration, -}; - -use alloy::{ - consensus::BlobTransactionSidecar, - eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, - primitives::U256, - providers::{}, - rpc::types::FeeHistory, - transports::http::{}, -}; -use futures::{stream, StreamExt, TryStreamExt}; -use itertools::{izip, Itertools}; -use services::{ - historical_fees::port::l1::{ SequentialBlockFees}, - types::{ - BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, - TransactionResponse, - }, - Result, -}; +use services::Result; mod aws; +mod blob_encoder; mod error; mod fee_api_helpers; +mod http; mod metrics; mod websocket; -mod blob_encoder; -mod http; - pub use alloy::primitives::Address; -pub use blob_encoder::BlobEncoder; pub use aws::*; -pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; +pub use blob_encoder::BlobEncoder; pub use http::Provider as HttpClient; +pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; diff --git a/packages/adapters/eth/src/websocket.rs b/packages/adapters/eth/src/websocket.rs index 0bcd9821..49d5c1d7 100644 --- a/packages/adapters/eth/src/websocket.rs +++ b/packages/adapters/eth/src/websocket.rs @@ -9,7 +9,7 @@ use alloy::{ signers::{local::PrivateKeySigner, Signature}, }; use delegate::delegate; -use futures::{stream, StreamExt, TryStreamExt}; +use futures::{StreamExt, TryStreamExt}; use serde::Deserialize; use services::{ historical_fees::port::l1::SequentialBlockFees, @@ -19,17 +19,13 @@ use services::{ }, Result, }; -use static_assertions::const_assert; use url::Url; use self::{ connection::WsConnection, health_tracking_middleware::{EthApi, HealthTrackingMiddleware}, }; -use crate::{ - fee_api_helpers::{self, batch_requests}, - http, AwsClient, AwsConfig, -}; +use crate::{fee_api_helpers::batch_requests, AwsClient, AwsConfig}; mod connection; mod health_tracking_middleware; diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index e653e59c..aed1e83c 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -31,7 +31,7 @@ use url::Url; use super::{health_tracking_middleware::EthApi, Signers}; use crate::{ - blob_encoder::{self, BlobEncoder}, + blob_encoder::{self}, error::{Error, Result}, }; @@ -485,9 +485,8 @@ mod tests { use alloy::{node_bindings::Anvil, signers::local::PrivateKeySigner}; use services::{block_bundler::port::l1::FragmentEncoder, types::nonempty}; - use crate::blob_encoder; - use super::*; + use crate::blob_encoder; #[test] fn calculates_correctly_the_commit_height() { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 73c924ed..07c92e55 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,4 +1,4 @@ mod fee_algo; -pub use fee_algo::{Config as AlgoConfig, FeeThresholds, Percentage}; +pub use fee_algo::{Config as AlgoConfig, FeeThresholds, Percentage, SmaFeeAlgo}; pub mod port; pub mod service; From aefe371a6bbe5db09035a275cf66cf127187ffd2 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 31 Dec 2024 10:48:04 +0100 Subject: [PATCH 065/136] change the reported statistics --- fee_algo_simulation/src/main.rs | 198 ++++++++++++++++++++++++++------ 1 file changed, 165 insertions(+), 33 deletions(-) diff --git a/fee_algo_simulation/src/main.rs b/fee_algo_simulation/src/main.rs index a3a2e39a..a87a47e6 100644 --- a/fee_algo_simulation/src/main.rs +++ b/fee_algo_simulation/src/main.rs @@ -86,6 +86,9 @@ struct FeeParams { end_premium_percentage: Option, always_acceptable_fee: Option, + // Number of blobs per transaction + num_blobs: Option, + // How many L2 blocks behind are we? If none is given, default 0 num_l2_blocks_behind: Option, } @@ -104,9 +107,27 @@ struct FeeDataPoint { acceptable: bool, } +/// Statistics struct +#[derive(Debug, Serialize)] +struct FeeStats { + #[serde(rename = "percentageAcceptable")] + percentage_acceptable: f64, // Percentage of acceptable blocks + #[serde(rename = "percentile95GapSize")] + percentile_95_gap_size: u64, // 95th percentile of gap sizes in blocks + #[serde(rename = "longestUnacceptableStreak")] + longest_unacceptable_streak: u64, // Longest consecutive unacceptable blocks +} + +/// Complete response struct +#[derive(Debug, Serialize)] +struct FeeResponse { + data: Vec, + stats: FeeStats, +} + /// GET /fees /// -/// Returns JSON: each item is { blockHeight, currentFee, shortFee, longFee, acceptable } +/// Returns JSON: { data: [...], stats: {...} } async fn get_fees( State(state): State, Query(params): Query, @@ -123,14 +144,26 @@ async fn get_fees( let max_l2 = params.max_l2_blocks_behind.unwrap_or(8 * 3600); let start_discount = params.start_discount_percentage.unwrap_or(0.10); let end_premium = params.end_premium_percentage.unwrap_or(0.20); - let always_acceptable_fee = params - .always_acceptable_fee - .map(|v| v.parse().unwrap()) - .unwrap_or(1_000_000_000_000_000); + + let always_acceptable_fee = match params.always_acceptable_fee { + Some(v) => match v.parse::() { + Ok(val) => val, + Err(_) => { + return ( + StatusCode::BAD_REQUEST, + "Invalid always_acceptable_fee value", + ) + .into_response() + } + }, + None => 1_000_000_000_000_000, + }; + + let num_blobs = params.num_blobs.unwrap_or(6); // default to 6 blobs + let num_l2_blocks_behind = params.num_l2_blocks_behind.unwrap_or(0); // 2) Build an SmaFeeAlgo config from user’s inputs - // Notice we reuse your “fee_algo::Config” (renamed FeeAlgoConfig in this file). let config = AlgoConfig { sma_periods: SmaPeriods { short: match NonZeroU64::new(short) { @@ -176,28 +209,37 @@ async fn get_fees( for block_fees in seq_fees.into_iter() { let block_height = block_fees.height; - let current_fee = calculate_blob_tx_fee(6, &block_fees.fees); + let current_fee = calculate_blob_tx_fee(num_blobs, &block_fees.fees); - // fetch the shortTerm + longTerm SMA at exactly this height - let short_term_sma = state + // Fetch the shortTerm + longTerm SMA at exactly this height + let short_term_sma = match state .historical_fees .calculate_sma(last_n_blocks(block_height, config.sma_periods.short)) .await - .unwrap(); + { + Ok(f) => f, + Err(_) => Fees::default(), + }; - let long_term_sma = state + let long_term_sma = match state .historical_fees .calculate_sma(last_n_blocks(block_height, config.sma_periods.long)) .await - .unwrap(); + { + Ok(f) => f, + Err(_) => Fees::default(), + }; - let short_fee = calculate_blob_tx_fee(6, &short_term_sma); - let long_fee = calculate_blob_tx_fee(6, &long_term_sma); + let short_fee = calculate_blob_tx_fee(num_blobs, &short_term_sma); + let long_fee = calculate_blob_tx_fee(num_blobs, &long_term_sma); - let acceptable = sma_algo - .fees_acceptable(6, num_l2_blocks_behind, block_height) + let acceptable = match sma_algo + .fees_acceptable(num_blobs, num_l2_blocks_behind, block_height) .await - .unwrap(); + { + Ok(decision) => decision, + Err(_) => false, // or handle error differently + }; data.push(FeeDataPoint { block_height, @@ -208,8 +250,56 @@ async fn get_fees( }); } + // 6) Calculate statistics + let total_blocks = data.len() as f64; + let acceptable_blocks = data.iter().filter(|d| d.acceptable).count() as f64; + let percentage_acceptable = if total_blocks > 0.0 { + (acceptable_blocks / total_blocks) * 100.0 + } else { + 0.0 + }; + + // 7) Calculate gap sizes (streaks of unacceptable blocks) + let mut gap_sizes = Vec::new(); + let mut current_gap = 0; + + for d in &data { + if !d.acceptable { + current_gap += 1; + } else if current_gap > 0 { + gap_sizes.push(current_gap); + current_gap = 0; + } + } + + // Push the last gap if it ends with an unacceptable streak + if current_gap > 0 { + gap_sizes.push(current_gap); + } + + // 8) Calculate the 95th percentile of gap sizes + let percentile_95_gap_size = if !gap_sizes.is_empty() { + let mut sorted_gaps = gap_sizes.clone(); + sorted_gaps.sort_unstable(); + let index = ((sorted_gaps.len() as f64) * 0.95).ceil() as usize - 1; + sorted_gaps[index.min(sorted_gaps.len() - 1)] + } else { + 0 + }; + + // 9) Find the longest unacceptable streak + let longest_unacceptable_streak = gap_sizes.iter().cloned().max().unwrap_or(0); + + let stats = FeeStats { + percentage_acceptable, + percentile_95_gap_size, + longest_unacceptable_streak, + }; + + let response = FeeResponse { data, stats }; + // Return as JSON - Json(data).into_response() + Json(response).into_response() } /// Helper: compute [current_block - (n-1) .. current_block] @@ -226,6 +316,28 @@ async fn index_html() -> Html<&'static str> { Fee Simulator - Multiple Series + Shading +

Fee Simulator - Multiple Series + Shading

@@ -258,6 +370,10 @@ async fn index_html() -> Html<&'static str> { + + +
+ @@ -271,7 +387,14 @@ async fn index_html() -> Html<&'static str> { -
+
+ +
+

Statistics

+
Percentage of Acceptable Blocks: 0%
+
95th Percentile of Gap Sizes: 0 blocks
+
Longest Unacceptable Streak: 0 blocks
+
- -"#, +"#, ) } From 871d981a665cecef452e04e8a313fe1315f4b534 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 31 Dec 2024 11:34:16 +0100 Subject: [PATCH 069/136] refactoring --- Cargo.lock | 2 + fee_algo_simulation/Cargo.toml | 2 + fee_algo_simulation/src/handlers.rs | 274 ++++++++++ fee_algo_simulation/src/index.html | 300 +++++++++++ fee_algo_simulation/src/main.rs | 671 +------------------------ fee_algo_simulation/src/models.rs | 73 +++ fee_algo_simulation/src/state.rs | 10 + fee_algo_simulation/src/utils.rs | 60 +++ packages/adapters/eth/src/http.rs | 2 +- packages/adapters/eth/src/websocket.rs | 1 - 10 files changed, 749 insertions(+), 646 deletions(-) create mode 100644 fee_algo_simulation/src/handlers.rs create mode 100644 fee_algo_simulation/src/index.html create mode 100644 fee_algo_simulation/src/models.rs create mode 100644 fee_algo_simulation/src/state.rs create mode 100644 fee_algo_simulation/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index c199b736..f96fa9d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2895,6 +2895,8 @@ dependencies = [ "serde_json", "services", "tokio", + "tracing", + "tracing-subscriber", "xdg", ] diff --git a/fee_algo_simulation/Cargo.toml b/fee_algo_simulation/Cargo.toml index 969f37d9..5dd2c28c 100644 --- a/fee_algo_simulation/Cargo.toml +++ b/fee_algo_simulation/Cargo.toml @@ -18,6 +18,8 @@ itertools = { workspace = true, features = ["use_std"] } # TODO: move this to workspace level axum = { version = "0.7" } eth = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["fmt", "json"] } # TODO: segfault features tokio = { workspace = true, features = [ "macros", diff --git a/fee_algo_simulation/src/handlers.rs b/fee_algo_simulation/src/handlers.rs new file mode 100644 index 00000000..8a408c58 --- /dev/null +++ b/fee_algo_simulation/src/handlers.rs @@ -0,0 +1,274 @@ +use std::num::{NonZeroU32, NonZeroU64}; +use std::time::Duration; + +use super::models::{FeeDataPoint, FeeParams, FeeResponse, FeeStats}; +use super::state::AppState; +use super::utils::last_n_blocks; + +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{Html, IntoResponse}; +use axum::Json; +use services::historical_fees::port::l1::Api; +use services::historical_fees::service::calculate_blob_tx_fee; + +use tracing::{error, info}; + +/// Handler for the `/fees` endpoint. +pub async fn get_fees( + State(state): State, + Query(params): Query, +) -> impl IntoResponse { + // Resolve user inputs or use defaults + let ending_height = params.ending_height.unwrap_or(21_514_918); + let amount_of_blocks = params + .amount_of_blocks + .unwrap_or(state.num_blocks_per_month); + + let short = params.short.unwrap_or(25); // default short + let long = params.long.unwrap_or(300); // default long + + let max_l2 = params.max_l2_blocks_behind.unwrap_or(8 * 3600); + let start_discount = params.start_discount_percentage.unwrap_or(0.10); + let end_premium = params.end_premium_percentage.unwrap_or(0.20); + + let always_acceptable_fee = match params.always_acceptable_fee { + Some(v) => match v.parse::() { + Ok(val) => val, + Err(_) => { + return ( + axum::http::StatusCode::BAD_REQUEST, + "Invalid always_acceptable_fee value", + ) + .into_response() + } + }, + None => 1_000_000_000_000_000, + }; + + let num_blobs = params.num_blobs.unwrap_or(6); // default to 6 blobs + + let num_l2_blocks_behind = params.num_l2_blocks_behind.unwrap_or(0); + + // Build SmaFeeAlgo config from user’s inputs + let config = services::state_committer::AlgoConfig { + sma_periods: services::historical_fees::service::SmaPeriods { + short: match NonZeroU64::new(short) { + Some(nz) => nz, + None => NonZeroU64::new(1).unwrap(), + }, + long: match NonZeroU64::new(long) { + Some(nz) => nz, + None => NonZeroU64::new(1).unwrap(), + }, + }, + fee_thresholds: services::state_committer::FeeThresholds { + max_l2_blocks_behind: match NonZeroU32::new(max_l2) { + Some(nz) => nz, + None => NonZeroU32::new(1).unwrap(), + }, + start_discount_percentage: match services::state_committer::Percentage::try_from( + start_discount, + ) { + Ok(p) => p, + Err(_) => services::state_committer::Percentage::ZERO, + }, + end_premium_percentage: match services::state_committer::Percentage::try_from( + end_premium, + ) { + Ok(p) => p, + Err(_) => services::state_committer::Percentage::ZERO, + }, + always_acceptable_fee, + }, + }; + + let sma_algo = + services::state_committer::SmaFeeAlgo::new(state.historical_fees.clone(), config); + + // Determine which blocks to fetch + let start_height = ending_height.saturating_sub(amount_of_blocks); + let range = start_height..=ending_height; + + // Fetch fees from the caching API + let fees_res = state.caching_api.fees(range).await; + let seq_fees = match fees_res { + Ok(fees) => fees, + Err(_) => { + error!( + "Failed to fetch fees for range {}-{}", + start_height, ending_height + ); + return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to fetch fees").into_response(); + } + }; + + // Fetch the last block's time + let last_block_height = seq_fees.last().height; + let last_block_time = match state + .caching_api + .inner() + .get_block_time(last_block_height) + .await + { + Ok(Some(t)) => t, // Assuming `t` is a UNIX timestamp in seconds + Ok(None) => { + error!( + "Failed to retrieve the last block's time for block height {}", + last_block_height + ); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to retrieve the last block's time", + ) + .into_response(); + } + Err(e) => { + error!("Error while fetching the last block's time: {:?}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "Error while fetching the last block's time", + ) + .into_response(); + } + }; + + info!("Last block time: {}", last_block_time); + + // Prepare data points + let mut data = Vec::with_capacity(seq_fees.len()); + + for block_fees in seq_fees.into_iter() { + let block_height = block_fees.height; + let current_fee_wei = calculate_blob_tx_fee(num_blobs, &block_fees.fees); + + // Fetch the shortTerm + longTerm SMA at exactly this height + let short_term_sma = match state + .historical_fees + .calculate_sma(last_n_blocks(block_height, config.sma_periods.short)) + .await + { + Ok(f) => f, + Err(e) => { + error!( + "Error calculating short-term SMA for block {}: {:?}", + block_height, e + ); + services::historical_fees::port::l1::Fees::default() + } + }; + + let long_term_sma = match state + .historical_fees + .calculate_sma(last_n_blocks(block_height, config.sma_periods.long)) + .await + { + Ok(f) => f, + Err(e) => { + error!( + "Error calculating long-term SMA for block {}: {:?}", + block_height, e + ); + services::historical_fees::port::l1::Fees::default() + } + }; + + let short_fee_wei = calculate_blob_tx_fee(num_blobs, &short_term_sma); + let long_fee_wei = calculate_blob_tx_fee(num_blobs, &long_term_sma); + + let acceptable = match sma_algo + .fees_acceptable(num_blobs, num_l2_blocks_behind, block_height) + .await + { + Ok(decision) => decision, + Err(e) => { + error!( + "Error determining fee acceptability for block {}: {:?}", + block_height, e + ); + false // or handle error differently + } + }; + + // Calculate the time for this block + let block_gap = last_block_height - block_height; + let block_time = last_block_time - Duration::from_secs(12 * block_gap); + let block_time_str = block_time.to_rfc3339(); // ISO 8601 format + + // Convert fees from wei to ETH with 4 decimal places + let current_fee_eth = (current_fee_wei as f64) / 1e18; + let short_fee_eth = (short_fee_wei as f64) / 1e18; + let long_fee_eth = (long_fee_wei as f64) / 1e18; + + data.push(FeeDataPoint { + block_height, + block_time: block_time_str, + current_fee: format!("{:.4}", current_fee_eth), + short_fee: format!("{:.4}", short_fee_eth), + long_fee: format!("{:.4}", long_fee_eth), + acceptable, + }); + } + + // Calculate statistics + let stats = calculate_statistics(&data); + + let response = FeeResponse { data, stats }; + + // Return as JSON + Json(response).into_response() +} + +/// Handler for the root `/` endpoint, serving the HTML page. +pub async fn index_html() -> Html<&'static str> { + // The HTML content is stored in `index.html` and included at compile time. + Html(include_str!("./index.html")) +} + +/// Calculates statistics from the fee data. +fn calculate_statistics(data: &Vec) -> FeeStats { + let total_blocks = data.len() as f64; + let acceptable_blocks = data.iter().filter(|d| d.acceptable).count() as f64; + let percentage_acceptable = if total_blocks > 0.0 { + (acceptable_blocks / total_blocks) * 100.0 + } else { + 0.0 + }; + + // Calculate gap sizes (streaks of unacceptable blocks) + let mut gap_sizes = Vec::new(); + let mut current_gap = 0; + + for d in data { + if !d.acceptable { + current_gap += 1; + } else if current_gap > 0 { + gap_sizes.push(current_gap); + current_gap = 0; + } + } + + // Push the last gap if it ends with an unacceptable streak + if current_gap > 0 { + gap_sizes.push(current_gap); + } + + // Calculate the 95th percentile of gap sizes + let percentile_95_gap_size = if !gap_sizes.is_empty() { + let mut sorted_gaps = gap_sizes.clone(); + sorted_gaps.sort_unstable(); + let index = ((sorted_gaps.len() as f64) * 0.95).ceil() as usize - 1; + sorted_gaps[index.min(sorted_gaps.len() - 1)] + } else { + 0 + }; + + // Find the longest unacceptable streak + let longest_unacceptable_streak = gap_sizes.iter().cloned().max().unwrap_or(0); + + FeeStats { + percentage_acceptable, + percentile_95_gap_size, + longest_unacceptable_streak, + } +} diff --git a/fee_algo_simulation/src/index.html b/fee_algo_simulation/src/index.html new file mode 100644 index 00000000..a9af1042 --- /dev/null +++ b/fee_algo_simulation/src/index.html @@ -0,0 +1,300 @@ + + + + + Fee Simulator - Multiple Series + Shading + + + + +

Fee Simulator - Multiple Series + Shading

+ +
+ + + + + + +
+ + + + + +
+ + + + + + + + + +
+ + + + + + +
+ + + + +
+ + + + + + + + +
+ +
+ +
+

Statistics

+
Percentage of Acceptable Blocks: 0%
+
95th Percentile of Gap Sizes: 0 blocks
+
Longest Unacceptable Streak: 0 blocks
+
Loading...
+
+ + + + diff --git a/fee_algo_simulation/src/main.rs b/fee_algo_simulation/src/main.rs index e3673a40..3627b49f 100644 --- a/fee_algo_simulation/src/main.rs +++ b/fee_algo_simulation/src/main.rs @@ -1,666 +1,48 @@ -use std::{ - net::SocketAddr, - num::{NonZeroU32, NonZeroU64}, - ops::RangeInclusive, - path::PathBuf, - time::Duration, -}; +use std::net::SocketAddr; use anyhow::Result; -use axum::{ - extract::{Query, State}, - http::StatusCode, - response::{Html, IntoResponse}, - routing::get, - Json, Router, -}; -use serde::{Deserialize, Serialize}; -use services::{ - historical_fees::{ - port::{ - cache::CachingApi, - l1::{Api, BlockFees, Fees}, - }, - service::{calculate_blob_tx_fee, HistoricalFees, SmaPeriods}, - }, - state_committer::{AlgoConfig, FeeThresholds, Percentage, SmaFeeAlgo}, - types::{DateTime, Utc}, -}; -use xdg::BaseDirectories; +use axum::{routing::get, Router}; +use services::historical_fees::service::HistoricalFees; -#[derive(Debug, Serialize, Deserialize, Default)] -struct SavedFees { - fees: Vec, -} - -const URL: &str = "https://eth.llamarpc.com"; - -/// Same fee_cache.json location logic -fn fee_file() -> PathBuf { - let xdg = BaseDirectories::with_prefix("fee_simulation").unwrap(); - if let Some(cache) = xdg.find_cache_file("fee_cache.json") { - cache - } else { - xdg.place_data_file("fee_cache.json").unwrap() - } -} - -/// Load from disk -fn load_cache() -> Vec<(u64, Fees)> { - let Ok(contents) = std::fs::read_to_string(fee_file()) else { - return vec![]; - }; - let fees: SavedFees = serde_json::from_str(&contents).unwrap_or_default(); - fees.fees.into_iter().map(|f| (f.height, f.fees)).collect() -} - -/// Save to disk -fn save_cache(cache: impl IntoIterator) -> anyhow::Result<()> { - let fees = SavedFees { - fees: cache - .into_iter() - .map(|(height, fees)| BlockFees { height, fees }) - .collect(), - }; - std::fs::write(fee_file(), serde_json::to_string(&fees)?)?; - Ok(()) -} - -/// Shared state across routes -#[derive(Clone)] -struct AppState { - caching_api: CachingApi, - historical_fees: HistoricalFees>, - num_blocks_per_month: u64, -} - -/// Query params for /fees -#[derive(Debug, Deserialize)] -struct FeeParams { - ending_height: Option, - amount_of_blocks: Option, - - // Fee Algo settings - short: Option, - long: Option, - max_l2_blocks_behind: Option, - start_discount_percentage: Option, - end_premium_percentage: Option, - always_acceptable_fee: Option, - - // Number of blobs per transaction - num_blobs: Option, - - // How many L2 blocks behind are we? If none is given, default 0 - num_l2_blocks_behind: Option, -} - -/// Response struct for each fee data point -#[derive(Debug, Serialize)] -struct FeeDataPoint { - #[serde(rename = "blockHeight")] - block_height: u64, - #[serde(rename = "blockTime")] - block_time: String, // ISO 8601 format - #[serde(rename = "currentFee")] - current_fee: String, // ETH with 4 decimal places - #[serde(rename = "shortFee")] - short_fee: String, // ETH with 4 decimal places - #[serde(rename = "longFee")] - long_fee: String, // ETH with 4 decimal places - acceptable: bool, -} - -/// Statistics struct -#[derive(Debug, Serialize)] -struct FeeStats { - #[serde(rename = "percentageAcceptable")] - percentage_acceptable: f64, // Percentage of acceptable blocks - #[serde(rename = "percentile95GapSize")] - percentile_95_gap_size: u64, // 95th percentile of gap sizes in blocks - #[serde(rename = "longestUnacceptableStreak")] - longest_unacceptable_streak: u64, // Longest consecutive unacceptable blocks -} - -/// Complete response struct -#[derive(Debug, Serialize)] -struct FeeResponse { - data: Vec, - stats: FeeStats, -} - -/// GET /fees -/// -/// Returns JSON: { data: [...], stats: {...} } -async fn get_fees( - State(state): State, - Query(params): Query, -) -> impl IntoResponse { - // 1) Resolve user inputs or use defaults - let ending_height = params.ending_height.unwrap_or(21_514_918); - let amount_of_blocks = params - .amount_of_blocks - .unwrap_or(state.num_blocks_per_month); - - let short = params.short.unwrap_or(25); // default short - let long = params.long.unwrap_or(300); // default long - - let max_l2 = params.max_l2_blocks_behind.unwrap_or(8 * 3600); - let start_discount = params.start_discount_percentage.unwrap_or(0.10); - let end_premium = params.end_premium_percentage.unwrap_or(0.20); - - let always_acceptable_fee = match params.always_acceptable_fee { - Some(v) => match v.parse::() { - Ok(val) => val, - Err(_) => { - return ( - StatusCode::BAD_REQUEST, - "Invalid always_acceptable_fee value", - ) - .into_response() - } - }, - None => 1_000_000_000_000_000, - }; - - let num_blobs = params.num_blobs.unwrap_or(6); // default to 6 blobs - - let num_l2_blocks_behind = params.num_l2_blocks_behind.unwrap_or(0); - - // 2) Build an SmaFeeAlgo config from user’s inputs - let config = AlgoConfig { - sma_periods: SmaPeriods { - short: match NonZeroU64::new(short) { - Some(nz) => nz, - None => NonZeroU64::new(1).unwrap(), - }, - long: match NonZeroU64::new(long) { - Some(nz) => nz, - None => NonZeroU64::new(1).unwrap(), - }, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: match NonZeroU32::new(max_l2) { - Some(nz) => nz, - None => NonZeroU32::new(1).unwrap(), - }, - start_discount_percentage: match Percentage::try_from(start_discount) { - Ok(p) => p, - Err(_) => Percentage::ZERO, - }, - end_premium_percentage: match Percentage::try_from(end_premium) { - Ok(p) => p, - Err(_) => Percentage::ZERO, - }, - always_acceptable_fee, - }, - }; - - let sma_algo = SmaFeeAlgo::new(state.historical_fees.clone(), config); - - // 3) Determine which blocks to fetch - let start_height = ending_height.saturating_sub(amount_of_blocks); - let range = start_height..=ending_height; - - // 4) Actually fetch from the caching API - let fees_res = state.caching_api.fees(range).await; - let Ok(seq_fees) = fees_res else { - return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to fetch fees").into_response(); - }; - - // 5) Fetch the last block's time - // Assuming CachingApi provides access to the underlying client - // You might need to adjust this based on your actual CachingApi implementation - let last_block_height = seq_fees.last().height; - let last_block_time = state - .caching_api - .inner() - .get_block_time(last_block_height) - .await - .unwrap() - .unwrap(); - - // 6) Prepare data points - let mut data = Vec::with_capacity(seq_fees.len()); - - for block_fees in seq_fees.into_iter() { - let block_height = block_fees.height; - let current_fee_wei = calculate_blob_tx_fee(num_blobs, &block_fees.fees); - - // Fetch the shortTerm + longTerm SMA at exactly this height - let short_term_sma = state - .historical_fees - .calculate_sma(last_n_blocks(block_height, config.sma_periods.short)) - .await - .unwrap(); - - let long_term_sma = state - .historical_fees - .calculate_sma(last_n_blocks(block_height, config.sma_periods.long)) - .await - .unwrap(); - - let short_fee_wei = calculate_blob_tx_fee(num_blobs, &short_term_sma); - let long_fee_wei = calculate_blob_tx_fee(num_blobs, &long_term_sma); - - let acceptable = sma_algo - .fees_acceptable(num_blobs, num_l2_blocks_behind, block_height) - .await - .unwrap(); - - // Calculate the time for this block - let block_gap = last_block_height - block_height; - - let block_time = last_block_time - Duration::from_secs(12 * block_gap); - let block_time_str = block_time.to_rfc3339(); // ISO 8601 format - - // Convert fees from wei to ETH with 4 decimal places - let current_fee_eth = (current_fee_wei as f64) / 1e18; - let short_fee_eth = (short_fee_wei as f64) / 1e18; - let long_fee_eth = (long_fee_wei as f64) / 1e18; - - data.push(FeeDataPoint { - block_height, - block_time: block_time_str, - current_fee: format!("{:.4}", current_fee_eth), - short_fee: format!("{:.4}", short_fee_eth), - long_fee: format!("{:.4}", long_fee_eth), - acceptable, - }); - } - - // 7) Calculate statistics - let total_blocks = data.len() as f64; - let acceptable_blocks = data.iter().filter(|d| d.acceptable).count() as f64; - let percentage_acceptable = if total_blocks > 0.0 { - (acceptable_blocks / total_blocks) * 100.0 - } else { - 0.0 - }; - - // 8) Calculate gap sizes (streaks of unacceptable blocks) - let mut gap_sizes = Vec::new(); - let mut current_gap = 0; - - for d in &data { - if !d.acceptable { - current_gap += 1; - } else if current_gap > 0 { - gap_sizes.push(current_gap); - current_gap = 0; - } - } - - // Push the last gap if it ends with an unacceptable streak - if current_gap > 0 { - gap_sizes.push(current_gap); - } - - // 9) Calculate the 95th percentile of gap sizes - let percentile_95_gap_size = if !gap_sizes.is_empty() { - let mut sorted_gaps = gap_sizes.clone(); - sorted_gaps.sort_unstable(); - let index = ((sorted_gaps.len() as f64) * 0.95).ceil() as usize - 1; - sorted_gaps[index.min(sorted_gaps.len() - 1)] - } else { - 0 - }; - - // 10) Find the longest unacceptable streak - let longest_unacceptable_streak = gap_sizes.iter().cloned().max().unwrap_or(0); - - let stats = FeeStats { - percentage_acceptable, - percentile_95_gap_size, - longest_unacceptable_streak, - }; - - let response = FeeResponse { data, stats }; - - // Return as JSON - Json(response).into_response() -} - -/// Helper: compute [current_block - (n-1) .. current_block] -fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { - current_block.saturating_sub(n.get().saturating_sub(1))..=current_block -} - -/// The HTML page at GET / -async fn index_html() -> Html<&'static str> { - Html( - r#" - - - - Fee Simulator - Multiple Series + Shading - - - - -

Fee Simulator - Multiple Series + Shading

- -
- - - - - - -
- - - - - -
- - - - - - - - - -
- - - - - - -
- - - - -
- - - - - - - - -
- -
- -
-

Statistics

-
Percentage of Acceptable Blocks: 0%
-
95th Percentile of Gap Sizes: 0 blocks
-
Longest Unacceptable Streak: 0 blocks
-
Loading...
-
- - - -"#, - ) -} +mod handlers; +mod models; +mod state; +mod utils; #[tokio::main] async fn main() -> Result<()> { - // 1) Create your ETH HTTP client - let client = eth::HttpClient::new(URL).unwrap(); + // Initialize tracing subscriber for logging + tracing_subscriber::fmt::init(); + + // Initialize the HTTP client for Ethereum RPC + let client = eth::HttpClient::new(models::URL).unwrap(); - // 2) ~1 month = 259200 blocks (approx) if 12s per block + // Calculate the number of blocks per month (~259200 blocks) let num_blocks_per_month = 30 * 24 * 3600 / 12; // 259200 blocks - // 3) Build your CachingApi & import any existing cache - let caching_api = CachingApi::new(client, num_blocks_per_month * 2); - caching_api.import(load_cache()).await; + // Build the CachingApi and import any existing cache + let caching_api = utils::CachingApiBuilder::new(client, num_blocks_per_month * 2) + .build() + .await?; + caching_api.import(utils::load_cache()).await; - // 4) Build your HistoricalFees + // Build HistoricalFees service let historical_fees = HistoricalFees::new(caching_api.clone()); - // 5) Bundle everything into state - let state = AppState { + // Bundle everything into shared application state + let state = state::AppState { caching_api: caching_api.clone(), historical_fees, num_blocks_per_month: num_blocks_per_month as u64, }; - // 6) Axum router: serve front-end + fees endpoint + // Set up Axum router with routes and shared state let app = Router::new() - .route("/", get(index_html)) - .route("/fees", get(get_fees)) + .route("/", get(handlers::index_html)) + .route("/fees", get(handlers::get_fees)) .with_state(state); - // 7) Run server + // Define the server address let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); println!("Server listening on http://{}", addr); @@ -670,7 +52,8 @@ async fn main() -> Result<()> { .unwrap(); axum::serve(listener, app).await.unwrap(); - save_cache(caching_api.export().await)?; + // Save cache on shutdown + utils::save_cache(caching_api.export().await)?; Ok(()) } diff --git a/fee_algo_simulation/src/models.rs b/fee_algo_simulation/src/models.rs new file mode 100644 index 00000000..a71af1aa --- /dev/null +++ b/fee_algo_simulation/src/models.rs @@ -0,0 +1,73 @@ +use serde::{Deserialize, Serialize}; +use services::historical_fees::port::l1::BlockFees; + +/// Ethereum RPC URL. +pub const URL: &str = "https://eth.llamarpc.com"; + +/// Structure for saving fees to cache. +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SavedFees { + pub fees: Vec, +} + +/// Query parameters for the `/fees` endpoint. +#[derive(Debug, Deserialize)] +pub struct FeeParams { + pub ending_height: Option, + pub amount_of_blocks: Option, + + // Fee Algo settings + pub short: Option, + pub long: Option, + pub max_l2_blocks_behind: Option, + pub start_discount_percentage: Option, + pub end_premium_percentage: Option, + pub always_acceptable_fee: Option, + + // Number of blobs per transaction + pub num_blobs: Option, + + // How many L2 blocks behind are we? If none is given, default 0 + pub num_l2_blocks_behind: Option, +} + +/// Response struct for each fee data point. +#[derive(Debug, Serialize)] +pub struct FeeDataPoint { + #[serde(rename = "blockHeight")] + pub block_height: u64, + + #[serde(rename = "blockTime")] + pub block_time: String, // ISO 8601 format + + #[serde(rename = "currentFee")] + pub current_fee: String, // ETH with 4 decimal places + + #[serde(rename = "shortFee")] + pub short_fee: String, // ETH with 4 decimal places + + #[serde(rename = "longFee")] + pub long_fee: String, // ETH with 4 decimal places + + pub acceptable: bool, +} + +/// Statistics struct. +#[derive(Debug, Serialize)] +pub struct FeeStats { + #[serde(rename = "percentageAcceptable")] + pub percentage_acceptable: f64, // Percentage of acceptable blocks + + #[serde(rename = "percentile95GapSize")] + pub percentile_95_gap_size: u64, // 95th percentile of gap sizes in blocks + + #[serde(rename = "longestUnacceptableStreak")] + pub longest_unacceptable_streak: u64, // Longest consecutive unacceptable blocks +} + +/// Complete response struct. +#[derive(Debug, Serialize)] +pub struct FeeResponse { + pub data: Vec, + pub stats: FeeStats, +} diff --git a/fee_algo_simulation/src/state.rs b/fee_algo_simulation/src/state.rs new file mode 100644 index 00000000..3d2408e1 --- /dev/null +++ b/fee_algo_simulation/src/state.rs @@ -0,0 +1,10 @@ +use super::HistoricalFees; +use services::historical_fees::port::cache::CachingApi; + +/// Shared state across routes. +#[derive(Clone)] +pub struct AppState { + pub caching_api: CachingApi, + pub historical_fees: HistoricalFees>, + pub num_blocks_per_month: u64, +} diff --git a/fee_algo_simulation/src/utils.rs b/fee_algo_simulation/src/utils.rs new file mode 100644 index 00000000..cf17689f --- /dev/null +++ b/fee_algo_simulation/src/utils.rs @@ -0,0 +1,60 @@ +use super::models::SavedFees; +use anyhow::Result; +use services::historical_fees::port::cache::CachingApi; +use std::{ops::RangeInclusive, path::PathBuf}; +use xdg::BaseDirectories; + +/// Path to the fee cache file. +pub fn fee_file() -> PathBuf { + let xdg = BaseDirectories::with_prefix("fee_simulation").unwrap(); + if let Some(cache) = xdg.find_cache_file("fee_cache.json") { + cache + } else { + xdg.place_data_file("fee_cache.json").unwrap() + } +} + +/// Load fees from the cache file. +pub fn load_cache() -> Vec<(u64, services::historical_fees::port::l1::Fees)> { + let Ok(contents) = std::fs::read_to_string(fee_file()) else { + return vec![]; + }; + let fees: SavedFees = serde_json::from_str(&contents).unwrap_or_default(); + fees.fees.into_iter().map(|f| (f.height, f.fees)).collect() +} + +/// Save fees to the cache file. +pub fn save_cache( + cache: impl IntoIterator, +) -> anyhow::Result<()> { + let fees = SavedFees { + fees: cache + .into_iter() + .map(|(height, fees)| services::historical_fees::port::l1::BlockFees { height, fees }) + .collect(), + }; + std::fs::write(fee_file(), serde_json::to_string(&fees)?)?; + Ok(()) +} + +/// Helper to create and configure CachingApi. +pub struct CachingApiBuilder { + client: eth::HttpClient, + cache_size: usize, +} + +impl CachingApiBuilder { + pub fn new(client: eth::HttpClient, cache_size: usize) -> Self { + Self { client, cache_size } + } + + pub async fn build(self) -> Result> { + let caching_api = CachingApi::new(self.client, self.cache_size); + Ok(caching_api) + } +} + +/// Helper to calculate the last N blocks as a range. +pub fn last_n_blocks(current_block: u64, n: std::num::NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block +} diff --git a/packages/adapters/eth/src/http.rs b/packages/adapters/eth/src/http.rs index 0f69eb54..4251da90 100644 --- a/packages/adapters/eth/src/http.rs +++ b/packages/adapters/eth/src/http.rs @@ -1,4 +1,4 @@ -use std::{ops::RangeInclusive, time}; +use std::ops::RangeInclusive; use alloy::{ providers::{Provider as AlloyProvider, ProviderBuilder, RootProvider}, diff --git a/packages/adapters/eth/src/websocket.rs b/packages/adapters/eth/src/websocket.rs index 49d5c1d7..8fc4d745 100644 --- a/packages/adapters/eth/src/websocket.rs +++ b/packages/adapters/eth/src/websocket.rs @@ -9,7 +9,6 @@ use alloy::{ signers::{local::PrivateKeySigner, Signature}, }; use delegate::delegate; -use futures::{StreamExt, TryStreamExt}; use serde::Deserialize; use services::{ historical_fees::port::l1::SequentialBlockFees, From 0600329a836fa404da9364343b85f9013894d210 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 31 Dec 2024 11:45:50 +0100 Subject: [PATCH 070/136] add some bootstrap --- fee_algo_simulation/src/index.html | 214 +++++++++++++++++++++-------- 1 file changed, 154 insertions(+), 60 deletions(-) diff --git a/fee_algo_simulation/src/index.html b/fee_algo_simulation/src/index.html index a9af1042..5354f6e6 100644 --- a/fee_algo_simulation/src/index.html +++ b/fee_algo_simulation/src/index.html @@ -3,11 +3,21 @@ Fee Simulator - Multiple Series + Shading + + + -

Fee Simulator - Multiple Series + Shading

- -
- - - - - - -
- - - - - -
- - - - - - - - - -
- - - - - - -
- - - - -
- - - - - - - - +
+

Fee Simulator - Multiple Series + Shading

+ + +
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ +
+
+ + +
e.g., 8 hours worth
+
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ +
+
+ + +
Value in Wei (e.g., 1 ETH = 1,000,000,000,000,000,000 Wei)
+
+
+ + +
+ + + + + + +
+
+
+
+ + +
+
+
+
+
+ + +
+
+

Statistics

+
+
+ +
+
+ +
+
+ +
+
+
+ +

Loading...

+
+
+
-
- -
-

Statistics

-
Percentage of Acceptable Blocks: 0%
-
95th Percentile of Gap Sizes: 0 blocks
-
Longest Unacceptable Streak: 0 blocks
-
Loading...
-
+ + - +