diff --git a/Cargo.lock b/Cargo.lock index 3b82182..4a739a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "thirdweb-core", "thiserror 2.0.12", "tokio", "tracing", diff --git a/executors/Cargo.toml b/executors/Cargo.toml index 2148d9e..283ef56 100644 --- a/executors/Cargo.toml +++ b/executors/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] hex = "0.4.3" alloy = { version = "1.0.8", features = ["serde"] } +thirdweb-core = { version = "0.1.0", path = "../thirdweb-core" } hmac = "0.12.1" reqwest = "0.12.15" serde = "1.0.219" diff --git a/executors/src/eoa/events.rs b/executors/src/eoa/events.rs new file mode 100644 index 0000000..b9d62be --- /dev/null +++ b/executors/src/eoa/events.rs @@ -0,0 +1,134 @@ +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; +use twmq::job::RequeuePosition; + +use crate::{ + eoa::{ + store::{SubmittedTransaction, TransactionData}, + worker::{ConfirmedTransactionWithRichReceipt, EoaExecutorWorkerError}, + }, + webhook::envelope::{ + BareWebhookNotificationEnvelope, SerializableFailData, SerializableNackData, + SerializableSuccessData, StageEvent, + }, +}; + +pub struct EoaExecutorEvent { + pub transaction_data: TransactionData, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaSendAttemptNackData { + pub nonce: u64, + pub error: EoaExecutorWorkerError, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EoaExecutorStage { + SendAttempt, + TransactionReplaced, + TransactionConfirmed, +} + +impl Display for EoaExecutorStage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EoaExecutorStage::SendAttempt => write!(f, "send_attempt"), + EoaExecutorStage::TransactionReplaced => write!(f, "transaction_replaced"), + EoaExecutorStage::TransactionConfirmed => write!(f, "transaction_confirmed"), + } + } +} + +const EXECUTOR_NAME: &str = "eoa"; + +impl EoaExecutorEvent { + pub fn send_attempt_success_envelope( + &self, + submitted_transaction: SubmittedTransaction, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: submitted_transaction.clone(), + }, + } + } + + pub fn send_attempt_nack_envelope( + &self, + nonce: u64, + error: EoaExecutorWorkerError, + attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), + event_type: StageEvent::Nack, + payload: SerializableNackData { + error: EoaSendAttemptNackData { + nonce, + error: error.clone(), + }, + delay_ms: None, + position: RequeuePosition::Last, + attempt_number, + max_attempts: None, + next_retry_at: None, + }, + } + } + + pub fn transaction_replaced_envelope( + &self, + replaced_transaction: SubmittedTransaction, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::TransactionReplaced.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: replaced_transaction.clone(), + }, + } + } + + pub fn transaction_confirmed_envelope( + &self, + confirmed_transaction: ConfirmedTransactionWithRichReceipt, + ) -> BareWebhookNotificationEnvelope> + { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::TransactionConfirmed.to_string(), + event_type: StageEvent::Success, + payload: SerializableSuccessData { + result: confirmed_transaction.clone(), + }, + } + } + + pub fn transaction_failed_envelope( + &self, + error: EoaExecutorWorkerError, + final_attempt_number: u32, + ) -> BareWebhookNotificationEnvelope> { + BareWebhookNotificationEnvelope { + transaction_id: self.transaction_data.transaction_id.clone(), + executor_name: EXECUTOR_NAME.to_string(), + stage_name: EoaExecutorStage::SendAttempt.to_string(), + event_type: StageEvent::Failure, + payload: SerializableFailData { + error: error.clone(), + final_attempt_number, + }, + } + } +} diff --git a/executors/src/eoa/mod.rs b/executors/src/eoa/mod.rs index c7186f7..89dd7e2 100644 --- a/executors/src/eoa/mod.rs +++ b/executors/src/eoa/mod.rs @@ -1,6 +1,8 @@ pub mod error_classifier; +pub mod events; pub mod store; pub mod worker; + pub use error_classifier::{EoaErrorMapper, EoaExecutionError, RecoveryStrategy}; pub use store::{EoaExecutorStore, EoaTransactionRequest}; pub use worker::{EoaExecutorWorker, EoaExecutorWorkerJobData}; diff --git a/executors/src/eoa/store.rs b/executors/src/eoa/store.rs deleted file mode 100644 index d377300..0000000 --- a/executors/src/eoa/store.rs +++ /dev/null @@ -1,2156 +0,0 @@ -use alloy::consensus::{Signed, Transaction, TypedTransaction}; -use alloy::network::AnyTransactionReceipt; -use alloy::primitives::{Address, B256, Bytes, U256}; -use chrono; -use engine_core::chain::RpcCredentials; -use engine_core::credentials::SigningCredential; -use engine_core::execution_options::WebhookOptions; -use engine_core::transaction::TransactionTypeData; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::future::Future; -use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; - -pub trait SafeRedisTransaction: Send + Sync { - fn name(&self) -> &str; - fn operation(&self, pipeline: &mut Pipeline); - fn validation( - &self, - conn: &mut ConnectionManager, - ) -> impl Future> + Send; - fn watch_keys(&self) -> Vec; -} - -struct MovePendingToBorrowedWithRecycledNonce { - recycled_key: String, - pending_key: String, - transaction_id: String, - borrowed_key: String, - nonce: u64, - prepared_tx_json: String, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonce { - fn name(&self) -> &str { - "pending->borrowed with recycled nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove nonce from recycled set (we know it exists) - pipeline.zrem(&self.recycled_key, self.nonce); - // Remove transaction from pending (we know it exists) - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.recycled_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check if nonce exists in recycled set - let nonce_score: Option = conn.zscore(&self.recycled_key, self.nonce).await?; - if nonce_score.is_none() { - return Err(TransactionStoreError::NonceNotInRecycledSet { nonce: self.nonce }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MovePendingToBorrowedWithNewNonce { - optimistic_key: String, - pending_key: String, - nonce: u64, - prepared_tx_json: String, - transaction_id: String, - borrowed_key: String, - eoa: Address, - chain_id: u64, -} - -impl SafeRedisTransaction for MovePendingToBorrowedWithNewNonce { - fn name(&self) -> &str { - "pending->borrowed with new nonce" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Increment optimistic nonce - pipeline.incr(&self.optimistic_key, 1); - // Remove transaction from pending - pipeline.lrem(&self.pending_key, 0, &self.transaction_id); - // Store borrowed transaction - pipeline.hset( - &self.borrowed_key, - self.nonce.to_string(), - &self.prepared_tx_json, - ); - } - - fn watch_keys(&self) -> Vec { - vec![self.optimistic_key.clone(), self.pending_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Check current optimistic nonce - let current_optimistic: Option = conn.get(&self.optimistic_key).await?; - let current_nonce = match current_optimistic { - Some(nonce) => nonce, - None => { - return Err(TransactionStoreError::NonceSyncRequired { - eoa: self.eoa, - chain_id: self.chain_id, - }); - } - }; - - if current_nonce != self.nonce { - return Err(TransactionStoreError::OptimisticNonceChanged { - expected: self.nonce, - actual: current_nonce, - }); - } - - // Check if transaction exists in pending - let pending_transactions: Vec = conn.lrange(&self.pending_key, 0, -1).await?; - if !pending_transactions.contains(&self.transaction_id) { - return Err(TransactionStoreError::TransactionNotInPendingQueue { - transaction_id: self.transaction_id.clone(), - }); - } - - Ok(()) - } -} - -struct MoveBorrowedToSubmitted { - nonce: u64, - hash: String, - transaction_id: String, - borrowed_key: String, - submitted_key: String, - hash_to_id_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToSubmitted { - fn name(&self) -> &str { - "borrowed->submitted" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add to submitted with hash:id format - let hash_id_value = format!("{}:{}", self.hash, self.transaction_id); - pipeline.zadd(&self.submitted_key, &hash_id_value, self.nonce); - - // Still maintain hash-to-ID mapping for backward compatibility and external lookups - pipeline.set(&self.hash_to_id_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -struct MoveBorrowedToRecycled { - nonce: u64, - transaction_id: String, - borrowed_key: String, - recycled_key: String, - pending_key: String, -} - -impl SafeRedisTransaction for MoveBorrowedToRecycled { - fn name(&self) -> &str { - "borrowed->recycled" - } - - fn operation(&self, pipeline: &mut Pipeline) { - // Remove from borrowed (we know it exists) - pipeline.hdel(&self.borrowed_key, self.nonce.to_string()); - - // Add nonce to recycled set (with timestamp as score) - pipeline.zadd(&self.recycled_key, self.nonce, self.nonce); - - // Add transaction back to pending - pipeline.lpush(&self.pending_key, &self.transaction_id); - } - - fn watch_keys(&self) -> Vec { - vec![self.borrowed_key.clone()] - } - - async fn validation(&self, conn: &mut ConnectionManager) -> Result<(), TransactionStoreError> { - // Validate that borrowed transaction actually exists - let borrowed_tx: Option = conn - .hget(&self.borrowed_key, self.nonce.to_string()) - .await?; - if borrowed_tx.is_none() { - return Err(TransactionStoreError::TransactionNotInBorrowedState { - transaction_id: self.transaction_id.clone(), - nonce: self.nonce, - }); - } - Ok(()) - } -} - -/// The actual user request data -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(rename_all = "camelCase")] -pub struct EoaTransactionRequest { - pub transaction_id: String, - pub chain_id: u64, - - pub from: Address, - pub to: Option
, - pub value: U256, - pub data: Bytes, - - #[serde(alias = "gas")] - pub gas_limit: Option, - - pub webhook_options: Option>, - - pub signing_credential: SigningCredential, - pub rpc_credentials: RpcCredentials, - - #[serde(flatten)] - pub transaction_type_data: Option, -} - -/// Active attempt for a transaction (full alloy transaction + metadata) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionAttempt { - pub transaction_id: String, - pub details: Signed, - pub sent_at: u64, // Unix timestamp in milliseconds - pub attempt_number: u32, -} - -/// Transaction data for a transaction_id -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TransactionData { - pub transaction_id: String, - pub user_request: EoaTransactionRequest, - pub receipt: Option, - pub attempts: Vec, -} - -pub struct BorrowedTransaction { - pub transaction_id: String, - pub data: Signed, - pub borrowed_at: chrono::DateTime, -} - -/// Transaction store focused on transaction_id operations and nonce indexing -pub struct EoaExecutorStore { - pub redis: ConnectionManager, - pub namespace: Option, -} - -impl EoaExecutorStore { - pub fn new(redis: ConnectionManager, namespace: Option) -> Self { - Self { redis, namespace } - } - - /// Name of the key for the transaction data - /// - /// Transaction data is stored as a Redis HSET with the following fields: - /// - "user_request": JSON string containing EoaTransactionRequest - /// - "receipt": JSON string containing AnyTransactionReceipt (optional) - /// - "status": String status ("confirmed", "failed", etc.) - /// - "completed_at": String Unix timestamp (optional) - fn transaction_data_key_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), - None => format!("eoa_executor:_tx_data:{transaction_id}"), - } - } - - /// Name of the list for transaction attempts - /// - /// Attempts are stored as a separate Redis LIST where each element is a JSON blob - /// of a TransactionAttempt. This allows efficient append operations. - fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), - None => format!("eoa_executor:tx_attempts:{transaction_id}"), - } - } - - /// Name of the list for pending transactions - fn pending_transactions_list_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:pending_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:pending_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the zset for submitted transactions. nonce -> hash:id - /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) - fn submitted_transactions_zset_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:submitted_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:submitted_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the key that maps transaction hash to transaction id - fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), - None => format!("eoa_executor:tx_hash_to_id:{hash}"), - } - } - - /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` - /// - /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. - /// - /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted - /// - /// If there's no crash, happy path moves borrowed transactions back to pending or submitted - fn borrowed_transactions_hashmap_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - None => format!("eoa_executor:borrowed_txs:{chain_id}:{eoa}"), - } - } - - /// Name of the set that contains recycled nonces. - /// - /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), - /// - /// we add the nonce to this set. - /// - /// These nonces are used with priority, before any other nonces. - fn recycled_nonces_set_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - None => format!("eoa_executor:recycled_nonces:{chain_id}:{eoa}"), - } - } - - /// Optimistic nonce key name. - /// - /// This is used for optimistic nonce tracking. - /// - /// We store the nonce of the last successfuly sent transaction for each EOA. - /// - /// We increment this nonce for each new transaction. - /// - /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. - fn optimistic_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:optimistic_nonce:{chain_id}:{eoa}"), - } - } - - /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. - /// - /// This is a cache for the actual transaction count, which is fetched from the RPC. - /// - /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) - /// - /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. - fn last_transaction_count_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - None => format!("eoa_executor:last_tx_nonce:{chain_id}:{eoa}"), - } - } - - /// EOA health key name. - /// - /// EOA health stores: - /// - cached balance, the timestamp of the last balance fetch - /// - timestamp of the last successful transaction confirmation - /// - timestamp of the last 5 nonce resets - fn eoa_health_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:health:{chain_id}:{eoa}"), - None => format!("eoa_executor:health:{chain_id}:{eoa}"), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EoaHealth { - pub balance: U256, - /// Update the balance threshold when we see out of funds errors - pub balance_threshold: U256, - pub balance_fetched_at: u64, - pub last_confirmation_at: u64, - pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection - pub nonce_resets: Vec, // Last 5 reset timestamps -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BorrowedTransactionData { - pub transaction_id: String, - pub signed_transaction: Signed, - pub hash: String, - pub borrowed_at: u64, -} - -/// Type of nonce allocation for transaction processing -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NonceType { - /// Nonce was recycled from a previously failed transaction - Recycled(u64), - /// Nonce was incremented from the current optimistic counter - Incremented(u64), -} - -impl NonceType { - /// Get the nonce value regardless of type - pub fn nonce(&self) -> u64 { - match self { - NonceType::Recycled(nonce) => *nonce, - NonceType::Incremented(nonce) => *nonce, - } - } - - /// Check if this is a recycled nonce - pub fn is_recycled(&self) -> bool { - matches!(self, NonceType::Recycled(_)) - } - - /// Check if this is an incremented nonce - pub fn is_incremented(&self) -> bool { - matches!(self, NonceType::Incremented(_)) - } -} - -impl EoaExecutorStore { - // ========== BOILERPLATE REDUCTION PATTERN ========== - // - // This implementation uses a helper method `execute_with_watch_and_retry` to reduce - // boilerplate in atomic Redis operations. The pattern separates: - // 1. Validation phase: async closure that checks preconditions - // 2. Pipeline phase: sync closure that builds Redis commands - // - // Benefits: - // - Eliminates ~80 lines of boilerplate per method - // - Centralizes retry logic, lock checking, and error handling - // - Makes individual methods focus on business logic - // - Reduces chance of bugs in WATCH/MULTI/EXEC handling - // - // See examples in: - // - atomic_move_pending_to_borrowed_with_recycled_nonce_v2() - // - atomic_move_pending_to_borrowed_with_new_nonce() - // - move_borrowed_to_submitted() - // - move_borrowed_to_recycled() - - /// Aggressively acquire EOA lock, forcefully taking over from stalled workers - pub async fn acquire_eoa_lock_aggressively( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // First try normal acquisition - let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; - if acquired { - return Ok(()); - } - // Lock exists, forcefully take it over - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Forcefully taking over EOA lock from stalled worker" - ); - // Force set - no expiry, only released by explicit takeover - let _: () = conn.set(&lock_key, worker_id).await?; - Ok(()) - } - - /// Release EOA lock following the spec's finally pattern - pub async fn release_eoa_lock( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - ) -> Result<(), TransactionStoreError> { - // Use existing utility method that handles all the atomic lock checking - match self - .with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - pipeline.del(&lock_key); - }) - .await - { - Ok(()) => { - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Successfully released EOA lock" - ); - Ok(()) - } - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was already taken over, which is fine for release - tracing::debug!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - "Lock already released or taken over by another worker" - ); - Ok(()) - } - Err(e) => { - // Other errors shouldn't fail the worker, just log - tracing::warn!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, - error = %e, - "Failed to release EOA lock" - ); - Ok(()) - } - } - } - - /// Helper to execute atomic operations with proper retry logic and watch handling - /// - /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: - /// - Retry logic with exponential backoff - /// - Lock ownership validation - /// - WATCH key management - /// - Error handling and UNWATCH cleanup - /// - /// ## Usage: - /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. - /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. - /// - /// ## Example: - /// ```rust - /// let safe_tx = MovePendingToBorrowedWithNewNonce { - /// nonce: expected_nonce, - /// prepared_tx_json, - /// transaction_id, - /// borrowed_key, - /// optimistic_key, - /// pending_key, - /// eoa, - /// chain_id, - /// }; - /// - /// self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx).await?; - /// ``` - /// - /// ## When to use this helper: - /// - Operations that implement `SafeRedisTransaction` trait - /// - Need atomic WATCH/MULTI/EXEC with retry logic - /// - Want centralized lock checking and error handling - /// - /// ## When NOT to use this helper: - /// - Simple operations that can use `with_lock_check` instead - /// - Operations that don't need WATCH on multiple keys - /// - Read-only operations that don't modify state - async fn execute_with_watch_and_retry( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - safe_tx: &impl SafeRedisTransaction, - ) -> Result<(), TransactionStoreError> { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, - safe_tx.name(), - eoa, - chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = safe_tx.name(), - "Retrying atomic operation" - ); - } - - // WATCH all specified keys including lock - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in safe_tx.watch_keys() { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Execute validation - match safe_tx.validation(&mut conn).await { - Ok(()) => { - // Build and execute pipeline - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - safe_tx.operation(&mut pipeline); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // State changed, retry - retry_count += 1; - continue; - } - } - } - Err(e) => { - // Validation failed, unwatch and return error - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(e); - } - } - } - } - - /// Example of how to refactor a complex method using the helper to reduce boilerplate - /// This shows the pattern for atomic_move_pending_to_borrowed_with_recycled_nonce - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let safe_tx = MovePendingToBorrowedWithRecycledNonce { - recycled_key: self.recycled_nonces_set_name(eoa, chain_id), - pending_key: self.pending_transactions_list_name(eoa, chain_id), - transaction_id: transaction_id.to_string(), - borrowed_key: self.borrowed_transactions_hashmap_name(eoa, chain_id), - nonce, - prepared_tx_json: serde_json::to_string(prepared_tx)?, - }; - - self.execute_with_watch_and_retry(eoa, chain_id, worker_id, &safe_tx) - .await?; - - Ok(()) - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let prepared_tx_json = serde_json::to_string(prepared_tx)?; - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MovePendingToBorrowedWithNewNonce { - nonce: expected_nonce, - prepared_tx_json, - transaction_id, - borrowed_key, - optimistic_key, - pending_key, - eoa, - chain_id, - }, - ) - .await - } - - /// Generic helper that handles WATCH + retry logic for atomic operations - /// The operation closure receives a mutable connection and should: - /// 1. Perform any validation (return early errors if needed) - /// 2. Build and execute the pipeline - /// 3. Return the result - pub async fn with_atomic_operation( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - watch_keys: Vec, - operation_name: &str, - operation: F, - ) -> Result - where - F: Fn(&mut ConnectionManager) -> Fut, - Fut: std::future::Future>, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for {} on {}:{}", - MAX_RETRIES, operation_name, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - operation = operation_name, - "Retrying atomic operation" - ); - } - - // WATCH all specified keys (lock is always included) - let mut watch_cmd = twmq::redis::cmd("WATCH"); - watch_cmd.arg(&lock_key); - for key in &watch_keys { - watch_cmd.arg(key); - } - let _: () = watch_cmd.query_async(&mut conn).await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Execute operation (includes validation and pipeline execution) - match operation(&mut conn).await { - Ok(result) => return Ok(result), - Err(TransactionStoreError::LockLost { .. }) => { - // Lock was lost during operation, propagate immediately - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - Err(TransactionStoreError::WatchFailed) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, retry - retry_count += 1; - continue; - } - Err(other_error) => { - // Other errors propagate immediately (validation failures, etc.) - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(other_error); - } - } - } - } - - /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC - pub async fn with_lock_check( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - operation: F, - ) -> Result - where - F: Fn(&mut Pipeline) -> R, - T: From, - { - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for lock check on {}:{}", - MAX_RETRIES, eoa, chain_id - ), - }); - } - - // Exponential backoff after first retry - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - tracing::debug!( - retry_count = retry_count, - delay_ms = delay_ms, - eoa = %eoa, - chain_id = chain_id, - "Retrying lock check operation" - ); - } - - // WATCH the EOA lock - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .query_async(&mut conn) - .await?; - - // Check if we still own the lock - let current_owner: Option = conn.get(&lock_key).await?; - match current_owner { - Some(owner) if owner == worker_id => { - // We still own it, proceed - } - _ => { - // Lost ownership - immediately fail - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - } - - // Build pipeline with operation - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - let result = operation(&mut pipeline); - - // Execute with WATCH protection - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(T::from(result)), - Err(_) => { - // WATCH failed, check if it was our lock or someone else's - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Our lock is fine, someone else's WATCH failed - retry - retry_count += 1; - continue; - } - } - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; - let mut result = Vec::new(); - - for (_nonce_str, transaction_json) in borrowed_map { - let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; - result.push(borrowed_data); - } - - Ok(result) - } - - /// Atomically move borrowed transaction to submitted state - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_submitted( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let hash = hash.to_string(); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToSubmitted { - nonce, - hash: hash.to_string(), - transaction_id, - borrowed_key, - submitted_key, - hash_to_id_key, - }, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - /// Returns error if transaction not found in borrowed state - pub async fn move_borrowed_to_recycled( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let borrowed_key = self.borrowed_transactions_hashmap_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let transaction_id = transaction_id.to_string(); - - self.execute_with_watch_and_retry( - eoa, - chain_id, - worker_id, - &MoveBorrowedToRecycled { - nonce, - transaction_id, - borrowed_key, - recycled_key, - pending_key, - }, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - eoa: Address, - chain_id: u64, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all entries with nonce < below_nonce - let results: Vec<(String, u64)> = conn - .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) - .await?; - - let mut parsed_results = Vec::new(); - for (hash_id_value, nonce) in results { - // Parse hash:id format - if let Some((hash, transaction_id)) = hash_id_value.split_once(':') { - parsed_results.push((nonce, hash.to_string(), transaction_id.to_string())); - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(transaction_id) = - self.get_transaction_id_for_hash(&hash_id_value).await? - { - parsed_results.push((nonce, hash_id_value, transaction_id)); - } - } - } - - Ok(parsed_results) - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - eoa: Address, - chain_id: u64, - nonce: u64, - ) -> Result, TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Get all members with the exact nonce - let members: Vec = conn - .zrangebyscore(&submitted_key, nonce, nonce) - .await - .map_err(|e| TransactionStoreError::RedisError { - message: format!("Failed to get transaction IDs for nonce {}: {}", nonce, e), - })?; - - let mut transaction_ids = Vec::new(); - for value in members { - // Parse the value as hash:id format, with fallback to old format - if let Some((_, transaction_id)) = value.split_once(':') { - // New format: hash:id - transaction_ids.push(transaction_id.to_string()); - } else { - // Old format: just hash - look up transaction ID - if let Some(transaction_id) = self.get_transaction_id_for_hash(&value).await? { - transaction_ids.push(transaction_id); - } - } - } - - Ok(transaction_ids) - } - - /// Remove all hashes for a transaction and requeue it - /// Returns error if no hashes found for this transaction in submitted state - /// NOTE: This method keeps the original boilerplate pattern because it needs to pass - /// complex data (transaction_hashes) from validation to pipeline phase. - /// The helper pattern works best for simple validation that doesn't need to pass data. - pub async fn fail_and_requeue_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let lock_key = self.eoa_lock_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let mut retry_count = 0; - - loop { - if retry_count >= MAX_RETRIES { - return Err(TransactionStoreError::InternalError { - message: format!( - "Exceeded max retries ({}) for fail and requeue transaction {}:{} tx:{}", - MAX_RETRIES, eoa, chain_id, transaction_id - ), - }); - } - - if retry_count > 0 { - let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); - tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; - } - - // WATCH lock and submitted state - let _: () = twmq::redis::cmd("WATCH") - .arg(&lock_key) - .arg(&submitted_key) - .query_async(&mut conn) - .await?; - - // Check lock ownership - let current_owner: Option = conn.get(&lock_key).await?; - if current_owner.as_deref() != Some(worker_id) { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - - // Find all hashes for this transaction that actually exist in submitted - let all_hash_id_values: Vec = conn.zrange(&submitted_key, 0, -1).await?; - let mut transaction_hashes = Vec::new(); - - for hash_id_value in all_hash_id_values { - // Parse hash:id format - if let Some((hash, tx_id)) = hash_id_value.split_once(':') { - if tx_id == transaction_id { - transaction_hashes.push(hash.to_string()); - } - } else { - // Fallback for old format (just hash) - look up transaction ID - if let Some(tx_id) = self.get_transaction_id_for_hash(&hash_id_value).await? { - if tx_id == transaction_id { - transaction_hashes.push(hash_id_value); - } - } - } - } - - if transaction_hashes.is_empty() { - let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; - return Err(TransactionStoreError::TransactionNotInSubmittedState { - transaction_id: transaction_id.to_string(), - }); - } - - // Transaction has hashes in submitted, proceed with atomic removal and requeue - let mut pipeline = twmq::redis::pipe(); - pipeline.atomic(); - - // Remove all hash:id values for this transaction (we know they exist) - for hash in &transaction_hashes { - // Remove the hash:id value from the zset - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Also remove the separate hash-to-ID mapping for backward compatibility - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - pipeline.del(&hash_to_id_key); - } - - // Add back to pending - pipeline.lpush(&pending_key, transaction_id); - - match pipeline - .query_async::>(&mut conn) - .await - { - Ok(_) => return Ok(()), // Success - Err(_) => { - // WATCH failed, check if it was our lock - let still_own_lock: Option = conn.get(&lock_key).await?; - if still_own_lock.as_deref() != Some(worker_id) { - return Err(TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.to_string(), - }); - } - // Submitted state changed, retry - retry_count += 1; - continue; - } - } - } - } - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let health_key = self.eoa_health_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let health_json: Option = conn.get(&health_key).await?; - if let Some(json) = health_json { - let health: EoaHealth = serde_json::from_str(&json)?; - Ok(Some(health)) - } else { - Ok(None) - } - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - let health_json = serde_json::to_string(health)?; - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, &health_json); - }) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - pipeline.set(&tx_count_key, transaction_count); - }) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces( - &self, - eoa: Address, - chain_id: u64, - ) -> Result, TransactionStoreError> { - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; - Ok(nonces) - } - - /// Peek at pending transactions without removing them (safe for planning) - pub async fn peek_pending_transactions( - &self, - eoa: Address, - chain_id: u64, - limit: u64, - ) -> Result, TransactionStoreError> { - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Use LRANGE to peek without removing - let transaction_ids: Vec = - conn.lrange(&pending_key, 0, (limit as isize) - 1).await?; - Ok(transaction_ids) - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - eoa: Address, - chain_id: u64, - max_inflight: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let last_tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - // Read both values atomically to avoid race conditions - let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() - .get(&optimistic_key) - .get(&last_tx_count_key) - .query_async(&mut conn) - .await?; - - let optimistic = match optimistic_nonce { - Some(nonce) => nonce, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - let last_count = match last_tx_count { - Some(count) => count, - None => return Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - }; - - let current_inflight = optimistic.saturating_sub(last_count); - let available_budget = max_inflight.saturating_sub(current_inflight); - - Ok(available_budget) - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let current: Option = conn.get(&optimistic_key).await?; - match current { - Some(nonce) => Ok(nonce), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Lock key name for EOA processing - fn eoa_lock_key_name(&self, eoa: Address, chain_id: u64) -> String { - match &self.namespace { - Some(ns) => format!("{ns}:eoa_executor:lock:{chain_id}:{eoa}"), - None => format!("eoa_executor:lock:{chain_id}:{eoa}"), - } - } - - /// Get transaction ID for a given hash - pub async fn get_transaction_id_for_hash( - &self, - hash: &str, - ) -> Result, TransactionStoreError> { - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let mut conn = self.redis.clone(); - - let transaction_id: Option = conn.get(&hash_to_id_key).await?; - Ok(transaction_id) - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - let tx_data_key = self.transaction_data_key_name(transaction_id); - let mut conn = self.redis.clone(); - - // Get the hash data (the transaction data is stored as a hash) - let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; - - if hash_data.is_empty() { - return Ok(None); - } - - // Extract user_request from the hash data - let user_request_json = hash_data.get("user_request").ok_or_else(|| { - TransactionStoreError::TransactionNotFound { - transaction_id: transaction_id.to_string(), - } - })?; - - let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; - - // Extract receipt if present - let receipt = hash_data - .get("receipt") - .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); - - // Extract attempts from separate list - let attempts_key = self.transaction_attempts_list_name(transaction_id); - let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; - let mut attempts = Vec::new(); - for attempt_json in attempts_json_list { - if let Ok(attempt) = serde_json::from_str::(&attempt_json) { - attempts.push(attempt); - } - } - - Ok(Some(TransactionData { - transaction_id: transaction_id.to_string(), - user_request, - receipt, - attempts, - })) - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); - let tx_data_key = self.transaction_data_key_name(transaction_id); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // Remove this hash:id from submitted - let hash_id_value = format!("{}:{}", hash, transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove hash mapping - pipeline.del(&hash_to_id_key); - - // Update transaction data with success - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", receipt); - pipeline.hset(&tx_data_key, "status", "confirmed"); - }) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - let new_hash = signed_transaction.hash().to_string(); - let nonce = signed_transaction.nonce(); - - // Create new attempt - let new_attempt = TransactionAttempt { - transaction_id: transaction_id.to_string(), - details: signed_transaction, - sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - attempt_number: 0, // Will be set correctly when reading all attempts - }; - - // Serialize the new attempt - let attempt_json = serde_json::to_string(&new_attempt)?; - - // Get key names - let attempts_list_key = self.transaction_attempts_list_name(transaction_id); - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); - let hash_id_value = format!("{}:{}", new_hash, transaction_id); - - // Now perform the atomic update - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - // Add new hash:id to submitted (keeping old ones) - pipeline.zadd(&submitted_key, &hash_id_value, nonce); - - // Still maintain separate hash-to-ID mapping for backward compatibility - pipeline.set(&hash_to_id_key, transaction_id); - - // Simply push the new attempt to the attempts list - pipeline.lpush(&attempts_list_key, &attempt_json); - }) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_fail_and_requeue_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - if failures.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Remove all hash:id values from submitted - for failure in &failures { - let hash_id_value = format!("{}:{}", failure.hash, failure.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&failure.hash); - pipeline.del(&hash_to_id_key); - } - - // Add unique transaction IDs back to pending (avoid duplicates) - let mut unique_tx_ids = std::collections::HashSet::new(); - for failure in &failures { - unique_tx_ids.insert(&failure.transaction_id); - } - - for transaction_id in unique_tx_ids { - pipeline.lpush(&pending_key, transaction_id); - } - }) - .await - } - - /// Efficiently batch succeed multiple transactions - /// This avoids hash-to-ID lookups since we already have both pieces of information - pub async fn batch_succeed_transactions( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - if successes.is_empty() { - return Ok(()); - } - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let submitted_key = self.submitted_transactions_zset_name(eoa, chain_id); - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - for success in &successes { - // Remove hash:id from submitted - let hash_id_value = format!("{}:{}", success.hash, success.transaction_id); - pipeline.zrem(&submitted_key, &hash_id_value); - - // Remove separate hash-to-ID mapping - let hash_to_id_key = self.transaction_hash_to_id_key_name(&success.hash); - pipeline.del(&hash_to_id_key); - - // Update transaction data with success (following existing Redis hash pattern) - let tx_data_key = self.transaction_data_key_name(&success.transaction_id); - pipeline.hset(&tx_data_key, "completed_at", now); - pipeline.hset(&tx_data_key, "receipt", &success.receipt_data); - pipeline.hset(&tx_data_key, "status", "confirmed"); - } - }) - .await - } - - // ========== SEND FLOW ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - - let count: Option = conn.get(&tx_count_key).await?; - match count { - Some(count) => Ok(count), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce( - &self, - eoa: Address, - chain_id: u64, - ) -> Result { - // Check recycled nonces first - let recycled = self.peek_recycled_nonces(eoa, chain_id).await?; - if !recycled.is_empty() { - return Ok(NonceType::Recycled(recycled[0])); - } - - // Get next optimistic nonce - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let mut conn = self.redis.clone(); - let current_optimistic: Option = conn.get(&optimistic_key).await?; - - match current_optimistic { - Some(nonce) => Ok(NonceType::Incremented(nonce)), - None => Err(TransactionStoreError::NonceSyncRequired { eoa, chain_id }), - } - } - - /// Synchronize nonces with the chain - /// - /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce - pub async fn synchronize_nonces_with_chain( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - // First, read current health data - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.last_nonce_movement_at = now; - health.last_confirmation_at = now; - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let tx_count_key = self.last_transaction_count_key_name(eoa, chain_id); - - // Update cached transaction count - pipeline.set(&tx_count_key, current_chain_tx_count); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - }) - .await - } - - /// Reset nonces to specified value - /// - /// This is called when we have too many recycled nonces and detect something wrong - /// We want to start fresh, with the chain nonce as the new optimistic nonce - pub async fn reset_nonces( - &self, - eoa: Address, - chain_id: u64, - worker_id: &str, - current_chain_tx_count: u64, - ) -> Result<(), TransactionStoreError> { - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let current_health = self.check_eoa_health(eoa, chain_id).await?; - - // Prepare health update if health data exists - let health_update = if let Some(mut health) = current_health { - health.nonce_resets.push(now); - Some(serde_json::to_string(&health)?) - } else { - None - }; - - self.with_lock_check(eoa, chain_id, worker_id, |pipeline| { - let optimistic_key = self.optimistic_transaction_count_key_name(eoa, chain_id); - let cached_nonce_key = self.last_transaction_count_key_name(eoa, chain_id); - let recycled_key = self.recycled_nonces_set_name(eoa, chain_id); - - // Update health data only if it exists - if let Some(ref health_json) = health_update { - let health_key = self.eoa_health_key_name(eoa, chain_id); - pipeline.set(&health_key, health_json); - } - - // Reset the optimistic nonce - pipeline.set(&optimistic_key, current_chain_tx_count); - - // Reset the cached nonce - pipeline.set(&cached_nonce_key, current_chain_tx_count); - - // Reset the recycled nonces - pipeline.del(recycled_key); - }) - .await - } - - /// Add a transaction to the pending queue and store its data - /// This is called when a new transaction request comes in for an EOA - pub async fn add_transaction( - &self, - transaction_request: EoaTransactionRequest, - ) -> Result<(), TransactionStoreError> { - let transaction_id = &transaction_request.transaction_id; - let eoa = transaction_request.from; - let chain_id = transaction_request.chain_id; - - let tx_data_key = self.transaction_data_key_name(transaction_id); - let pending_key = self.pending_transactions_list_name(eoa, chain_id); - - // Store transaction data as JSON in the user_request field of the hash - let user_request_json = serde_json::to_string(&transaction_request)?; - let now = chrono::Utc::now().timestamp_millis().max(0) as u64; - - let mut conn = self.redis.clone(); - - // Use a pipeline to atomically store data and add to pending queue - let mut pipeline = twmq::redis::pipe(); - - // Store transaction data - pipeline.hset(&tx_data_key, "user_request", &user_request_json); - pipeline.hset(&tx_data_key, "status", "pending"); - pipeline.hset(&tx_data_key, "created_at", now); - - // Add to pending queue - pipeline.lpush(&pending_key, transaction_id); - - pipeline.query_async::<()>(&mut conn).await?; - - Ok(()) - } -} - -// Additional error types -#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] -#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] -pub enum TransactionStoreError { - #[error("Redis error: {message}")] - RedisError { message: String }, - - #[error("Serialization error: {message}")] - DeserError { message: String, text: String }, - - #[error("Transaction not found: {transaction_id}")] - TransactionNotFound { transaction_id: String }, - - #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] - LockLost { - eoa: Address, - chain_id: u64, - worker_id: String, - }, - - #[error("Internal error - worker should quit: {message}")] - InternalError { message: String }, - - #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] - TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, - - #[error("Hash {hash} not found in submitted transactions")] - HashNotInSubmittedState { hash: String }, - - #[error("Transaction {transaction_id} has no hashes in submitted state")] - TransactionNotInSubmittedState { transaction_id: String }, - - #[error("Nonce {nonce} not available in recycled set")] - NonceNotInRecycledSet { nonce: u64 }, - - #[error("Transaction {transaction_id} not found in pending queue")] - TransactionNotInPendingQueue { transaction_id: String }, - - #[error("Optimistic nonce changed: expected {expected}, found {actual}")] - OptimisticNonceChanged { expected: u64, actual: u64 }, - - #[error("WATCH failed - state changed during operation")] - WatchFailed, - - #[error( - "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" - )] - NonceSyncRequired { eoa: Address, chain_id: u64 }, -} - -impl From for TransactionStoreError { - fn from(error: twmq::redis::RedisError) -> Self { - TransactionStoreError::RedisError { - message: error.to_string(), - } - } -} - -impl From for TransactionStoreError { - fn from(error: serde_json::Error) -> Self { - TransactionStoreError::DeserError { - message: error.to_string(), - text: error.to_string(), - } - } -} - -const MAX_RETRIES: u32 = 10; -const RETRY_BASE_DELAY_MS: u64 = 10; - -/// Scoped transaction store for a specific EOA, chain, and worker -/// -/// This wrapper eliminates the need to repeatedly pass EOA, chain_id, and worker_id -/// to every method call. It provides the same interface as TransactionStore but with -/// these parameters already bound. -/// -/// ## Usage: -/// ```rust -/// let scoped = ScopedTransactionStore::build(store, eoa, chain_id, worker_id).await?; -/// -/// // Much cleaner method calls: -/// scoped.peek_pending_transactions(limit).await?; -/// scoped.move_borrowed_to_submitted(nonce, hash, tx_id, attempt).await?; -/// ``` -pub struct ScopedEoaExecutorStore<'a> { - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, -} - -impl<'a> ScopedEoaExecutorStore<'a> { - /// Build a scoped transaction store for a specific EOA, chain, and worker - /// - /// This acquires the lock for the given EOA/chain. - /// If the lock is not acquired, returns a LockLost error. - #[tracing::instrument(skip_all, fields(eoa = %eoa, chain_id = chain_id, worker_id = %worker_id))] - pub async fn build( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Result { - // 1. ACQUIRE LOCK AGGRESSIVELY - tracing::info!("Acquiring EOA lock aggressively"); - store - .acquire_eoa_lock_aggressively(eoa, chain_id, &worker_id) - .await - .map_err(|e| { - tracing::error!("Failed to acquire EOA lock: {}", e); - TransactionStoreError::LockLost { - eoa, - chain_id, - worker_id: worker_id.clone(), - } - })?; - - Ok(Self { - store, - eoa, - chain_id, - worker_id, - }) - } - - /// Create a scoped store without lock validation (for read-only operations) - pub fn new_unchecked( - store: &'a EoaExecutorStore, - eoa: Address, - chain_id: u64, - worker_id: String, - ) -> Self { - Self { - store, - eoa, - chain_id, - worker_id, - } - } - - // ========== ATOMIC OPERATIONS ========== - - /// Atomically move specific transaction from pending to borrowed with recycled nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_recycled_nonce( - &self, - transaction_id: &str, - nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_recycled_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - nonce, - prepared_tx, - ) - .await - } - - /// Atomically move specific transaction from pending to borrowed with new nonce allocation - pub async fn atomic_move_pending_to_borrowed_with_new_nonce( - &self, - transaction_id: &str, - expected_nonce: u64, - prepared_tx: &BorrowedTransactionData, - ) -> Result<(), TransactionStoreError> { - self.store - .atomic_move_pending_to_borrowed_with_new_nonce( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - expected_nonce, - prepared_tx, - ) - .await - } - - /// Peek all borrowed transactions without removing them - pub async fn peek_borrowed_transactions( - &self, - ) -> Result, TransactionStoreError> { - self.store - .peek_borrowed_transactions(self.eoa, self.chain_id) - .await - } - - /// Atomically move borrowed transaction to submitted state - pub async fn move_borrowed_to_submitted( - &self, - nonce: u64, - hash: &str, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_submitted( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - hash, - transaction_id, - ) - .await - } - - /// Atomically move borrowed transaction back to recycled nonces and pending queue - pub async fn move_borrowed_to_recycled( - &self, - nonce: u64, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .move_borrowed_to_recycled( - self.eoa, - self.chain_id, - &self.worker_id, - nonce, - transaction_id, - ) - .await - } - - /// Get all hashes below a certain nonce from submitted transactions - /// Returns (nonce, hash, transaction_id) tuples - pub async fn get_hashes_below_nonce( - &self, - below_nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_hashes_below_nonce(self.eoa, self.chain_id, below_nonce) - .await - } - - /// Get all transaction IDs for a specific nonce - pub async fn get_transaction_ids_for_nonce( - &self, - nonce: u64, - ) -> Result, TransactionStoreError> { - self.store - .get_transaction_ids_for_nonce(self.eoa, self.chain_id, nonce) - .await - } - - /// Remove all hashes for a transaction and requeue it - pub async fn fail_and_requeue_transaction( - &self, - transaction_id: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .fail_and_requeue_transaction(self.eoa, self.chain_id, &self.worker_id, transaction_id) - .await - } - - /// Efficiently batch fail and requeue multiple transactions - pub async fn batch_fail_and_requeue_transactions( - &self, - failures: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_fail_and_requeue_transactions(self.eoa, self.chain_id, &self.worker_id, failures) - .await - } - - /// Efficiently batch succeed multiple transactions - pub async fn batch_succeed_transactions( - &self, - successes: Vec, - ) -> Result<(), TransactionStoreError> { - self.store - .batch_succeed_transactions(self.eoa, self.chain_id, &self.worker_id, successes) - .await - } - - // ========== EOA HEALTH & NONCE MANAGEMENT ========== - - /// Check EOA health (balance, etc.) - pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { - self.store.check_eoa_health(self.eoa, self.chain_id).await - } - - /// Update EOA health data - pub async fn update_health_data( - &self, - health: &EoaHealth, - ) -> Result<(), TransactionStoreError> { - self.store - .update_health_data(self.eoa, self.chain_id, &self.worker_id, health) - .await - } - - /// Update cached transaction count - pub async fn update_cached_transaction_count( - &self, - transaction_count: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .update_cached_transaction_count( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_count, - ) - .await - } - - /// Peek recycled nonces without removing them - pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { - self.store - .peek_recycled_nonces(self.eoa, self.chain_id) - .await - } - - /// Peek at pending transactions without removing them - pub async fn peek_pending_transactions( - &self, - limit: u64, - ) -> Result, TransactionStoreError> { - self.store - .peek_pending_transactions(self.eoa, self.chain_id, limit) - .await - } - - /// Get inflight budget (how many new transactions can be sent) - pub async fn get_inflight_budget( - &self, - max_inflight: u64, - ) -> Result { - self.store - .get_inflight_budget(self.eoa, self.chain_id, max_inflight) - .await - } - - /// Get current optimistic nonce (without incrementing) - pub async fn get_optimistic_nonce(&self) -> Result { - self.store - .get_optimistic_nonce(self.eoa, self.chain_id) - .await - } - - /// Mark transaction as successful and remove from submitted - pub async fn succeed_transaction( - &self, - transaction_id: &str, - hash: &str, - receipt: &str, - ) -> Result<(), TransactionStoreError> { - self.store - .succeed_transaction( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - hash, - receipt, - ) - .await - } - - /// Add a gas bump attempt (new hash) to submitted transactions - pub async fn add_gas_bump_attempt( - &self, - transaction_id: &str, - signed_transaction: Signed, - ) -> Result<(), TransactionStoreError> { - self.store - .add_gas_bump_attempt( - self.eoa, - self.chain_id, - &self.worker_id, - transaction_id, - signed_transaction, - ) - .await - } - - pub async fn synchronize_nonces_with_chain( - &self, - nonce: u64, - ) -> Result<(), TransactionStoreError> { - self.store - .synchronize_nonces_with_chain(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - pub async fn reset_nonces(&self, nonce: u64) -> Result<(), TransactionStoreError> { - self.store - .reset_nonces(self.eoa, self.chain_id, &self.worker_id, nonce) - .await - } - - // ========== READ-ONLY OPERATIONS ========== - - /// Get cached transaction count - pub async fn get_cached_transaction_count(&self) -> Result { - self.store - .get_cached_transaction_count(self.eoa, self.chain_id) - .await - } - - /// Peek next available nonce (recycled or new) - pub async fn peek_next_available_nonce(&self) -> Result { - self.store - .peek_next_available_nonce(self.eoa, self.chain_id) - .await - } - - // ========== ACCESSORS ========== - - /// Get the EOA address this store is scoped to - pub fn eoa(&self) -> Address { - self.eoa - } - - /// Get the chain ID this store is scoped to - pub fn chain_id(&self) -> u64 { - self.chain_id - } - - /// Get the worker ID this store is scoped to - pub fn worker_id(&self) -> &str { - &self.worker_id - } - - /// Get a reference to the underlying transaction store - pub fn inner(&self) -> &EoaExecutorStore { - self.store - } - - /// Get transaction data by transaction ID - pub async fn get_transaction_data( - &self, - transaction_id: &str, - ) -> Result, TransactionStoreError> { - self.store.get_transaction_data(transaction_id).await - } -} diff --git a/executors/src/eoa/store/atomic.rs b/executors/src/eoa/store/atomic.rs new file mode 100644 index 0000000..cea982d --- /dev/null +++ b/executors/src/eoa/store/atomic.rs @@ -0,0 +1,587 @@ +use std::sync::Arc; + +use alloy::{ + consensus::{Signed, TypedTransaction}, + primitives::Address, +}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::{ + eoa::{ + EoaExecutorStore, + store::{ + BorrowedTransactionData, ConfirmedTransaction, EoaHealth, TransactionAttempt, + TransactionStoreError, + borrowed::{BorrowedProcessingReport, ProcessBorrowedTransactions, SubmissionResult}, + pending::{ + MovePendingToBorrowedWithIncrementedNonces, MovePendingToBorrowedWithRecycledNonces, + }, + submitted::{CleanSubmittedTransactions, CleanupReport, SubmittedTransaction}, + }, + }, + webhook::WebhookJobHandler, +}; + +const MAX_RETRIES: u32 = 10; +const RETRY_BASE_DELAY_MS: u64 = 10; + +pub trait SafeRedisTransaction: Send + Sync { + type ValidationData; + type OperationResult; + + fn name(&self) -> &str; + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult; + fn validation( + &self, + conn: &mut ConnectionManager, + ) -> impl Future> + Send; + fn watch_keys(&self) -> Vec; +} + +/// Atomic transaction store that owns the base store and provides atomic operations +/// +/// This store is created by calling `acquire_lock()` on the base store and provides +/// access to both atomic (lock-protected) and non-atomic operations. +/// +/// ## Usage: +/// ```rust +/// let base_store = EoaExecutorStore::new(redis, namespace, ); +/// let atomic_store = base_store.acquire_lock(worker_id).await?; +/// +/// // Atomic operations: +/// atomic_store.move_borrowed_to_submitted(nonce, hash, tx_id).await?; +/// +/// // Non-atomic operations via deref: +/// atomic_store.peek_pending_transactions(limit).await?; +/// ``` +pub struct AtomicEoaExecutorStore { + pub store: EoaExecutorStore, + pub worker_id: String, +} + +impl std::ops::Deref for AtomicEoaExecutorStore { + type Target = EoaExecutorStore; + + fn deref(&self) -> &Self::Target { + &self.store + } +} + +impl AtomicEoaExecutorStore { + /// Get the EOA address this store is scoped to + pub fn eoa(&self) -> Address { + self.store.eoa + } + + /// Get the chain ID this store is scoped to + pub fn chain_id(&self) -> u64 { + self.store.chain_id + } + + /// Get the worker ID this store is scoped to + pub fn worker_id(&self) -> &str { + &self.worker_id + } + + /// Release EOA lock following the spec's finally pattern + pub async fn release_eoa_lock(self) -> Result { + // Use existing utility method that handles all the atomic lock checking + match self + .with_lock_check(|pipeline| { + let lock_key = self.eoa_lock_key_name(); + pipeline.del(&lock_key); + }) + .await + { + Ok(()) => { + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Successfully released EOA lock" + ); + Ok(self.store) + } + Err(TransactionStoreError::LockLost { .. }) => { + // Lock was already taken over, which is fine for release + tracing::debug!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + "Lock already released or taken over by another worker" + ); + Ok(self.store) + } + Err(e) => { + // Other errors shouldn't fail the worker, just log + tracing::warn!( + eoa = %self.eoa(), + chain_id = %self.chain_id(), + worker_id = %self.worker_id(), + error = %e, + "Failed to release EOA lock" + ); + Ok(self.store) + } + } + } + + /// Atomically move multiple pending transactions to borrowed state using incremented nonces + /// + /// The transactions must have sequential nonces starting from the current optimistic count. + /// This operation validates nonce ordering and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_incremented_nonces( + &self, + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithIncrementedNonces { + transactions, + keys: &self.keys, + eoa: self.eoa, + chain_id: self.chain_id, + }) + .await + } + + /// Atomically move multiple pending transactions to borrowed state using recycled nonces + /// + /// All nonces must exist in the recycled nonces set. This operation validates nonce + /// availability and atomically moves all transactions. + pub async fn atomic_move_pending_to_borrowed_with_recycled_nonces( + &self, + transactions: &[BorrowedTransactionData], + ) -> Result { + self.execute_with_watch_and_retry(&MovePendingToBorrowedWithRecycledNonces { + transactions, + keys: &self.keys, + }) + .await + } + + /// Wrapper that executes operations with lock validation using WATCH/MULTI/EXEC + pub async fn with_lock_check(&self, operation: F) -> Result + where + F: Fn(&mut Pipeline) -> R, + T: From, + { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for lock check on {}:{}", + MAX_RETRIES, + self.eoa(), + self.chain_id() + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa(), + chain_id = self.chain_id(), + "Retrying lock check operation" + ); + } + + // WATCH the EOA lock + let _: () = twmq::redis::cmd("WATCH") + .arg(&lock_key) + .query_async(&mut conn) + .await?; + + // Check if we still own the lock + let current_owner: Option = conn.get(&lock_key).await?; + match current_owner { + Some(owner) if owner == self.worker_id() => { + // We still own it, proceed + } + _ => { + // Lost ownership - immediately fail + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(self.eoa_lock_lost_error()); + } + } + + // Build pipeline with operation + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = operation(&mut pipeline); + + // Execute with WATCH protection + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(T::from(result)), + Err(_) => { + // WATCH failed, check if it was our lock or someone else's + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(self.eoa_lock_lost_error()); + } + // Our lock is fine, someone else's WATCH failed - retry + retry_count += 1; + continue; + } + } + } + } + + /// Helper to execute atomic operations with proper retry logic and watch handling + /// + /// This helper centralizes all the boilerplate for WATCH/MULTI/EXEC operations: + /// - Retry logic with exponential backoff + /// - Lock ownership validation + /// - WATCH key management + /// - Error handling and UNWATCH cleanup + /// + /// ## Usage: + /// Implement the `SafeRedisTransaction` trait for your operation, then call this method. + /// The trait separates validation (async) from pipeline operations (sync) for clean patterns. + /// + /// ## Example: + /// ```rust + /// let safe_tx = MovePendingToBorrowedWithNewNonce { + /// nonce: expected_nonce, + /// prepared_tx_json, + /// transaction_id, + /// borrowed_key, + /// optimistic_key, + /// pending_key, + /// eoa, + /// chain_id, + /// }; + /// + /// self.execute_with_watch_and_retry(, worker_id, &safe_tx).await?; + /// ``` + /// + /// ## When to use this helper: + /// - Operations that implement `SafeRedisTransaction` trait + /// - Need atomic WATCH/MULTI/EXEC with retry logic + /// - Want centralized lock checking and error handling + /// + /// ## When NOT to use this helper: + /// - Simple operations that can use `with_lock_check` instead + /// - Operations that don't need WATCH on multiple keys + /// - Read-only operations that don't modify state + async fn execute_with_watch_and_retry( + &self, + safe_tx: &T, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + let mut retry_count = 0; + + loop { + if retry_count >= MAX_RETRIES { + return Err(TransactionStoreError::InternalError { + message: format!( + "Exceeded max retries ({}) for {} on {}:{}", + MAX_RETRIES, + safe_tx.name(), + self.eoa, + self.chain_id + ), + }); + } + + // Exponential backoff after first retry + if retry_count > 0 { + let delay_ms = RETRY_BASE_DELAY_MS * (1 << (retry_count - 1).min(6)); + tokio::time::sleep(tokio::time::Duration::from_millis(delay_ms)).await; + tracing::debug!( + retry_count = retry_count, + delay_ms = delay_ms, + eoa = %self.eoa, + chain_id = self.chain_id, + operation = safe_tx.name(), + "Retrying atomic operation" + ); + } + + // WATCH all specified keys including lock + let mut watch_cmd = twmq::redis::cmd("WATCH"); + watch_cmd.arg(&lock_key); + for key in safe_tx.watch_keys() { + watch_cmd.arg(key); + } + let _: () = watch_cmd.query_async(&mut conn).await?; + + // Check lock ownership + let current_owner: Option = conn.get(&lock_key).await?; + if current_owner.as_deref() != Some(self.worker_id()) { + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + + // Execute validation + match safe_tx.validation(&mut conn).await { + Ok(validation_data) => { + // Build and execute pipeline + let mut pipeline = twmq::redis::pipe(); + pipeline.atomic(); + let result = safe_tx.operation(&mut pipeline, validation_data); + + match pipeline + .query_async::>(&mut conn) + .await + { + Ok(_) => return Ok(result), // Success + Err(_) => { + // WATCH failed, check if it was our lock + let still_own_lock: Option = conn.get(&lock_key).await?; + if still_own_lock.as_deref() != Some(self.worker_id()) { + return Err(TransactionStoreError::LockLost { + eoa: self.eoa, + chain_id: self.chain_id, + worker_id: self.worker_id().to_string(), + }); + } + // State changed, retry + retry_count += 1; + continue; + } + } + } + Err(e) => { + // Validation failed, unwatch and return error + let _: () = twmq::redis::cmd("UNWATCH").query_async(&mut conn).await?; + return Err(e); + } + } + } + } + + /// Update EOA health data + pub async fn update_health_data( + &self, + health: &EoaHealth, + ) -> Result<(), TransactionStoreError> { + let health_json = serde_json::to_string(health)?; + self.with_lock_check(|pipeline| { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, &health_json); + }) + .await + } + + /// Synchronize nonces with the chain + /// + /// Part of standard nonce management flow, called in the confirm stage when chain nonce advances, and we need to update our cached nonce + pub async fn update_cached_transaction_count( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // First, read current health data + let current_health = self.check_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.last_nonce_movement_at = now; + health.last_confirmation_at = now; + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let tx_count_key = self.last_transaction_count_key_name(); + + // Update cached transaction count + pipeline.set(&tx_count_key, current_chain_tx_count); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + }) + .await + } + + /// Add a gas bump attempt (new hash) to submitted transactions + pub async fn add_gas_bump_attempt( + &self, + submitted_transaction: &SubmittedTransaction, + signed_transaction: Signed, + ) -> Result<(), TransactionStoreError> { + let new_hash = signed_transaction.hash().to_string(); + + // Create new attempt + let new_attempt = TransactionAttempt { + transaction_id: submitted_transaction.transaction_id.clone(), + details: signed_transaction, + sent_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + attempt_number: 0, // Will be set correctly when reading all attempts + }; + + // Serialize the new attempt + let attempt_json = serde_json::to_string(&new_attempt)?; + + // Get key names + let attempts_list_key = + self.transaction_attempts_list_name(&submitted_transaction.transaction_id); + let submitted_key = self.submitted_transactions_zset_name(); + + let hash_to_id_key = self.transaction_hash_to_id_key_name(&new_hash); + + let (submitted_transaction_string, nonce) = + submitted_transaction.to_redis_string_with_nonce(); + + // Now perform the atomic update + self.with_lock_check(|pipeline| { + // Add new hash:id to submitted (keeping old ones) + pipeline.zadd(&submitted_key, &submitted_transaction_string, nonce); + + // Still maintain separate hash-to-ID mapping for backward compatibility + pipeline.set(&hash_to_id_key, &submitted_transaction.transaction_id); + + // Simply push the new attempt to the attempts list + pipeline.lpush(&attempts_list_key, &attempt_json); + }) + .await + } + + /// Reset nonces to specified value + /// + /// This is called when we have too many recycled nonces and detect something wrong + /// We want to start fresh, with the chain nonce as the new optimistic nonce + pub async fn reset_nonces( + &self, + current_chain_tx_count: u64, + ) -> Result<(), TransactionStoreError> { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let current_health = self.check_eoa_health().await?; + + // Prepare health update if health data exists + let health_update = if let Some(mut health) = current_health { + health.nonce_resets.push(now); + Some(serde_json::to_string(&health)?) + } else { + None + }; + + self.with_lock_check(|pipeline| { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let cached_nonce_key = self.last_transaction_count_key_name(); + let recycled_key = self.recycled_nonces_zset_name(); + + // Update health data only if it exists + if let Some(ref health_json) = health_update { + let health_key = self.eoa_health_key_name(); + pipeline.set(&health_key, health_json); + } + + // Reset the optimistic nonce + pipeline.set(&optimistic_key, current_chain_tx_count); + + // Reset the cached nonce + pipeline.set(&cached_nonce_key, current_chain_tx_count); + + // Reset the recycled nonces + pipeline.del(recycled_key); + }) + .await + } + + /// Fail a transaction that's in the borrowed state (we know the nonce) + pub async fn fail_borrowed_transaction( + &self, + transaction_id: &str, + nonce: u64, + failure_reason: &str, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from borrowed state using the known nonce + pipeline.hdel(&borrowed_key, nonce.to_string()); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + }) + .await + } + + /// Fail a transaction that's in the pending state (remove from pending and fail) + /// This is used for deterministic failures during preparation that should not retry + pub async fn fail_pending_transaction( + &self, + transaction_id: &str, + failure_reason: &str, + webhook_queue: Arc>, + ) -> Result<(), TransactionStoreError> { + self.with_lock_check(|pipeline| { + let pending_key = self.pending_transactions_zset_name(); + let tx_data_key = self.transaction_data_key_name(transaction_id); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Remove from pending state + pipeline.zrem(&pending_key, transaction_id); + + // Update transaction data with failure + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", failure_reason); + pipeline.hset(&tx_data_key, "status", "failed"); + + // TODO: Queue webhook event for failed transaction + // let webhook_job = WebhookJobHandler::new(...); + // webhook_queue.push(webhook_job); + }) + .await + } + + pub async fn clean_submitted_transactions( + &self, + confirmed_transactions: &[ConfirmedTransaction], + last_confirmed_nonce: u64, + ) -> Result { + self.execute_with_watch_and_retry(&CleanSubmittedTransactions { + confirmed_transactions, + last_confirmed_nonce, + keys: &self.keys, + }) + .await + } + + /// Process borrowed transactions with given submission results + /// This method moves transactions from borrowed state to submitted/pending/failed states + /// based on the submission results, and queues appropriate webhook events + pub async fn process_borrowed_transactions( + &self, + results: Vec, + webhook_queue: Arc>, + ) -> Result { + self.execute_with_watch_and_retry(&ProcessBorrowedTransactions { + results, + keys: &self.keys, + webhook_queue, + }) + .await + } +} diff --git a/executors/src/eoa/store/borrowed.rs b/executors/src/eoa/store/borrowed.rs new file mode 100644 index 0000000..3eceb35 --- /dev/null +++ b/executors/src/eoa/store/borrowed.rs @@ -0,0 +1,383 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use alloy::consensus::Transaction; +use alloy::primitives::Address; +use serde::{Deserialize, Serialize}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; +use twmq::{Queue, hooks::TransactionContext}; + +use crate::eoa::{ + events::EoaExecutorEvent, + store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionData, TransactionStoreError, + atomic::SafeRedisTransaction, submitted::SubmittedTransaction, + }, + worker::EoaExecutorWorkerError, +}; +use crate::webhook::{WebhookJobHandler, queue_webhook_envelopes}; + +/// Error information for NACK operations (retryable errors) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubmissionErrorNack { + pub transaction_id: String, + pub error: EoaExecutorWorkerError, + pub user_data: Option, +} + +/// Error information for FAIL operations (permanent failures) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubmissionErrorFail { + pub transaction_id: String, + pub error: EoaExecutorWorkerError, + pub user_data: Option, +} + +/// Result of a submission attempt +#[derive(Debug, Clone)] +pub enum SubmissionResult { + Success(SubmittedTransaction), + Nack(SubmissionErrorNack), + Fail(SubmissionErrorFail), +} + +/// Internal representation where all user data is guaranteed to be present +#[derive(Debug, Clone)] +pub enum SubmissionResultWithUserData { + Success(SubmittedTransaction, TransactionData), + Nack(SubmissionErrorNack, TransactionData), + Fail(SubmissionErrorFail, TransactionData), +} + +impl SubmissionResultWithUserData { + fn transaction_id(&self) -> &str { + match self { + SubmissionResultWithUserData::Success(tx, _) => &tx.transaction_id, + SubmissionResultWithUserData::Nack(err, _) => &err.transaction_id, + SubmissionResultWithUserData::Fail(err, _) => &err.transaction_id, + } + } + + fn user_data(&self) -> &TransactionData { + match self { + SubmissionResultWithUserData::Success(_, data) => data, + SubmissionResultWithUserData::Nack(_, data) => data, + SubmissionResultWithUserData::Fail(_, data) => data, + } + } +} + +/// Batch operation to process borrowed transactions +pub struct ProcessBorrowedTransactions<'a> { + pub results: Vec, + pub keys: &'a EoaExecutorStoreKeys, + pub webhook_queue: Arc>, +} + +#[derive(Debug, Default)] +pub struct BorrowedProcessingReport { + pub total_processed: usize, + pub moved_to_submitted: usize, + pub moved_to_pending: usize, + pub failed_transactions: usize, + pub webhook_events_queued: usize, +} + +impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> { + type ValidationData = ( + Vec, + Vec, + ); + type OperationResult = BorrowedProcessingReport; + + fn name(&self) -> &str { + "process borrowed transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.borrowed_transactions_hashmap_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + // Collect all transaction IDs that need user data + let mut transactions_needing_data = Vec::new(); + let mut results_with_partial_data = Vec::new(); + + for result in &self.results { + match result { + SubmissionResult::Success(tx) => { + transactions_needing_data.push(tx.transaction_id.clone()); + results_with_partial_data.push(result.clone()); + } + SubmissionResult::Nack(err) => { + if err.user_data.is_none() { + transactions_needing_data.push(err.transaction_id.clone()); + } + results_with_partial_data.push(result.clone()); + } + SubmissionResult::Fail(err) => { + if err.user_data.is_none() { + transactions_needing_data.push(err.transaction_id.clone()); + } + results_with_partial_data.push(result.clone()); + } + } + } + + // Batch fetch missing user data + let mut user_data_map = HashMap::new(); + for transaction_id in transactions_needing_data { + let data_key = self.keys.transaction_data_key_name(&transaction_id); + if let Some(data_json) = conn.get::<&str, Option>(&data_key).await? { + let transaction_data: TransactionData = serde_json::from_str(&data_json)?; + user_data_map.insert(transaction_id, transaction_data); + } + } + + // Get all borrowed transactions to validate they exist + let borrowed_transactions_map: HashMap = conn + .hgetall(self.keys.borrowed_transactions_hashmap_name()) + .await?; + + let borrowed_transactions: Vec = borrowed_transactions_map + .into_iter() + .filter_map(|(nonce_str, data_json)| { + let borrowed_data: BorrowedTransactionData = + serde_json::from_str(&data_json).ok()?; + Some(borrowed_data) + }) + .collect(); + + // Convert to results with guaranteed user data + let mut results_with_user_data = Vec::new(); + for result in results_with_partial_data { + match result { + SubmissionResult::Success(tx) => { + if let Some(user_data) = user_data_map.get(&tx.transaction_id) { + results_with_user_data + .push(SubmissionResultWithUserData::Success(tx, user_data.clone())); + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: tx.transaction_id.clone(), + }); + } + } + SubmissionResult::Nack(mut err) => { + let user_data = if let Some(data) = err.user_data.take() { + data + } else if let Some(data) = user_data_map.get(&err.transaction_id) { + data.clone() + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: err.transaction_id.clone(), + }); + }; + results_with_user_data.push(SubmissionResultWithUserData::Nack(err, user_data)); + } + SubmissionResult::Fail(mut err) => { + let user_data = if let Some(data) = err.user_data.take() { + data + } else if let Some(data) = user_data_map.get(&err.transaction_id) { + data.clone() + } else { + return Err(TransactionStoreError::TransactionNotFound { + transaction_id: err.transaction_id.clone(), + }); + }; + results_with_user_data.push(SubmissionResultWithUserData::Fail(err, user_data)); + } + } + } + + Ok((results_with_user_data, borrowed_transactions)) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + validation_data: Self::ValidationData, + ) -> Self::OperationResult { + let (results_with_user_data, borrowed_transactions) = validation_data; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Create borrowed transactions lookup by transaction_id + let borrowed_by_id: HashMap = borrowed_transactions + .iter() + .map(|tx| (tx.transaction_id.clone(), tx)) + .collect(); + + let mut report = BorrowedProcessingReport::default(); + + for result in &results_with_user_data { + let transaction_id = result.transaction_id(); + let user_data = result.user_data(); + + // Find the corresponding borrowed transaction to get the nonce + let borrowed_tx = match borrowed_by_id.get(transaction_id) { + Some(tx) => tx, + None => { + // Transaction not in borrowed state, skip + continue; + } + }; + + let nonce = borrowed_tx.signed_transaction.nonce(); + + // We'll set attempt_number to 1 for simplicity in the operation phase + // The actual attempt tracking is handled by the attempts list + let attempt_number = 1; + + // Define attempts_key for all match arms + let attempts_key = self.keys.transaction_attempts_list_name(transaction_id); + + match result { + SubmissionResultWithUserData::Success(tx, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Add to submitted + let (submitted_tx_redis_string, nonce) = tx.to_redis_string_with_nonce(); + pipeline.zadd( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + nonce, + ); + + // Update hash-to-ID mapping + let hash_to_id_key = self.keys.transaction_hash_to_id_key_name(&tx.hash); + pipeline.set(&hash_to_id_key, &tx.transaction_id); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(&tx.transaction_id); + pipeline.hset(&tx_data_key, "status", "submitted"); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = event.send_attempt_success_envelope(tx.clone()); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for success: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_submitted += 1; + } + SubmissionResultWithUserData::Nack(err, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Add back to pending + pipeline.zadd( + self.keys.pending_transactions_zset_name(), + &err.transaction_id, + now, + ); + + // Update transaction data status + let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + pipeline.hset(&tx_data_key, "status", "pending"); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = + event.send_attempt_nack_envelope(nonce, err.error.clone(), attempt_number); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for nack: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.moved_to_pending += 1; + } + SubmissionResultWithUserData::Fail(err, user_data) => { + // Remove from borrowed + pipeline.hdel( + self.keys.borrowed_transactions_hashmap_name(), + nonce.to_string(), + ); + + // Update transaction data with failure + let tx_data_key = self.keys.transaction_data_key_name(&err.transaction_id); + pipeline.hset(&tx_data_key, "status", "failed"); + pipeline.hset(&tx_data_key, "completed_at", now); + pipeline.hset(&tx_data_key, "failure_reason", err.error.to_string()); + + // Add attempt to attempts list + let attempt_json = + serde_json::to_string(&borrowed_tx.signed_transaction).unwrap(); + pipeline.lpush(&attempts_key, &attempt_json); + + // Queue webhook event + let event = EoaExecutorEvent { + transaction_data: user_data.clone(), + }; + let envelope = + event.transaction_failed_envelope(err.error.clone(), attempt_number); + if let Some(webhook_options) = &user_data.user_request.webhook_options { + let mut tx_context = self + .webhook_queue + .transaction_context_from_pipeline(pipeline); + if let Err(e) = queue_webhook_envelopes( + envelope, + webhook_options.clone(), + &mut tx_context, + self.webhook_queue.clone(), + ) { + tracing::error!("Failed to queue webhook for fail: {}", e); + } else { + report.webhook_events_queued += 1; + } + } + + report.failed_transactions += 1; + } + } + } + + report.total_processed = results_with_user_data.len(); + report + } +} diff --git a/executors/src/eoa/store/error.rs b/executors/src/eoa/store/error.rs new file mode 100644 index 0000000..275541f --- /dev/null +++ b/executors/src/eoa/store/error.rs @@ -0,0 +1,22 @@ +use crate::eoa::EoaExecutorStore; +use crate::eoa::store::TransactionStoreError; +use crate::eoa::store::atomic::AtomicEoaExecutorStore; + +impl AtomicEoaExecutorStore { + pub fn eoa_lock_lost_error(&self) -> TransactionStoreError { + TransactionStoreError::LockLost { + eoa: self.eoa(), + chain_id: self.chain_id(), + worker_id: self.worker_id().to_string(), + } + } +} + +impl EoaExecutorStore { + pub fn nonce_sync_required_error(&self) -> TransactionStoreError { + TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + } + } +} diff --git a/executors/src/eoa/store/mod.rs b/executors/src/eoa/store/mod.rs new file mode 100644 index 0000000..a0d3874 --- /dev/null +++ b/executors/src/eoa/store/mod.rs @@ -0,0 +1,759 @@ +use alloy::consensus::{Signed, Transaction, TypedTransaction}; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::{Address, Bytes, U256}; +use chrono; +use engine_core::chain::RpcCredentials; +use engine_core::credentials::SigningCredential; +use engine_core::execution_options::WebhookOptions; +use engine_core::transaction::TransactionTypeData; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use twmq::redis::{AsyncCommands, aio::ConnectionManager}; + +mod atomic; +mod borrowed; +mod pending; +mod submitted; + +pub mod error; +pub use atomic::AtomicEoaExecutorStore; +pub use borrowed::{ + BorrowedProcessingReport, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, +}; +pub use submitted::{CleanupReport, SubmittedTransaction}; + +use crate::eoa::store::submitted::SubmittedTransactionStringWithNonce; + +pub const NO_OP_TRANSACTION_ID: &str = "noop"; + +#[derive(Debug, Clone)] +pub struct ReplacedTransaction { + pub hash: String, + pub transaction_id: String, +} + +#[derive(Debug, Clone)] +pub struct ConfirmedTransaction { + pub hash: String, + pub transaction_id: String, + pub receipt_data: String, +} + +#[derive(Debug, Clone)] +pub struct PendingTransaction { + pub transaction_id: String, + pub queued_at: u64, +} + +impl From<(String, u64)> for PendingTransaction { + fn from((transaction_id, queued_at): (String, u64)) -> Self { + Self { + transaction_id, + queued_at, + } + } +} + +/// The actual user request data +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct EoaTransactionRequest { + pub transaction_id: String, + pub chain_id: u64, + + pub from: Address, + pub to: Option
, + pub value: U256, + pub data: Bytes, + + #[serde(alias = "gas")] + pub gas_limit: Option, + + pub webhook_options: Option>, + + pub signing_credential: SigningCredential, + pub rpc_credentials: RpcCredentials, + + #[serde(flatten)] + pub transaction_type_data: Option, +} + +/// Active attempt for a transaction (full alloy transaction + metadata) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionAttempt { + pub transaction_id: String, + pub details: Signed, + pub sent_at: u64, // Unix timestamp in milliseconds + pub attempt_number: u32, +} + +/// Transaction data for a transaction_id +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionData { + pub transaction_id: String, + pub user_request: EoaTransactionRequest, + pub receipt: Option, + pub attempts: Vec, + pub created_at: u64, // Unix timestamp in milliseconds +} + +pub struct BorrowedTransaction { + pub transaction_id: String, + pub data: Signed, + pub borrowed_at: chrono::DateTime, +} + +/// Transaction store focused on transaction_id operations and nonce indexing +pub struct EoaExecutorStore { + pub redis: ConnectionManager, + pub keys: EoaExecutorStoreKeys, +} + +pub struct EoaExecutorStoreKeys { + pub eoa: Address, + pub chain_id: u64, + pub namespace: Option, +} + +impl EoaExecutorStoreKeys { + pub fn new(eoa: Address, chain_id: u64, namespace: Option) -> Self { + Self { + eoa, + chain_id, + namespace, + } + } + + /// Lock key name for EOA processing + pub fn eoa_lock_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:lock:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key for the transaction data + /// + /// Transaction data is stored as a Redis HSET with the following fields: + /// - "user_request": JSON string containing EoaTransactionRequest + /// - "receipt": JSON string containing AnyTransactionReceipt (optional) + /// - "status": String status ("confirmed", "failed", etc.) + /// - "completed_at": String Unix timestamp (optional) + /// - "created_at": String Unix timestamp (optional) + /// - "failure_reason": String failure reason (optional) + pub fn transaction_data_key_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_data:{transaction_id}"), + None => format!("eoa_executor:_tx_data:{transaction_id}"), + } + } + + /// Name of the list for transaction attempts + /// + /// Attempts are stored as a separate Redis LIST where each element is a JSON blob + /// of a TransactionAttempt. This allows efficient append operations. + pub fn transaction_attempts_list_name(&self, transaction_id: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_attempts:{transaction_id}"), + None => format!("eoa_executor:tx_attempts:{transaction_id}"), + } + } + + /// Name of the zset for pending transactions + /// + /// zset contains the `transaction_id` scored by the queued_at timestamp (unix timestamp in milliseconds) + pub fn pending_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:pending_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:pending_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the zset for submitted transactions. nonce -> hash:id + /// + /// Same transaction might appear multiple times in the zset with different nonces/gas prices (and thus different hashes) + pub fn submitted_transactions_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:submitted_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:submitted_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the key that maps transaction hash to transaction id + pub fn transaction_hash_to_id_key_name(&self, hash: &str) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:tx_hash_to_id:{hash}"), + None => format!("eoa_executor:tx_hash_to_id:{hash}"), + } + } + + /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData` + /// + /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap. + /// + /// On worker recovery, if any borrowed transactions are found, we rebroadcast them and move back to pending or submitted + /// + /// If there's no crash, happy path moves borrowed transactions back to pending or submitted + pub fn borrowed_transactions_hashmap_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:borrowed_txs:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:borrowed_txs:{}:{}", self.chain_id, self.eoa), + } + } + + /// Name of the set that contains recycled nonces. + /// + /// If a transaction was submitted but failed (ie, we know with certainty it didn't enter the mempool), + /// + /// we add the nonce to this set. + /// + /// These nonces are used with priority, before any other nonces. + pub fn recycled_nonces_zset_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:recycled_nonces:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Optimistic nonce key name. + /// + /// This is used for optimistic nonce tracking. + /// + /// We store the nonce of the last successfuly sent transaction for each EOA. + /// + /// We increment this nonce for each new transaction. + /// + /// !IMPORTANT! When sending a transaction, we use this nonce as the assigned nonce, NOT the incremented nonce. + pub fn optimistic_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!( + "eoa_executor:optimistic_nonce:{}:{}", + self.chain_id, self.eoa + ), + } + } + + /// Name of the key that contains the nonce of the last fetched ONCHAIN transaction count for each EOA. + /// + /// This is a cache for the actual transaction count, which is fetched from the RPC. + /// + /// The nonce for the NEXT transaction is the ONCHAIN transaction count (NOT + 1) + /// + /// Eg: transaction count is 0, so we use nonce 0 for sending the next transaction. Once successful, transaction count will be 1. + pub fn last_transaction_count_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!( + "{ns}:eoa_executor:last_tx_nonce:{}:{}", + self.chain_id, self.eoa + ), + None => format!("eoa_executor:last_tx_nonce:{}:{}", self.chain_id, self.eoa), + } + } + + /// EOA health key name. + /// + /// EOA health stores: + /// - cached balance, the timestamp of the last balance fetch + /// - timestamp of the last successful transaction confirmation + /// - timestamp of the last 5 nonce resets + pub fn eoa_health_key_name(&self) -> String { + match &self.namespace { + Some(ns) => format!("{ns}:eoa_executor:health:{}:{}", self.chain_id, self.eoa), + None => format!("eoa_executor:health:{}:{}", self.chain_id, self.eoa), + } + } +} + +impl EoaExecutorStore { + pub fn new( + redis: ConnectionManager, + namespace: Option, + eoa: Address, + chain_id: u64, + ) -> Self { + Self { + redis, + keys: EoaExecutorStoreKeys { + eoa, + chain_id, + namespace, + }, + } + } +} + +impl std::ops::Deref for EoaExecutorStore { + type Target = EoaExecutorStoreKeys; + fn deref(&self) -> &Self::Target { + &self.keys + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EoaHealth { + pub balance: U256, + /// Update the balance threshold when we see out of funds errors + pub balance_threshold: U256, + pub balance_fetched_at: u64, + pub last_confirmation_at: u64, + pub last_nonce_movement_at: u64, // Track when nonce last moved for gas bump detection + pub nonce_resets: Vec, // Last 5 reset timestamps +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BorrowedTransactionData { + pub transaction_id: String, + pub signed_transaction: Signed, + pub queued_at: u64, + pub hash: String, + pub borrowed_at: u64, +} + +impl Into for &BorrowedTransactionData { + fn into(self) -> SubmittedTransaction { + SubmittedTransaction { + nonce: self.signed_transaction.nonce(), + hash: self.signed_transaction.hash().to_string(), + transaction_id: self.transaction_id.clone(), + queued_at: self.queued_at, + } + } +} + +/// Type of nonce allocation for transaction processing +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NonceType { + /// Nonce was recycled from a previously failed transaction + Recycled(u64), + /// Nonce was incremented from the current optimistic counter + Incremented(u64), +} + +impl NonceType { + /// Get the nonce value regardless of type + pub fn nonce(&self) -> u64 { + match self { + NonceType::Recycled(nonce) => *nonce, + NonceType::Incremented(nonce) => *nonce, + } + } + + /// Check if this is a recycled nonce + pub fn is_recycled(&self) -> bool { + matches!(self, NonceType::Recycled(_)) + } + + /// Check if this is an incremented nonce + pub fn is_incremented(&self) -> bool { + matches!(self, NonceType::Incremented(_)) + } +} + +impl EoaExecutorStore { + /// Aggressively acquire EOA lock, forcefully taking over from stalled workers + /// + /// Creates an AtomicEoaExecutorStore that owns the lock. + pub async fn acquire_eoa_lock_aggressively( + self, + worker_id: &str, + ) -> Result { + let lock_key = self.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + + // First try normal acquisition + let acquired: bool = conn.set_nx(&lock_key, worker_id).await?; + if acquired { + return Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }); + } + // Lock exists, forcefully take it over + tracing::warn!( + eoa = %self.eoa, + chain_id = %self.chain_id, + worker_id = %worker_id, + "Forcefully taking over EOA lock from stalled worker" + ); + // Force set - no expiry, only released by explicit takeover + let _: () = conn.set(&lock_key, worker_id).await?; + Ok(AtomicEoaExecutorStore { + store: self, + worker_id: worker_id.to_string(), + }) + } + + /// Peek all borrowed transactions without removing them + pub async fn peek_borrowed_transactions( + &self, + ) -> Result, TransactionStoreError> { + let borrowed_key = self.borrowed_transactions_hashmap_name(); + let mut conn = self.redis.clone(); + + let borrowed_map: HashMap = conn.hgetall(&borrowed_key).await?; + let mut result = Vec::new(); + + for (_nonce_str, transaction_json) in borrowed_map { + let borrowed_data: BorrowedTransactionData = serde_json::from_str(&transaction_json)?; + result.push(borrowed_data); + } + + Ok(result) + } + + /// Get all hashes below a certain nonce from submitted transactions + /// Returns (nonce, hash, transaction_id) tuples + pub async fn get_submitted_transactions_below_nonce( + &self, + below_nonce: u64, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Get all entries with nonce < below_nonce + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, 0, below_nonce - 1) + .await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Get all transaction IDs for a specific nonce + pub async fn get_submitted_transactions_for_nonce( + &self, + nonce: u64, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let results: Vec = conn + .zrangebyscore_withscores(&submitted_key, nonce, nonce) + .await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&results); + + Ok(submitted_txs) + } + + /// Check EOA health (balance, etc.) + pub async fn check_eoa_health(&self) -> Result, TransactionStoreError> { + let mut conn = self.redis.clone(); + + let health_json: Option = conn.get(self.eoa_health_key_name()).await?; + if let Some(json) = health_json { + let health: EoaHealth = serde_json::from_str(&json)?; + Ok(Some(health)) + } else { + Ok(None) + } + } + + /// Peek recycled nonces without removing them + pub async fn peek_recycled_nonces(&self) -> Result, TransactionStoreError> { + let recycled_key = self.recycled_nonces_zset_name(); + let mut conn = self.redis.clone(); + + let nonces: Vec = conn.zrange(&recycled_key, 0, -1).await?; + Ok(nonces) + } + + /// Peek at pending transactions without removing them (safe for planning) + pub async fn peek_pending_transactions( + &self, + limit: u64, + ) -> Result, TransactionStoreError> { + let pending_key = self.pending_transactions_zset_name(); + let mut conn = self.redis.clone(); + + // Use ZRANGE to peek without removing + let transaction_ids: Vec<(String, u64)> = conn + .zrange_withscores(&pending_key, 0, (limit - 1) as isize) + .await?; + + Ok(transaction_ids + .into_iter() + .map(PendingTransaction::from) + .collect()) + } + + /// Get inflight budget (how many new transactions can be sent) + pub async fn get_inflight_budget( + &self, + max_inflight: u64, + ) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let last_tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + // Read both values atomically to avoid race conditions + let (optimistic_nonce, last_tx_count): (Option, Option) = twmq::redis::pipe() + .get(&optimistic_key) + .get(&last_tx_count_key) + .query_async(&mut conn) + .await?; + + let optimistic = match optimistic_nonce { + Some(nonce) => nonce, + None => return Err(self.nonce_sync_required_error()), + }; + let last_count = match last_tx_count { + Some(count) => count, + None => return Err(self.nonce_sync_required_error()), + }; + + let current_inflight = optimistic.saturating_sub(last_count); + let available_budget = max_inflight.saturating_sub(current_inflight); + + Ok(available_budget) + } + + /// Get current optimistic nonce (without incrementing) + pub async fn get_optimistic_transaction_count(&self) -> Result { + let optimistic_key = self.optimistic_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let current: Option = conn.get(&optimistic_key).await?; + match current { + Some(nonce) => Ok(nonce), + None => Err(self.nonce_sync_required_error()), + } + } + /// Get transaction ID for a given hash + pub async fn get_transaction_id_for_hash( + &self, + hash: &str, + ) -> Result, TransactionStoreError> { + let hash_to_id_key = self.transaction_hash_to_id_key_name(hash); + let mut conn = self.redis.clone(); + + let transaction_id: Option = conn.get(&hash_to_id_key).await?; + Ok(transaction_id) + } + + /// Get transaction data by transaction ID + pub async fn get_transaction_data( + &self, + transaction_id: &str, + ) -> Result, TransactionStoreError> { + let tx_data_key = self.transaction_data_key_name(transaction_id); + let mut conn = self.redis.clone(); + + // Get the hash data (the transaction data is stored as a hash) + let hash_data: HashMap = conn.hgetall(&tx_data_key).await?; + + if hash_data.is_empty() { + return Ok(None); + } + + // Extract user_request from the hash data + let user_request_json = hash_data.get("user_request").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + let user_request: EoaTransactionRequest = serde_json::from_str(user_request_json)?; + + // Extract receipt if present + let receipt = hash_data + .get("receipt") + .and_then(|receipt_str| serde_json::from_str(receipt_str).ok()); + + let created_at = hash_data.get("created_at").ok_or_else(|| { + TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + } + })?; + + // todo: in case of non-existent created_at, we should return a default value + let created_at = + created_at + .parse::() + .map_err(|_| TransactionStoreError::TransactionNotFound { + transaction_id: transaction_id.to_string(), + })?; + + // Extract attempts from separate list + let attempts_key = self.transaction_attempts_list_name(transaction_id); + let attempts_json_list: Vec = conn.lrange(&attempts_key, 0, -1).await?; + let mut attempts = Vec::new(); + for attempt_json in attempts_json_list { + if let Ok(attempt) = serde_json::from_str::(&attempt_json) { + attempts.push(attempt); + } + } + + Ok(Some(TransactionData { + transaction_id: transaction_id.to_string(), + created_at, + user_request, + receipt, + attempts, + })) + } + + /// Get cached transaction count + pub async fn get_cached_transaction_count(&self) -> Result { + let tx_count_key = self.last_transaction_count_key_name(); + let mut conn = self.redis.clone(); + + let count: Option = conn.get(&tx_count_key).await?; + match count { + Some(count) => Ok(count), + None => Err(self.nonce_sync_required_error()), + } + } + + /// Add a transaction to the pending queue and store its data + /// This is called when a new transaction request comes in for an EOA + pub async fn add_transaction( + &self, + transaction_request: EoaTransactionRequest, + ) -> Result<(), TransactionStoreError> { + let transaction_id = &transaction_request.transaction_id; + + let tx_data_key = self.transaction_data_key_name(transaction_id); + let pending_key = self.pending_transactions_zset_name(); + + // Store transaction data as JSON in the user_request field of the hash + let user_request_json = serde_json::to_string(&transaction_request)?; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + let mut conn = self.redis.clone(); + + // Use a pipeline to atomically store data and add to pending queue + let mut pipeline = twmq::redis::pipe(); + + // Store transaction data + pipeline.hset(&tx_data_key, "user_request", &user_request_json); + pipeline.hset(&tx_data_key, "status", "pending"); + pipeline.hset(&tx_data_key, "created_at", now); + + // Add to pending queue + pipeline.zadd(&pending_key, transaction_id, now); + + pipeline.query_async::<()>(&mut conn).await?; + + Ok(()) + } + + /// Get count of submitted transactions awaiting confirmation + pub async fn get_submitted_transactions_count(&self) -> Result { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let count: u64 = conn.zcard(&submitted_key).await?; + Ok(count) + } + + /// Get the submitted transactions for the highest nonce value + /// + /// Internally submissions are stored in a zset by nonce -> hash:id + /// + /// This will return all hash:id pairs for the highest nonce + #[tracing::instrument(skip_all)] + pub async fn get_highest_submitted_nonce_tranasactions( + &self, + ) -> Result, TransactionStoreError> { + let submitted_key = self.submitted_transactions_zset_name(); + let mut conn = self.redis.clone(); + + let highest_nonce_txs: Vec = + conn.zrange_withscores(&submitted_key, -1, -1).await?; + + let submitted_txs: Vec = + SubmittedTransaction::from_redis_strings(&highest_nonce_txs); + + Ok(submitted_txs) + } +} + +// Additional error types +#[derive(Debug, thiserror::Error, Serialize, Deserialize, Clone)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE", tag = "errorCode")] +pub enum TransactionStoreError { + #[error("Redis error: {message}")] + RedisError { message: String }, + + #[error("Serialization error: {message}")] + DeserError { message: String, text: String }, + + #[error("Transaction not found: {transaction_id}")] + TransactionNotFound { transaction_id: String }, + + #[error("Lost EOA lock: {eoa}:{chain_id} worker: {worker_id}")] + LockLost { + eoa: Address, + chain_id: u64, + worker_id: String, + }, + + #[error("Internal error - worker should quit: {message}")] + InternalError { message: String }, + + #[error("Transaction {transaction_id} not in borrowed state for nonce {nonce}")] + TransactionNotInBorrowedState { transaction_id: String, nonce: u64 }, + + #[error("Hash {hash} not found in submitted transactions")] + HashNotInSubmittedState { hash: String }, + + #[error("Transaction {transaction_id} has no hashes in submitted state")] + TransactionNotInSubmittedState { transaction_id: String }, + + #[error("Nonce {nonce} not available in recycled set")] + NonceNotInRecycledSet { nonce: u64 }, + + #[error("Transaction {transaction_id} not found in pending queue")] + TransactionNotInPendingQueue { transaction_id: String }, + + #[error("Optimistic nonce changed: expected {expected}, found {actual}")] + OptimisticNonceChanged { expected: u64, actual: u64 }, + + #[error("WATCH failed - state changed during operation")] + WatchFailed, + + #[error( + "Nonce synchronization required for {eoa}:{chain_id} - no cached transaction count available" + )] + NonceSyncRequired { eoa: Address, chain_id: u64 }, +} + +impl From for TransactionStoreError { + fn from(error: twmq::redis::RedisError) -> Self { + TransactionStoreError::RedisError { + message: error.to_string(), + } + } +} + +impl From for TransactionStoreError { + fn from(error: serde_json::Error) -> Self { + TransactionStoreError::DeserError { + message: error.to_string(), + text: error.to_string(), + } + } +} diff --git a/executors/src/eoa/store/pending.rs b/executors/src/eoa/store/pending.rs new file mode 100644 index 0000000..08da0f5 --- /dev/null +++ b/executors/src/eoa/store/pending.rs @@ -0,0 +1,257 @@ +use std::collections::HashSet; + +use alloy::{consensus::Transaction, primitives::Address}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::store::{ + BorrowedTransactionData, EoaExecutorStoreKeys, TransactionStoreError, + atomic::SafeRedisTransaction, +}; + +/// Atomic operation to move pending transactions to borrowed state using incremented nonces +/// +/// This operation validates that: +/// 1. The nonces in the vector are sequential with no gaps +/// 2. The lowest nonce matches the current optimistic transaction count +/// 3. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes transactions from pending queue +/// 2. Adds transactions to borrowed state +/// 3. Updates optimistic transaction count to highest nonce + 1 +pub struct MovePendingToBorrowedWithIncrementedNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, + pub eoa: Address, + pub chain_id: u64, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithIncrementedNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with incremented nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.optimistic_transaction_count_key_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get current optimistic nonce + let current_optimistic: Option = conn + .get(self.keys.optimistic_transaction_count_key_name()) + .await?; + let current_nonce = + current_optimistic.ok_or_else(|| TransactionStoreError::NonceSyncRequired { + eoa: self.eoa, + chain_id: self.chain_id, + })?; + + // Extract and validate nonces + let mut nonces: Vec = self + .transactions + .iter() + .map(|tx| tx.signed_transaction.nonce()) + .collect(); + nonces.sort(); + + // Check that nonces are sequential with no gaps + for (i, &nonce) in nonces.iter().enumerate() { + let expected_nonce = current_nonce + i as u64; + if nonce != expected_nonce { + return Err(TransactionStoreError::InternalError { + message: format!( + "Non-sequential nonces detected: expected {}, found {} at position {}", + expected_nonce, nonce, i + ), + }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let optimistic_key = self.keys.optimistic_transaction_count_key_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + let nonce = tx.signed_transaction.nonce(); + + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + } + + // Update optimistic tx count to highest nonce + 1 + if let Some(last_tx) = self.transactions.last() { + let new_optimistic_tx_count = last_tx.signed_transaction.nonce() + 1; + pipeline.set(&optimistic_key, new_optimistic_tx_count); + } + + self.transactions.len() + } +} + +/// Atomic operation to move pending transactions to borrowed state using recycled nonces +/// +/// This operation validates that: +/// 1. All nonces exist in the recycled nonces set +/// 2. All transactions exist in the pending queue +/// +/// Then atomically: +/// 1. Removes nonces from recycled set +/// 2. Removes transactions from pending queue +/// 3. Adds transactions to borrowed state +pub struct MovePendingToBorrowedWithRecycledNonces<'a> { + pub transactions: &'a [BorrowedTransactionData], + pub keys: &'a EoaExecutorStoreKeys, +} + +impl SafeRedisTransaction for MovePendingToBorrowedWithRecycledNonces<'_> { + type ValidationData = Vec; // serialized borrowed transactions + type OperationResult = usize; // number of transactions processed + + fn name(&self) -> &str { + "pending->borrowed with recycled nonces" + } + + fn watch_keys(&self) -> Vec { + vec![ + self.keys.recycled_nonces_zset_name(), + self.keys.borrowed_transactions_hashmap_name(), + ] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + if self.transactions.is_empty() { + return Err(TransactionStoreError::InternalError { + message: "Cannot process empty transaction list".to_string(), + }); + } + + // Get all recycled nonces + let recycled_nonces: HashSet = conn + .zrange(self.keys.recycled_nonces_zset_name(), 0, -1) + .await?; + + // Verify all nonces are in recycled set + for tx in self.transactions { + let nonce = tx.signed_transaction.nonce(); + if !recycled_nonces.contains(&nonce) { + return Err(TransactionStoreError::NonceNotInRecycledSet { nonce }); + } + } + + // Verify all transactions exist in pending queue using batched ZSCORE calls + if !self.transactions.is_empty() { + let mut pipe = twmq::redis::pipe(); + for tx in self.transactions { + pipe.zscore( + self.keys.pending_transactions_zset_name(), + &tx.transaction_id, + ); + } + let scores: Vec> = pipe.query_async(conn).await?; + + for (tx, score) in self.transactions.iter().zip(scores.iter()) { + if score.is_none() { + return Err(TransactionStoreError::TransactionNotInPendingQueue { + transaction_id: tx.transaction_id.clone(), + }); + } + } + } + + // Pre-serialize all borrowed transaction data + let mut serialized_transactions = Vec::with_capacity(self.transactions.len()); + for tx in self.transactions { + let borrowed_json = + serde_json::to_string(tx).map_err(|e| TransactionStoreError::InternalError { + message: format!("Failed to serialize borrowed transaction: {}", e), + })?; + serialized_transactions.push(borrowed_json); + } + + Ok(serialized_transactions) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + serialized_transactions: Self::ValidationData, + ) -> Self::OperationResult { + let recycled_key = self.keys.recycled_nonces_zset_name(); + let pending_key = self.keys.pending_transactions_zset_name(); + let borrowed_key = self.keys.borrowed_transactions_hashmap_name(); + + for (tx, borrowed_json) in self.transactions.iter().zip(serialized_transactions.iter()) { + let nonce = tx.signed_transaction.nonce(); + + // Remove nonce from recycled set + pipeline.zrem(&recycled_key, nonce); + + // Remove from pending queue + pipeline.zrem(&pending_key, &tx.transaction_id); + + // Add to borrowed state + pipeline.hset(&borrowed_key, nonce.to_string(), borrowed_json); + } + + self.transactions.len() + } +} diff --git a/executors/src/eoa/store/submitted.rs b/executors/src/eoa/store/submitted.rs new file mode 100644 index 0000000..943001c --- /dev/null +++ b/executors/src/eoa/store/submitted.rs @@ -0,0 +1,278 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; +use twmq::redis::{AsyncCommands, Pipeline, aio::ConnectionManager}; + +use crate::eoa::store::{ + ConfirmedTransaction, EoaExecutorStoreKeys, TransactionStoreError, atomic::SafeRedisTransaction, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmittedTransaction { + pub nonce: u64, + pub hash: String, + pub transaction_id: String, + pub queued_at: u64, +} + +pub type SubmittedTransactionStringWithNonce = (String, u64); + +impl SubmittedTransaction { + pub fn from_redis_strings(redis_strings: &[SubmittedTransactionStringWithNonce]) -> Vec { + redis_strings + .iter() + .filter_map(|tx| { + let parts: Vec<&str> = tx.0.split(':').collect(); + if parts.len() == 3 { + if let Ok(queued_at) = parts[2].parse::() { + Some(SubmittedTransaction { + hash: parts[0].to_string(), + transaction_id: parts[1].to_string(), + nonce: tx.1, + queued_at, + }) + } else { + tracing::error!("Invalid queued_at timestamp: {}", tx.0); + None + } + } else { + tracing::error!( + "Invalid transaction format, expected 3 parts separated by ':': {}", + tx.0 + ); + None + } + }) + .collect() + } + + /// Returns the string representation of the submitted transaction with the nonce + /// + /// This is used to add the transaction to the submitted state in Redis + /// + /// The format is: + /// + /// ```text + /// hash:transaction_id:queued_at + /// ``` + /// + /// The nonce is the value of the transaction in the submitted state, and is used as the score of the submitted zset + pub fn to_redis_string_with_nonce(&self) -> SubmittedTransactionStringWithNonce { + ( + format!("{}:{}:{}", self.hash, self.transaction_id, self.queued_at), + self.nonce, + ) + } +} + +pub struct CleanSubmittedTransactions<'a> { + pub last_confirmed_nonce: u64, + pub confirmed_transactions: &'a [ConfirmedTransaction], + pub keys: &'a EoaExecutorStoreKeys, +} + +#[derive(Debug, Default)] +pub struct CleanupReport { + pub total_hashes_processed: usize, + pub unique_transaction_ids: usize, + pub noop_count: usize, + pub moved_to_success: usize, + pub moved_to_pending: usize, + + /// Any transaction ID values that have multiple nonces in the submitted state + pub cross_nonce_violations: Vec<(String, Vec)>, // (transaction_id, nonces) + + /// Any nonces that have multiple confirmations (very rare, indicates re-org) + pub per_nonce_violations: Vec<(u64, Vec)>, // (nonce, confirmed_hashes) + + /// Any nonces that have no confirmations (transactions we sent got replaced by a different one uknown to us) + pub nonces_without_receipts: Vec<(u64, Vec)>, // (nonce, hashes) +} + +/// This operation takes a list of confirmed transactions and the last confirmed nonce +/// +/// It will fetch all submitted transactions with a nonce less than or equal to the last confirmed nonce. +/// For each nonce: +/// - it will go through all the hashes for that nonce +/// - if the hash is in the confirmed transactions, it will be removed from submitted to success +/// - if the hash is not in the confirmed transactions, it will be removed from submitted to pending +/// +/// It will also deduplicate transactions by ID, so if any of the hashes for that ID are in the confirmed transactions, +/// this hash will not be moved back to pending. +/// +/// ***IMPORTANT***: This should not happen with different nonces. A transaction ID should only appear once in the submitted state. +/// Multiple submissions for the same transaction ID with different nonces can cause duplicate transactions +/// Multiple submissions for the same transaction ID with the same nonce is fine, because this indicated gas bumps. +impl SafeRedisTransaction for CleanSubmittedTransactions<'_> { + type ValidationData = Vec; + type OperationResult = CleanupReport; + + fn name(&self) -> &str { + "clean submitted transactions" + } + + fn watch_keys(&self) -> Vec { + vec![self.keys.submitted_transactions_zset_name()] + } + + async fn validation( + &self, + conn: &mut ConnectionManager, + ) -> Result { + let submitted_txs: Vec = conn + .zrangebyscore_withscores( + self.keys.submitted_transactions_zset_name(), + 0, + self.last_confirmed_nonce as isize, + ) + .await?; + + let submitted_txs = SubmittedTransaction::from_redis_strings(&submitted_txs); + Ok(submitted_txs) + } + + fn operation( + &self, + pipeline: &mut Pipeline, + submitted_txs: Self::ValidationData, + ) -> Self::OperationResult { + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + + // Build confirmed lookups + let confirmed_hashes: HashSet<&str> = self + .confirmed_transactions + .iter() + .map(|tx| tx.hash.as_str()) + .collect(); + + let confirmed_ids: BTreeMap<&str, &ConfirmedTransaction> = self + .confirmed_transactions + .iter() + .map(|tx| (tx.transaction_id.as_str(), tx)) + .collect(); + + // Detect violations and get grouped data + let (_, _, mut report) = detect_violations(&submitted_txs, &confirmed_hashes); + + // Process every hash and track unique IDs + let mut processed_ids = HashSet::new(); + + let mut replaced_transactions = Vec::with_capacity(submitted_txs.len()); + + for tx in &submitted_txs { + // Clean up this hash from Redis (happens for ALL hashes) + let (submitted_tx_redis_string, _nonce) = tx.clone().to_redis_string_with_nonce(); + + pipeline.zrem( + self.keys.submitted_transactions_zset_name(), + &submitted_tx_redis_string, + ); + pipeline.del(self.keys.transaction_hash_to_id_key_name(&tx.hash)); + + // Process each unique transaction_id once + if processed_ids.insert(&tx.transaction_id) { + match ( + tx.transaction_id.as_str(), + confirmed_ids.get(tx.transaction_id.as_str()), + ) { + // if the transaction id is noop, we don't do anything + ("noop", _) => report.noop_count += 1, + + // in case of a valid ID, we check if it's in the confirmed transactions + // if it is confirmed, we succeed it and queue success jobs + (id, Some(confirmed_tx)) => { + let data_key_name = self.keys.transaction_data_key_name(id); + pipeline.hset(&data_key_name, "status", "confirmed"); + pipeline.hset(&data_key_name, "completed_at", now); + pipeline.hset(&data_key_name, "receipt", confirmed_tx.receipt_data.clone()); + + // TODO: + // queue success jobs here + + report.moved_to_success += 1; + } + + // if the ID is not in the confirmed transactions, we queue it for pending + _ => { + replaced_transactions.push((&tx.transaction_id, tx.queued_at)); + report.moved_to_pending += 1; + } + } + } + } + + pipeline.zadd_multiple( + self.keys.pending_transactions_zset_name(), + &replaced_transactions, + ); + + // Finalize report stats + report.total_hashes_processed = submitted_txs.len(); + report.unique_transaction_ids = processed_ids.len(); + + report + } +} + +fn detect_violations<'a>( + submitted_txs: &'a [SubmittedTransaction], + confirmed_hashes: &'a HashSet<&str>, +) -> ( + HashMap<&'a str, Vec>, + BTreeMap>, + CleanupReport, +) { + let mut report = CleanupReport::default(); + let mut txs_by_nonce: BTreeMap> = BTreeMap::new(); + let mut transaction_id_to_nonces: HashMap<&str, Vec> = HashMap::new(); + + // Group data + for tx in submitted_txs { + txs_by_nonce.entry(tx.nonce).or_default().push(tx); + transaction_id_to_nonces + .entry(&tx.transaction_id) + .or_default() + .push(tx.nonce); + } + + // Check cross-nonce violations + for (transaction_id, nonces) in &transaction_id_to_nonces { + let mut unique_nonces = nonces.clone(); + unique_nonces.sort(); + unique_nonces.dedup(); + if unique_nonces.len() > 1 { + report + .cross_nonce_violations + .push((transaction_id.to_string(), unique_nonces)); + } + } + + // Check per-nonce violations + for (nonce, txs) in &txs_by_nonce { + let confirmed_hashes_for_nonce: Vec = txs + .iter() + .filter(|tx| confirmed_hashes.contains(tx.hash.as_str())) + .map(|tx| tx.hash.clone()) + .collect(); + + if confirmed_hashes_for_nonce.len() > 1 { + report + .per_nonce_violations + .push((*nonce, confirmed_hashes_for_nonce)); + } + } + + // Check nonces without receipts + for (nonce, txs) in &txs_by_nonce { + let has_confirmed = txs + .iter() + .any(|tx| confirmed_hashes.contains(tx.hash.as_str())); + if !has_confirmed { + let hashes: Vec = txs.iter().map(|tx| tx.hash.clone()).collect(); + report.nonces_without_receipts.push((*nonce, hashes)); + } + } + + (transaction_id_to_nonces, txs_by_nonce, report) +} diff --git a/executors/src/eoa/worker.rs b/executors/src/eoa/worker.rs index f4939ba..7c488af 100644 --- a/executors/src/eoa/worker.rs +++ b/executors/src/eoa/worker.rs @@ -4,7 +4,7 @@ use alloy::consensus::{ }; use alloy::network::{TransactionBuilder, TransactionBuilder7702}; use alloy::primitives::{Address, B256, Bytes, U256}; -use alloy::providers::Provider; +use alloy::providers::{PendingTransactionBuilder, Provider}; use alloy::rpc::types::TransactionRequest as AlloyTransactionRequest; use alloy::signers::Signature; use alloy::transports::{RpcError, TransportErrorKind}; @@ -12,7 +12,7 @@ use engine_core::error::EngineError; use engine_core::signer::AccountSigner; use engine_core::transaction::TransactionTypeData; use engine_core::{ - chain::{Chain, ChainService, RpcCredentials}, + chain::{Chain, ChainService}, credentials::SigningCredential, error::{AlloyRpcErrorToEngineError, RpcErrorKind}, signer::{EoaSigner, EoaSigningOptions}, @@ -20,7 +20,11 @@ use engine_core::{ use hex; use serde::{Deserialize, Serialize}; use std::{sync::Arc, time::Duration}; +use thirdweb_core::iaw::IAWError; use tokio::time::sleep; +use twmq::Queue; +use twmq::redis::AsyncCommands; +use twmq::redis::aio::ConnectionManager; use twmq::{ DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable, error::TwmqError, @@ -29,9 +33,12 @@ use twmq::{ }; use crate::eoa::store::{ - BorrowedTransactionData, EoaExecutorStore, EoaHealth, EoaTransactionRequest, - ScopedEoaExecutorStore, TransactionData, TransactionStoreError, + AtomicEoaExecutorStore, BorrowedTransactionData, CleanupReport, ConfirmedTransaction, + EoaExecutorStore, EoaExecutorStoreKeys, EoaHealth, EoaTransactionRequest, PendingTransaction, + ReplacedTransaction, SubmissionErrorFail, SubmissionErrorNack, SubmissionResult, + SubmittedTransaction, TransactionData, TransactionStoreError, }; +use crate::webhook::WebhookJobHandler; // ========== SPEC-COMPLIANT CONSTANTS ========== const MAX_INFLIGHT_PER_EOA: u64 = 100; // Default from spec @@ -41,13 +48,17 @@ const MIN_TRANSACTIONS_PER_EOA: u64 = 1; // Fleet management from spec const HEALTH_CHECK_INTERVAL: u64 = 300; // 5 minutes in seconds const NONCE_STALL_TIMEOUT: u64 = 300_000; // 5 minutes in milliseconds - after this time, attempt gas bump +// Retry constants for preparation phase +const MAX_PREPARATION_RETRIES: u32 = 3; +const PREPARATION_RETRY_DELAY_MS: u64 = 100; + // ========== JOB DATA ========== #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct EoaExecutorWorkerJobData { pub eoa_address: Address, pub chain_id: u64, - pub worker_id: String, + pub noop_signing_credential: SigningCredential, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -83,12 +94,18 @@ pub enum EoaExecutorWorkerError { #[error("Transaction build failed: {message}")] TransactionBuildFailed { message: String }, - #[error("RPC error: {message}")] + #[error("RPC error encountered during generic operation: {message}")] RpcError { message: String, inner_error: EngineError, }, + #[error("Error encountered when broadcasting transaction: {message}")] + TransactionSendError { + message: String, + inner_error: EngineError, + }, + #[error("Signature parsing failed: {message}")] SignatureParsingFailed { message: String }, @@ -133,13 +150,13 @@ impl UserCancellable for EoaExecutorWorkerError { // ========== SIMPLE ERROR CLASSIFICATION ========== #[derive(Debug)] -enum SendErrorClassification { +pub enum SendErrorClassification { PossiblySent, // "nonce too low", "already known" etc DeterministicFailure, // Invalid signature, malformed tx, insufficient funds etc } #[derive(PartialEq, Eq, Debug)] -enum SendContext { +pub enum SendContext { Rebroadcast, InitialBroadcast, } @@ -227,48 +244,45 @@ fn is_retryable_rpc_error(kind: &RpcErrorKind) -> bool { } } -// ========== PREPARED TRANSACTION ========== -#[derive(Debug, Clone)] -struct PreparedTransaction { - transaction_id: String, - signed_tx: Signed, - nonce: u64, -} - -// ========== CONFIRMATION FLOW DATA STRUCTURES ========== -#[derive(Debug, Clone)] -struct PendingTransaction { - nonce: u64, - hash: String, - transaction_id: String, -} - -#[derive(Debug, Clone)] -struct ConfirmedTransaction { - nonce: u64, - hash: String, - transaction_id: String, - receipt: alloy::rpc::types::TransactionReceipt, -} - -#[derive(Debug, Clone)] -struct FailedTransaction { - hash: String, - transaction_id: String, -} - -// ========== STORE BATCH OPERATION TYPES ========== -#[derive(Debug, Clone)] -pub struct TransactionSuccess { - pub hash: String, - pub transaction_id: String, - pub receipt_data: String, +fn is_retryable_preparation_error(error: &EoaExecutorWorkerError) -> bool { + match error { + EoaExecutorWorkerError::RpcError { inner_error, .. } => { + // extract the RpcErrorKind from the inner error + if let EngineError::RpcError { kind, .. } = inner_error { + is_retryable_rpc_error(kind) + } else { + false + } + } + EoaExecutorWorkerError::ChainServiceError { .. } => true, // Network related + EoaExecutorWorkerError::StoreError { inner_error, .. } => { + matches!(inner_error, TransactionStoreError::RedisError { .. }) + } + EoaExecutorWorkerError::TransactionSimulationFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::TransactionBuildFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::SigningError { inner_error, .. } => match inner_error { + // if vault error, it's not retryable + EngineError::VaultError { .. } => false, + // if iaw error, it's retryable only if it's a network error + EngineError::IawError { error, .. } => matches!(error, IAWError::NetworkError { .. }), + _ => false, + }, + EoaExecutorWorkerError::TransactionNotFound { .. } => false, // Deterministic + EoaExecutorWorkerError::InternalError { .. } => false, // Deterministic + EoaExecutorWorkerError::UserCancelled => false, // Deterministic + EoaExecutorWorkerError::TransactionSendError { .. } => false, // Different context + EoaExecutorWorkerError::SignatureParsingFailed { .. } => false, // Deterministic + EoaExecutorWorkerError::WorkRemaining { .. } => false, // Different context + } } -#[derive(Debug, Clone)] -pub struct TransactionFailure { +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConfirmedTransactionWithRichReceipt { + pub nonce: u64, pub hash: String, pub transaction_id: String, + pub receipt: alloy::rpc::types::TransactionReceipt, } // ========== MAIN WORKER ========== @@ -293,7 +307,11 @@ where CS: ChainService + Send + Sync + 'static, { pub chain_service: Arc, - pub store: Arc, + pub webhook_queue: Arc>, + + pub redis: ConnectionManager, + pub namespace: Option, + pub eoa_signer: Arc, pub max_inflight: u64, // Note: Spec uses MAX_INFLIGHT_PER_EOA constant pub max_recycled_nonces: u64, // Note: Spec uses MAX_RECYCLED_THRESHOLD constant @@ -325,12 +343,13 @@ where .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // 2. CREATE SCOPED STORE (acquires lock) - let scoped = ScopedEoaExecutorStore::build( - &self.store, + let scoped = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), data.eoa_address, data.chain_id, - data.worker_id.clone(), ) + .acquire_eoa_lock_aggressively(&job.lease_token) .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; @@ -349,13 +368,7 @@ where _success_data: SuccessHookData<'_, Self::Output>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on success - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; + self.release_eoa_lock(&job.job.data).await; } async fn on_nack( @@ -364,13 +377,7 @@ where _nack_data: NackHookData<'_, Self::ErrorData>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on nack - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; + self.release_eoa_lock(&job.job.data).await; } async fn on_fail( @@ -379,13 +386,56 @@ where _fail_data: FailHookData<'_, Self::ErrorData>, _tx: &mut TransactionContext<'_>, ) { - // Release EOA lock on fail - self.release_eoa_lock( - job.job.data.eoa_address, - job.job.data.chain_id, - &job.job.data.worker_id, - ) - .await; + self.release_eoa_lock(&job.job.data).await; + } +} + +impl SubmissionResult { + /// Convert a send result to a SubmissionResult for batch processing + /// This handles the specific RpcError type from alloy + pub fn from_send_result( + borrowed_transaction: &BorrowedTransactionData, + send_result: Result>, + send_context: SendContext, + user_data: Option, + chain: &impl Chain, + ) -> Self { + match send_result { + Ok(_) => SubmissionResult::Success(borrowed_transaction.into()), + Err(ref rpc_error) => { + match classify_send_error(rpc_error, send_context) { + SendErrorClassification::PossiblySent => { + SubmissionResult::Success(borrowed_transaction.into()) + } + SendErrorClassification::DeterministicFailure => { + // Transaction failed, should be retried + let engine_error = rpc_error.to_engine_error(chain); + let error = EoaExecutorWorkerError::TransactionSendError { + message: format!("Transaction send failed: {}", rpc_error), + inner_error: engine_error, + }; + SubmissionResult::Nack(SubmissionErrorNack { + transaction_id: borrowed_transaction.transaction_id.clone(), + error, + user_data, + }) + } + } + } + } + } + + /// Helper method for when we need to create a failure result + pub fn from_failure( + transaction_id: String, + error: EoaExecutorWorkerError, + user_data: Option, + ) -> Self { + SubmissionResult::Fail(SubmissionErrorFail { + transaction_id, + error, + user_data, + }) } } @@ -396,7 +446,7 @@ where /// Execute the main EOA worker workflow async fn execute_main_workflow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> JobResult { // 1. CRASH RECOVERY @@ -406,7 +456,7 @@ where .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // 2. CONFIRM FLOW - let (confirmed, failed) = self + let confirmations_report = self .confirm_flow(scoped, chain) .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; @@ -433,13 +483,17 @@ where .await .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)? .len(); + let submitted_count = scoped + .get_submitted_transactions_count() + .await + .map_err_nack(Some(Duration::from_secs(10)), RequeuePosition::Last)?; // NACK here is a yield, when you think of the queue as a distributed EOA scheduler - if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 { + if pending_count > 0 || borrowed_count > 0 || recycled_count > 0 || submitted_count > 0 { return Err(EoaExecutorWorkerError::WorkRemaining { message: format!( - "Work remaining: {} pending, {} borrowed, {} recycled", - pending_count, borrowed_count, recycled_count + "Work remaining: {} pending, {} borrowed, {} recycled, {} submitted", + pending_count, borrowed_count, recycled_count, submitted_count ), }) .map_err_nack(Some(Duration::from_secs(2)), RequeuePosition::Last); @@ -448,19 +502,26 @@ where // Only succeed if no work remains Ok(EoaExecutorWorkerResult { recovered_transactions: recovered, - confirmed_transactions: confirmed, - failed_transactions: failed, + confirmed_transactions: confirmations_report.moved_to_success as u32, + failed_transactions: confirmations_report.moved_to_pending as u32, sent_transactions: sent, }) } /// Release EOA lock following the spec's finally pattern - async fn release_eoa_lock(&self, eoa: Address, chain_id: u64, worker_id: &str) { - if let Err(e) = self.store.release_eoa_lock(eoa, chain_id, worker_id).await { + async fn release_eoa_lock(&self, job_data: &EoaExecutorWorkerJobData) { + let keys = EoaExecutorStoreKeys::new( + job_data.eoa_address, + job_data.chain_id, + self.namespace.clone(), + ); + + let lock_key = keys.eoa_lock_key_name(); + let mut conn = self.redis.clone(); + if let Err(e) = conn.del::<&str, ()>(&lock_key).await { tracing::error!( - eoa = %eoa, - chain_id = %chain_id, - worker_id = %worker_id, + eoa = %job_data.eoa_address, + chain_id = %job_data.chain_id, error = %e, "Failed to release EOA lock" ); @@ -471,7 +532,7 @@ where #[tracing::instrument(skip_all)] async fn recover_borrowed_state( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let mut borrowed_transactions = scoped.peek_borrowed_transactions().await?; @@ -511,74 +572,54 @@ where let rebroadcast_results = futures::future::join_all(rebroadcast_futures).await; - // Process results sequentially for Redis state changes - let mut recovered_count = 0; - for (borrowed, send_result) in rebroadcast_results { - let nonce = borrowed.signed_transaction.nonce(); + // Convert results to SubmissionResult for batch processing + let submission_results: Vec = rebroadcast_results + .into_iter() + .map(|(borrowed, send_result)| { + SubmissionResult::from_send_result( + borrowed, + send_result, + SendContext::Rebroadcast, + None, // We'll let the batch operation fetch user data + chain, + ) + }) + .collect(); - match send_result { - Ok(_) => { - // Transaction was sent successfully - scoped - .move_borrowed_to_submitted( - nonce, - &format!("{:?}", borrowed.hash), - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted"); - } - Err(e) => { - match classify_send_error(&e, SendContext::Rebroadcast) { - SendErrorClassification::PossiblySent => { - // Transaction possibly sent, move to submitted - scoped - .move_borrowed_to_submitted( - nonce, - &format!("{:?}", borrowed.hash), - &borrowed.transaction_id, - ) - .await?; - tracing::info!(transaction_id = %borrowed.transaction_id, nonce = nonce, "Moved recovered transaction to submitted (possibly sent)"); - } - SendErrorClassification::DeterministicFailure => { - // Transaction is broken, recycle nonce and requeue - scoped - .move_borrowed_to_recycled(nonce, &borrowed.transaction_id) - .await?; - tracing::warn!(transaction_id = %borrowed.transaction_id, nonce = nonce, error = %e, "Recycled failed transaction"); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - self.update_balance_threshold(scoped, chain).await?; - } + // TODO: Implement post-processing analysis for balance threshold updates and nonce resets + // Currently we lose the granular error handling that was in the individual atomic operations. + // Consider: + // 1. Analyzing submission_results for specific error patterns + // 2. Calling update_balance_threshold if needed + // 3. Detecting nonce reset conditions + // 4. Or move this logic into the batch processor itself + + // Process all results in one batch operation + let report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; - // Check if this should trigger nonce reset - if should_trigger_nonce_reset(&e) { - tracing::warn!( - eoa = %scoped.eoa(), - chain_id = %scoped.chain_id(), - "Nonce too high error detected, may need nonce synchronization" - ); - // The next confirm_flow will fetch fresh nonce and auto-sync - } - } - } - } - } + // TODO: Handle post-processing updates here if needed + // For now, we skip the individual error analysis that was done in the old atomic approach - recovered_count += 1; - } + tracing::info!( + "Recovered {} transactions: {} submitted, {} recycled, {} failed", + report.total_processed, + report.moved_to_submitted, + report.moved_to_pending, + report.failed_transactions + ); - Ok(recovered_count) + Ok(report.total_processed as u32) } // ========== CONFIRM FLOW ========== #[tracing::instrument(skip_all)] async fn confirm_flow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, - ) -> Result<(u32, u32), EoaExecutorWorkerError> { + ) -> Result { // Get fresh on-chain transaction count let current_chain_nonce = chain .provider() @@ -603,6 +644,8 @@ where Ok(cached_nonce) => cached_nonce, }; + let submitted_count = scoped.get_submitted_transactions_count().await?; + // no nonce progress if current_chain_nonce == cached_nonce { let current_health = self.get_eoa_health(scoped, chain).await?; @@ -610,7 +653,8 @@ where // No nonce progress - check if we should attempt gas bumping for stalled nonce let time_since_movement = now.saturating_sub(current_health.last_nonce_movement_at); - if time_since_movement > NONCE_STALL_TIMEOUT { + // if there are waiting transactions, we can attempt a gas bump + if time_since_movement > NONCE_STALL_TIMEOUT && submitted_count > 0 { tracing::info!( time_since_movement = time_since_movement, stall_timeout = NONCE_STALL_TIMEOUT, @@ -631,7 +675,7 @@ where } tracing::debug!("No nonce progress, skipping confirm flow"); - return Ok((0, 0)); + return Ok(CleanupReport::default()); } tracing::info!( @@ -641,111 +685,69 @@ where ); // Get all pending transactions below the current chain nonce - let pending_txs = self - .get_pending_transactions_below_nonce(scoped, current_chain_nonce) + let waiting_txs = scoped + .get_submitted_transactions_below_nonce(current_chain_nonce) .await?; - if pending_txs.is_empty() { - tracing::debug!("No pending transactions to confirm"); - return Ok((0, 0)); + if waiting_txs.is_empty() { + tracing::debug!("No waiting transactions to confirm"); + return Ok(CleanupReport::default()); } // Fetch receipts and categorize transactions - let (confirmed_txs, failed_txs) = self - .fetch_and_categorize_transactions(chain, pending_txs) + let (confirmed_txs, replaced_txs) = self + .fetch_confirmed_transaction_receipts(chain, waiting_txs) .await; // Process confirmed transactions - let confirmed_count = if !confirmed_txs.is_empty() { - let successes: Vec = confirmed_txs - .into_iter() - .map(|tx| { - let receipt_data = match serde_json::to_string(&tx.receipt) { - Ok(receipt_json) => receipt_json, - Err(e) => { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - error = %e, - "Failed to serialize receipt as JSON, using debug format" - ); - format!("{:?}", tx.receipt) - } - }; - - tracing::info!( - transaction_id = %tx.transaction_id, - nonce = tx.nonce, - hash = %tx.hash, - "Transaction confirmed" - ); - - TransactionSuccess { - hash: tx.hash, - transaction_id: tx.transaction_id, - receipt_data, + let successes: Vec = confirmed_txs + .into_iter() + .map(|tx| { + let receipt_data = match serde_json::to_string(&tx.receipt) { + Ok(receipt_json) => receipt_json, + Err(e) => { + tracing::warn!( + transaction_id = %tx.transaction_id, + hash = %tx.hash, + error = %e, + "Failed to serialize receipt as JSON, using debug format" + ); + format!("{:?}", tx.receipt) } - }) - .collect(); + }; - let count = successes.len() as u32; - scoped.batch_succeed_transactions(successes).await?; - count - } else { - 0 - }; + tracing::info!( + transaction_id = %tx.transaction_id, + nonce = tx.nonce, + hash = %tx.hash, + "Transaction confirmed" + ); - // Process failed transactions - let failed_count = if !failed_txs.is_empty() { - let failures: Vec = failed_txs - .into_iter() - .map(|tx| { - tracing::warn!( - transaction_id = %tx.transaction_id, - hash = %tx.hash, - "Transaction failed, requeued" - ); - TransactionFailure { - hash: tx.hash, - transaction_id: tx.transaction_id, - } - }) - .collect(); + ConfirmedTransaction { + hash: tx.hash, + transaction_id: tx.transaction_id, + receipt_data, + } + }) + .collect(); - let count = failures.len() as u32; - scoped.batch_fail_and_requeue_transactions(failures).await?; - count - } else { - 0 - }; + let report = scoped + .clean_submitted_transactions(&successes, current_chain_nonce - 1) + .await?; // Update cached transaction count scoped .update_cached_transaction_count(current_chain_nonce) .await?; - // Synchronize nonces to ensure consistency - if let Err(e) = self - .store - .synchronize_nonces_with_chain( - scoped.eoa(), - scoped.chain_id(), - scoped.worker_id(), - current_chain_nonce, - ) - .await - { - tracing::warn!(error = %e, "Failed to synchronize nonces with chain"); - } - - Ok((confirmed_count, failed_count)) + Ok(report) } // ========== SEND FLOW ========== #[tracing::instrument(skip_all)] async fn send_flow( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { // 1. Get EOA health (initializes if needed) and check if we should update balance @@ -753,30 +755,33 @@ where let now = chrono::Utc::now().timestamp_millis().max(0) as u64; // Update balance if it's stale - if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { - let balance = chain - .provider() - .get_balance(scoped.eoa()) - .await - .map_err(|e| { - let engine_error = e.to_engine_error(chain); - EoaExecutorWorkerError::RpcError { - message: format!("Failed to get balance: {}", engine_error), - inner_error: engine_error, - } - })?; + // TODO: refactor this, very ugly + if health.balance <= health.balance_threshold { + if now - health.balance_fetched_at > HEALTH_CHECK_INTERVAL { + let balance = chain + .provider() + .get_balance(scoped.eoa()) + .await + .map_err(|e| { + let engine_error = e.to_engine_error(chain); + EoaExecutorWorkerError::RpcError { + message: format!("Failed to get balance: {}", engine_error), + inner_error: engine_error, + } + })?; - health.balance = balance; - health.balance_fetched_at = now; - scoped.update_health_data(&health).await?; - } + health.balance = balance; + health.balance_fetched_at = now; + scoped.update_health_data(&health).await?; + } - if health.balance <= health.balance_threshold { - tracing::warn!( - "EOA has insufficient balance (<= {} wei), skipping send flow", - health.balance_threshold - ); - return Ok(0); + if health.balance <= health.balance_threshold { + tracing::warn!( + "EOA has insufficient balance (<= {} wei), skipping send flow", + health.balance_threshold + ); + return Ok(0); + } } let mut total_sent = 0; @@ -805,7 +810,7 @@ where async fn process_recycled_nonces( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let recycled_nonces = scoped.peek_recycled_nonces().await?; @@ -814,253 +819,190 @@ where return Ok(0); } - // Get pending transactions (one per recycled nonce) - let pending_txs = scoped - .peek_pending_transactions(recycled_nonces.len() as u64) - .await?; + let mut total_sent = 0; + let mut remaining_nonces = recycled_nonces; + + // Loop to handle preparation failures and refill with new transactions + while !remaining_nonces.is_empty() { + // Get pending transactions to match with recycled nonces + let pending_txs = scoped + .peek_pending_transactions(remaining_nonces.len() as u64) + .await?; + + if pending_txs.is_empty() { + tracing::debug!("No pending transactions available for recycled nonces"); + break; + } - // 1. SEQUENTIAL REDIS: Collect nonce-transaction pairs - let mut nonce_tx_pairs = Vec::new(); - for (i, nonce) in recycled_nonces.into_iter().enumerate() { - if let Some(tx_id) = pending_txs.get(i) { - // Get transaction data - if let Some(tx_data) = scoped.get_transaction_data(tx_id).await? { - nonce_tx_pairs.push((nonce, tx_id.clone(), tx_data)); + // Pair recycled nonces with pending transactions + let mut build_tasks = Vec::new(); + let mut nonce_tx_pairs = Vec::new(); + + for (i, nonce) in remaining_nonces.iter().enumerate() { + if let Some(p_tx) = pending_txs.get(i) { + build_tasks.push(self.build_and_sign_single_transaction_with_retries( + scoped, p_tx, *nonce, chain, + )); + nonce_tx_pairs.push((*nonce, p_tx.clone())); } else { - tracing::warn!("Transaction data not found for {}", tx_id); - continue; + // No more pending transactions for this recycled nonce + tracing::debug!("No pending transaction for recycled nonce {}", nonce); + break; } - } else { - // No pending transactions - skip recycled nonces without pending transactions - tracing::debug!("No pending transaction for recycled nonce {}", nonce); - continue; } - } - if nonce_tx_pairs.is_empty() { - return Ok(0); - } + if build_tasks.is_empty() { + break; + } - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_futures: Vec<_> = nonce_tx_pairs - .iter() - .map(|(nonce, transaction_id, tx_data)| async move { - let prepared = self - .build_and_sign_transaction(tx_data, *nonce, chain) - .await; - (*nonce, transaction_id, prepared) - }) - .collect(); + // Build and sign all transactions in parallel + let prepared_results = futures::future::join_all(build_tasks).await; - let build_results = futures::future::join_all(build_futures).await; - - // 3. SEQUENTIAL REDIS: Move successfully built transactions to borrowed state - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (nonce, transaction_id, build_result) in build_results { - match build_result { - Ok(signed_tx) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: transaction_id.clone(), - signed_transaction: signed_tx.clone(), - hash: signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - // Try to atomically move from pending to borrowed with recycled nonce - match scoped - .atomic_move_pending_to_borrowed_with_recycled_nonce( - transaction_id, - nonce, - &borrowed_data, - ) - .await - { - Ok(()) => { - let prepared = PreparedTransaction { - transaction_id: transaction_id.clone(), - signed_tx, - nonce, - }; - prepared_txs.push(prepared); - } - Err(TransactionStoreError::NonceNotInRecycledSet { .. }) => { - tracing::debug!("Nonce {} was consumed by another worker", nonce); - continue; - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!("Transaction {} already processed", transaction_id); - continue; - } - Err(e) => { - tracing::error!("Failed to move {} to borrowed: {}", transaction_id, e); - continue; - } + // Separate successful preparations from failures + let mut prepared_txs = Vec::new(); + let mut failed_tx_ids = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (i, result) in prepared_results.into_iter().enumerate() { + match result { + Ok(borrowed_data) => { + prepared_txs.push(borrowed_data); } - } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, + .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + + let (_nonce, pending_tx) = &nonce_tx_pairs[i]; + tracing::warn!( + "Failed to build recycled transaction {}: {}", + pending_tx.transaction_id, + e + ); + + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + failed_tx_ids.push(pending_tx.transaction_id.clone()); } } + } + } - tracing::warn!("Failed to build transaction {}: {}", transaction_id, e); - continue; + // Fail deterministic failures from pending state + for tx_id in failed_tx_ids { + if let Err(e) = scoped + .fail_pending_transaction( + &tx_id, + "Deterministic preparation failure", + self.webhook_queue.clone(), + ) + .await + { + tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); + } + } + + // Update balance threshold if needed + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold(scoped, chain).await { + tracing::error!( + "Failed to update balance threshold after build failures: {}", + e + ); } } - } - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e + if prepared_txs.is_empty() { + // No successful preparations, try again with more pending transactions + // Remove the nonces we couldn't use from our list + remaining_nonces = remaining_nonces + .into_iter() + .skip(nonce_tx_pairs.len()) + .collect(); + continue; + } + + // Move prepared transactions to borrowed state with recycled nonces + let moved_count = scoped + .atomic_move_pending_to_borrowed_with_recycled_nonces(&prepared_txs) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = prepared_txs.len(), + "Moved transactions to borrowed state using recycled nonces" + ); + + // Actually send the transactions to the blockchain + let send_tasks: Vec<_> = prepared_txs + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { chain.provider().send_tx_envelope(signed_tx.into()).await } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // Process send results and update states + let mut submission_results = Vec::new(); + for (i, send_result) in send_results.into_iter().enumerate() { + let borrowed_tx = &prepared_txs[i]; + let user_data = scoped + .get_transaction_data(&borrowed_tx.transaction_id) + .await?; + + let submission_result = SubmissionResult::from_send_result( + borrowed_tx, + send_result, + SendContext::InitialBroadcast, + user_data, + chain, ); + submission_results.push(submission_result); } - } - if prepared_txs.is_empty() { - return Ok(0); - } + // Use batch processing to handle all submission results + let processing_report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; - // 4. PARALLEL SEND: Send all transactions in parallel - let send_futures: Vec<_> = prepared_txs - .iter() - .map(|prepared| async move { - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); - let send_results = futures::future::join_all(send_futures).await; + total_sent += processing_report.moved_to_submitted; - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(_) => { - // Transaction sent successfully - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent recycled transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "Recycled transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "Recycled transaction failed, re-recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} back to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } - } - } + // Remove the nonces we successfully processed from our list + remaining_nonces = remaining_nonces.into_iter().skip(moved_count).collect(); + + // If we didn't use all available nonces, we ran out of pending transactions + if moved_count < nonce_tx_pairs.len() { + break; } } - Ok(sent_count) + Ok(total_sent as u32) } async fn process_new_transactions( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, budget: u64, ) -> Result { @@ -1068,253 +1010,240 @@ where return Ok(0); } - // 1. SEQUENTIAL REDIS: Get pending transactions - let pending_txs = scoped.peek_pending_transactions(budget).await?; - if pending_txs.is_empty() { - return Ok(0); - } + let mut total_sent = 0; + let mut remaining_budget = budget; + + // Loop to handle preparation failures and refill with new transactions + while remaining_budget > 0 { + // 1. Get pending transactions + let pending_txs = scoped.peek_pending_transactions(remaining_budget).await?; + if pending_txs.is_empty() { + break; + } - let optimistic_nonce = scoped.get_optimistic_nonce().await?; + let optimistic_nonce = scoped.get_optimistic_transaction_count().await?; + + // 2. Build and sign all transactions in parallel + let build_tasks: Vec<_> = pending_txs + .iter() + .enumerate() + .map(|(i, tx)| { + let expected_nonce = optimistic_nonce + i as u64; + self.build_and_sign_single_transaction_with_retries( + scoped, + tx, + expected_nonce, + chain, + ) + }) + .collect(); - // 2. PARALLEL BUILD/SIGN: Build and sign all transactions in parallel - let build_tasks: Vec<_> = pending_txs - .iter() - .enumerate() - .map(|(i, tx_id)| { - let expected_nonce = optimistic_nonce + i as u64; - self.build_and_sign_single_transaction(scoped, tx_id, expected_nonce, chain) - }) - .collect(); + let prepared_results = futures::future::join_all(build_tasks).await; - let prepared_results = futures::future::join_all(build_tasks).await; - - // 3. SEQUENTIAL REDIS: Move successful transactions to borrowed state (maintain nonce order) - let mut prepared_txs = Vec::new(); - let mut balance_threshold_update_needed = false; - - for (i, result) in prepared_results.into_iter().enumerate() { - match result { - Ok(prepared) => { - let borrowed_data = BorrowedTransactionData { - transaction_id: prepared.transaction_id.clone(), - signed_transaction: prepared.signed_tx.clone(), - hash: prepared.signed_tx.hash().to_string(), - borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, - }; - - match scoped - .atomic_move_pending_to_borrowed_with_new_nonce( - &prepared.transaction_id, - prepared.nonce, - &borrowed_data, - ) - .await - { - Ok(()) => prepared_txs.push(prepared), - Err(TransactionStoreError::OptimisticNonceChanged { .. }) => { - tracing::debug!( - "Nonce changed for transaction {}, skipping", - prepared.transaction_id - ); - break; // Stop processing if nonce changed - } - Err(TransactionStoreError::TransactionNotInPendingQueue { .. }) => { - tracing::debug!( - "Transaction {} already processed, skipping", - prepared.transaction_id - ); - continue; - } - Err(e) => { - tracing::error!( - "Failed to move transaction {} to borrowed: {}", - prepared.transaction_id, - e - ); - continue; - } + // 3. Separate successful preparations from failures + let mut prepared_txs = Vec::new(); + let mut failed_tx_ids = Vec::new(); + let mut balance_threshold_update_needed = false; + + for (i, result) in prepared_results.into_iter().enumerate() { + match result { + Ok(borrowed_data) => { + prepared_txs.push(borrowed_data); } - } - Err(e) => { - // Accumulate balance threshold issues instead of updating immediately - if let EoaExecutorWorkerError::TransactionSimulationFailed { - inner_error, .. - } = &e - { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + Err(e) => { + // Track balance threshold issues + if let EoaExecutorWorkerError::TransactionSimulationFailed { + inner_error, + .. + } = &e + { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } + } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { + if should_update_balance_threshold(inner_error) { + balance_threshold_update_needed = true; + } } - } else if let EoaExecutorWorkerError::RpcError { inner_error, .. } = &e { - if should_update_balance_threshold(inner_error) { - balance_threshold_update_needed = true; + + let pending_tx = &pending_txs[i]; + tracing::warn!( + "Failed to build transaction {}: {}", + pending_tx.transaction_id, + e + ); + + // For deterministic build failures, fail the transaction immediately + if !is_retryable_preparation_error(&e) { + failed_tx_ids.push(pending_tx.transaction_id.clone()); } } + } + } - tracing::warn!("Failed to build transaction {}: {}", pending_txs[i], e); - // Individual transaction failure doesn't stop the worker - continue; + // 4. Fail deterministic failures from pending state + for tx_id in failed_tx_ids { + if let Err(e) = scoped + .fail_pending_transaction( + &tx_id, + "Deterministic preparation failure", + self.webhook_queue.clone(), + ) + .await + { + tracing::error!("Failed to fail pending transaction {}: {}", tx_id, e); } } - } - // Update balance threshold once if any build failures were due to balance issues - if balance_threshold_update_needed { - if let Err(e) = self.update_balance_threshold(scoped, chain).await { - tracing::error!( - "Failed to update balance threshold after parallel build failures: {}", - e + // Update balance threshold if needed + if balance_threshold_update_needed { + if let Err(e) = self.update_balance_threshold(scoped, chain).await { + tracing::error!( + "Failed to update balance threshold after build failures: {}", + e + ); + } + } + + if prepared_txs.is_empty() { + // No successful preparations, try again with remaining budget + remaining_budget = remaining_budget.saturating_sub(pending_txs.len() as u64); + continue; + } + + // 5. Move prepared transactions to borrowed state + let moved_count = scoped + .atomic_move_pending_to_borrowed_with_incremented_nonces(&prepared_txs) + .await?; + + tracing::debug!( + moved_count = moved_count, + total_prepared = prepared_txs.len(), + "Moved transactions to borrowed state using incremented nonces" + ); + + // 6. Actually send the transactions to the blockchain + let send_tasks: Vec<_> = prepared_txs + .iter() + .map(|borrowed_tx| { + let signed_tx = borrowed_tx.signed_transaction.clone(); + async move { chain.provider().send_tx_envelope(signed_tx.into()).await } + }) + .collect(); + + let send_results = futures::future::join_all(send_tasks).await; + + // 7. Process send results and update states + let mut submission_results = Vec::new(); + for (i, send_result) in send_results.into_iter().enumerate() { + let borrowed_tx = &prepared_txs[i]; + let user_data = scoped + .get_transaction_data(&borrowed_tx.transaction_id) + .await?; + + let submission_result = SubmissionResult::from_send_result( + borrowed_tx, + send_result, + SendContext::InitialBroadcast, + user_data, + chain, ); + submission_results.push(submission_result); } - } - if prepared_txs.is_empty() { - return Ok(0); + // 8. Use batch processing to handle all submission results + let processing_report = scoped + .process_borrowed_transactions(submission_results, self.webhook_queue.clone()) + .await?; + + tracing::debug!( + "Processed {} borrowed transactions: {} moved to submitted, {} moved to pending, {} failed", + processing_report.total_processed, + processing_report.moved_to_submitted, + processing_report.moved_to_pending, + processing_report.failed_transactions + ); + + total_sent += processing_report.moved_to_submitted; + remaining_budget = remaining_budget.saturating_sub(moved_count as u64); + + // If we didn't use all our budget, we ran out of pending transactions + if moved_count < pending_txs.len() { + break; + } } - // 4. PARALLEL SEND (but ordered): Send all transactions in parallel but in nonce order - let send_futures: Vec<_> = prepared_txs - .iter() - .enumerate() - .map(|(i, prepared)| async move { - // Add delay for ordering (except first transaction) - if i > 0 { - sleep(Duration::from_millis(50)).await; // 50ms delay between consecutive nonces - } + Ok(total_sent as u32) + } - let result = chain - .provider() - .send_tx_envelope(prepared.signed_tx.clone().into()) - .await; - (prepared, result) - }) - .collect(); + // ========== TRANSACTION BUILDING & SENDING ========== + async fn build_and_sign_single_transaction_with_retries( + &self, + scoped: &AtomicEoaExecutorStore, + pending_transaction: &PendingTransaction, + nonce: u64, + chain: &impl Chain, + ) -> Result { + let mut last_error = None; - let send_results = futures::future::join_all(send_futures).await; + // Internal retry loop for retryable errors + for attempt in 0..=MAX_PREPARATION_RETRIES { + if attempt > 0 { + // Simple exponential backoff + let delay = PREPARATION_RETRY_DELAY_MS * (2_u64.pow(attempt - 1)); + sleep(Duration::from_millis(delay)).await; - // 5. SEQUENTIAL REDIS: Process results and update states - let mut sent_count = 0; - for (prepared, send_result) in send_results { - match send_result { - Ok(_) => { - // Transaction sent successfully - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - hash = ?prepared.signed_tx.hash(), - "Successfully sent new transaction" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - Err(e) => { - match classify_send_error(&e, SendContext::InitialBroadcast) { - SendErrorClassification::PossiblySent => { - // Move to submitted state - match scoped - .move_borrowed_to_submitted( - prepared.nonce, - &format!("{:?}", prepared.signed_tx.hash()), - &prepared.transaction_id, - ) - .await - { - Ok(()) => { - sent_count += 1; - tracing::info!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - "New transaction possibly sent" - ); - } - Err(e) => { - tracing::error!( - "Failed to move {} to submitted: {}", - prepared.transaction_id, - e - ); - } - } - } - SendErrorClassification::DeterministicFailure => { - // Recycle nonce and requeue transaction - match scoped - .move_borrowed_to_recycled(prepared.nonce, &prepared.transaction_id) - .await - { - Ok(()) => { - tracing::warn!( - transaction_id = %prepared.transaction_id, - nonce = prepared.nonce, - error = %e, - "New transaction failed, recycled nonce" - ); - - if should_update_balance_threshold(&e.to_engine_error(chain)) { - if let Err(e) = - self.update_balance_threshold(scoped, chain).await - { - tracing::error!( - "Failed to update balance threshold: {}", - e - ); - } - } - - if should_trigger_nonce_reset(&e) { - tracing::warn!( - nonce = prepared.nonce, - "Nonce too high error detected, may need nonce synchronization" - ); - } - } - Err(e) => { - tracing::error!( - "Failed to move {} to recycled: {}", - prepared.transaction_id, - e - ); - } - } - } + tracing::debug!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + "Retrying transaction preparation" + ); + } + + match self + .build_and_sign_single_transaction(scoped, pending_transaction, nonce, chain) + .await + { + Ok(result) => return Ok(result), + Err(error) => { + if is_retryable_preparation_error(&error) && attempt < MAX_PREPARATION_RETRIES { + tracing::warn!( + transaction_id = %pending_transaction.transaction_id, + attempt = attempt, + error = %error, + "Retryable error during transaction preparation, will retry" + ); + last_error = Some(error); + continue; + } else { + // Either deterministic error or exceeded max retries + return Err(error); } } } } - Ok(sent_count) + // This should never be reached, but just in case + Err( + last_error.unwrap_or_else(|| EoaExecutorWorkerError::InternalError { + message: "Unexpected error in retry loop".to_string(), + }), + ) } - // ========== TRANSACTION BUILDING & SENDING ========== async fn build_and_sign_single_transaction( &self, - scoped: &ScopedEoaExecutorStore<'_>, - transaction_id: &str, + scoped: &AtomicEoaExecutorStore, + pending_transaction: &PendingTransaction, nonce: u64, chain: &impl Chain, - ) -> Result { + ) -> Result { // Get transaction data let tx_data = scoped - .get_transaction_data(transaction_id) + .get_transaction_data(&pending_transaction.transaction_id) .await? .ok_or_else(|| EoaExecutorWorkerError::TransactionNotFound { - transaction_id: transaction_id.to_string(), + transaction_id: pending_transaction.transaction_id.clone(), })?; // Build and sign transaction @@ -1322,25 +1251,28 @@ where .build_and_sign_transaction(&tx_data, nonce, chain) .await?; - Ok(PreparedTransaction { - transaction_id: transaction_id.to_string(), - signed_tx, - nonce, + Ok(BorrowedTransactionData { + transaction_id: pending_transaction.transaction_id.clone(), + hash: signed_tx.hash().to_string(), + signed_transaction: signed_tx, + borrowed_at: chrono::Utc::now().timestamp_millis().max(0) as u64, + queued_at: pending_transaction.queued_at, }) } async fn send_noop_transaction( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, nonce: u64, - ) -> Result { + credential: SigningCredential, + ) -> Result { // Create a minimal transaction to consume the recycled nonce // Send 0 ETH to self with minimal gas let eoa = scoped.eoa(); // Build no-op transaction (send 0 to self) - let mut tx_request = AlloyTransactionRequest::default() + let tx_request = AlloyTransactionRequest::default() .with_from(eoa) .with_to(eoa) // Send to self .with_value(U256::ZERO) // Send 0 value @@ -1349,43 +1281,24 @@ where .with_nonce(nonce) .with_gas_limit(21000); // Minimal gas for basic transfer - // Estimate gas to ensure the transaction is valid - match chain.provider().estimate_gas(tx_request.clone()).await { - Ok(gas_limit) => { - tx_request = tx_request.with_gas_limit(gas_limit); + let tx_request = self.estimate_gas_fees(chain, tx_request).await?; + let built_tx = tx_request.build_typed_tx().map_err(|e| { + EoaExecutorWorkerError::TransactionBuildFailed { + message: format!("Failed to build typed transaction for no-op: {e:?}"), } - Err(e) => { - tracing::warn!( - nonce = nonce, - error = %e, - "Failed to estimate gas for no-op transaction" - ); - return Ok(false); - } - } + })?; - // Build typed transaction - let typed_tx = match tx_request.build_typed_tx() { - Ok(tx) => tx, - Err(e) => { - tracing::warn!( - nonce = nonce, - error = ?e, - "Failed to build typed transaction for no-op" - ); - return Ok(false); - } - }; + let tx = self.sign_transaction(eoa, credential, built_tx).await?; - // Get signing credential from health or use default approach - // For no-op transactions, we need to find a valid signing credential - // This is a limitation of the current design - no-op transactions - // need access to signing credentials which are transaction-specific - tracing::warn!( - nonce = nonce, - "No-op transaction requires signing credential access - recycled nonce will remain unconsumed" - ); - Ok(false) + chain + .provider() + .send_tx_envelope(tx.into()) + .await + .map_err(|e| EoaExecutorWorkerError::TransactionSendError { + message: format!("Failed to send no-op transaction: {e:?}"), + inner_error: e.to_engine_error(chain), + }) + .map(|pending| pending.tx_hash().to_string()) } // ========== GAS BUMP METHODS ========== @@ -1393,7 +1306,7 @@ where /// Attempt to gas bump a stalled transaction for the next expected nonce async fn attempt_gas_bump_for_stalled_nonce( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, expected_nonce: u64, ) -> Result { @@ -1403,9 +1316,11 @@ where ); // Get all transaction IDs for this nonce - let transaction_ids = scoped.get_transaction_ids_for_nonce(expected_nonce).await?; + let submitted_transactions = scoped + .get_submitted_transactions_for_nonce(expected_nonce) + .await?; - if transaction_ids.is_empty() { + if submitted_transactions.is_empty() { tracing::debug!( nonce = expected_nonce, "No transactions found for stalled nonce" @@ -1417,7 +1332,7 @@ where let mut newest_transaction: Option<(String, TransactionData)> = None; let mut newest_submitted_at = 0u64; - for transaction_id in transaction_ids { + for SubmittedTransaction { transaction_id, .. } in submitted_transactions { if let Some(tx_data) = scoped.get_transaction_data(&transaction_id).await? { // Find the most recent attempt for this transaction if let Some(latest_attempt) = tx_data.attempts.last() { @@ -1473,7 +1388,14 @@ where } }; let bumped_typed_tx = self.apply_gas_bump_to_typed_transaction(typed_tx, 120); // 20% increase - let bumped_tx = match self.sign_transaction(bumped_typed_tx, &tx_data).await { + let bumped_tx = match self + .sign_transaction( + tx_data.user_request.from, + tx_data.user_request.signing_credential, + bumped_typed_tx, + ) + .await + { Ok(tx) => tx, Err(e) => { tracing::warn!( @@ -1488,7 +1410,15 @@ where // Record the gas bump attempt scoped - .add_gas_bump_attempt(&transaction_id, bumped_tx.clone()) + .add_gas_bump_attempt( + &SubmittedTransaction { + nonce: expected_nonce, + hash: bumped_tx.hash().to_string(), + transaction_id: transaction_id.to_string(), + queued_at: tx_data.created_at, + }, + bumped_tx.clone(), + ) .await?; // Send the bumped transaction @@ -1524,7 +1454,7 @@ where /// This method ensures the health data is always available for the worker async fn get_eoa_health( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result { let store_health = scoped.check_eoa_health().await?; @@ -1568,7 +1498,7 @@ where #[tracing::instrument(skip_all, fields(eoa = %scoped.eoa(), chain_id = %chain.chain_id()))] async fn update_balance_threshold( &self, - scoped: &ScopedEoaExecutorStore<'_>, + scoped: &AtomicEoaExecutorStore, chain: &impl Chain, ) -> Result<(), EoaExecutorWorkerError> { let mut health = self.get_eoa_health(scoped, chain).await?; @@ -1591,36 +1521,17 @@ where Ok(()) } - // ========== CONFIRMATION FLOW HELPERS ========== - - /// Get pending transactions below the given nonce - async fn get_pending_transactions_below_nonce( - &self, - scoped: &ScopedEoaExecutorStore<'_>, - nonce: u64, - ) -> Result, EoaExecutorWorkerError> { - let pending_hashes = scoped.get_hashes_below_nonce(nonce).await?; - - let pending_txs = pending_hashes - .into_iter() - .map(|(nonce, hash, transaction_id)| PendingTransaction { - nonce, - hash, - transaction_id, - }) - .collect(); - - Ok(pending_txs) - } - - /// Fetch receipts for all pending transactions and categorize them - async fn fetch_and_categorize_transactions( + /// Fetch receipts for all submitted transactions and categorize them + async fn fetch_confirmed_transaction_receipts( &self, chain: &impl Chain, - pending_txs: Vec, - ) -> (Vec, Vec) { + submitted_txs: Vec, + ) -> ( + Vec, + Vec, + ) { // Fetch all receipts in parallel - let receipt_futures: Vec<_> = pending_txs + let receipt_futures: Vec<_> = submitted_txs .iter() .filter_map(|tx| match tx.hash.parse::() { Ok(hash_bytes) => Some(async move { @@ -1643,7 +1554,7 @@ where for (tx, receipt_result) in receipt_results { match receipt_result { Ok(Some(receipt)) => { - confirmed_txs.push(ConfirmedTransaction { + confirmed_txs.push(ConfirmedTransactionWithRichReceipt { nonce: tx.nonce, hash: tx.hash.clone(), transaction_id: tx.transaction_id.clone(), @@ -1651,7 +1562,7 @@ where }); } Ok(None) | Err(_) => { - failed_txs.push(FailedTransaction { + failed_txs.push(ReplacedTransaction { hash: tx.hash.clone(), transaction_id: tx.transaction_id.clone(), }); @@ -1855,21 +1766,18 @@ where async fn sign_transaction( &self, + from: Address, + credential: SigningCredential, typed_tx: TypedTransaction, - tx_data: &TransactionData, ) -> Result, EoaExecutorWorkerError> { let signing_options = EoaSigningOptions { - from: tx_data.user_request.from, - chain_id: Some(tx_data.user_request.chain_id), + from, + chain_id: typed_tx.chain_id(), }; let signature = self .eoa_signer - .sign_transaction( - signing_options, - typed_tx.clone(), - tx_data.user_request.signing_credential.clone(), - ) + .sign_transaction(signing_options, typed_tx.clone(), credential) .await .map_err(|engine_error| EoaExecutorWorkerError::SigningError { message: format!("Failed to sign transaction: {}", engine_error), @@ -1892,7 +1800,12 @@ where chain: &impl Chain, ) -> Result, EoaExecutorWorkerError> { let typed_tx = self.build_typed_transaction(tx_data, nonce, chain).await?; - self.sign_transaction(typed_tx, tx_data).await + self.sign_transaction( + tx_data.user_request.from, + tx_data.user_request.signing_credential.clone(), + typed_tx, + ) + .await } fn apply_gas_bump_to_typed_transaction( diff --git a/executors/src/webhook/envelope.rs b/executors/src/webhook/envelope.rs index 1887615..b4b3e06 100644 --- a/executors/src/webhook/envelope.rs +++ b/executors/src/webhook/envelope.rs @@ -41,6 +41,35 @@ pub struct WebhookNotificationEnvelope { pub delivery_target_url: Option, } +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BareWebhookNotificationEnvelope { + pub transaction_id: String, + pub event_type: StageEvent, + pub executor_name: String, + pub stage_name: String, + pub payload: T, +} + +impl BareWebhookNotificationEnvelope { + pub fn into_webhook_notification_envelope( + self, + timestamp: u64, + delivery_target_url: String, + ) -> WebhookNotificationEnvelope { + WebhookNotificationEnvelope { + notification_id: Uuid::new_v4().to_string(), + transaction_id: self.transaction_id, + timestamp, + executor_name: self.executor_name, + stage_name: self.stage_name, + event_type: self.event_type, + payload: self.payload, + delivery_target_url: Some(delivery_target_url), + } + } +} + // --- Serializable Hook Data Wrappers --- // These wrap the hook data to make them serializable (removing lifetimes) #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/executors/src/webhook/mod.rs b/executors/src/webhook/mod.rs index 3408d26..271c4b6 100644 --- a/executors/src/webhook/mod.rs +++ b/executors/src/webhook/mod.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use engine_core::execution_options::WebhookOptions; use hex; use hmac::{Hmac, Mac}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; @@ -9,7 +10,9 @@ use serde::{Deserialize, Serialize}; use twmq::error::TwmqError; use twmq::hooks::TransactionContext; use twmq::job::{BorrowedJob, JobError, JobResult, RequeuePosition, ToJobResult}; -use twmq::{DurableExecution, FailHookData, NackHookData, SuccessHookData, UserCancellable}; +use twmq::{DurableExecution, FailHookData, NackHookData, Queue, SuccessHookData, UserCancellable}; + +use crate::webhook::envelope::{BareWebhookNotificationEnvelope, WebhookNotificationEnvelope}; pub mod envelope; @@ -113,7 +116,10 @@ impl DurableExecution for WebhookJobHandler { type JobData = WebhookJobPayload; #[tracing::instrument(skip_all, fields(queue = "webhook", job_id = job.job.id))] - async fn process(&self, job: &BorrowedJob) -> JobResult { + async fn process( + &self, + job: &BorrowedJob, + ) -> JobResult { let payload = &job.job.data; let mut request_headers = HeaderMap::new(); @@ -423,3 +429,77 @@ impl DurableExecution for WebhookJobHandler { ); } } + +pub fn queue_webhook_envelopes( + envelope: BareWebhookNotificationEnvelope, + webhook_options: Vec, + tx: &mut TransactionContext<'_>, + webhook_queue: Arc>, +) -> Result<(), TwmqError> { + let now = chrono::Utc::now().timestamp().max(0) as u64; + let serialised_webhook_envelopes = + webhook_options + .iter() + .map(|webhook_option| { + let webhook_notification_envelope = envelope + .clone() + .into_webhook_notification_envelope(now, webhook_option.url.clone()); + let serialised_envelope = serde_json::to_string(&webhook_notification_envelope)?; + Ok(( + serialised_envelope, + webhook_notification_envelope, + webhook_option.clone(), + )) + }) + .collect::, WebhookOptions)>, + serde_json::Error, + >>()?; + + let webhook_payloads = serialised_webhook_envelopes + .into_iter() + .map( + |(serialised_envelope, webhook_notification_envelope, webhook_option)| { + let payload = WebhookJobPayload { + url: webhook_option.url, + body: serialised_envelope, + headers: Some( + [ + ("Content-Type".to_string(), "application/json".to_string()), + ( + "User-Agent".to_string(), + format!("{}/{}", envelope.executor_name, envelope.stage_name), + ), + ] + .into_iter() + .collect(), + ), + hmac_secret: webhook_option.secret, // TODO: Add HMAC support if needed + http_method: Some("POST".to_string()), + }; + (payload, webhook_notification_envelope) + }, + ) + .collect::>(); + + for (payload, webhook_notification_envelope) in webhook_payloads { + let mut webhook_job = webhook_queue.clone().job(payload); + webhook_job.options.id = format!( + "{}_{}_webhook", + webhook_notification_envelope.transaction_id, + webhook_notification_envelope.notification_id + ); + + tx.queue_job(webhook_job)?; + tracing::info!( + transaction_id = %webhook_notification_envelope.transaction_id, + executor = %webhook_notification_envelope.executor_name, + stage = %webhook_notification_envelope.stage_name, + event = ?webhook_notification_envelope.event_type, + notification_id = %webhook_notification_envelope.notification_id, + "Queued webhook notification" + ); + } + + Ok(()) +} diff --git a/server/src/execution_router/mod.rs b/server/src/execution_router/mod.rs index 446ea07..9280dd1 100644 --- a/server/src/execution_router/mod.rs +++ b/server/src/execution_router/mod.rs @@ -26,7 +26,7 @@ use engine_executors::{ transaction_registry::TransactionRegistry, webhook::WebhookJobHandler, }; -use twmq::{Queue, error::TwmqError}; +use twmq::{Queue, error::TwmqError, redis::aio::ConnectionManager}; use vault_sdk::VaultClient; use vault_types::{ RegexRule, Rule, @@ -37,11 +37,12 @@ use vault_types::{ use crate::chains::ThirdwebChainService; pub struct ExecutionRouter { + pub redis: ConnectionManager, + pub namespace: Option, pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -404,13 +405,20 @@ impl ExecutionRouter { data: transaction.data.clone(), gas_limit: transaction.gas_limit, webhook_options: webhook_options.clone(), - signing_credential, + signing_credential: signing_credential.clone(), rpc_credentials, transaction_type_data: transaction.transaction_type_data.clone(), }; + let eoa_executor_store = EoaExecutorStore::new( + self.redis.clone(), + self.namespace.clone(), + eoa_execution_options.from, + base_execution_options.chain_id, + ); + // Add transaction to the store - self.eoa_executor_store + eoa_executor_store .add_transaction(eoa_transaction_request) .await .map_err(|e| TwmqError::Runtime { @@ -429,10 +437,7 @@ impl ExecutionRouter { let eoa_job_data = EoaExecutorWorkerJobData { eoa_address: eoa_execution_options.from, chain_id: base_execution_options.chain_id, - worker_id: format!( - "eoa_{}_{}", - eoa_execution_options.from, base_execution_options.chain_id - ), + noop_signing_credential: signing_credential, }; // Create idempotent job for this EOA:chain - only one will exist diff --git a/server/src/main.rs b/server/src/main.rs index 995317b..cd4dc4c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -51,9 +51,10 @@ async fn main() -> anyhow::Result<()> { iaw_client: iaw_client.clone(), }); let eoa_signer = Arc::new(EoaSigner::new(vault_client.clone(), iaw_client)); + let redis_client = twmq::redis::Client::open(config.redis.url.as_str())?; let queue_manager = QueueManager::new( - &config.redis, + redis_client.clone(), &config.queue, chains.clone(), signer.clone(), @@ -74,11 +75,12 @@ async fn main() -> anyhow::Result<()> { .build()?; let execution_router = ExecutionRouter { + namespace: config.queue.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, webhook_queue: queue_manager.webhook_queue.clone(), external_bundler_send_queue: queue_manager.external_bundler_send_queue.clone(), userop_confirm_queue: queue_manager.userop_confirm_queue.clone(), eoa_executor_queue: queue_manager.eoa_executor_queue.clone(), - eoa_executor_store: queue_manager.eoa_executor_store.clone(), eip7702_send_queue: queue_manager.eip7702_send_queue.clone(), eip7702_confirm_queue: queue_manager.eip7702_confirm_queue.clone(), transaction_registry: queue_manager.transaction_registry.clone(), diff --git a/server/src/queue/manager.rs b/server/src/queue/manager.rs index 04e216e..20fe806 100644 --- a/server/src/queue/manager.rs +++ b/server/src/queue/manager.rs @@ -5,7 +5,7 @@ use alloy::transports::http::reqwest; use engine_core::error::EngineError; use engine_executors::{ eip7702_executor::{confirm::Eip7702ConfirmationHandler, send::Eip7702SendHandler}, - eoa::{EoaExecutorStore, EoaExecutorWorker}, + eoa::EoaExecutorWorker, external_bundler::{ confirm::UserOpConfirmationHandler, deployment::{RedisDeploymentCache, RedisDeploymentLock}, @@ -16,17 +16,13 @@ use engine_executors::{ }; use twmq::{Queue, queue::QueueOptions, shutdown::ShutdownHandle}; -use crate::{ - chains::ThirdwebChainService, - config::{QueueConfig, RedisConfig}, -}; +use crate::{chains::ThirdwebChainService, config::QueueConfig}; pub struct QueueManager { pub webhook_queue: Arc>, pub external_bundler_send_queue: Arc>>, pub userop_confirm_queue: Arc>>, pub eoa_executor_queue: Arc>>, - pub eoa_executor_store: Arc, pub eip7702_send_queue: Arc>>, pub eip7702_confirm_queue: Arc>>, pub transaction_registry: Arc, @@ -48,27 +44,18 @@ const EOA_EXECUTOR_QUEUE_NAME: &str = "eoa_executor"; impl QueueManager { pub async fn new( - redis_config: &RedisConfig, + redis_client: twmq::redis::Client, queue_config: &QueueConfig, chain_service: Arc, userop_signer: Arc, eoa_signer: Arc, ) -> Result { - // Create Redis clients - let redis_client = twmq::redis::Client::open(redis_config.url.as_str())?; - // Create transaction registry let transaction_registry = Arc::new(TransactionRegistry::new( redis_client.get_connection_manager().await?, queue_config.execution_namespace.clone(), )); - // Create EOA executor store - let eoa_executor_store = Arc::new(EoaExecutorStore::new( - redis_client.get_connection_manager().await?, - queue_config.execution_namespace.clone(), - )); - // Create deployment cache and lock let deployment_cache = RedisDeploymentCache::new(redis_client.clone()).await?; let deployment_lock = RedisDeploymentLock::new(redis_client.clone()).await?; @@ -221,8 +208,10 @@ impl QueueManager { // Create EOA executor queue let eoa_executor_handler = EoaExecutorWorker { chain_service: chain_service.clone(), - store: eoa_executor_store.clone(), eoa_signer: eoa_signer.clone(), + webhook_queue: webhook_queue.clone(), + namespace: queue_config.execution_namespace.clone(), + redis: redis_client.get_connection_manager().await?, max_inflight: 100, max_recycled_nonces: 50, }; @@ -241,7 +230,6 @@ impl QueueManager { external_bundler_send_queue, userop_confirm_queue, eoa_executor_queue, - eoa_executor_store, eip7702_send_queue, eip7702_confirm_queue, transaction_registry, diff --git a/twmq/src/lib.rs b/twmq/src/lib.rs index be6733c..177fc0e 100644 --- a/twmq/src/lib.rs +++ b/twmq/src/lib.rs @@ -161,6 +161,15 @@ impl Queue { } } + /// Create a TransactionContext from an existing Redis pipeline + /// This allows queueing jobs atomically within an existing transaction + pub fn transaction_context_from_pipeline<'a>( + &self, + pipeline: &'a mut redis::Pipeline, + ) -> hooks::TransactionContext<'a> { + hooks::TransactionContext::new(pipeline, self.name.clone()) + } + // Get queue name pub fn name(&self) -> &str { &self.name