From e3d28d2ba22d1767d7082e57971ee20720ce46b4 Mon Sep 17 00:00:00 2001 From: Maksim Vykhota Date: Thu, 4 Apr 2024 22:35:46 +0300 Subject: [PATCH 1/3] Feat(evm-block-recovery): Add `account-ledger` routine --- Cargo.lock | 1 + evm-utils/evm-block-recovery/Cargo.toml | 1 + evm-utils/evm-block-recovery/src/cli.rs | 43 ++++- evm-utils/evm-block-recovery/src/error.rs | 7 + evm-utils/evm-block-recovery/src/main.rs | 11 +- evm-utils/evm-block-recovery/src/routines.rs | 6 +- .../src/routines/account_ledger.rs | 151 ++++++++++++++++++ .../src/routines/scratchpad.rs | 6 - 8 files changed, 206 insertions(+), 20 deletions(-) create mode 100644 evm-utils/evm-block-recovery/src/routines/account_ledger.rs delete mode 100644 evm-utils/evm-block-recovery/src/routines/scratchpad.rs diff --git a/Cargo.lock b/Cargo.lock index 41b6bfe3c2..5c32ef3eb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2068,6 +2068,7 @@ dependencies = [ "serde_json", "solana-client", "solana-evm-loader-program", + "solana-program 1.10.41", "solana-sdk 1.10.41", "solana-storage-bigtable", "solana-transaction-status", diff --git a/evm-utils/evm-block-recovery/Cargo.toml b/evm-utils/evm-block-recovery/Cargo.toml index c3bbbca0a1..cefcbeb2e1 100644 --- a/evm-utils/evm-block-recovery/Cargo.toml +++ b/evm-utils/evm-block-recovery/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" solana-client = { path = "../../client" } solana-evm-loader-program = { path = "../programs/evm_loader" } +solana-program = { path = "../../sdk/program" } solana-sdk = { path = "../../sdk", version = "=1.10.41" } solana-storage-bigtable = { path = "../../storage-bigtable", version = "=1.10.41" } solana-transaction-status = { path = "../../transaction-status", version = "=1.10.41" } diff --git a/evm-utils/evm-block-recovery/src/cli.rs b/evm-utils/evm-block-recovery/src/cli.rs index 93080fb809..463e86482f 100644 --- a/evm-utils/evm-block-recovery/src/cli.rs +++ b/evm-utils/evm-block-recovery/src/cli.rs @@ -51,7 +51,8 @@ pub enum Command { ScanEvmStateRoots(ScanEvmStateRootsArgs), - ScratchPad, + /// Get ledger for the provided address + AccountLedger(AccountLedgerArgs), /// Generetes Shell Completions for this Utility Completion(CompletionArgs), @@ -257,10 +258,46 @@ pub struct ScanEvmStateRootsArgs { pub rangemap_json: PathBuf, } +#[derive(clap::Args)] +pub struct AccountLedgerArgs { + /// Address to search for + #[clap(short, long, value_name = "PUBKEY")] + pub address: String, + + /// Start with the first signature older than this one + #[clap(short, long, value_name = "SIGNATURE")] + pub before_signature: Option, + + /// End with the last signature more recent than this one + #[clap(short, long, value_name = "SIGNATURE")] + pub until_signature: Option, + + /// Stop after this many signatures + #[clap(short, long, value_name = "NUM")] + pub limit: usize, + + /// Export CSV results to file + #[clap(short, long, value_hint = clap::ValueHint::FilePath, value_name = "FILEPATH")] + pub output: PathBuf, + + /// CSV column separator + #[clap(short, long, value_name = "STRING", value_enum, default_value_t = CsvSeparator::Semicolon)] + pub separator: CsvSeparator, +} + +#[derive(Clone, clap::ValueEnum)] +pub enum CsvSeparator { + /// Use Comma as separator in CSV output + Comma, + /// Use Semicolon as separator in CSV output + Semicolon, + /// Use Tabulation as separator in CSV output + Tab, +} + #[derive(clap::Args)] pub struct CompletionArgs { /// Which shell completions to generate - #[arg(value_enum)] - #[clap(long, value_name = "STRING")] + #[clap(long, value_enum, value_name = "STRING")] pub shell: clap_complete::Shell, } diff --git a/evm-utils/evm-block-recovery/src/error.rs b/evm-utils/evm-block-recovery/src/error.rs index f44a4049fd..64a1bc9699 100644 --- a/evm-utils/evm-block-recovery/src/error.rs +++ b/evm-utils/evm-block-recovery/src/error.rs @@ -85,6 +85,12 @@ pub enum AppError { limit: usize, }, + #[error("Unable to fetch Signatures")] + GetSignatures { + #[source] + source: solana_storage_bigtable::Error, + }, + #[error("Unable to write block to bigtable")] UploadEvmBlock(#[source] solana_storage_bigtable::Error), @@ -144,6 +150,7 @@ impl AppError { AppError::TokioTaskJoin(_) => 1021, AppError::IO(_) => 1022, AppError::Storage(_) => 1023, + AppError::GetSignatures { .. } => 1024, } } } diff --git a/evm-utils/evm-block-recovery/src/main.rs b/evm-utils/evm-block-recovery/src/main.rs index 1173bf810e..0b36c33810 100644 --- a/evm-utils/evm-block-recovery/src/main.rs +++ b/evm-utils/evm-block-recovery/src/main.rs @@ -34,13 +34,8 @@ async fn main() { builder.target(Target::Stderr); builder.init(); - match dotenv { - Ok(path) => { - log::info!(r#""{}" successfully loaded"#, path.display()) - } - Err(e) => { - log::warn!(r#"".env" file not found: {e:?}""#) - } + if let Ok(path) = dotenv { + log::info!(r#""{}" successfully loaded"#, path.display()) } let execution_result = match cli.subcommand { @@ -54,7 +49,7 @@ async fn main() { RepeatEvm(args) => repeat_evm(args).await, RepeatNative(args) => repeat_native(args).await, ScanEvmStateRoots(ref args) => scan_evm_state_roots::command(args).await, - ScratchPad => scratchpad::command().await, + AccountLedger(args) => account_ledger(cli.creds, cli.instance, args).await, Completion(args) => completion(args), }; diff --git a/evm-utils/evm-block-recovery/src/routines.rs b/evm-utils/evm-block-recovery/src/routines.rs index 966bc9a06a..12981548ea 100644 --- a/evm-utils/evm-block-recovery/src/routines.rs +++ b/evm-utils/evm-block-recovery/src/routines.rs @@ -1,3 +1,4 @@ +pub(crate) mod account_ledger; pub(crate) mod check_evm; pub(crate) mod check_native; pub(crate) mod compare; @@ -7,12 +8,11 @@ pub(crate) mod repeat; pub(crate) mod restore_chain; pub(crate) mod scan_evm_state_roots; pub(crate) mod upload; -// a tmp placeholder to test out various new commands or approaches -// it's convenient, as all potentially required dependencies have been imported -pub(crate) mod scratchpad; use crate::error::AppError; + pub use { + account_ledger::account_ledger, check_evm::check_evm, check_native::check_native, compare::compare_native, diff --git a/evm-utils/evm-block-recovery/src/routines/account_ledger.rs b/evm-utils/evm-block-recovery/src/routines/account_ledger.rs new file mode 100644 index 0000000000..afa0c8017b --- /dev/null +++ b/evm-utils/evm-block-recovery/src/routines/account_ledger.rs @@ -0,0 +1,151 @@ +use std::str::FromStr; + +use solana_sdk::{message::VersionedMessage, pubkey::Pubkey, signature::Signature, system_program}; +use solana_transaction_status::TransactionWithStatusMeta; + +use crate::{ + cli::{AccountLedgerArgs, CsvSeparator}, + error::{AppError, RoutineResult}, + ledger, +}; + +// NOTE: remove panics if routine as service embedding is required + +pub async fn account_ledger( + creds: Option, + instance: String, + args: AccountLedgerArgs, +) -> RoutineResult { + let AccountLedgerArgs { + address, + before_signature, + until_signature, + limit, + output, + separator, + } = args; + + let before_signature = before_signature.map(|x| { + solana_sdk::signature::Signature::from_str(&x).expect(&format!("Invalid signature: {}", x)) + }); + + let until_signature = until_signature.map(|x| { + solana_sdk::signature::Signature::from_str(&x).expect(&format!("Invalid signature: {}", x)) + }); + + let ledger = ledger::with_params(creds, instance).await?; + + let address = Pubkey::from_str(&address).unwrap(); + + let signatures = ledger + .get_confirmed_signatures_for_address( + &address, + before_signature.as_ref(), + until_signature.as_ref(), + limit, + ) + .await + .map_err(|source| AppError::GetSignatures { source })?; + + log::info!("{} signatures discovered", signatures.len()); + + struct LedgerEntry { + signature: Signature, + is_transfer: bool, + counterparty: Pubkey, + block_time: i64, + pre_balance: u64, + post_balance: u64, + } + + let mut result_ledger: Vec = Vec::with_capacity(signatures.len()); + let mut txs_count = 0; + + for (status, _tx_slot_pos) in signatures { + let signature = status.signature; + let block_time = status.block_time.expect("Block has no timestamp"); + + if txs_count % 100 == 0 { + log::info!("{txs_count} transactions processed..."); + } + + let transaction = ledger + .get_confirmed_transaction(&signature) + .await + .unwrap() + .unwrap(); + + txs_count += 1; + + let tx_with_meta = match &transaction.tx_with_meta { + TransactionWithStatusMeta::Complete(complete) => complete.clone(), + TransactionWithStatusMeta::MissingMetadata(_) => { + panic!("Transaction {} has no metadata!", &signature) + } + }; + + let message = match transaction.get_transaction().message { + VersionedMessage::Legacy(legacy) => legacy, + VersionedMessage::V0(_v0) => panic!( + "Unsupported `message` version for transaction {}", + &signature + ), + }; + + let mut account_keys = message.account_keys.clone(); + + let balance_idx = account_keys + .iter() + .position(|a| a == &address) + .expect("Account should be present in account_keys"); + + account_keys.retain(|e| e != &system_program::ID && e != &address); + + let pre_balance = tx_with_meta.meta.pre_balances[balance_idx]; + let post_balance = tx_with_meta.meta.post_balances[balance_idx]; + + let is_transfer = account_keys.len() == 1; + let counterparty = account_keys[0]; + + result_ledger.push(LedgerEntry { + signature, + is_transfer, + counterparty, + block_time, + pre_balance, + post_balance, + }); + } + + let separator = match separator { + CsvSeparator::Comma => ',', + CsvSeparator::Semicolon => ';', + CsvSeparator::Tab => '\t', + }; + + let mut output_buffer = String::new(); + + output_buffer.push_str(&format!( + "Signature{separator}Is Transfer{separator}Counterparty{separator}Unix Timestamp (s){separator}Change (VLX){separator}Post Balance (VLX)\n" + )); + + for entry in result_ledger { + let balance_change = + (entry.post_balance as i64 - entry.pre_balance as i64) as f64 / 1_000_000_000.; + let post_balance = entry.post_balance as f64 / 1_000_000_000.; + + output_buffer.push_str(&format!( + "{}{separator}{}{separator}{}{separator}{}{separator}{}{separator}{}\n", + entry.signature, + entry.is_transfer, + entry.counterparty, + entry.block_time, + balance_change, + post_balance + )); + } + + std::fs::write(output, output_buffer).expect("Write output ledger file failed"); + + Ok(()) +} diff --git a/evm-utils/evm-block-recovery/src/routines/scratchpad.rs b/evm-utils/evm-block-recovery/src/routines/scratchpad.rs deleted file mode 100644 index f5ba8081d9..0000000000 --- a/evm-utils/evm-block-recovery/src/routines/scratchpad.rs +++ /dev/null @@ -1,6 +0,0 @@ -use crate::error::AppError; - -pub async fn command() -> Result<(), AppError> { - println!("scratchpad"); - Ok(()) -} From e7ec5e5abde62192ef660fb6347d9b62f7cd487d Mon Sep 17 00:00:00 2001 From: Maksim Vykhota Date: Thu, 11 Apr 2024 19:05:51 +0300 Subject: [PATCH 2/3] Feat(evm-block-recovery): Add `native-by-evm` routine --- evm-utils/evm-block-recovery/src/cli.rs | 14 +++- evm-utils/evm-block-recovery/src/main.rs | 1 + evm-utils/evm-block-recovery/src/routines.rs | 2 + .../src/routines/native_by_evm.rs | 78 +++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 evm-utils/evm-block-recovery/src/routines/native_by_evm.rs diff --git a/evm-utils/evm-block-recovery/src/cli.rs b/evm-utils/evm-block-recovery/src/cli.rs index 463e86482f..54128dfde3 100644 --- a/evm-utils/evm-block-recovery/src/cli.rs +++ b/evm-utils/evm-block-recovery/src/cli.rs @@ -34,7 +34,7 @@ pub enum Command { /// Checks content of Native Block CheckNative(CheckNativeArgs), - /// Checks content of Evm Block + /// Checks content of EVM Block CheckEvm(CheckEvmArgs), /// Compares difference of Native Block sets @@ -51,9 +51,12 @@ pub enum Command { ScanEvmStateRoots(ScanEvmStateRootsArgs), - /// Get ledger for the provided address + /// Gets Ledger for the provided Native Address AccountLedger(AccountLedgerArgs), + /// Gets underlying Native Transaction for provided EVM transaction + NativeByEvm(NativeByEvmArgs), + /// Generetes Shell Completions for this Utility Completion(CompletionArgs), } @@ -295,6 +298,13 @@ pub enum CsvSeparator { Tab, } +#[derive(clap::Args)] +pub struct NativeByEvmArgs { + /// EVM Transaction Hash + #[clap(long, value_enum, value_name = "H256")] + pub hash: String, +} + #[derive(clap::Args)] pub struct CompletionArgs { /// Which shell completions to generate diff --git a/evm-utils/evm-block-recovery/src/main.rs b/evm-utils/evm-block-recovery/src/main.rs index 0b36c33810..6567edab0f 100644 --- a/evm-utils/evm-block-recovery/src/main.rs +++ b/evm-utils/evm-block-recovery/src/main.rs @@ -50,6 +50,7 @@ async fn main() { RepeatNative(args) => repeat_native(args).await, ScanEvmStateRoots(ref args) => scan_evm_state_roots::command(args).await, AccountLedger(args) => account_ledger(cli.creds, cli.instance, args).await, + NativeByEvm(args) => native_by_evm(cli.creds, cli.instance, args).await, Completion(args) => completion(args), }; diff --git a/evm-utils/evm-block-recovery/src/routines.rs b/evm-utils/evm-block-recovery/src/routines.rs index 12981548ea..c059ae53f3 100644 --- a/evm-utils/evm-block-recovery/src/routines.rs +++ b/evm-utils/evm-block-recovery/src/routines.rs @@ -4,6 +4,7 @@ pub(crate) mod check_native; pub(crate) mod compare; pub(crate) mod completion; pub(crate) mod find; +pub(crate) mod native_by_evm; pub(crate) mod repeat; pub(crate) mod restore_chain; pub(crate) mod scan_evm_state_roots; @@ -18,6 +19,7 @@ pub use { compare::compare_native, completion::completion, find::{find_evm, find_native}, + native_by_evm::native_by_evm, repeat::{repeat_evm, repeat_native}, restore_chain::restore_chain, upload::upload, diff --git a/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs b/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs new file mode 100644 index 0000000000..68e4bb7f57 --- /dev/null +++ b/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs @@ -0,0 +1,78 @@ +use evm_state::H256; +use solana_sdk::{evm_loader::check_id as is_evm_loader, evm_state::check_id as is_evm_state}; +use solana_transaction_status::TransactionWithStatusMeta; +use std::str::FromStr; + +use crate::{cli::NativeByEvmArgs, error::RoutineResult, ledger}; + +// NOTE: remove panics if routine as service embedding is required + +pub async fn native_by_evm( + creds: Option, + instance: String, + args: NativeByEvmArgs, +) -> RoutineResult { + let NativeByEvmArgs { hash } = args; + + let hash = H256::from_str(&hash).unwrap(); + + let ledger = ledger::with_params(creds, instance).await?; + + let receipt = ledger.get_evm_confirmed_receipt(&hash).await.unwrap(); + + let (block_number, index) = match receipt { + Some(receipt) => (receipt.block_number, receipt.index), + None => { + panic!("Receipt for transaction {hash} does not exist"); + } + }; + + let evm_header = ledger + .get_evm_confirmed_block_header(block_number) + .await + .unwrap(); + + let native_block = ledger + .get_confirmed_block(evm_header.native_chain_slot) + .await + .unwrap(); + + let evm_related_txs = native_block + .transactions + .into_iter() + .filter(is_evm_related_tx) + .collect::>(); + + let result_tx = &evm_related_txs[index as usize - 1]; + + let logs = &result_tx.get_status_meta().unwrap().log_messages.unwrap(); + + let mut evm_hash_found = false; + + for log in logs { + if log.contains(&format!("{hash:?}")) { + evm_hash_found = true; + break; + } + } + + let signature = result_tx.transaction_signature(); + + log::info!("EVM Hash: {hash:?}"); + log::info!("Native Transaction Signature: {signature}"); + if !evm_hash_found { + log::warn!("Logs does NOT contain EVM hash: {hash:?}"); + log::debug!("{logs:?}"); + } + + Ok(()) +} + +fn is_evm_related_tx(tx: &TransactionWithStatusMeta) -> bool { + let tx_keys = tx.account_keys(); + + let contains_evm_state_key = tx_keys.iter().find(|key| is_evm_state(key)).is_some(); + let contains_evm_loadr_key = tx_keys.iter().find(|key| is_evm_loader(key)).is_some(); + + contains_evm_state_key && contains_evm_loadr_key +} From 8d2dbbbd44d2c10d8e25cc478de1d770e055e114 Mon Sep 17 00:00:00 2001 From: Maksim Vykhota Date: Mon, 13 May 2024 18:59:32 +0300 Subject: [PATCH 3/3] Fix: clarify logs --- evm-utils/evm-block-recovery/src/routines/native_by_evm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs b/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs index 68e4bb7f57..df1b2518b0 100644 --- a/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs +++ b/evm-utils/evm-block-recovery/src/routines/native_by_evm.rs @@ -58,10 +58,10 @@ pub async fn native_by_evm( let signature = result_tx.transaction_signature(); - log::info!("EVM Hash: {hash:?}"); + log::info!("EVM Transaction Hash: {hash:?}"); log::info!("Native Transaction Signature: {signature}"); if !evm_hash_found { - log::warn!("Logs does NOT contain EVM hash: {hash:?}"); + log::warn!("Logs do NOT contain EVM Transaction Hash: {hash:?}"); log::debug!("{logs:?}"); }