From 2a485d839699d97a851e57ec2cc9c6bc71da26ab Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Mon, 22 Jan 2024 09:58:39 +0100 Subject: [PATCH 1/3] Initital implementation of overlayed backend --- src/backend/mod.rs | 3 + src/backend/overlayed.rs | 265 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 src/backend/overlayed.rs diff --git a/src/backend/mod.rs b/src/backend/mod.rs index fe8ca30f3..f925ec027 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -11,6 +11,9 @@ //! pushing/poping layers are dealt by extern functions), layers are handled //! internally inside a backend. +mod overlayed; + +pub use self::overlayed::OverlayedBackend; pub use evm_interpreter::{RuntimeBackend, RuntimeBaseBackend, RuntimeEnvironment}; /// Backend with layers that can transactionally be committed or discarded. diff --git a/src/backend/overlayed.rs b/src/backend/overlayed.rs new file mode 100644 index 000000000..f7385941c --- /dev/null +++ b/src/backend/overlayed.rs @@ -0,0 +1,265 @@ +use crate::{ + ExitError, ExitException, Log, MergeStrategy, RuntimeBackend, RuntimeBaseBackend, + TransactionalBackend, +}; +use alloc::collections::{BTreeMap, BTreeSet}; +use core::mem; +use primitive_types::{H160, H256, U256}; + +pub struct OverlayedBackend { + backend: B, + substate: Box, + accessed: BTreeSet<(H160, Option)>, +} + +impl OverlayedBackend { + pub fn new(backend: B) -> Self { + Self { + backend, + substate: Box::new(Substate::new()), + accessed: BTreeSet::new(), + } + } +} + +impl RuntimeBaseBackend for OverlayedBackend { + fn balance(&self, address: H160) -> U256 { + if let Some(balance) = self.substate.known_balance(address) { + balance + } else { + self.backend.balance(address) + } + } + + fn code(&self, address: H160) -> Vec { + if let Some(code) = self.substate.known_code(address) { + code + } else { + self.backend.code(address) + } + } + + fn storage(&self, address: H160, index: H256) -> H256 { + if let Some(value) = self.substate.known_storage(address, index) { + value + } else { + self.backend.storage(address, index) + } + } + + fn exists(&self, address: H160) -> bool { + if let Some(exists) = self.substate.known_exists(address) { + exists + } else { + self.backend.exists(address) + } + } + + fn nonce(&self, address: H160) -> U256 { + if let Some(nonce) = self.substate.known_nonce(address) { + nonce + } else { + self.backend.nonce(address) + } + } +} + +impl RuntimeBackend for OverlayedBackend { + fn original_storage(&self, address: H160, index: H256) -> H256 { + self.backend.storage(address, index) + } + + fn deleted(&self, address: H160) -> bool { + self.substate.deleted(address) + } + + fn is_cold(&self, address: H160, index: Option) -> bool { + !self.accessed.contains(&(address, index)) + } + + fn mark_hot(&mut self, address: H160, index: Option) { + self.accessed.insert((address, index)); + } + + fn set_storage(&mut self, address: H160, index: H256, value: H256) -> Result<(), ExitError> { + self.substate.storages.insert((address, index), value); + Ok(()) + } + + fn log(&mut self, log: Log) -> Result<(), ExitError> { + self.substate.logs.push(log); + Ok(()) + } + + fn mark_delete(&mut self, address: H160) { + self.substate.deletes.insert(address); + } + + fn reset_storage(&mut self, address: H160) { + self.substate.storage_resets.insert(address); + } + + fn set_code(&mut self, address: H160, code: Vec) -> Result<(), ExitError> { + self.substate.codes.insert(address, code); + Ok(()) + } + + fn reset_balance(&mut self, address: H160) { + self.substate.balances.insert(address, U256::zero()); + } + + fn deposit(&mut self, target: H160, value: U256) { + let current_balance = self.balance(target); + self.substate + .balances + .insert(target, current_balance.saturating_add(value)); + } + + fn withdrawal(&mut self, source: H160, value: U256) -> Result<(), ExitError> { + let current_balance = self.balance(source); + if current_balance < value { + return Err(ExitException::OutOfFund.into()); + } + let new_balance = current_balance - value; + self.substate.balances.insert(source, new_balance); + Ok(()) + } + + fn inc_nonce(&mut self, address: H160) -> Result<(), ExitError> { + let new_nonce = self.nonce(address).saturating_add(U256::from(1)); + self.substate.nonces.insert(address, new_nonce); + Ok(()) + } +} + +impl TransactionalBackend for OverlayedBackend { + fn push_substate(&mut self) { + let mut parent = Box::new(Substate::new()); + mem::swap(&mut parent, &mut self.substate); + self.substate.parent = Some(parent); + } + + fn pop_substate(&mut self, strategy: MergeStrategy) { + let mut child = self.substate.parent.take().expect("uneven substate pop"); + mem::swap(&mut child, &mut self.substate); + let child = child; + + match strategy { + MergeStrategy::Commit => { + for log in child.logs { + self.substate.logs.push(log); + } + for (address, balance) in child.balances { + self.substate.balances.insert(address, balance); + } + for (address, code) in child.codes { + self.substate.codes.insert(address, code); + } + for (address, nonce) in child.nonces { + self.substate.nonces.insert(address, nonce); + } + for address in child.storage_resets { + self.substate.storage_resets.insert(address); + } + for ((address, key), value) in child.storages { + self.substate.storages.insert((address, key), value); + } + for address in child.deletes { + self.substate.deletes.insert(address); + } + } + MergeStrategy::Revert | MergeStrategy::Discard => {} + } + } +} + +struct Substate { + parent: Option>, + logs: Vec, + balances: BTreeMap, + codes: BTreeMap>, + nonces: BTreeMap, + storage_resets: BTreeSet, + storages: BTreeMap<(H160, H256), H256>, + deletes: BTreeSet, +} + +impl Substate { + pub fn new() -> Self { + Self { + parent: None, + logs: Vec::new(), + balances: Default::default(), + codes: Default::default(), + nonces: Default::default(), + storage_resets: Default::default(), + storages: Default::default(), + deletes: Default::default(), + } + } + + pub fn known_balance(&self, address: H160) -> Option { + if let Some(balance) = self.balances.get(&address) { + Some(*balance) + } else if let Some(parent) = self.parent.as_ref() { + parent.known_balance(address) + } else { + None + } + } + + pub fn known_code(&self, address: H160) -> Option> { + if let Some(code) = self.codes.get(&address) { + Some(code.clone()) + } else if let Some(parent) = self.parent.as_ref() { + parent.known_code(address) + } else { + None + } + } + + pub fn known_nonce(&self, address: H160) -> Option { + if let Some(nonce) = self.nonces.get(&address) { + Some(*nonce) + } else if let Some(parent) = self.parent.as_ref() { + parent.known_nonce(address) + } else { + None + } + } + + pub fn known_storage(&self, address: H160, key: H256) -> Option { + if let Some(value) = self.storages.get(&(address, key)) { + Some(*value) + } else if self.storage_resets.contains(&address) { + Some(H256::default()) + } else if let Some(parent) = self.parent.as_ref() { + parent.known_storage(address, key) + } else { + None + } + } + + pub fn known_exists(&self, address: H160) -> Option { + if self.balances.contains_key(&address) + || self.nonces.contains_key(&address) + || self.codes.contains_key(&address) + { + Some(true) + } else if let Some(parent) = self.parent.as_ref() { + parent.known_exists(address) + } else { + None + } + } + + pub fn deleted(&self, address: H160) -> bool { + if self.deletes.contains(&address) { + true + } else if let Some(parent) = self.parent.as_ref() { + parent.deleted(address) + } else { + false + } + } +} From fa1a92c467c8ef381eeadad2409597ee4b3178d1 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 23 Jan 2024 01:07:10 +0100 Subject: [PATCH 2/3] Integrate overlayed backend to jsontests and fix no_std --- interpreter/Cargo.toml | 1 + interpreter/src/runtime.rs | 2 + jsontests/src/hash.rs | 1 - jsontests/src/in_memory.rs | 199 +++++++------------------------------ jsontests/src/run.rs | 41 ++++---- src/backend/mod.rs | 2 +- src/backend/overlayed.rs | 84 +++++++++++++++- 7 files changed, 141 insertions(+), 189 deletions(-) diff --git a/interpreter/Cargo.toml b/interpreter/Cargo.toml index 54375c80d..199483d04 100644 --- a/interpreter/Cargo.toml +++ b/interpreter/Cargo.toml @@ -15,6 +15,7 @@ scale-codec = { package = "parity-scale-codec", version = "3.2", default-feature scale-info = { version = "2.3", default-features = false, features = ["derive"], optional = true } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } sha3 = { version = "0.10", default-features = false } +auto_impl = "1.1" [dev-dependencies] hex = "0.4" diff --git a/interpreter/src/runtime.rs b/interpreter/src/runtime.rs index 163c6ee9b..7dbce8a49 100644 --- a/interpreter/src/runtime.rs +++ b/interpreter/src/runtime.rs @@ -75,6 +75,7 @@ pub struct Log { pub data: Vec, } +#[auto_impl::auto_impl(&, Box)] pub trait RuntimeEnvironment { /// Get environmental block hash. fn block_hash(&self, number: U256) -> H256; @@ -96,6 +97,7 @@ pub trait RuntimeEnvironment { fn chain_id(&self) -> U256; } +#[auto_impl::auto_impl(&, Box)] pub trait RuntimeBaseBackend { /// Get balance of address. fn balance(&self, address: H160) -> U256; diff --git a/jsontests/src/hash.rs b/jsontests/src/hash.rs index e803ab842..7a7e91e52 100644 --- a/jsontests/src/hash.rs +++ b/jsontests/src/hash.rs @@ -66,7 +66,6 @@ impl rlp::Decodable for TrieAccount { pub fn state_root(backend: &InMemoryBackend) -> H256 { let tree = backend - .current_layer() .state .iter() .map(|(address, account)| { diff --git a/jsontests/src/in_memory.rs b/jsontests/src/in_memory.rs index 1a94b08b5..05fc87e69 100644 --- a/jsontests/src/in_memory.rs +++ b/jsontests/src/in_memory.rs @@ -1,12 +1,6 @@ -use evm::{ - ExitError, ExitException, Log, MergeStrategy, RuntimeBackend, RuntimeBaseBackend, - RuntimeEnvironment, TransactionalBackend, -}; +use evm::{backend::OverlayedChangeSet, RuntimeBaseBackend, RuntimeEnvironment}; use primitive_types::{H160, H256, U256}; -use std::{ - collections::{BTreeMap, BTreeSet}, - mem, -}; +use std::collections::BTreeMap; #[derive(Clone, Debug)] pub struct InMemoryEnvironment { @@ -27,7 +21,6 @@ pub struct InMemoryAccount { pub code: Vec, pub nonce: U256, pub storage: BTreeMap, - pub original_storage: BTreeMap, } #[derive(Clone, Debug)] @@ -36,38 +29,42 @@ pub struct InMemorySuicideInfo { } #[derive(Clone, Debug)] -pub struct InMemoryLayer { +pub struct InMemoryBackend { + pub environment: InMemoryEnvironment, pub state: BTreeMap, - pub logs: Vec, - pub suicides: Vec, - pub hots: BTreeSet<(H160, Option)>, } -impl InMemoryLayer { - pub fn clear_pending(&mut self) { - self.hots.clear(); +impl InMemoryBackend { + pub fn apply_overlayed(&mut self, changeset: &OverlayedChangeSet) { + for (address, balance) in changeset.balances.clone() { + self.state.entry(address).or_default().balance = balance; + } - let mut suicides = Vec::new(); - mem::swap(&mut suicides, &mut self.suicides); - for suicide in suicides { - self.state.remove(&suicide.address); + for (address, code) in changeset.codes.clone() { + self.state.entry(address).or_default().code = code; } - } -} -#[derive(Clone, Debug)] -pub struct InMemoryBackend { - pub environment: InMemoryEnvironment, - pub layers: Vec, -} + for (address, nonce) in changeset.nonces.clone() { + self.state.entry(address).or_default().nonce = nonce; + } -impl InMemoryBackend { - pub fn current_layer(&self) -> &InMemoryLayer { - self.layers.last().expect("current layer exists") - } + for address in changeset.storage_resets.clone() { + self.state.entry(address).or_default().storage = BTreeMap::new(); + } + + for ((address, key), value) in changeset.storages.clone() { + let account = self.state.entry(address).or_default(); - pub fn current_layer_mut(&mut self) -> &mut InMemoryLayer { - self.layers.last_mut().expect("current layer exists") + if value == H256::default() { + account.storage.remove(&key); + } else { + account.storage.insert(key, value); + } + } + + for address in changeset.deletes.clone() { + self.state.remove(&address); + } } } @@ -115,8 +112,7 @@ impl RuntimeEnvironment for InMemoryBackend { impl RuntimeBaseBackend for InMemoryBackend { fn balance(&self, address: H160) -> U256 { - self.current_layer() - .state + self.state .get(&address) .cloned() .unwrap_or(Default::default()) @@ -124,8 +120,7 @@ impl RuntimeBaseBackend for InMemoryBackend { } fn code(&self, address: H160) -> Vec { - self.current_layer() - .state + self.state .get(&address) .cloned() .unwrap_or(Default::default()) @@ -133,12 +128,11 @@ impl RuntimeBaseBackend for InMemoryBackend { } fn exists(&self, address: H160) -> bool { - self.current_layer().state.get(&address).is_some() + self.state.get(&address).is_some() } fn storage(&self, address: H160, index: H256) -> H256 { - self.current_layer() - .state + self.state .get(&address) .cloned() .unwrap_or(Default::default()) @@ -149,133 +143,10 @@ impl RuntimeBaseBackend for InMemoryBackend { } fn nonce(&self, address: H160) -> U256 { - self.current_layer() - .state + self.state .get(&address) .cloned() .unwrap_or(Default::default()) .nonce } } - -impl RuntimeBackend for InMemoryBackend { - fn original_storage(&self, address: H160, index: H256) -> H256 { - self.current_layer() - .state - .get(&address) - .cloned() - .unwrap_or(Default::default()) - .original_storage - .get(&index) - .cloned() - .unwrap_or(H256::default()) - } - - fn deleted(&self, address: H160) -> bool { - self.current_layer() - .suicides - .iter() - .any(|suicide| suicide.address == address) - } - - fn is_cold(&self, address: H160, index: Option) -> bool { - !self.current_layer().hots.contains(&(address, index)) - } - - fn mark_hot(&mut self, address: H160, index: Option) { - self.current_layer_mut().hots.insert((address, index)); - } - - fn set_storage(&mut self, address: H160, index: H256, value: H256) -> Result<(), ExitError> { - let entry = self.current_layer_mut().state.entry(address).or_default(); - - if value == H256::default() { - entry.storage.remove(&index); - } else { - entry.storage.insert(index, value); - } - Ok(()) - } - - fn log(&mut self, log: Log) -> Result<(), ExitError> { - self.current_layer_mut().logs.push(log); - Ok(()) - } - - fn mark_delete(&mut self, address: H160) { - self.current_layer_mut() - .suicides - .push(InMemorySuicideInfo { address }); - } - - fn reset_storage(&mut self, address: H160) { - self.current_layer_mut() - .state - .entry(address) - .or_default() - .storage = Default::default(); - } - - fn set_code(&mut self, address: H160, code: Vec) -> Result<(), ExitError> { - self.current_layer_mut() - .state - .entry(address) - .or_default() - .code = code; - - Ok(()) - } - - fn reset_balance(&mut self, address: H160) { - self.current_layer_mut() - .state - .entry(address) - .or_default() - .balance = U256::zero(); - } - - fn withdrawal(&mut self, source: H160, value: U256) -> Result<(), ExitError> { - let source = self.current_layer_mut().state.entry(source).or_default(); - if source.balance < value { - return Err(ExitException::OutOfFund.into()); - } - source.balance -= value; - Ok(()) - } - - fn deposit(&mut self, target: H160, value: U256) { - if value == U256::zero() { - return; - } - - self.current_layer_mut() - .state - .entry(target) - .or_default() - .balance += value; - } - - fn inc_nonce(&mut self, address: H160) -> Result<(), ExitError> { - let entry = self.current_layer_mut().state.entry(address).or_default(); - entry.nonce = entry.nonce.saturating_add(U256::one()); - Ok(()) - } -} - -impl TransactionalBackend for InMemoryBackend { - fn push_substate(&mut self) { - let layer = self.current_layer().clone(); - self.layers.push(layer); - } - - fn pop_substate(&mut self, strategy: MergeStrategy) { - let layer = self.layers.pop().expect("current layer exist"); - - match strategy { - MergeStrategy::Commit => { - *self.current_layer_mut() = layer; - } - MergeStrategy::Discard | MergeStrategy::Revert => (), - } - } -} diff --git a/jsontests/src/run.rs b/jsontests/src/run.rs index 835abbdeb..5fa6e8c6a 100644 --- a/jsontests/src/run.rs +++ b/jsontests/src/run.rs @@ -1,6 +1,7 @@ use crate::error::{Error, TestError}; -use crate::in_memory::{InMemoryAccount, InMemoryBackend, InMemoryEnvironment, InMemoryLayer}; +use crate::in_memory::{InMemoryAccount, InMemoryBackend, InMemoryEnvironment}; use crate::types::{Fork, TestCompletionStatus, TestData, TestExpectException, TestMulti}; +use evm::backend::OverlayedBackend; use evm::standard::{Config, Etable, EtableResolver, Invoker, TransactArgs}; use evm::utils::u256_to_h256; use evm::Capture; @@ -136,7 +137,6 @@ pub fn run_test( balance: account.balance, code: account.code.0, nonce: account.nonce, - original_storage: storage.clone(), storage, }, ) @@ -164,26 +164,27 @@ pub fn run_test( .collect(), }; - let mut run_backend = InMemoryBackend { + let initial_accessed = { + let mut hots = BTreeSet::new(); + for i in 1..10 { + hots.insert((u256_to_h256(U256::from(i)).into(), None)); + } + hots + }; + + let base_backend = InMemoryBackend { environment: env, - layers: vec![InMemoryLayer { - state, - logs: Vec::new(), - suicides: Vec::new(), - hots: { - let mut hots = BTreeSet::new(); - for i in 1..10 { - hots.insert((u256_to_h256(U256::from(i)).into(), None)); - } - hots - }, - }], + state, }; - let mut step_backend = run_backend.clone(); + + let mut run_backend = OverlayedBackend::new(&base_backend, initial_accessed.clone()); + let mut step_backend = OverlayedBackend::new(&base_backend, initial_accessed.clone()); // Run let run_result = evm::transact(args.clone(), Some(4), &mut run_backend, &invoker); - run_backend.layers[0].clear_pending(); + let run_changeset = run_backend.deconstruct().1; + let mut run_backend = base_backend.clone(); + run_backend.apply_overlayed(&run_changeset); // Step if debug { @@ -204,7 +205,9 @@ pub fn run_test( } }, ); - step_backend.layers[0].clear_pending(); + let step_changeset = step_backend.deconstruct().1; + let mut step_backend = base_backend.clone(); + step_backend.apply_overlayed(&step_changeset); } let state_root = crate::hash::state_root(&run_backend); @@ -219,7 +222,7 @@ pub fn run_test( if state_root != test.post.hash { if debug { - for (address, account) in &run_backend.layers[0].state { + for (address, account) in &run_backend.state { println!( "address: {:?}, balance: {}, nonce: {}, code: 0x{}, storage: {:?}", address, diff --git a/src/backend/mod.rs b/src/backend/mod.rs index f925ec027..96436d450 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -13,7 +13,7 @@ mod overlayed; -pub use self::overlayed::OverlayedBackend; +pub use self::overlayed::{OverlayedBackend, OverlayedChangeSet}; pub use evm_interpreter::{RuntimeBackend, RuntimeBaseBackend, RuntimeEnvironment}; /// Backend with layers that can transactionally be committed or discarded. diff --git a/src/backend/overlayed.rs b/src/backend/overlayed.rs index f7385941c..7ffd43df9 100644 --- a/src/backend/overlayed.rs +++ b/src/backend/overlayed.rs @@ -1,11 +1,26 @@ use crate::{ ExitError, ExitException, Log, MergeStrategy, RuntimeBackend, RuntimeBaseBackend, - TransactionalBackend, + RuntimeEnvironment, TransactionalBackend, +}; +use alloc::{ + boxed::Box, + collections::{BTreeMap, BTreeSet}, + vec::Vec, }; -use alloc::collections::{BTreeMap, BTreeSet}; use core::mem; use primitive_types::{H160, H256, U256}; +#[derive(Clone, Debug)] +pub struct OverlayedChangeSet { + pub logs: Vec, + pub balances: BTreeMap, + pub codes: BTreeMap>, + pub nonces: BTreeMap, + pub storage_resets: BTreeSet, + pub storages: BTreeMap<(H160, H256), H256>, + pub deletes: BTreeSet, +} + pub struct OverlayedBackend { backend: B, substate: Box, @@ -13,13 +28,66 @@ pub struct OverlayedBackend { } impl OverlayedBackend { - pub fn new(backend: B) -> Self { + pub fn new(backend: B, accessed: BTreeSet<(H160, Option)>) -> Self { Self { backend, substate: Box::new(Substate::new()), - accessed: BTreeSet::new(), + accessed, } } + + pub fn deconstruct(self) -> (B, OverlayedChangeSet) { + ( + self.backend, + OverlayedChangeSet { + logs: self.substate.logs, + balances: self.substate.balances, + codes: self.substate.codes, + nonces: self.substate.nonces, + storage_resets: self.substate.storage_resets, + storages: self.substate.storages, + deletes: self.substate.deletes, + }, + ) + } +} + +impl RuntimeEnvironment for OverlayedBackend { + fn block_hash(&self, number: U256) -> H256 { + self.backend.block_hash(number) + } + + fn block_number(&self) -> U256 { + self.backend.block_number() + } + + fn block_coinbase(&self) -> H160 { + self.backend.block_coinbase() + } + + fn block_timestamp(&self) -> U256 { + self.backend.block_timestamp() + } + + fn block_difficulty(&self) -> U256 { + self.backend.block_difficulty() + } + + fn block_randomness(&self) -> Option { + self.backend.block_randomness() + } + + fn block_gas_limit(&self) -> U256 { + self.backend.block_gas_limit() + } + + fn block_base_fee_per_gas(&self) -> U256 { + self.backend.block_base_fee_per_gas() + } + + fn chain_id(&self) -> U256 { + self.backend.chain_id() + } } impl RuntimeBaseBackend for OverlayedBackend { @@ -109,6 +177,10 @@ impl RuntimeBackend for OverlayedBackend { } fn deposit(&mut self, target: H160, value: U256) { + if value == U256::zero() { + return; + } + let current_balance = self.balance(target); self.substate .balances @@ -116,6 +188,10 @@ impl RuntimeBackend for OverlayedBackend { } fn withdrawal(&mut self, source: H160, value: U256) -> Result<(), ExitError> { + if value == U256::zero() { + return Ok(()); + } + let current_balance = self.balance(source); if current_balance < value { return Err(ExitException::OutOfFund.into()); From def71a68994fe4f6216e68c02de00b2d621109c1 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 23 Jan 2024 01:10:04 +0100 Subject: [PATCH 3/3] Set rust version to stable 1.75.0 --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 1a15b5bce..9465ad536 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly" +channel = "1.75.0" profile = "minimal" components = [ "rustfmt", "clippy" ]