From f912b89be07c452da1ee251b7e7c1ba4a50c17d6 Mon Sep 17 00:00:00 2001 From: Rami <2418646+hashcashier@users.noreply.github.com> Date: Sat, 18 Jan 2025 04:12:34 +0200 Subject: [PATCH] feat(driver): Multi-block derivation (#888) Co-authored-by: refcell --- Cargo.lock | 1 + bin/client/src/interop.rs | 2 + bin/client/src/single.rs | 23 +++++----- crates/driver/Cargo.toml | 5 ++- crates/driver/src/core.rs | 37 +++++++++------- crates/proof-sdk/proof/src/hint.rs | 38 +++++++++++++++- .../proof-sdk/proof/src/l1/chain_provider.rs | 14 ++---- crates/proof-sdk/proof/src/l1/pipeline.rs | 5 ++- .../proof-sdk/proof/src/l2/chain_provider.rs | 44 +++++++++++-------- crates/proof-sdk/proof/src/sync.rs | 8 +++- 10 files changed, 110 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58adff10e..a0f97fce8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2434,6 +2434,7 @@ dependencies = [ "maili-protocol", "op-alloy-consensus", "op-alloy-rpc-types-engine", + "spin", "thiserror 2.0.11", "tracing", ] diff --git a/bin/client/src/interop.rs b/bin/client/src/interop.rs index fbac5d12d..b39e743b3 100644 --- a/bin/client/src/interop.rs +++ b/bin/client/src/interop.rs @@ -128,6 +128,8 @@ where let cursor = new_pipeline_cursor(&boot.rollup_config, safe_head, &mut l1_provider, &mut l2_provider) .await?; + l2_provider.set_cursor(cursor.clone()); + let cfg = Arc::new(boot.rollup_config.clone()); let pipeline = OraclePipeline::new( cfg.clone(), diff --git a/bin/client/src/single.rs b/bin/client/src/single.rs index d63a9a2c1..74b1fa83f 100644 --- a/bin/client/src/single.rs +++ b/bin/client/src/single.rs @@ -6,9 +6,7 @@ use alloy_primitives::B256; use core::fmt::Debug; use kona_driver::{Driver, DriverError}; use kona_executor::{ExecutorError, KonaHandleRegister, TrieDBProvider}; -use kona_preimage::{ - CommsClient, HintWriterClient, PreimageKey, PreimageKeyType, PreimageOracleClient, -}; +use kona_preimage::{CommsClient, HintWriterClient, PreimageKeyType, PreimageOracleClient}; use kona_proof::{ errors::OracleProviderError, executor::KonaExecutor, @@ -109,6 +107,8 @@ where let cursor = new_pipeline_cursor(&boot.rollup_config, safe_head, &mut l1_provider, &mut l2_provider) .await?; + l2_provider.set_cursor(cursor.clone()); + let cfg = Arc::new(boot.rollup_config.clone()); let pipeline = OraclePipeline::new( cfg.clone(), @@ -152,25 +152,22 @@ where /// Fetches the safe head hash of the L2 chain based on the agreed upon L2 output root in the /// [BootInfo]. -async fn fetch_safe_head_hash( +pub async fn fetch_safe_head_hash( caching_oracle: &O, boot_info: &BootInfo, ) -> Result where O: CommsClient, { - caching_oracle - .write(&HintType::StartingL2Output.encode_with(&[boot_info.agreed_l2_output_root.as_ref()])) - .await - .map_err(OracleProviderError::Preimage)?; let mut output_preimage = [0u8; 128]; - caching_oracle - .get_exact( - PreimageKey::new(*boot_info.agreed_l2_output_root, PreimageKeyType::Keccak256), + HintType::StartingL2Output + .get_exact_preimage( + caching_oracle, + boot_info.agreed_l2_output_root, + PreimageKeyType::Keccak256, &mut output_preimage, ) - .await - .map_err(OracleProviderError::Preimage)?; + .await?; output_preimage[96..128].try_into().map_err(OracleProviderError::SliceConversion) } diff --git a/crates/driver/Cargo.toml b/crates/driver/Cargo.toml index 09238c10f..4d412d9bb 100644 --- a/crates/driver/Cargo.toml +++ b/crates/driver/Cargo.toml @@ -29,6 +29,7 @@ op-alloy-consensus.workspace = true op-alloy-rpc-types-engine.workspace = true # Misc -tracing.workspace = true -thiserror .workspace = true async-trait.workspace = true +spin.workspace = true +thiserror .workspace = true +tracing.workspace = true diff --git a/crates/driver/src/core.rs b/crates/driver/src/core.rs index 249e36c7f..ade07225f 100644 --- a/crates/driver/src/core.rs +++ b/crates/driver/src/core.rs @@ -1,6 +1,7 @@ //! The driver of the kona derivation pipeline. -use alloc::vec::Vec; +use crate::{DriverError, DriverPipeline, DriverResult, Executor, PipelineCursor, TipCursor}; +use alloc::{sync::Arc, vec::Vec}; use alloy_consensus::{BlockBody, Sealable}; use alloy_primitives::B256; use alloy_rlp::Decodable; @@ -14,8 +15,7 @@ use maili_genesis::RollupConfig; use maili_protocol::L2BlockInfo; use op_alloy_consensus::{OpBlock, OpTxEnvelope, OpTxType}; use op_alloy_rpc_types_engine::OpAttributesWithParent; - -use crate::{DriverError, DriverPipeline, DriverResult, Executor, PipelineCursor, TipCursor}; +use spin::RwLock; /// The Rollup Driver entrypoint. #[derive(Debug)] @@ -32,7 +32,7 @@ where /// A pipeline abstraction. pub pipeline: DP, /// Cursor to keep track of the L2 tip - pub cursor: PipelineCursor, + pub cursor: Arc>, /// The Executor. pub executor: E, } @@ -44,7 +44,7 @@ where P: Pipeline + SignalReceiver + Send + Sync + Debug, { /// Creates a new [Driver]. - pub const fn new(cursor: PipelineCursor, executor: E, pipeline: DP) -> Self { + pub const fn new(cursor: Arc>, executor: E, pipeline: DP) -> Self { Self { _marker: core::marker::PhantomData, _marker2: core::marker::PhantomData, @@ -76,20 +76,22 @@ where ) -> DriverResult<(u64, B256, B256), E::Error> { loop { // Check if we have reached the target block number. + let pipeline_cursor = self.cursor.read(); + let tip_cursor = pipeline_cursor.tip(); if let Some(tb) = target { - if self.cursor.l2_safe_head().block_info.number >= tb { + if tip_cursor.l2_safe_head.block_info.number >= tb { info!(target: "client", "Derivation complete, reached L2 safe head."); return Ok(( - self.cursor.l2_safe_head().block_info.number, - self.cursor.l2_safe_head().block_info.hash, - *self.cursor.l2_safe_head_output_root(), + tip_cursor.l2_safe_head.block_info.number, + tip_cursor.l2_safe_head.block_info.hash, + tip_cursor.l2_safe_head_output_root, )); } } let OpAttributesWithParent { mut attributes, .. } = match self .pipeline - .produce_payload(*self.cursor.l2_safe_head()) + .produce_payload(tip_cursor.l2_safe_head) .await { Ok(attrs) => attrs, @@ -99,7 +101,7 @@ where // Adjust the target block number to the current safe head, as no more blocks // can be produced. if target.is_some() { - target = Some(self.cursor.l2_safe_head().block_info.number); + target = Some(tip_cursor.l2_safe_head.block_info.number); }; continue; } @@ -109,7 +111,7 @@ where } }; - self.executor.update_safe_head(self.cursor.l2_safe_head_header().clone()); + self.executor.update_safe_head(tip_cursor.l2_safe_head_header.clone()); let header = match self.executor.execute_payload(attributes.clone()).await { Ok(header) => header, Err(e) => { @@ -133,7 +135,7 @@ where }); // Retry the execution. - self.executor.update_safe_head(self.cursor.l2_safe_head_header().clone()); + self.executor.update_safe_head(tip_cursor.l2_safe_head_header.clone()); match self.executor.execute_payload(attributes.clone()).await { Ok(header) => header, Err(e) => { @@ -166,18 +168,21 @@ where }, }; - // Get the pipeline origin and update the cursor. + // Get the pipeline origin and update the tip cursor. let origin = self.pipeline.origin().ok_or(PipelineError::MissingOrigin.crit())?; let l2_info = L2BlockInfo::from_block_and_genesis( &block, &self.pipeline.rollup_config().genesis, )?; - let cursor = TipCursor::new( + let tip_cursor = TipCursor::new( l2_info, header.clone().seal_slow(), self.executor.compute_output_root().map_err(DriverError::Executor)?, ); - self.cursor.advance(origin, cursor); + + // Advance the derivation pipeline cursor + drop(pipeline_cursor); + self.cursor.write().advance(origin, tip_cursor); } } } diff --git a/crates/proof-sdk/proof/src/hint.rs b/crates/proof-sdk/proof/src/hint.rs index f77fadc30..00bda9b3f 100644 --- a/crates/proof-sdk/proof/src/hint.rs +++ b/crates/proof-sdk/proof/src/hint.rs @@ -1,12 +1,13 @@ //! This module contains the [HintType] enum. -use crate::errors::HintParsingError; +use crate::errors::{HintParsingError, OracleProviderError}; use alloc::{ string::{String, ToString}, vec::Vec, }; -use alloy_primitives::{hex, Bytes}; +use alloy_primitives::{hex, Bytes, B256}; use core::fmt::Display; +use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; /// A [Hint] is parsed in the format ` `, where `` is a string that /// represents the type of hint, and `` is the data associated with the hint (bytes @@ -80,6 +81,39 @@ impl HintType { let concatenated = hex::encode(data.iter().copied().flatten().copied().collect::>()); alloc::format!("{} {}", self, concatenated) } + /// Retrieves a preimage through an oracle + pub async fn get_preimage( + &self, + oracle: &T, + image: B256, + preimage_key_type: PreimageKeyType, + ) -> Result, OracleProviderError> { + oracle + .write(&self.encode_with(&[image.as_ref()])) + .await + .map_err(OracleProviderError::Preimage)?; + oracle + .get(PreimageKey::new(*image, preimage_key_type)) + .await + .map_err(OracleProviderError::Preimage) + } + /// Retrieves a preimage through an oracle + pub async fn get_exact_preimage( + &self, + oracle: &T, + image: B256, + preimage_key_type: PreimageKeyType, + buf: &mut [u8], + ) -> Result<(), OracleProviderError> { + oracle + .write(&self.encode_with(&[image.as_ref()])) + .await + .map_err(OracleProviderError::Preimage)?; + oracle + .get_exact(PreimageKey::new(*image, preimage_key_type), buf) + .await + .map_err(OracleProviderError::Preimage) + } } impl TryFrom<&str> for HintType { diff --git a/crates/proof-sdk/proof/src/l1/chain_provider.rs b/crates/proof-sdk/proof/src/l1/chain_provider.rs index de07a5e38..15842ff8f 100644 --- a/crates/proof-sdk/proof/src/l1/chain_provider.rs +++ b/crates/proof-sdk/proof/src/l1/chain_provider.rs @@ -33,18 +33,10 @@ impl ChainProvider for OracleL1ChainProvider { type Error = OracleProviderError; async fn header_by_hash(&mut self, hash: B256) -> Result { - // Send a hint for the block header. - self.oracle - .write(&HintType::L1BlockHeader.encode_with(&[hash.as_ref()])) - .await - .map_err(OracleProviderError::Preimage)?; - // Fetch the header RLP from the oracle. - let header_rlp = self - .oracle - .get(PreimageKey::new(*hash, PreimageKeyType::Keccak256)) - .await - .map_err(OracleProviderError::Preimage)?; + let header_rlp = HintType::L1BlockHeader + .get_preimage(self.oracle.as_ref(), hash, PreimageKeyType::Keccak256) + .await?; // Decode the header RLP into a Header. Header::decode(&mut header_rlp.as_slice()).map_err(OracleProviderError::Rlp) diff --git a/crates/proof-sdk/proof/src/l1/pipeline.rs b/crates/proof-sdk/proof/src/l1/pipeline.rs index dd23f835b..9388937ba 100644 --- a/crates/proof-sdk/proof/src/l1/pipeline.rs +++ b/crates/proof-sdk/proof/src/l1/pipeline.rs @@ -21,6 +21,7 @@ use kona_preimage::CommsClient; use maili_genesis::{RollupConfig, SystemConfig}; use maili_protocol::{BlockInfo, L2BlockInfo}; use op_alloy_rpc_types_engine::OpAttributesWithParent; +use spin::RwLock; /// An oracle-backed derivation pipeline. pub type OracleDerivationPipeline = DerivationPipeline< @@ -73,7 +74,7 @@ where /// Constructs a new oracle-backed derivation pipeline. pub fn new( cfg: Arc, - sync_start: PipelineCursor, + sync_start: Arc>, caching_oracle: Arc, blob_provider: B, chain_provider: OracleL1ChainProvider, @@ -92,7 +93,7 @@ where .l2_chain_provider(l2_chain_provider) .chain_provider(chain_provider) .builder(attributes) - .origin(sync_start.origin()) + .origin(sync_start.read().origin()) .build(); Self { pipeline, caching_oracle } } diff --git a/crates/proof-sdk/proof/src/l2/chain_provider.rs b/crates/proof-sdk/proof/src/l2/chain_provider.rs index 4e10b2862..c4f0a96cf 100644 --- a/crates/proof-sdk/proof/src/l2/chain_provider.rs +++ b/crates/proof-sdk/proof/src/l2/chain_provider.rs @@ -8,12 +8,14 @@ use alloy_primitives::{Address, Bytes, B256}; use alloy_rlp::Decodable; use async_trait::async_trait; use kona_derive::traits::L2ChainProvider; +use kona_driver::PipelineCursor; use kona_executor::TrieDBProvider; use kona_mpt::{OrderedListWalker, TrieHinter, TrieNode, TrieProvider}; use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; use maili_genesis::{RollupConfig, SystemConfig}; use maili_protocol::{to_system_config, BatchValidationProvider, L2BlockInfo}; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; +use spin::RwLock; /// The oracle-backed L2 chain provider for the client program. #[derive(Debug, Clone)] @@ -24,12 +26,27 @@ pub struct OracleL2ChainProvider { rollup_config: RollupConfig, /// The preimage oracle client. oracle: Arc, + /// The derivation pipeline cursor + cursor: Option>>, } impl OracleL2ChainProvider { /// Creates a new [OracleL2ChainProvider] with the given boot information and oracle client. pub const fn new(l2_head: B256, rollup_config: RollupConfig, oracle: Arc) -> Self { - Self { l2_head, rollup_config, oracle } + Self { l2_head, rollup_config, oracle, cursor: None } + } + + /// Updates the derivation pipeline cursor + pub fn set_cursor(&mut self, cursor: Arc>) { + self.cursor = Some(cursor); + } + + /// Fetches the latest known safe head block hash according to the derivation pipeline cursor + /// or uses the initial l2_head value if no cursor is set. + pub async fn l2_safe_head(&self) -> Result { + self.cursor + .as_ref() + .map_or(Ok(self.l2_head), |cursor| Ok(cursor.read().l2_safe_head().block_info.hash)) } } @@ -37,7 +54,8 @@ impl OracleL2ChainProvider { /// Returns a [Header] corresponding to the given L2 block number, by walking back from the /// L2 safe head. async fn header_by_number(&mut self, block_number: u64) -> Result { - let mut header = self.header_by_hash(self.l2_head)?; + // Fetch the starting block header. + let mut header = self.header_by_hash(self.l2_safe_head().await?)?; // Check if the block number is in range. If not, we can fail early. if block_number > header.number { @@ -151,32 +169,20 @@ impl TrieDBProvider for OracleL2ChainProvider { fn bytecode_by_hash(&self, hash: B256) -> Result { // Fetch the bytecode preimage from the caching oracle. crate::block_on(async move { - self.oracle - .write(&HintType::L2Code.encode_with(&[hash.as_ref()])) - .await - .map_err(OracleProviderError::Preimage)?; - - self.oracle - .get(PreimageKey::new(*hash, PreimageKeyType::Keccak256)) + HintType::L2Code + .get_preimage(self.oracle.as_ref(), hash, PreimageKeyType::Keccak256) .await .map(Into::into) - .map_err(OracleProviderError::Preimage) }) } fn header_by_hash(&self, hash: B256) -> Result { // Fetch the header from the caching oracle. crate::block_on(async move { - self.oracle - .write(&HintType::L2BlockHeader.encode_with(&[hash.as_ref()])) - .await - .map_err(OracleProviderError::Preimage)?; + let header_bytes = HintType::L2BlockHeader + .get_preimage(self.oracle.as_ref(), hash, PreimageKeyType::Keccak256) + .await?; - let header_bytes = self - .oracle - .get(PreimageKey::new(*hash, PreimageKeyType::Keccak256)) - .await - .map_err(OracleProviderError::Preimage)?; Header::decode(&mut header_bytes.as_slice()).map_err(OracleProviderError::Rlp) }) } diff --git a/crates/proof-sdk/proof/src/sync.rs b/crates/proof-sdk/proof/src/sync.rs index 9865c207c..ef765faa8 100644 --- a/crates/proof-sdk/proof/src/sync.rs +++ b/crates/proof-sdk/proof/src/sync.rs @@ -4,6 +4,7 @@ use crate::{ errors::OracleProviderError, l1::OracleL1ChainProvider, l2::OracleL2ChainProvider, FlushableCache, }; +use alloc::sync::Arc; use alloy_consensus::{Header, Sealed}; use alloy_primitives::B256; use core::fmt::Debug; @@ -12,6 +13,7 @@ use kona_driver::{PipelineCursor, TipCursor}; use kona_preimage::CommsClient; use maili_protocol::BatchValidationProvider; use maili_registry::RollupConfig; +use spin::RwLock; /// Constructs a [`PipelineCursor`] from the caching oracle, boot info, and providers. pub async fn new_pipeline_cursor( @@ -19,7 +21,7 @@ pub async fn new_pipeline_cursor( safe_header: Sealed
, chain_provider: &mut OracleL1ChainProvider, l2_chain_provider: &mut OracleL2ChainProvider, -) -> Result +) -> Result>, OracleProviderError> where O: CommsClient + FlushableCache + FlushableCache + Send + Sync + Debug, { @@ -39,5 +41,7 @@ where let mut cursor = PipelineCursor::new(channel_timeout, origin); let tip = TipCursor::new(safe_head_info, safe_header, B256::ZERO); cursor.advance(origin, tip); - Ok(cursor) + + // Wrap the cursor in a shared read-write lock + Ok(Arc::new(RwLock::new(cursor))) }