Merge pull request #56 from ethereum-optimism/refcell/span-batches

feat(derive): Span Batch Types

crates/derive/Cargo.toml

@@ -26,6 +26,7 @@ serde = { version = "1.0.197", default-features = false, features = ["derive"],
 
 [dev-dependencies]
 tokio = { version = "1.36", features = ["full"] }
+proptest = "1.4.0"
 
 [features]
 serde = ["dep:serde", "alloy-primitives/serde"]

crates/derive/src/stages/channel_reader.rs

@@ -8,7 +8,7 @@ use crate::{
 use alloc::vec::Vec;
 use anyhow::anyhow;
 use core::fmt::Debug;
-use miniz_oxide::inflate::decompress_to_vec;
+use miniz_oxide::inflate::decompress_to_vec_zlib;
 
 /// [ChannelReader] is a stateful stage that does the following:
 #[derive(Debug)]
@@ -45,7 +45,7 @@ where
         match self
             .next_batch
             .as_mut()
-            .unwrap()
+            .expect("Cannot be None")
             .next_batch()
             .ok_or(StageError::NotEnoughData)
         {
@@ -88,15 +88,26 @@ pub(crate) struct BatchReader {
     data: Option<Vec<u8>>,
     /// Decompressed data.
     decompressed: Vec<u8>,
+    /// The current cursor in the `decompressed` data.
+    cursor: usize,
 }
 
 impl BatchReader {
     /// Pulls out the next batch from the reader.
     pub(crate) fn next_batch(&mut self) -> Option<Batch> {
+        // If the data is not already decompressed, decompress it.
         if let Some(data) = self.data.take() {
-            self.decompressed = decompress_to_vec(&data).ok()?;
+            let decompressed_data = decompress_to_vec_zlib(&data).ok()?;
+            self.decompressed = decompressed_data;
         }
-        let batch = Batch::decode(&mut self.decompressed.as_ref()).ok()?;
+
+        // Decompress and RLP decode the batch data, before finally decoding the batch itself.
+        let mut decompressed_reader = self.decompressed.as_slice();
+        let batch = Batch::decode(&mut decompressed_reader).ok()?;
+
+        // Advance the cursor on the reader.
+        self.cursor += self.decompressed.len() - decompressed_reader.len();
+
         Some(batch)
     }
 }
@@ -106,6 +117,39 @@ impl From<&[u8]> for BatchReader {
         Self {
             data: Some(data.to_vec()),
             decompressed: Vec::new(),
+            cursor: 0,
+        }
+    }
+}
+
+impl From<Vec<u8>> for BatchReader {
+    fn from(data: Vec<u8>) -> Self {
+        Self {
+            data: Some(data),
+            decompressed: Vec::new(),
+            cursor: 0,
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::{stages::channel_reader::BatchReader, types::BatchType};
+    use alloc::vec;
+    use miniz_oxide::deflate::compress_to_vec_zlib;
+
+    // TODO(clabby): More tests here for multiple batches, integration w/ channel bank, etc.
+
+    #[test]
+    fn test_batch_reader() {
+        let raw_data = include_bytes!("../../testdata/raw_batch.hex");
+        let mut typed_data = vec![BatchType::Span as u8];
+        typed_data.extend_from_slice(raw_data.as_slice());
+
+        let compressed_raw_data = compress_to_vec_zlib(typed_data.as_slice(), 5);
+        let mut reader = BatchReader::from(compressed_raw_data);
+        reader.next_batch().unwrap();
+
+        assert_eq!(reader.cursor, typed_data.len());
+    }
+}

crates/derive/src/types/eips/eip1559/helpers.rs → crates/derive/src/types/alloy/eips/eip1559/helpers.rs

@@ -63,7 +63,7 @@ pub fn calc_next_block_base_fee(
 
 #[cfg(test)]
 mod tests {
-    use crate::types::eips::eip1559::{MIN_PROTOCOL_BASE_FEE, MIN_PROTOCOL_BASE_FEE_U256};
+    use crate::types::eip1559::{MIN_PROTOCOL_BASE_FEE, MIN_PROTOCOL_BASE_FEE_U256};
 
     use super::*;
 

crates/derive/src/types/header.rs → crates/derive/src/types/alloy/header.rs

@@ -1,9 +1,7 @@
 use crate::types::{
-    eips::{
-        eip1559::{calc_next_block_base_fee, BaseFeeParams},
-        eip4844::{calc_blob_gasprice, calc_excess_blob_gas},
-    },
-    network::Sealable,
+    eip1559::{calc_next_block_base_fee, BaseFeeParams},
+    eip4844::{calc_blob_gasprice, calc_excess_blob_gas},
+    Sealable,
 };
 use alloc::vec::Vec;
 use alloy_primitives::{b256, keccak256, Address, BlockNumber, Bloom, Bytes, B256, B64, U256};

crates/derive/src/types/alloy/mod.rs

@@ -0,0 +1,19 @@
+//! This module contains `alloy` types that have been ported from various alloy crates to support `no_std`.
+
+mod transaction;
+pub use transaction::{TxDeposit, TxEip1559, TxEip2930, TxEip4844, TxEnvelope, TxLegacy, TxType};
+
+mod network;
+pub use network::{Receipt as NetworkReceipt, Sealable, Sealed, Signed, Transaction, TxKind};
+
+mod header;
+pub use header::{Header, EMPTY_OMMER_ROOT_HASH, EMPTY_ROOT_HASH};
+
+mod receipt;
+pub use receipt::{Receipt, ReceiptWithBloom};
+
+mod eips;
+pub use eips::{
+    calc_blob_gasprice, calc_excess_blob_gas, calc_next_block_base_fee, eip1559, eip2718, eip2930,
+    eip4788, eip4844,
+};

crates/derive/src/types/network/mod.rs → crates/derive/src/types/alloy/network/mod.rs

@@ -2,7 +2,7 @@
 
 #![allow(unused, unreachable_pub)]
 
-use crate::types::eips::eip2718::Eip2718Envelope;
+use crate::types::eip2718::Eip2718Envelope;
 use alloc::vec::Vec;
 use alloy_primitives::B256;
 

crates/derive/src/types/network/transaction/signed.rs → crates/derive/src/types/alloy/network/transaction/signed.rs

@@ -1,4 +1,4 @@
-use crate::types::network::Transaction;
+use crate::types::Transaction;
 use alloc::{vec, vec::Vec};
 use alloy_primitives::{Signature, B256};
 use alloy_rlp::BufMut;

crates/derive/src/types/receipt.rs → crates/derive/src/types/alloy/receipt.rs

@@ -1,11 +1,10 @@
 //! This module contains the receipt types used within the derivation pipeline.
 
-use core::cmp::Ordering;
-
-use crate::types::transaction::TxType;
+use crate::types::TxType;
 use alloc::vec::Vec;
 use alloy_primitives::{Bloom, Log};
 use alloy_rlp::{length_of_length, Buf, BufMut, BytesMut, Decodable, Encodable};
+use core::cmp::Ordering;
 
 /// Receipt containing result of transaction execution.
 #[derive(Clone, Debug, PartialEq, Eq, Default)]

crates/derive/src/types/transaction/deposit.rs → crates/derive/src/types/alloy/transaction/deposit.rs

@@ -1,7 +1,4 @@
-use crate::types::{
-    network::{Signed, Transaction, TxKind},
-    transaction::TxType,
-};
+use crate::types::{Signed, Transaction, TxKind, TxType};
 use alloc::vec::Vec;
 use alloy_primitives::{keccak256, Address, Bytes, ChainId, Signature, B256, U256};
 use alloy_rlp::{

crates/derive/src/types/transaction/eip1559.rs → crates/derive/src/types/alloy/transaction/eip1559.rs

@@ -1,8 +1,4 @@
-use crate::types::{
-    eips::eip2930::AccessList,
-    network::{Signed, Transaction, TxKind},
-    transaction::TxType,
-};
+use crate::types::{eip2930::AccessList, Signed, Transaction, TxKind, TxType};
 use alloc::vec::Vec;
 use alloy_primitives::{keccak256, Bytes, ChainId, Signature, U256};
 use alloy_rlp::{length_of_length, BufMut, Decodable, Encodable, Header};

crates/derive/src/types/transaction/eip2930.rs → crates/derive/src/types/alloy/transaction/eip2930.rs

@@ -1,8 +1,4 @@
-use crate::types::{
-    eips::eip2930::AccessList,
-    network::{Signed, Transaction, TxKind},
-    transaction::TxType,
-};
+use crate::types::{eip2930::AccessList, Signed, Transaction, TxKind, TxType};
 use alloc::vec::Vec;
 use alloy_primitives::{keccak256, Bytes, ChainId, Signature, U256};
 use alloy_rlp::{length_of_length, BufMut, Decodable, Encodable, Header};

crates/derive/src/types/transaction/eip4844.rs → crates/derive/src/types/alloy/transaction/eip4844.rs

@@ -1,7 +1,5 @@
 use crate::types::{
-    eips::{eip2930::AccessList, eip4844::DATA_GAS_PER_BLOB},
-    network::{Signed, Transaction, TxKind},
-    transaction::TxType,
+    eip2930::AccessList, eip4844::DATA_GAS_PER_BLOB, Signed, Transaction, TxKind, TxType,
 };
 use alloc::vec::Vec;
 use alloy_primitives::{keccak256, Bytes, ChainId, Signature, B256, U256};

crates/derive/src/types/transaction/envelope.rs → crates/derive/src/types/alloy/transaction/envelope.rs

@@ -1,7 +1,6 @@
 use crate::types::{
-    eips::eip2718::{Decodable2718, Eip2718Error, Encodable2718},
-    network::Signed,
-    transaction::{TxDeposit, TxEip1559, TxEip2930, TxEip4844, TxLegacy},
+    eip2718::{Decodable2718, Eip2718Error, Encodable2718},
+    Signed, TxDeposit, TxEip1559, TxEip2930, TxEip4844, TxLegacy,
 };
 use alloy_rlp::{length_of_length, Decodable, Encodable};
 

crates/derive/src/types/transaction/legacy.rs → crates/derive/src/types/alloy/transaction/legacy.rs

@@ -1,4 +1,4 @@
-use crate::types::network::{Signed, Transaction, TxKind};
+use crate::types::{Signed, Transaction, TxKind};
 use alloc::vec::Vec;
 use alloy_primitives::{keccak256, Bytes, ChainId, Signature, U256};
 use alloy_rlp::{length_of_length, BufMut, Decodable, Encodable, Header, Result};