From bccf249ddf828bdac3616b7f1d797ecd3643fe22 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 15:06:27 +0800 Subject: [PATCH 01/69] init --- benches/ycsb.rs | 15 ++++++++------- src/key.rs | 2 +- src/lib.rs | 17 +---------------- src/lsm/core.rs | 1 - src/mvcc/mod.rs | 0 src/persistent/file_object.rs | 2 -- src/state/states.rs | 1 - 7 files changed, 10 insertions(+), 28 deletions(-) create mode 100644 src/mvcc/mod.rs diff --git a/benches/ycsb.rs b/benches/ycsb.rs index 57cf7be..e762231 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -1,17 +1,18 @@ -use better_mini_lsm::fibonacci; -use better_mini_lsm::persistent::LocalFs; -use better_mini_lsm::sst::SstOptions; -use better_mini_lsm::state::{LsmStorageState, Map}; -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use itertools::Itertools; -use maplit::hashmap; use std::collections::HashMap; use std::sync::Arc; + +use criterion::{Criterion, criterion_group, criterion_main}; +use itertools::Itertools; +use maplit::hashmap; use tempfile::tempdir; use ycsb::db::DB; use ycsb::properties::Properties; use ycsb::workload::CoreWorkload; +use better_mini_lsm::persistent::LocalFs; +use better_mini_lsm::sst::SstOptions; +use better_mini_lsm::state::{LsmStorageState, Map}; + #[derive(Clone)] struct LsmStorageStateBench(Arc>); diff --git a/src/key.rs b/src/key.rs index cfe761c..db093ef 100644 --- a/src/key.rs +++ b/src/key.rs @@ -3,7 +3,7 @@ use std::fmt::Debug; use std::ops::Bound; #[derive(PartialEq)] -pub struct Key>(T); +pub struct Key(T); pub type KeySlice<'a> = Key<&'a [u8]>; pub type KeyVec = Key>; diff --git a/src/lib.rs b/src/lib.rs index 9283cc0..f2319de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,20 +15,5 @@ mod lsm; mod manifest; mod test_utils; mod utils; +pub mod mvcc; -pub async fn fibonacci(n: u64) -> u64 { - let mut a = 0; - let mut b = 1; - - match n { - 0 => b, - _ => { - for _ in 0..n { - let c = a + b; - a = b; - b = c; - } - b - } - } -} diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 2567152..830d176 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -82,7 +82,6 @@ impl Lsm

{ .take_while(|signal| ready(matches!(signal, Trigger))) .for_each(|_| async { let lock = state.state_lock().lock().await; - // println!("trigger compaction"); state .force_compact(&lock) .await diff --git a/src/mvcc/mod.rs b/src/mvcc/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/persistent/file_object.rs b/src/persistent/file_object.rs index 1eacee5..b3dfd64 100644 --- a/src/persistent/file_object.rs +++ b/src/persistent/file_object.rs @@ -45,7 +45,6 @@ impl Persistent for LocalFs { /// Create a new file object (day 2) and write the file to the disk (day 4). async fn create_sst(&self, id: usize, data: Vec) -> anyhow::Result { - println!("create sst {}", id); let size = data.len().try_into()?; let path = self.build_sst_path(id); let file = spawn_blocking(move || { @@ -77,7 +76,6 @@ impl Persistent for LocalFs { } async fn open_wal_handle(&self, id: usize) -> anyhow::Result { - println!("open wal {}", id); let path = self.build_wal_path(id); let file = tokio::fs::OpenOptions::new() .create(true) diff --git a/src/state/states.rs b/src/state/states.rs index 1b32eed..21c140a 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -410,7 +410,6 @@ mod test { storage.put_for_test(b"1", b"2333").await.unwrap(); } let num_imm_memtables = storage.inner.load().imm_memtables().len(); - println!("num_imm_memtables: {}", num_imm_memtables); assert!(num_imm_memtables >= 1, "no memtable frozen?"); for _ in 0..1000 { storage.delete_for_test(b"1").await.unwrap(); From 2bc1f692e4705b01e49cc9e515ca9572c52632fc Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 15:59:16 +0800 Subject: [PATCH 02/69] add tests --- benches/ycsb.rs | 2 +- src/lib.rs | 3 +- src/mvcc/mod.rs | 1 + src/mvcc/transaction.rs | 75 ++++++++++++++ src/sst/builder.rs | 66 +++++++++++- src/state/states.rs | 217 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 src/mvcc/transaction.rs diff --git a/benches/ycsb.rs b/benches/ycsb.rs index e762231..604f80f 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use criterion::{Criterion, criterion_group, criterion_main}; +use criterion::{criterion_group, criterion_main, Criterion}; use itertools::Itertools; use maplit::hashmap; use tempfile::tempdir; diff --git a/src/lib.rs b/src/lib.rs index f2319de..93c02f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,6 @@ mod wal; mod lsm; mod manifest; +pub mod mvcc; mod test_utils; mod utils; -pub mod mvcc; - diff --git a/src/mvcc/mod.rs b/src/mvcc/mod.rs index e69de29..37f0806 100644 --- a/src/mvcc/mod.rs +++ b/src/mvcc/mod.rs @@ -0,0 +1 @@ +pub mod transaction; diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs new file mode 100644 index 0000000..a102de3 --- /dev/null +++ b/src/mvcc/transaction.rs @@ -0,0 +1,75 @@ +use crate::entry::Entry; +use crate::iterators::LockedLsmIter; +use crate::persistent::Persistent; +use crate::state::{LsmStorageState, LsmStorageStateInner, Map}; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use futures::{stream, Stream}; +use std::collections::{Bound, HashSet}; +use std::future::Future; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use tokio::sync::Mutex; + +pub struct Transaction { + pub(crate) read_ts: u64, + pub(crate) inner: Arc>, + pub(crate) local_storage: Arc>, + pub(crate) committed: Arc, + /// Write set and read set + pub(crate) key_hashes: Option, HashSet)>>, +} + +impl Map for Transaction

{ + type Error = anyhow::Error; + + async fn get(&self, key: &[u8]) -> Result, Self::Error> { + todo!() + } + + async fn put( + &self, + key: impl Into + Send, + value: impl Into + Send, + ) -> Result<(), Self::Error> { + todo!() + } + + async fn delete(&self, key: impl Into + Send) -> Result<(), Self::Error> { + todo!() + } +} + +impl Drop for Transaction

{ + fn drop(&mut self) { + // commit + todo!() + } +} + +impl Transaction

{ + pub fn scan<'a>( + &self, + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, + ) -> anyhow::Result>> { + todo!(); + Ok(stream::empty()) + } +} + +#[cfg(test)] +impl Transaction

{ + pub async fn get_for_test(&self, key: &[u8]) -> anyhow::Result> { + self.get(key).await + } + + pub async fn put_for_test(&self, key: &[u8], value: &[u8]) -> anyhow::Result<()> { + self.put(Bytes::copy_from_slice(key), Bytes::copy_from_slice(value)) + .await + } + + pub async fn delete_for_test(&self, key: &[u8]) -> anyhow::Result<()> { + self.delete(Bytes::copy_from_slice(key)).await + } +} diff --git a/src/sst/builder.rs b/src/sst/builder.rs index c37aa68..8b4eb43 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -188,12 +188,17 @@ pub async fn generate_sst(dir: &TempDir) -> SsTable { #[cfg(test)] mod tests { + use crate::block::BlockCache; + use bytes::Bytes; + use std::sync::Arc; use tempfile::tempdir; use crate::key::KeySlice; - use crate::persistent::LocalFs; + use crate::persistent::{LocalFs, Persistent}; use crate::sst::builder::{key_of, num_of_keys, value_of}; + use crate::sst::iterator::SsTableIterator; use crate::sst::{SsTable, SsTableBuilder}; + use futures::stream::StreamExt; #[tokio::test] async fn test_sst_build_single_key() { @@ -243,6 +248,65 @@ mod tests { ); } + #[tokio::test] + async fn test_sst_build_multi_version_simple() { + let mut builder = SsTableBuilder::new(16); + builder.add( + KeySlice::for_testing_from_slice_with_ts(b"233", 233), + b"233333", + ); + builder.add( + KeySlice::for_testing_from_slice_with_ts(b"233", 0), + b"2333333", + ); + let dir = tempdir().unwrap(); + builder.build_for_test(&dir, 1).await.unwrap(); + } + + // todo: add test + // #[tokio::test] + // async fn test_sst_build_multi_version_hard() { + // let dir = tempdir().unwrap(); + // let persistent = LocalFs::new(dir.path().to_path_buf()); + // let data = generate_test_data(); + // let _ = generate_sst_with_ts(1, &persistent, data.clone(), None).await; + // let sst = SsTable::open(1, None, &persistent).await.unwrap(); + // let sst_iter = SsTableIterator::create_and_seek_to_first(&sst) + // .map(|entry| { + // let entry = entry.unwrap(); + // let key = entry.key; + // todo!() + // }); + // + // } + + pub async fn generate_sst_with_ts( + id: usize, + persistent: &P, + data: Vec<((Bytes, u64), Bytes)>, + block_cache: Option>, + ) -> SsTable { + let mut builder = SsTableBuilder::new(128); + for ((key, ts), value) in data { + builder.add( + KeySlice::for_testing_from_slice_with_ts(&key[..], ts), + &value[..], + ); + } + builder.build(id, block_cache, persistent).await.unwrap() + } + + fn generate_test_data() -> Vec<((Bytes, u64), Bytes)> { + (0..100) + .map(|id| { + ( + (Bytes::from(format!("key{:05}", id / 5)), 5 - (id % 5)), + Bytes::from(format!("value{:05}", id)), + ) + }) + .collect() + } + fn get_builder() -> SsTableBuilder { let mut builder = SsTableBuilder::new(128); for idx in 0..num_of_keys() { diff --git a/src/state/states.rs b/src/state/states.rs index 21c140a..382b8c7 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -17,6 +17,7 @@ use crate::block::BlockCache; use crate::iterators::LockedLsmIter; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; +use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; @@ -77,6 +78,10 @@ where }; Ok(this) } + + pub fn new_txn(&self) -> anyhow::Result> { + todo!() + } } // KV store @@ -749,4 +754,216 @@ mod test { .build(); LsmStorageState::new(options, persistent).await } + + #[tokio::test] + async fn test_task2_memtable_mvcc() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let options = SstOptions::builder() + .target_sst_size(1024) + .block_size(4096) + .num_memtable_limit(1000) + .compaction_option(Default::default()) + .enable_wal(true) + .build(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); + + storage.put_for_test(b"a", b"1").await.unwrap(); + storage.put_for_test(b"b", b"1").await.unwrap(); + let snapshot1 = storage.new_txn().unwrap(); + storage.put_for_test(b"a", b"2").await.unwrap(); + let snapshot2 = storage.new_txn().unwrap(); + storage.delete_for_test(b"b").await.unwrap(); + storage.put_for_test(b"c", b"1").await.unwrap(); + let snapshot3 = storage.new_txn().unwrap(); + assert_eq!( + snapshot1.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!( + snapshot1.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!(snapshot1.get_for_test(b"c").await.unwrap(), None); + + { + let iter = snapshot1.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "1"), ("b", "1")]), + ) + .await; + } + + assert_eq!( + snapshot2.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"2")) + ); + assert_eq!( + snapshot2.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!(snapshot2.get_for_test(b"c").await.unwrap(), None); + + { + let iter = snapshot2.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "2"), ("b", "1")]), + ) + .await; + } + + assert_eq!( + snapshot3.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"2")) + ); + assert_eq!(snapshot3.get_for_test(b"b").await.unwrap(), None); + assert_eq!( + snapshot3.get_for_test(b"c").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + + { + let iter = snapshot3.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "2"), ("c", "1")]), + ) + .await; + } + + { + let guard = storage.state_lock.lock().await; + storage.force_freeze_memtable(&guard).await.unwrap(); + } + + storage.put_for_test(b"a", b"3").await.unwrap(); + storage.put_for_test(b"b", b"3").await.unwrap(); + let snapshot4 = storage.new_txn().unwrap(); + storage.put_for_test(b"a", b"4").await.unwrap(); + let snapshot5 = storage.new_txn().unwrap(); + storage.delete_for_test(b"b").await.unwrap(); + storage.put_for_test(b"c", b"5").await.unwrap(); + let snapshot6 = storage.new_txn().unwrap(); + assert_eq!( + snapshot1.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!( + snapshot1.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!(snapshot1.get_for_test(b"c").await.unwrap(), None); + + { + let iter = snapshot1.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "1"), ("b", "1")]), + ) + .await; + } + + assert_eq!( + snapshot2.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"2")) + ); + assert_eq!( + snapshot2.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + assert_eq!(snapshot2.get_for_test(b"c").await.unwrap(), None); + + { + let iter = snapshot2.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "2"), ("b", "1")]), + ) + .await; + } + + assert_eq!( + snapshot3.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"2")) + ); + assert_eq!(snapshot3.get_for_test(b"b").await.unwrap(), None); + assert_eq!( + snapshot3.get_for_test(b"c").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + + { + let iter = snapshot3.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "2"), ("c", "1")]), + ) + .await; + } + + assert_eq!( + snapshot4.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"3")) + ); + assert_eq!( + snapshot4.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"3")) + ); + assert_eq!( + snapshot4.get_for_test(b"c").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + + { + let iter = snapshot4.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "3"), ("b", "3"), ("c", "1")]), + ) + .await; + } + + assert_eq!( + snapshot5.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"4")) + ); + assert_eq!( + snapshot5.get_for_test(b"b").await.unwrap(), + Some(Bytes::from_static(b"3")) + ); + assert_eq!( + snapshot5.get_for_test(b"c").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + + { + let iter = snapshot4.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "4"), ("b", "3"), ("c", "1")]), + ) + .await; + } + + assert_eq!( + snapshot6.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"4")) + ); + assert_eq!(snapshot6.get_for_test(b"b").await.unwrap(), None); + assert_eq!( + snapshot6.get_for_test(b"c").await.unwrap(), + Some(Bytes::from_static(b"5")) + ); + + { + let iter = snapshot6.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "4"), ("c", "5")]), + ) + .await; + } + } } From ea9ef8926c13fedf474068d97795b0b29681a891 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 16:02:03 +0800 Subject: [PATCH 03/69] add test --- src/sst/builder.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 8b4eb43..b6f4ecd 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -280,6 +280,20 @@ mod tests { // // } + #[tokio::test] + async fn test_task3_sst_ts() { + let mut builder = SsTableBuilder::new(16); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"11", 1), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"22", 2), b"22"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"33", 3), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"44", 4), b"22"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"55", 5), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"66", 6), b"22"); + let dir = tempdir().unwrap(); + let sst = builder.build_for_test(&dir, 1).await.unwrap(); + assert_eq!(*sst.max_ts(), 6); + } + pub async fn generate_sst_with_ts( id: usize, persistent: &P, From c13c21aaa03c611b597b52c7ac952d35fbae2e35 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 16:10:45 +0800 Subject: [PATCH 04/69] add test --- src/state/states.rs | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/state/states.rs b/src/state/states.rs index 382b8c7..fb640d9 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -322,7 +322,7 @@ where #[cfg(test)] mod test { use std::collections::Bound; - use std::ops::Bound::{Included, Unbounded}; + use std::ops::Bound::{Excluded, Included, Unbounded}; use bytes::Bytes; use futures::StreamExt; @@ -757,6 +757,15 @@ mod test { #[tokio::test] async fn test_task2_memtable_mvcc() { + test_task2_memtable_mvcc_helper(false).await; + } + + #[tokio::test] + async fn test_task2_lsm_iterator_mvcc() { + test_task2_memtable_mvcc_helper(true).await; + } + + async fn test_task2_memtable_mvcc_helper(flush: bool) { let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); let options = SstOptions::builder() @@ -833,7 +842,7 @@ mod test { .await; } - { + if !flush { let guard = storage.state_lock.lock().await; storage.force_freeze_memtable(&guard).await.unwrap(); } @@ -846,6 +855,12 @@ mod test { storage.delete_for_test(b"b").await.unwrap(); storage.put_for_test(b"c", b"5").await.unwrap(); let snapshot6 = storage.new_txn().unwrap(); + + if flush { + let guard = storage.state_lock.lock().await; + storage.force_flush_imm_memtable(&guard).await.unwrap(); + } + assert_eq!( snapshot1.get_for_test(b"a").await.unwrap(), Some(Bytes::from_static(b"1")) @@ -965,5 +980,25 @@ mod test { ) .await; } + + if flush { + { + let iter = snapshot6.scan(Included(b"a"), Included(b"a")).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("a", "4")]), + ) + .await; + } + + { + let iter = snapshot6.scan(Excluded(b"a"), Excluded(b"c")).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([]), + ) + .await; + } + } } } From 7f8de7a8c79895ef9adf620c6a51671d73566e00 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 16:18:39 +0800 Subject: [PATCH 05/69] add test --- src/mvcc/mod.rs | 1 + src/mvcc/watermark.rs | 68 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 src/mvcc/watermark.rs diff --git a/src/mvcc/mod.rs b/src/mvcc/mod.rs index 37f0806..929f9a8 100644 --- a/src/mvcc/mod.rs +++ b/src/mvcc/mod.rs @@ -1 +1,2 @@ pub mod transaction; +mod watermark; diff --git a/src/mvcc/watermark.rs b/src/mvcc/watermark.rs new file mode 100644 index 0000000..2de10d0 --- /dev/null +++ b/src/mvcc/watermark.rs @@ -0,0 +1,68 @@ +use std::collections::BTreeMap; + +pub struct Watermark { + readers: BTreeMap, +} + +impl Watermark { + pub fn new() -> Self { + Self { + readers: BTreeMap::new(), + } + } + + pub fn add_reader(&mut self, ts: u64) {} + + pub fn remove_reader(&mut self, ts: u64) {} + + pub fn watermark(&self) -> Option { + todo!() + } + + fn num_retained_snapshots(&self) -> usize { + todo!() + } +} + +#[cfg(test)] +mod tests { + use crate::mvcc::watermark::Watermark; + + #[test] + fn test_task1_watermark() { + let mut watermark = Watermark::new(); + watermark.add_reader(0); + for i in 1..=1000 { + watermark.add_reader(i); + assert_eq!(watermark.watermark(), Some(0)); + assert_eq!(watermark.num_retained_snapshots(), i as usize + 1); + } + let mut cnt = 1001; + for i in 0..500 { + watermark.remove_reader(i); + assert_eq!(watermark.watermark(), Some(i + 1)); + cnt -= 1; + assert_eq!(watermark.num_retained_snapshots(), cnt); + } + for i in (501..=1000).rev() { + watermark.remove_reader(i); + assert_eq!(watermark.watermark(), Some(500)); + cnt -= 1; + assert_eq!(watermark.num_retained_snapshots(), cnt); + } + watermark.remove_reader(500); + assert_eq!(watermark.watermark(), None); + assert_eq!(watermark.num_retained_snapshots(), 0); + watermark.add_reader(2000); + watermark.add_reader(2000); + watermark.add_reader(2001); + assert_eq!(watermark.num_retained_snapshots(), 2); + assert_eq!(watermark.watermark(), Some(2000)); + watermark.remove_reader(2000); + assert_eq!(watermark.num_retained_snapshots(), 2); + assert_eq!(watermark.watermark(), Some(2000)); + watermark.remove_reader(2000); + assert_eq!(watermark.num_retained_snapshots(), 1); + assert_eq!(watermark.watermark(), Some(2001)); + } +} From e6a4a86ceda163a616223290c0bc3bd8b0998d78 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 16:26:57 +0800 Subject: [PATCH 06/69] add test --- src/mvcc/core.rs | 55 +++++++++++++++++++++++++++++++++++++++++++++ src/mvcc/mod.rs | 1 + src/state/states.rs | 32 ++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 src/mvcc/core.rs diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs new file mode 100644 index 0000000..fc99196 --- /dev/null +++ b/src/mvcc/core.rs @@ -0,0 +1,55 @@ +use crate::mvcc::transaction::Transaction; +use crate::mvcc::watermark::Watermark; +use crate::persistent::Persistent; +use crate::state::LsmStorageState; +use parking_lot::Mutex; +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; + +pub(crate) struct CommittedTxnData { + pub(crate) key_hashes: HashSet, + #[allow(dead_code)] + pub(crate) read_ts: u64, + #[allow(dead_code)] + pub(crate) commit_ts: u64, +} + +pub(crate) struct LsmMvccInner { + pub(crate) write_lock: Mutex<()>, + pub(crate) commit_lock: Mutex<()>, + pub(crate) ts: Arc>, + pub(crate) committed_txns: Arc>>, +} + +impl LsmMvccInner { + pub fn new(initial_ts: u64) -> Self { + Self { + write_lock: Mutex::new(()), + commit_lock: Mutex::new(()), + ts: Arc::new(Mutex::new((initial_ts, Watermark::new()))), + committed_txns: Arc::new(Mutex::new(BTreeMap::new())), + } + } + + pub fn latest_commit_ts(&self) -> u64 { + self.ts.lock().0 + } + + pub fn update_commit_ts(&self, ts: u64) { + self.ts.lock().0 = ts; + } + + /// All ts (strictly) below this ts can be garbage collected. + pub fn watermark(&self) -> u64 { + let ts = self.ts.lock(); + ts.1.watermark().unwrap_or(ts.0) + } + + pub fn new_txn( + &self, + inner: Arc>, + serializable: bool, + ) -> Arc> { + unimplemented!() + } +} diff --git a/src/mvcc/mod.rs b/src/mvcc/mod.rs index 929f9a8..8737ac0 100644 --- a/src/mvcc/mod.rs +++ b/src/mvcc/mod.rs @@ -1,2 +1,3 @@ +pub mod core; pub mod transaction; mod watermark; diff --git a/src/state/states.rs b/src/state/states.rs index fb640d9..7f62e73 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -17,6 +17,7 @@ use crate::block::BlockCache; use crate::iterators::LockedLsmIter; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; +use crate::mvcc::core::LsmMvccInner; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; @@ -34,6 +35,7 @@ pub struct LsmStorageState { pub(crate) persistent: P, pub(crate) options: SstOptions, pub(crate) sst_id: AtomicUsize, + mvcc: Option, } impl

Debug for LsmStorageState

@@ -75,6 +77,7 @@ where persistent, options, sst_id, + mvcc: None, // todo }; Ok(this) } @@ -1001,4 +1004,33 @@ mod test { } } } + + #[tokio::test] + async fn test_task2_snapshot_watermark() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let options = SstOptions::builder() + .target_sst_size(1024) + .block_size(4096) + .num_memtable_limit(1000) + .compaction_option(Default::default()) + .enable_wal(true) + .build(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); + + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + storage.put_for_test(b"233", b"23333").await.unwrap(); + let txn3 = storage.new_txn().unwrap(); + assert_eq!(storage.mvcc().as_ref().unwrap().watermark(), txn1.read_ts); + drop(txn1); + assert_eq!(storage.mvcc().as_ref().unwrap().watermark(), txn2.read_ts); + drop(txn2); + assert_eq!(storage.mvcc().as_ref().unwrap().watermark(), txn3.read_ts); + drop(txn3); + assert_eq!( + storage.mvcc().as_ref().unwrap().watermark(), + storage.mvcc().as_ref().unwrap().latest_commit_ts() + ); + } } From 4159641c7dc46b43369fe143eb2402a9c0c144ea Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 16:53:01 +0800 Subject: [PATCH 07/69] add test --- src/state/map.rs | 3 +- src/state/mod.rs | 1 + src/state/mut_op.rs | 5 ++ src/state/states.rs | 133 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/state/mut_op.rs diff --git a/src/state/map.rs b/src/state/map.rs index 28c8a10..85a9814 100644 --- a/src/state/map.rs +++ b/src/state/map.rs @@ -1,6 +1,7 @@ -use bytes::Bytes; use std::future::Future; +use bytes::Bytes; + pub trait Map { type Error; diff --git a/src/state/mod.rs b/src/state/mod.rs index 7e4cc5a..af330bc 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -1,5 +1,6 @@ mod inner; mod map; +mod mut_op; mod states; pub use inner::LsmStorageStateInner; diff --git a/src/state/mut_op.rs b/src/state/mut_op.rs new file mode 100644 index 0000000..fd0c610 --- /dev/null +++ b/src/state/mut_op.rs @@ -0,0 +1,5 @@ +#[derive(Debug)] +pub enum Op { + Put { key: T, value: T }, + Del(T), +} diff --git a/src/state/states.rs b/src/state/states.rs index 7f62e73..0368f22 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -23,6 +23,7 @@ use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::LsmStorageStateInner; +use crate::state::mut_op::Op; use crate::state::Map; use crate::utils::vec::pop; @@ -320,6 +321,13 @@ where async fn delete_for_test(&self, key: &[u8]) -> anyhow::Result<()> { self.delete(Bytes::copy_from_slice(key)).await } + + async fn write_batch_for_test( + &self, + records: impl IntoIterator>, + ) -> anyhow::Result<()> { + todo!() + } } #[cfg(test)] @@ -339,6 +347,7 @@ mod test { use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; use crate::sst::SstOptions; + use crate::state::mut_op::Op; use crate::state::states::LsmStorageState; #[tokio::test] @@ -1033,4 +1042,128 @@ mod test { storage.mvcc().as_ref().unwrap().latest_commit_ts() ); } + + // todo: add test + // #[tokio::test] + // async fn test_task3_mvcc_compaction() { + // use Op::{Put, Del}; + // let dir = tempdir().unwrap(); + // let persistent = LocalFs::new(dir.path().to_path_buf()); + // let options = SstOptions::builder() + // .target_sst_size(1024) + // .block_size(4096) + // .num_memtable_limit(1000) + // .compaction_option(Default::default()) + // .enable_wal(true) + // .build(); + // let storage = LsmStorageState::new(options, persistent).await.unwrap(); + // + // let snapshot0 = storage.new_txn().unwrap(); + // storage + // .write_batch_for_test(&[ + // Put {key: b"a", value: b"1"}, + // Put{key: b"b", value: b"1"}, + // ]).await + // .unwrap(); + // let snapshot1 = storage.new_txn().unwrap(); + // storage + // .write_batch_for_test(&[ + // Put{key: b"a", value: b"2"}, + // Put{key: b"d", value: b"2"}, + // ]).await + // .unwrap(); + // let snapshot2 = storage.new_txn().unwrap(); + // storage + // .write_batch_for_test(&[ + // Put{key: b"a", value: b"3"}, + // Del(b"d"), + // ]).await + // .unwrap(); + // let snapshot3 = storage.new_txn().unwrap(); + // storage + // .write_batch_for_test(&[ + // Put{key: b"c", value: b"4"}, + // Del(b"a"), + // ]).await + // .unwrap(); + // + // storage.force_flush().unwrap(); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("a"), Bytes::new()), + // (Bytes::from("a"), Bytes::from("3")), + // (Bytes::from("a"), Bytes::from("2")), + // (Bytes::from("a"), Bytes::from("1")), + // (Bytes::from("b"), Bytes::from("1")), + // (Bytes::from("c"), Bytes::from("4")), + // (Bytes::from("d"), Bytes::new()), + // (Bytes::from("d"), Bytes::from("2")), + // ], + // ); + // + // drop(snapshot0); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("a"), Bytes::new()), + // (Bytes::from("a"), Bytes::from("3")), + // (Bytes::from("a"), Bytes::from("2")), + // (Bytes::from("a"), Bytes::from("1")), + // (Bytes::from("b"), Bytes::from("1")), + // (Bytes::from("c"), Bytes::from("4")), + // (Bytes::from("d"), Bytes::new()), + // (Bytes::from("d"), Bytes::from("2")), + // ], + // ); + // + // drop(snapshot1); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("a"), Bytes::new()), + // (Bytes::from("a"), Bytes::from("3")), + // (Bytes::from("a"), Bytes::from("2")), + // (Bytes::from("b"), Bytes::from("1")), + // (Bytes::from("c"), Bytes::from("4")), + // (Bytes::from("d"), Bytes::new()), + // (Bytes::from("d"), Bytes::from("2")), + // ], + // ); + // + // drop(snapshot2); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("a"), Bytes::new()), + // (Bytes::from("a"), Bytes::from("3")), + // (Bytes::from("b"), Bytes::from("1")), + // (Bytes::from("c"), Bytes::from("4")), + // ], + // ); + // + // drop(snapshot3); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("b"), Bytes::from("1")), + // (Bytes::from("c"), Bytes::from("4")), + // ], + // ); + // } } From 3450d93e5134f94772b466c732b0591bdcacb569 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 1 Jun 2024 17:04:57 +0800 Subject: [PATCH 08/69] add test --- src/mvcc/transaction.rs | 11 ++-- src/state/states.rs | 128 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 7 deletions(-) diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index a102de3..12deec0 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -40,13 +40,6 @@ impl Map for Transaction

{ } } -impl Drop for Transaction

{ - fn drop(&mut self) { - // commit - todo!() - } -} - impl Transaction

{ pub fn scan<'a>( &self, @@ -56,6 +49,10 @@ impl Transaction

{ todo!(); Ok(stream::empty()) } + + pub fn commit(self) -> impl Future> + Send { + async { todo!() } + } } #[cfg(test)] diff --git a/src/state/states.rs b/src/state/states.rs index 0368f22..b1cfdef 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1166,4 +1166,132 @@ mod test { // ], // ); // } + + #[tokio::test] + async fn test_txn_integration() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let options = SstOptions::builder() + .target_sst_size(1024) + .block_size(4096) + .num_memtable_limit(1000) + .compaction_option(Default::default()) + .enable_wal(true) + .build(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); + + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn1.put_for_test(b"test1", b"233").await.unwrap(); + txn2.put_for_test(b"test2", b"233").await.unwrap(); + + { + let iter = txn1.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("test1", "233")]), + ) + .await; + } + + { + let iter = txn2.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("test2", "233")]), + ) + .await; + } + + let txn3 = storage.new_txn().unwrap(); + + { + let iter = txn3.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([]), + ) + .await; + } + + txn1.commit().await.unwrap(); + txn2.commit().await.unwrap(); + + { + let iter = txn3.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([]), + ) + .await; + } + + drop(txn3); + + // todo: add this test + // { + // let iter = storage.scan(Unbounded, Unbounded); + // assert_stream_eq( + // iter.iter().await.unwrap().map(Result::unwrap).map(Entry::into_tuple), + // build_tuple_stream([("test1", "233"), ("test2", "233")]), + // ) + // .await; + // } + + let txn4 = storage.new_txn().unwrap(); + + assert_eq!( + txn4.get_for_test(b"test1").await.unwrap(), + Some(Bytes::from("233")) + ); + assert_eq!( + txn4.get_for_test(b"test2").await.unwrap(), + Some(Bytes::from("233")) + ); + + { + let iter = txn4.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("test1", "233"), ("test2", "233")]), + ) + .await; + } + + txn4.put_for_test(b"test2", b"2333").await.unwrap(); + assert_eq!( + txn4.get_for_test(b"test1").await.unwrap(), + Some(Bytes::from("233")) + ); + assert_eq!( + txn4.get_for_test(b"test2").await.unwrap(), + Some(Bytes::from("2333")) + ); + + { + let iter = txn4.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("test1", "233"), ("test2", "2333")]), + ) + .await; + } + + txn4.delete_for_test(b"test2").await.unwrap(); + + assert_eq!( + txn4.get_for_test(b"test1").await.unwrap(), + Some(Bytes::from("233")) + ); + assert_eq!(txn4.get_for_test(b"test2").await.unwrap(), None); + + { + let iter = txn4.scan(Unbounded, Unbounded).unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream([("test1", "233")]), + ) + .await; + } + } } From 22537a324aa6683fe9df3ff27e35bbcf3de5f6b0 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 2 Jun 2024 11:43:14 +0800 Subject: [PATCH 09/69] support watermark --- src/mvcc/watermark.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/mvcc/watermark.rs b/src/mvcc/watermark.rs index 2de10d0..16e4b72 100644 --- a/src/mvcc/watermark.rs +++ b/src/mvcc/watermark.rs @@ -11,16 +11,28 @@ impl Watermark { } } - pub fn add_reader(&mut self, ts: u64) {} + pub fn add_reader(&mut self, ts: u64) { + self.readers + .entry(ts) + .and_modify(|count| *count = *count + 1) + .or_insert(1); + } - pub fn remove_reader(&mut self, ts: u64) {} + pub fn remove_reader(&mut self, ts: u64) { + let count = self.readers.get_mut(&ts).unwrap(); + if *count == 1 { + self.readers.remove(&ts); + } else { + *count = *count - 1; + } + } pub fn watermark(&self) -> Option { - todo!() + self.readers.keys().copied().next() } fn num_retained_snapshots(&self) -> usize { - todo!() + self.readers.len() } } From 631a4f5094418b3db373e18621c2097eb89c196b Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 2 Jun 2024 20:51:09 +0800 Subject: [PATCH 10/69] wip --- src/bound.rs | 21 ------ src/key.rs | 140 ++++++++++++++++++---------------------- src/memtable/mutable.rs | 1 + 3 files changed, 63 insertions(+), 99 deletions(-) diff --git a/src/bound.rs b/src/bound.rs index 4a9c297..5f6ba24 100644 --- a/src/bound.rs +++ b/src/bound.rs @@ -1,27 +1,6 @@ -use bytes::Bytes; -use nom::AsBytes; use std::collections::Bound; use std::ops::RangeBounds; -/// Create a bound of `Bytes` from a bound of `&[u8]`. -pub(crate) fn map_bound_own(bound: Bound<&[u8]>) -> Bound { - map_bound(bound, Bytes::copy_from_slice) -} - -pub(crate) fn map_bound_ref(bound: Bound<&Bytes>) -> Bound<&[u8]> { - map_bound(bound, |b| b.as_bytes()) -} - -/// todo: 用 Bound::map 替代 -pub(crate) fn map_bound U>(bound: Bound, f: F) -> Bound { - use Bound::{Excluded, Included, Unbounded}; - match bound { - Unbounded => Unbounded, - Included(x) => Included(f(x)), - Excluded(x) => Excluded(f(x)), - } -} - pub struct BytesBound<'a> { pub start: Bound<&'a [u8]>, pub end: Bound<&'a [u8]>, diff --git a/src/key.rs b/src/key.rs index db093ef..285bf3e 100644 --- a/src/key.rs +++ b/src/key.rs @@ -1,25 +1,41 @@ use bytes::Bytes; +use derive_new::new; +use nom::AsBytes; +use std::cmp::Ordering; use std::fmt::Debug; use std::ops::Bound; -#[derive(PartialEq)] -pub struct Key(T); +#[derive(PartialEq, Eq, Debug, new, Default, Clone, Copy)] +pub struct Key { + key: T, + timestamp: u64, +} pub type KeySlice<'a> = Key<&'a [u8]>; pub type KeyVec = Key>; pub type KeyBytes = Key; +impl Key { + pub fn map(self, f: impl FnOnce(T) -> U) -> Key { + Key::new(f(self.key), self.timestamp) + } + + pub fn as_ref(&self) -> Key<&T> { + Key::new(&self.key, self.timestamp) + } +} + impl> Key { pub fn into_inner(self) -> T { - self.0 + self.key } pub fn len(&self) -> usize { - self.0.as_ref().len() + self.key.as_ref().len() } pub fn is_empty(&self) -> bool { - self.0.as_ref().is_empty() + self.key.as_ref().is_empty() } pub fn for_testing_ts(self) -> u64 { @@ -28,139 +44,107 @@ impl> Key { } impl Key> { - pub fn new() -> Self { - Self(Vec::new()) - } - - /// Create a `KeyVec` from a `Vec`. Will be removed in week 3. - pub fn from_vec(key: Vec) -> Self { - Self(key) - } - - /// Clears the key and set ts to 0. - pub fn clear(&mut self) { - self.0.clear() - } - - /// Append a slice to the end of the key - pub fn append(&mut self, data: &[u8]) { - self.0.extend(data) - } - - /// Set the key from a slice without re-allocating. The signature will change in week 3. - pub fn set_from_slice(&mut self, key_slice: KeySlice) { - self.0.clear(); - self.0.extend(key_slice.0); - } - pub fn as_key_slice(&self) -> KeySlice { - Key(self.0.as_slice()) + self.as_ref().map(Vec::as_slice) } pub fn into_key_bytes(self) -> KeyBytes { - Key(self.0.into()) + self.map(Into::into) } /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(&self) -> &[u8] { - self.0.as_ref() + self.key.as_ref() } pub fn for_testing_key_ref(&self) -> &[u8] { - self.0.as_ref() + self.key.as_ref() } pub fn for_testing_from_vec_no_ts(key: Vec) -> Self { - Self(key) + Self::new(key, 0) } } impl Key { pub fn as_key_slice(&self) -> KeySlice { - Key(&self.0) + self.as_ref().map(|b| b.as_bytes()) } /// Create a `KeyBytes` from a `Bytes`. Will be removed in week 3. pub fn from_bytes(bytes: Bytes) -> KeyBytes { - Key(bytes) + todo!() + // Key(bytes) } /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(&self) -> &[u8] { - self.0.as_ref() + todo!() + // self.0.as_ref() } pub fn for_testing_from_bytes_no_ts(bytes: Bytes) -> KeyBytes { - Key(bytes) + todo!() + // Key(bytes) } pub fn for_testing_key_ref(&self) -> &[u8] { - self.0.as_ref() + todo!() + // self.0.as_ref() } } impl<'a> Key<&'a [u8]> { pub fn to_key_vec(self) -> KeyVec { - Key(self.0.to_vec()) + todo!() + // Key(self.0.to_vec()) } /// Create a key slice from a slice. Will be removed in week 3. pub fn from_slice(slice: &'a [u8]) -> Self { - Self(slice) + todo!() + // Self(slice) } /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(self) -> &'a [u8] { - self.0 + todo!() + // self.0 } pub fn for_testing_key_ref(self) -> &'a [u8] { - self.0 + todo!() + // self.0 } pub fn for_testing_from_slice_no_ts(slice: &'a [u8]) -> Self { - Self(slice) + todo!() + // Self(slice) } pub fn for_testing_from_slice_with_ts(slice: &'a [u8], _ts: u64) -> Self { - Self(slice) - } -} - -impl + Debug> Debug for Key { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl + Default> Default for Key { - fn default() -> Self { - Self(T::default()) + todo!() + // Self(slice) } } -impl + Eq> Eq for Key {} - -impl + Clone> Clone for Key { - fn clone(&self) -> Self { - Self(self.0.clone()) +impl PartialOrd for Key { + fn partial_cmp(&self, other: &Self) -> Option { + let key_order = self.key.partial_cmp(&other.key)?; + match key_order { + Ordering::Equal => self + .timestamp + .partial_cmp(&other.timestamp) + .map(Ordering::reverse), + _ => Some(key_order), + } } } -impl + Copy> Copy for Key {} - -impl + PartialOrd> PartialOrd for Key { - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) +impl Ord for Key { + fn cmp(&self, other: &Self) -> Ordering { + self.key + .cmp(&other.key) + .then_with(|| self.timestamp.cmp(&other.timestamp).reverse()) } } - -impl + Ord> Ord for Key { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.0.cmp(&other.0) - } -} - -pub fn bound_bytes_as_ref(b: &Bound) -> Bound<&[u8]> { - b.as_ref().map(Bytes::as_ref) -} diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index cecbc35..1664a65 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -15,6 +15,7 @@ use tracing_futures::Instrument; use crate::bound::BytesBound; use crate::iterators::NonEmptyStream; +use crate::key::KeyBytes; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::immutable::ImmutableMemTable; use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; From cd84f3e17b774f14c0c29f2157dd59ba4d5cd071 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 3 Jun 2024 13:21:17 +0800 Subject: [PATCH 11/69] wip --- src/bound.rs | 12 ++++---- src/entry.rs | 51 +++++++++++++++++++------------- src/key.rs | 12 ++++++++ src/memtable/immutable.rs | 3 +- src/memtable/iterator.rs | 32 ++++++++++---------- src/memtable/mutable.rs | 60 ++++++++++++++++++++++++++++---------- src/sst/bloom.rs | 1 + src/sst/builder.rs | 2 +- src/sst/iterator/concat.rs | 8 ++--- src/sst/iterator/iter.rs | 25 +++++++--------- src/sst/iterator/merged.rs | 8 ++--- src/sst/sstables.rs | 37 +++++++++-------------- src/wal.rs | 25 +++++++++++----- 13 files changed, 163 insertions(+), 113 deletions(-) diff --git a/src/bound.rs b/src/bound.rs index 5f6ba24..3936301 100644 --- a/src/bound.rs +++ b/src/bound.rs @@ -1,17 +1,19 @@ use std::collections::Bound; use std::ops::RangeBounds; +use crate::key::KeyBytes; + pub struct BytesBound<'a> { - pub start: Bound<&'a [u8]>, - pub end: Bound<&'a [u8]>, + pub start: Bound<&'a KeyBytes>, + pub end: Bound<&'a KeyBytes>, } -impl<'a> RangeBounds<[u8]> for BytesBound<'a> { - fn start_bound(&self) -> Bound<&[u8]> { +impl<'a> RangeBounds for BytesBound<'a> { + fn start_bound(&self) -> Bound<&KeyBytes> { self.start } - fn end_bound(&self) -> Bound<&[u8]> { + fn end_bound(&self) -> Bound<&KeyBytes> { self.end } } diff --git a/src/entry.rs b/src/entry.rs index 442a60c..a71b811 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,42 +1,53 @@ +use std::cmp::Ordering; + use bytes::Bytes; -use nom::AsBytes; -use std::cmp; +use crate::key::KeyBytes; + +pub type Entry = Keyed; +pub type InnerEntry = Keyed; -#[derive(Debug, Eq)] -pub struct Entry { - pub key: Bytes, - pub value: Bytes, +#[derive(Debug)] +pub struct Keyed { + pub key: K, + pub value: V, } -impl PartialEq for Entry { +impl Eq for Keyed {} + +impl PartialEq for Keyed { fn eq(&self, other: &Self) -> bool { - self.cmp(other) == cmp::Ordering::Equal + self.key.eq(&other.key) } } -impl PartialOrd for Entry { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) +impl PartialOrd for Keyed { + fn partial_cmp(&self, other: &Self) -> Option { + self.key.partial_cmp(&other.key) } } -impl Ord for Entry { - fn cmp(&self, other: &Self) -> cmp::Ordering { +impl Ord for Keyed { + fn cmp(&self, other: &Self) -> Ordering { self.key.cmp(&other.key) } } #[cfg(test)] -impl Entry { +impl Keyed { + pub fn into_tuple(self) -> (K, V) { + let Self { key, value } = self; + (key, value) + } + + +} + +#[cfg(test)] +impl Keyed { pub fn from_slice(key: &[u8], value: &[u8]) -> Self { Self { key: Bytes::copy_from_slice(key), value: Bytes::copy_from_slice(value), } } - - pub fn into_tuple(self) -> (Bytes, Bytes) { - let Self { key, value } = self; - (key, value) - } -} +} \ No newline at end of file diff --git a/src/key.rs b/src/key.rs index 285bf3e..d4388c4 100644 --- a/src/key.rs +++ b/src/key.rs @@ -1,3 +1,4 @@ +use std::borrow::Borrow; use bytes::Bytes; use derive_new::new; use nom::AsBytes; @@ -23,6 +24,17 @@ impl Key { pub fn as_ref(&self) -> Key<&T> { Key::new(&self.key, self.timestamp) } + + pub fn timestamp(&self) -> u64 { + self.timestamp + } +} + +#[cfg(test)] +impl Key { + pub fn new_no_ts(key: &[u8]) -> Self { + Self::new(Bytes::copy_from_slice(key), 0) + } } impl> Key { diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index e6498e8..bbdbec3 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -7,6 +7,7 @@ use crossbeam_skiplist::map; use deref_ext::DerefExt; use derive_new::new; use ref_cast::RefCast; +use crate::key::KeyBytes; use crate::memtable::iterator::MaybeEmptyMemTableIterRef; use crate::memtable::mutable::MemTable; @@ -48,7 +49,7 @@ impl ImmutableMemTable { self.0.get(key) } - pub fn iter(&self) -> impl Iterator> { + pub fn iter(&self) -> impl Iterator> { self.0.map().iter() } } diff --git a/src/memtable/iterator.rs b/src/memtable/iterator.rs index c90c858..3c88e7b 100644 --- a/src/memtable/iterator.rs +++ b/src/memtable/iterator.rs @@ -1,4 +1,3 @@ -use std::collections::Bound; use std::iter; use bytes::Bytes; @@ -6,46 +5,45 @@ use crossbeam_skiplist::map; use futures::stream; use crate::bound::BytesBound; -use crate::entry::Entry; +use crate::entry::InnerEntry; use crate::iterators::{MaybeEmptyStream, NonEmptyStream, OkIter}; +use crate::key::KeyBytes; pub type MemTableIterator<'a> = stream::Iter>>; type ClonedSkipMapRangeIter<'a> = - iter::Map, for<'b> fn(SkipMapRangeEntry<'b>) -> Entry>; + iter::Map, for<'b> fn(SkipMapRangeEntry<'b>) -> InnerEntry>; pub fn new_memtable_iter(iter: SkipMapRangeIter) -> MemTableIterator { - let iter = iter.map(convert_entry as for<'a> fn(map::Entry<'a, Bytes, Bytes>) -> _); + let iter = iter.map(convert_entry as for<'a> fn(map::Entry<'a, KeyBytes, Bytes>) -> _); stream::iter(OkIter::new(iter)) } -fn convert_entry(x: map::Entry<'_, Bytes, Bytes>) -> Entry { - Entry { +fn convert_entry(x: map::Entry<'_, KeyBytes, Bytes>) -> InnerEntry { + InnerEntry { key: x.key().clone(), value: x.value().clone(), } } -type SkipMapRangeIter<'a> = map::Range<'a, [u8], BytesBound<'a>, Bytes, Bytes>; +type SkipMapRangeIter<'a> = map::Range<'a, KeyBytes, BytesBound<'a>, KeyBytes, Bytes>; type SkipMapRangeEntry<'a> = map::Entry<'a, Bytes, Bytes>; -pub type NonEmptyMemTableIterRef<'a> = NonEmptyStream>; -pub type MaybeEmptyMemTableIterRef<'a> = MaybeEmptyStream>; +pub type NonEmptyMemTableIterRef<'a> = NonEmptyStream>; +pub type MaybeEmptyMemTableIterRef<'a> = MaybeEmptyStream>; #[cfg(test)] mod test { + use std::collections::Bound; + + use futures::{stream, Stream, StreamExt}; + use nom::AsBytes; + use crate::entry::Entry; - use crate::iterators::{ - create_merge_iter, create_merge_iter_from_non_empty_iters, eq, MergeIterator, - }; + use crate::iterators::create_merge_iter_from_non_empty_iters; use crate::memtable::MemTable; use crate::persistent::interface::WalHandle; use crate::persistent::wal_handle::WalFile; - use bytes::Bytes; - use futures::{stream, Stream, StreamExt}; - use nom::AsBytes; - use std::collections::Bound; - use std::vec; #[tokio::test] async fn test_task1_memtable_iter() { diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 1664a65..b4d9e47 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -15,7 +15,7 @@ use tracing_futures::Instrument; use crate::bound::BytesBound; use crate::iterators::NonEmptyStream; -use crate::key::KeyBytes; +use crate::key::{KeyBytes, KeySlice}; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::immutable::ImmutableMemTable; use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; @@ -31,7 +31,7 @@ use crate::wal::Wal; /// todo: MemTable 本质是 Map，可以抽象为 trait #[derive(Getters)] pub struct MemTable { - pub(self) map: SkipMap, + pub(self) map: SkipMap, wal: Option>, id: usize, @@ -62,7 +62,7 @@ impl MemTable { Self::new(id, SkipMap::new(), None) } - fn new(id: usize, map: SkipMap, wal: impl Into>>) -> Self { + fn new(id: usize, map: SkipMap, wal: impl Into>>) -> Self { Self { map, wal: wal.into(), @@ -108,25 +108,18 @@ impl MemTable { } /// Get a value by key. + /// todo: remote this method pub fn get(&self, key: &[u8]) -> Option { - self.map.get(key).map(|x| x.value().clone()) + self.get_with_ts(KeySlice::new(key, 0)) } /// Put a key-value pair into the mem-table. /// /// In week 1, day 1, simply put the key-value pair into the skipmap. /// In week 2, day 6, also flush the data to WAL. + /// todo: remote this method pub async fn put(&self, key: Bytes, value: Bytes) -> anyhow::Result<()> { - let size = key.len() + value.len(); - if let Some(wal) = self.wal.as_ref() { - wal.put(key.as_bytes(), value.as_bytes()) - .instrument(tracing::info_span!("wal_put")) - .await? - } - self.map.insert(key, value); - self.approximate_size.fetch_add(size, Ordering::Release); - - Ok(()) + self.put_with_ts(KeyBytes::new(key, 0), value) } pub async fn sync_wal(&self) -> anyhow::Result<()> { @@ -142,14 +135,49 @@ impl MemTable { } /// Get an iterator over a range of keys. + /// todo: remote this method pub async fn scan<'a>( &'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, ) -> anyhow::Result> { + self.scan_with_ts(lower.map(|k| KeySlice::new(k, 0)), upper.map(|k| KeySlice::new(k, 0))) + } +} + +// with transaction +impl MemTable { + pub fn get_with_ts(&self, key: KeySlice) -> Option { + // todo: 因为 rust 的 Borrow trait 不够 general，这里需要复制，如何避免？ + let key = key.map(Bytes::copy_from_slice); + self.map.get(&key).map(|x| x.value().clone()) + } + + pub async fn put_with_ts(&self, key: KeyBytes, value: Bytes) -> anyhow::Result<()> { + let size = key.len() + value.len(); + if let Some(wal) = self.wal.as_ref() { + wal.put(key.as_key_slice(), value.as_bytes()) + .instrument(tracing::info_span!("wal_put")) + .await? + } + self.map.insert(key, value); + self.approximate_size.fetch_add(size, Ordering::Release); + + Ok(()) + } + + pub async fn scan_with_ts<'a>( + &'a self, + lower: Bound>, + upper: Bound>, + ) -> anyhow::Result> { + // todo: 由于 rust 的 Borrow trait 的限制，这里只能 copy + let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); + let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); + let iter = self.map.range(BytesBound { - start: lower, - end: upper, + start: lower.as_ref(), + end: upper.as_ref(), }); let iter = new_memtable_iter(iter); NonEmptyStream::try_new(iter).await diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index 322740a..95665a0 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -2,6 +2,7 @@ use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; +use crate::key::KeySlice; /// Implements a bloom filter pub struct Bloom { diff --git a/src/sst/builder.rs b/src/sst/builder.rs index b6f4ecd..8b6ce69 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -145,7 +145,7 @@ impl SsTableBuilder { pub fn flush(&mut self, memtable: &ImmutableMemTable) { for entry in memtable.iter() { self.add( - KeySlice::from_slice(entry.key().as_bytes()), + entry.key().as_key_slice(), entry.value().as_bytes(), ); } diff --git a/src/sst/iterator/concat.rs b/src/sst/iterator/concat.rs index d234308..77098a9 100644 --- a/src/sst/iterator/concat.rs +++ b/src/sst/iterator/concat.rs @@ -4,7 +4,7 @@ use anyhow::Result; use bytes::Bytes; use futures::{stream, Stream, StreamExt}; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::key::KeySlice; use crate::persistent::SstHandle; use crate::sst::iterator::iter::SsTableIterator; @@ -14,7 +14,7 @@ use crate::sst::SsTable; /// iterators when initializing this iterator to reduce the overhead of seeking. // todo: 这里应该用 type alias impl trait 去除 Box -pub type SstConcatIterator<'a> = Box> + Send + Unpin + 'a>; +pub type SstConcatIterator<'a> = Box> + Send + Unpin + 'a>; pub fn create_sst_concat_and_seek_to_first( sstables: Vec<&SsTable>, @@ -42,8 +42,8 @@ where pub fn scan_sst_concat<'a, File, I>( sstables: I, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + lower: Bound>, + upper: Bound>, ) -> Result> where File: SstHandle + 'a, diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index c5b6e4e..4718446 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -10,7 +10,7 @@ use pin_project::pin_project; use tracing::info; use crate::block::BlockIterator; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::iterators::{iter_fut_iter_to_stream, split_first, MergeIterator, TwoMergeIterator}; use crate::key::{KeyBytes, KeySlice}; use crate::persistent::SstHandle; @@ -19,13 +19,13 @@ use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::{bloom, BlockMeta, SsTable}; // 暂时用 box，目前 rust 不能够方便地在 struct 中存 closure -type InnerIter<'a> = Pin> + Send + 'a>>; +type InnerIter<'a> = Pin> + Send + 'a>>; fn build_iter<'a, File>( table: &'a SsTable, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, -) -> impl Stream> + Send + 'a + lower: Bound>, + upper: Bound>, +) -> impl Stream> + Send + 'a where File: SstHandle, { @@ -134,7 +134,9 @@ pub struct SsTableIterator<'a, File> { impl<'a, File> SsTableIterator<'a, File> { pub fn may_contain(&self, key: &[u8]) -> bool { - bloom::may_contain(self.bloom, key) + true + // todo + // bloom::may_contain(self.bloom, key) } } @@ -145,18 +147,13 @@ where pub fn create_and_seek_to_first(table: &'a SsTable) -> Self { Self::scan(table, Bound::Unbounded, Bound::Unbounded) } - - // todo: 能不能删除 - pub fn create_and_seek_to_key(table: &'a SsTable, key: &'a [u8]) -> Self { - Self::scan(table, Bound::Included(key), Bound::Unbounded) - } } impl<'a, File> SsTableIterator<'a, File> where File: SstHandle, { - pub fn scan(table: &'a SsTable, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> Self { + pub fn scan(table: &'a SsTable, lower: Bound>, upper: Bound>) -> Self { let iter = build_iter(table, lower, upper); let this = Self { table, @@ -169,7 +166,7 @@ where // todo: 感觉没必要 impl Stream，使用 (Bloom, InnerIter) 比较好？ impl<'a, File> Stream for SsTableIterator<'a, File> { - type Item = anyhow::Result; + type Item = anyhow::Result; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.project(); @@ -179,7 +176,7 @@ impl<'a, File> Stream for SsTableIterator<'a, File> { } } -pub type BlockFallibleIter = either::Either>>; +pub type BlockFallibleIter = either::Either>>; pub type MergedSstIterator<'a, File> = TwoMergeIterator< Entry, diff --git a/src/sst/iterator/merged.rs b/src/sst/iterator/merged.rs index 00d54c7..eff3f1b 100644 --- a/src/sst/iterator/merged.rs +++ b/src/sst/iterator/merged.rs @@ -1,11 +1,11 @@ -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::iterators::{MergeIterator, TwoMergeIterator}; use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::iterator::iter::SsTableIterator; // todo: 用 MergeIterator vs MergeIteratorInner pub type MergedSstIterator<'a, File> = TwoMergeIterator< - Entry, - MergeIterator>, - MergeIterator>, + InnerEntry, + MergeIterator>, + MergeIterator>, >; diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index 7bb1a31..93f874e 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -17,7 +17,7 @@ use ordered_float::NotNan; use tokio::sync::RwLock; use tracing::error; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::iterators::merge::MergeIteratorInner; use crate::iterators::{ create_merge_iter, create_merge_iter_from_non_empty_iters, create_two_merge_iter, @@ -128,8 +128,8 @@ where pub async fn scan_sst<'a>( &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + lower: Bound>, + upper: Bound>, ) -> anyhow::Result> { let l0 = self.scan_l0(lower, upper).await; let levels = self.scan_levels(lower, upper).await; @@ -139,28 +139,18 @@ where pub async fn scan_l0<'a>( &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, - ) -> MergeIterator> { + lower: Bound>, + upper: Bound>, + ) -> MergeIterator> { let iters = self.build_l0_iter(lower, upper); let iters = stream::iter(iters); create_merge_iter(iters).await } - async fn scan_l02<'a>( - &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, - ) -> MergeIteratorInner> { - let iters = self.build_l0_iter(lower, upper); - let iters = stream::iter(iters); - MergeIteratorInner::create(iters).await - } - fn build_l0_iter<'a>( &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + lower: Bound>, + upper: Bound>, ) -> impl Iterator> + 'a { let iters = self .l0_sstables @@ -178,8 +168,8 @@ where async fn scan_levels<'a>( &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + lower: Bound>, + upper: Bound>, ) -> MergeIterator> { let iters = self.levels.iter().filter_map(move |ids| { let tables = ids.iter().map(|id| self.sstables.get(id).unwrap().as_ref()); @@ -251,13 +241,14 @@ where fn filter_sst_by_bloom( table: &SsTable, - lower: Bound<&[u8]>, - upper: Bound<&[u8]>, + lower: Bound, + upper: Bound, ) -> bool { use Bound::Included; if let (Included(lower), Included(upper)) = (lower, upper) { if lower == upper { - return bloom::may_contain(table.bloom.as_ref(), lower); + return true + // return bloom::may_contain(table.bloom.as_ref(), lower); } } true diff --git a/src/wal.rs b/src/wal.rs index 49cd7c1..c67a6f5 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -10,6 +10,7 @@ use tokio::fs::{File, OpenOptions}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Mutex; use tracing_futures::Instrument; +use crate::key::{KeyBytes, KeySlice}; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; @@ -33,7 +34,7 @@ impl Wal { pub async fn recover>( id: usize, persistent: &P, - ) -> anyhow::Result<(Self, SkipMap)> { + ) -> anyhow::Result<(Self, SkipMap)> { let mut file = persistent.open_wal_handle(id).await?; let data = { let mut data = Vec::new(); @@ -45,8 +46,12 @@ impl Wal { while data.has_remaining() { let key_len = data.get_u32(); let key = data.copy_to_bytes(key_len as usize); + let timestamp = data.get_u64(); + let key = KeyBytes::new(key, timestamp); + let value_len = data.get_u32(); let value = data.copy_to_bytes(value_len as usize); + map.insert(key, value); } let wal = Wal { @@ -55,16 +60,19 @@ impl Wal { Ok((wal, map)) } - pub async fn put<'a>(&'a self, key: &'a [u8], value: &'a [u8]) -> anyhow::Result<()> { + pub async fn put<'a>(&'a self, key: KeySlice<'a>, value: &'a [u8]) -> anyhow::Result<()> { let mut guard = self.file.lock().await; guard .write_u32(key.len() as u32) .instrument(tracing::info_span!("wal_put_write_key_len")) .await?; guard - .write_all(key) + .write_all(key.raw_ref()) .instrument(tracing::info_span!("wal_put_write_all_key")) .await?; + guard.write_u64(key.timestamp()) + .await?; + guard .write_u32(value.len() as u32) .instrument(tracing::info_span!("wal_put_write_value_len")) @@ -100,6 +108,7 @@ async fn get_file(path: impl AsRef) -> anyhow::Result { mod tests { use bytes::Bytes; use tempfile::tempdir; + use crate::key::KeyBytes; use crate::persistent::LocalFs; use crate::wal::Wal; @@ -122,11 +131,11 @@ mod tests { { let (wal, map) = Wal::recover(id, &persistent).await.unwrap(); - assert_eq!(map.get(&Bytes::from("111")).unwrap().value(), "a"); - assert_eq!(map.get(&Bytes::from("222")).unwrap().value(), "bb"); - assert_eq!(map.get(&Bytes::from("333")).unwrap().value(), "ccc"); - assert_eq!(map.get(&Bytes::from("4")).unwrap().value(), ""); - assert!(map.get(&Bytes::from("555")).is_none()); + assert_eq!(map.get(&KeyBytes::new_no_ts(b"111")).unwrap().value(), "a"); + assert_eq!(map.get(&KeyBytes::new_no_ts(b"222")).unwrap().value(), "bb"); + assert_eq!(map.get(&KeyBytes::new_no_ts(b"333")).unwrap().value(), "ccc"); + assert_eq!(map.get(&KeyBytes::new_no_ts(b"4")).unwrap().value(), ""); + assert!(map.get(&KeyBytes::new_no_ts(b"555")).is_none()); } } } From a680bdd205a617502ba5475112fab0b9943485d2 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 3 Jun 2024 13:28:44 +0800 Subject: [PATCH 12/69] wip --- src/iterators/lsm.rs | 26 ++++++++++++++++++++------ src/memtable/immutable.rs | 27 +++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index e69b348..d79f18e 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -7,12 +7,13 @@ use derive_new::new; use futures::{stream, StreamExt}; use tracing::error; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{ create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, NoDeletedIterator, TwoMergeIterator, }; +use crate::key::KeySlice; use crate::memtable::MemTableIterator; use crate::persistent::Persistent; use crate::sst::iterator::MergedSstIterator; @@ -26,11 +27,23 @@ type LsmIteratorInner<'a, File> = TwoMergeIterator< MergedSstIterator<'a, File>, >; -#[derive(new)] pub struct LockedLsmIter<'a, P: Persistent> { state: arc_swap::Guard>>, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + lower: Bound>, + upper: Bound>, +} + +impl<'a, P: Persistent> LockedLsmIter<'a, P> { + pub fn new( + state: arc_swap::Guard>>, + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, + timestamp: u64, + ) -> Self { + let lower = lower.map(|key| KeySlice::new(key, timestamp)); + let upper = upper.map(|key| KeySlice::new(key, timestamp)); + Self { state, lower, upper } + } } impl<'a, P> LockedLsmIter<'a, P> @@ -41,18 +54,19 @@ where let a = self.build_memtable_iter().await; let b = self.build_sst_iter().await?; let merge = create_two_merge_iter(a, b).await?; + // todo: dedup timestamps let iter = new_no_deleted_iter(merge); Ok(iter) } - pub async fn build_memtable_iter(&self) -> MergeIterator { + pub async fn build_memtable_iter(&self) -> MergeIterator { let memtable = self.state.memtable().deref().as_immutable_ref(); let imm_memtables = self.state.imm_memtables().as_slice(); let imm_memtables = imm_memtables.iter().map(Arc::as_ref); let tables = iter::once(memtable).chain(imm_memtables); let iters = stream::iter(tables).filter_map(move |table| async { table - .scan(self.lower, self.upper) + .scan_with_ts(self.lower, self.upper) .await .inspect_err(|e| error!(error = ?e)) .ok() diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index bbdbec3..29e6327 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -1,15 +1,19 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; +use std::sync::atomic::Ordering; use bytemuck::TransparentWrapper; use bytes::Bytes; use crossbeam_skiplist::map; use deref_ext::DerefExt; use derive_new::new; +use nom::AsBytes; use ref_cast::RefCast; -use crate::key::KeyBytes; +use crate::bound::BytesBound; +use crate::iterators::NonEmptyStream; +use crate::key::{KeyBytes, KeySlice}; -use crate::memtable::iterator::MaybeEmptyMemTableIterRef; +use crate::memtable::iterator::{MaybeEmptyMemTableIterRef, new_memtable_iter}; use crate::memtable::mutable::MemTable; use crate::persistent::interface::WalHandle; @@ -36,6 +40,7 @@ impl ImmutableMemTable { } } +// todo: remove it impl ImmutableMemTable { pub async fn scan<'a>( &'a self, @@ -54,6 +59,24 @@ impl ImmutableMemTable { } } +impl ImmutableMemTable { + pub fn get_with_ts(&self, key: KeySlice) -> Option { + self.0.get_with_ts(key) + } + + pub async fn put_with_ts(&self, key: KeyBytes, value: Bytes) -> anyhow::Result<()> { + self.0.put_with_ts(key, value) + } + + pub async fn scan_with_ts<'a>( + &'a self, + lower: Bound>, + upper: Bound>, + ) -> anyhow::Result> { + self.0.scan_with_ts(lower, upper) + } +} + impl From> for ImmutableMemTable { fn from(table: MemTable) -> Self { Self::new(table) From c87ccf004acedb2bc87fe0256fc43f5c626c0008 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 9 Jun 2024 21:01:31 +0800 Subject: [PATCH 13/69] wip --- Cargo.lock | 1 + Cargo.toml | 1 + src/entry.rs | 6 ++--- src/iterators/lsm.rs | 48 +++++++++++++++++++-------------------- src/key.rs | 11 ++++++++- src/memtable/immutable.rs | 8 +++---- src/memtable/mutable.rs | 5 +++- src/mvcc/iterator.rs | 48 +++++++++++++++++++++++++++++++++++++++ src/mvcc/mod.rs | 1 + src/sst/bloom.rs | 2 +- src/sst/builder.rs | 5 +--- src/sst/iterator/iter.rs | 6 ++++- src/sst/sstables.rs | 2 +- src/wal.rs | 12 ++++++---- 14 files changed, 109 insertions(+), 47 deletions(-) create mode 100644 src/mvcc/iterator.rs diff --git a/Cargo.lock b/Cargo.lock index f52842e..a24d2d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,6 +200,7 @@ dependencies = [ "maplit", "moka", "nom", + "num-traits", "ordered-float", "ouroboros", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 673f7c4..377d7d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ ordered-float = "4.2.2" getset = "0.1.2" derive_more = "1.0.0" tracing-futures = { version = "0.2.5", features = ["futures-03"] } +num-traits = "0.2.19" [dev-dependencies] tempfile = "3" diff --git a/src/entry.rs b/src/entry.rs index a71b811..ba707c9 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; -use bytes::Bytes; use crate::key::KeyBytes; +use bytes::Bytes; pub type Entry = Keyed; pub type InnerEntry = Keyed; @@ -38,8 +38,6 @@ impl Keyed { let Self { key, value } = self; (key, value) } - - } #[cfg(test)] @@ -50,4 +48,4 @@ impl Keyed { value: Bytes::copy_from_slice(value), } } -} \ No newline at end of file +} diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index d79f18e..ca8c9c9 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -4,7 +4,7 @@ use std::ops::Deref; use std::sync::Arc; use derive_new::new; -use futures::{stream, StreamExt}; +use futures::{stream, Stream, StreamExt}; use tracing::error; use crate::entry::{Entry, InnerEntry}; @@ -13,13 +13,15 @@ use crate::iterators::{ create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, NoDeletedIterator, TwoMergeIterator, }; -use crate::key::KeySlice; +use crate::key::{Key, KeySlice}; use crate::memtable::MemTableIterator; +use crate::mvcc::iterator::{build_time_dedup_iter, transform_bound}; use crate::persistent::Persistent; use crate::sst::iterator::MergedSstIterator; use crate::state::LsmStorageStateInner; -pub type LsmIterator<'a, File> = NoDeletedIterator, anyhow::Error>; +// pub type LsmIterator<'a, File> = NoDeletedIterator, anyhow::Error>; +pub type LsmIterator<'a> = Box> + Unpin + 'a>; type LsmIteratorInner<'a, File> = TwoMergeIterator< Entry, @@ -27,46 +29,41 @@ type LsmIteratorInner<'a, File> = TwoMergeIterator< MergedSstIterator<'a, File>, >; +#[derive(new)] pub struct LockedLsmIter<'a, P: Persistent> { state: arc_swap::Guard>>, - lower: Bound>, - upper: Bound>, -} - -impl<'a, P: Persistent> LockedLsmIter<'a, P> { - pub fn new( - state: arc_swap::Guard>>, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, - timestamp: u64, - ) -> Self { - let lower = lower.map(|key| KeySlice::new(key, timestamp)); - let upper = upper.map(|key| KeySlice::new(key, timestamp)); - Self { state, lower, upper } - } + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, + timestamp: u64, } impl<'a, P> LockedLsmIter<'a, P> where P: Persistent, { - pub async fn iter(&'a self) -> anyhow::Result> { + pub async fn iter(&'a self) -> anyhow::Result> { let a = self.build_memtable_iter().await; let b = self.build_sst_iter().await?; let merge = create_two_merge_iter(a, b).await?; + let time_dedup = build_time_dedup_iter(merge, self.timestamp); // todo: dedup timestamps - let iter = new_no_deleted_iter(merge); + let iter = new_no_deleted_iter(time_dedup); + let iter = Box::new(iter) as _; Ok(iter) } pub async fn build_memtable_iter(&self) -> MergeIterator { + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); + let memtable = self.state.memtable().deref().as_immutable_ref(); let imm_memtables = self.state.imm_memtables().as_slice(); let imm_memtables = imm_memtables.iter().map(Arc::as_ref); let tables = iter::once(memtable).chain(imm_memtables); let iters = stream::iter(tables).filter_map(move |table| async { table - .scan_with_ts(self.lower, self.upper) + .scan_with_ts(lower, upper) .await .inspect_err(|e| error!(error = ?e)) .ok() @@ -76,9 +73,10 @@ where } pub async fn build_sst_iter(&self) -> anyhow::Result> { - self.state - .sstables_state() - .scan_sst(self.lower, self.upper) - .await + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); + + self.state.sstables_state().scan_sst(lower, upper).await } } diff --git a/src/key.rs b/src/key.rs index d4388c4..b876934 100644 --- a/src/key.rs +++ b/src/key.rs @@ -1,7 +1,7 @@ -use std::borrow::Borrow; use bytes::Bytes; use derive_new::new; use nom::AsBytes; +use std::borrow::Borrow; use std::cmp::Ordering; use std::fmt::Debug; use std::ops::Bound; @@ -30,6 +30,15 @@ impl Key { } } +impl From<(T, u64)> for Key { + fn from(pair: (T, u64)) -> Self { + Self { + key: pair.0, + timestamp: pair.1, + } + } +} + #[cfg(test)] impl Key { pub fn new_no_ts(key: &[u8]) -> Self { diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 29e6327..59f657e 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -2,6 +2,9 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; use std::sync::atomic::Ordering; +use crate::bound::BytesBound; +use crate::iterators::NonEmptyStream; +use crate::key::{KeyBytes, KeySlice}; use bytemuck::TransparentWrapper; use bytes::Bytes; use crossbeam_skiplist::map; @@ -9,11 +12,8 @@ use deref_ext::DerefExt; use derive_new::new; use nom::AsBytes; use ref_cast::RefCast; -use crate::bound::BytesBound; -use crate::iterators::NonEmptyStream; -use crate::key::{KeyBytes, KeySlice}; -use crate::memtable::iterator::{MaybeEmptyMemTableIterRef, new_memtable_iter}; +use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; use crate::memtable::mutable::MemTable; use crate::persistent::interface::WalHandle; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index b4d9e47..72f13e3 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -141,7 +141,10 @@ impl MemTable { lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, ) -> anyhow::Result> { - self.scan_with_ts(lower.map(|k| KeySlice::new(k, 0)), upper.map(|k| KeySlice::new(k, 0))) + self.scan_with_ts( + lower.map(|k| KeySlice::new(k, 0)), + upper.map(|k| KeySlice::new(k, 0)), + ) } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs new file mode 100644 index 0000000..b73e4a2 --- /dev/null +++ b/src/mvcc/iterator.rs @@ -0,0 +1,48 @@ +use std::ops::Bound; + +use async_iter_ext::StreamTools; +use futures::{Stream, TryStreamExt}; +use num_traits::Bounded; + +pub fn build_time_dedup_iter( + s: S, + timestamp_upper: T, +) -> impl Stream> + Unpin +where + S: Stream> + Unpin, + A: PartialEq, + T: PartialOrd + Copy, +{ + let s = s + .try_filter(|(timestamp, _)| async { timestamp.le(×tamp_upper) }) + .dedup_by(|left, right| match (left, right) { + (Ok((left, _)), Ok((right, _))) => left.eq(right), + _ => false, + }); + s +} + +pub fn transform_bound( + lower: Bound, + upper: Bound, + timestamp: T, +) -> (Bound<(A, T)>, Bound<(A, T)>) +where + T: Bounded + Clone, +{ + use Bound::{Excluded, Included, Unbounded}; + + let lower = match lower { + Included(a) => Included((a, timestamp.clone())), + Excluded(a) => Excluded((a, T::min_value())), + Unbounded => Unbounded, + }; + + let upper = match upper { + Included(a) => Included((a, T::min_value())), + Excluded(a) => Excluded((a, T::max_value())), + Unbounded => Unbounded, + }; + + (lower, upper) +} diff --git a/src/mvcc/mod.rs b/src/mvcc/mod.rs index 8737ac0..ffa40cd 100644 --- a/src/mvcc/mod.rs +++ b/src/mvcc/mod.rs @@ -1,3 +1,4 @@ pub mod core; +pub mod iterator; pub mod transaction; mod watermark; diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index 95665a0..3f814a8 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -1,8 +1,8 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. +use crate::key::KeySlice; use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; -use crate::key::KeySlice; /// Implements a bloom filter pub struct Bloom { diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 8b6ce69..e34cd0e 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -144,10 +144,7 @@ impl SsTableBuilder { pub fn flush(&mut self, memtable: &ImmutableMemTable) { for entry in memtable.iter() { - self.add( - entry.key().as_key_slice(), - entry.value().as_bytes(), - ); + self.add(entry.key().as_key_slice(), entry.value().as_bytes()); } } } diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 4718446..4d3c3b6 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -153,7 +153,11 @@ impl<'a, File> SsTableIterator<'a, File> where File: SstHandle, { - pub fn scan(table: &'a SsTable, lower: Bound>, upper: Bound>) -> Self { + pub fn scan( + table: &'a SsTable, + lower: Bound>, + upper: Bound>, + ) -> Self { let iter = build_iter(table, lower, upper); let this = Self { table, diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index 93f874e..dc797d4 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -247,7 +247,7 @@ fn filter_sst_by_bloom( use Bound::Included; if let (Included(lower), Included(upper)) = (lower, upper) { if lower == upper { - return true + return true; // return bloom::may_contain(table.bloom.as_ref(), lower); } } diff --git a/src/wal.rs b/src/wal.rs index c67a6f5..3621923 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -4,13 +4,13 @@ use std::ops::DerefMut; use std::path::Path; use std::sync::Arc; +use crate::key::{KeyBytes, KeySlice}; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; use tokio::fs::{File, OpenOptions}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Mutex; use tracing_futures::Instrument; -use crate::key::{KeyBytes, KeySlice}; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; @@ -70,8 +70,7 @@ impl Wal { .write_all(key.raw_ref()) .instrument(tracing::info_span!("wal_put_write_all_key")) .await?; - guard.write_u64(key.timestamp()) - .await?; + guard.write_u64(key.timestamp()).await?; guard .write_u32(value.len() as u32) @@ -106,9 +105,9 @@ async fn get_file(path: impl AsRef) -> anyhow::Result { #[cfg(test)] mod tests { + use crate::key::KeyBytes; use bytes::Bytes; use tempfile::tempdir; - use crate::key::KeyBytes; use crate::persistent::LocalFs; use crate::wal::Wal; @@ -133,7 +132,10 @@ mod tests { assert_eq!(map.get(&KeyBytes::new_no_ts(b"111")).unwrap().value(), "a"); assert_eq!(map.get(&KeyBytes::new_no_ts(b"222")).unwrap().value(), "bb"); - assert_eq!(map.get(&KeyBytes::new_no_ts(b"333")).unwrap().value(), "ccc"); + assert_eq!( + map.get(&KeyBytes::new_no_ts(b"333")).unwrap().value(), + "ccc" + ); assert_eq!(map.get(&KeyBytes::new_no_ts(b"4")).unwrap().value(), ""); assert!(map.get(&KeyBytes::new_no_ts(b"555")).is_none()); } From cdad59d32fcec65a20578fc9e227dc090d6a5112 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 10 Jun 2024 16:04:19 +0800 Subject: [PATCH 14/69] wip --- src/block/blocks.rs | 4 +- src/block/iterator.rs | 4 +- src/entry.rs | 11 ++++- src/iterators/lsm.rs | 91 ++++++++++++++++++++++++--------------- src/key.rs | 4 ++ src/memtable/immutable.rs | 4 +- src/memtable/mutable.rs | 3 +- src/mvcc/iterator.rs | 11 ++--- src/sst/block_meta.rs | 14 +++++- src/sst/iterator/iter.rs | 58 +++++++++++-------------- src/sst/sstables.rs | 2 +- src/state/states.rs | 2 +- 12 files changed, 125 insertions(+), 83 deletions(-) diff --git a/src/block/blocks.rs b/src/block/blocks.rs index 3945b46..af06a8d 100644 --- a/src/block/blocks.rs +++ b/src/block/blocks.rs @@ -1,7 +1,7 @@ use crate::key::KeyBytes; use bytes::Bytes; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; /// A block is the smallest unit of read and caching in LSM tree. It is a collection of sorted key-value pairs. #[derive(Debug)] @@ -100,7 +100,7 @@ impl Block { (data, key) } - pub fn get_entry(&self, index: usize) -> Entry { + pub fn get_entry(&self, index: usize) -> InnerEntry { let (key, value) = self.get_entry_ref(index); let key = Bytes::copy_from_slice(key); let value = Bytes::copy_from_slice(value); diff --git a/src/block/iterator.rs b/src/block/iterator.rs index a9fa724..50553e2 100644 --- a/src/block/iterator.rs +++ b/src/block/iterator.rs @@ -1,5 +1,5 @@ use crate::block::blocks::Block; -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::key::{Key, KeySlice}; use std::sync::Arc; use tracing::info; @@ -47,7 +47,7 @@ impl BlockIterator { } impl Iterator for BlockIterator { - type Item = anyhow::Result; + type Item = anyhow::Result; fn next(&mut self) -> Option { if self.idx >= self.block.len() { diff --git a/src/entry.rs b/src/entry.rs index ba707c9..eafdf7a 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering; -use crate::key::KeyBytes; +use crate::key::{Key, KeyBytes}; use bytes::Bytes; pub type Entry = Keyed; @@ -32,7 +32,6 @@ impl Ord for Keyed { } } -#[cfg(test)] impl Keyed { pub fn into_tuple(self) -> (K, V) { let Self { key, value } = self; @@ -40,6 +39,14 @@ impl Keyed { } } +impl Keyed, V> { + pub fn into_timed_tuple(self) -> (Keyed, u64) { + let Self { key, value } = self; + let (key, timestamp) = key.into_tuple(); + (Keyed { key, value }, timestamp) + } +} + #[cfg(test)] impl Keyed { pub fn from_slice(key: &[u8], value: &[u8]) -> Self { diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index ca8c9c9..3d3c487 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -1,4 +1,6 @@ +use bytes::Bytes; use std::collections::Bound; +use std::future::Future; use std::iter; use std::ops::Deref; use std::sync::Arc; @@ -7,7 +9,7 @@ use derive_new::new; use futures::{stream, Stream, StreamExt}; use tracing::error; -use crate::entry::{Entry, InnerEntry}; +use crate::entry::{Entry, InnerEntry, Keyed}; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{ create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, @@ -20,8 +22,7 @@ use crate::persistent::Persistent; use crate::sst::iterator::MergedSstIterator; use crate::state::LsmStorageStateInner; -// pub type LsmIterator<'a, File> = NoDeletedIterator, anyhow::Error>; -pub type LsmIterator<'a> = Box> + Unpin + 'a>; +pub type LsmIterator<'a> = Box> + Unpin + Send + 'a>; type LsmIteratorInner<'a, File> = TwoMergeIterator< Entry, @@ -37,46 +38,68 @@ pub struct LockedLsmIter<'a, P: Persistent> { timestamp: u64, } +fn assert_raw_stream(s: &impl Stream>) {} + +fn assert_tuple_stream(s: &impl Stream, u64)>>) {} + +fn assert_result_stream(s: &impl Stream>>) {} + impl<'a, P> LockedLsmIter<'a, P> where P: Persistent, { - pub async fn iter(&'a self) -> anyhow::Result> { - let a = self.build_memtable_iter().await; - let b = self.build_sst_iter().await?; - let merge = create_two_merge_iter(a, b).await?; - let time_dedup = build_time_dedup_iter(merge, self.timestamp); - // todo: dedup timestamps - let iter = new_no_deleted_iter(time_dedup); - let iter = Box::new(iter) as _; - Ok(iter) + pub fn iter(&'a self) -> impl Future>> + Send { + async { + let a = self.build_memtable_iter().await; + assert_raw_stream(&a); + let b = self.build_sst_iter().await?; + assert_raw_stream(&b); + let merge = create_two_merge_iter(a, b).await?; + assert_raw_stream(&merge); + let merge = merge.map(|entry| entry.map(Keyed::into_timed_tuple)); + assert_tuple_stream(&merge); + let time_dedup = build_time_dedup_iter(merge, self.timestamp); + assert_result_stream(&time_dedup); + // todo: dedup timestamps + let iter = new_no_deleted_iter(time_dedup); + let iter = Box::new(iter) as _; + Ok(iter) + } } - pub async fn build_memtable_iter(&self) -> MergeIterator { - let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); - let lower = lower.map(Key::from); - let upper = upper.map(Key::from); + pub fn build_memtable_iter( + &self, + ) -> impl Future> + Send { + async { + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); - let memtable = self.state.memtable().deref().as_immutable_ref(); - let imm_memtables = self.state.imm_memtables().as_slice(); - let imm_memtables = imm_memtables.iter().map(Arc::as_ref); - let tables = iter::once(memtable).chain(imm_memtables); - let iters = stream::iter(tables).filter_map(move |table| async { - table - .scan_with_ts(lower, upper) - .await - .inspect_err(|e| error!(error = ?e)) - .ok() - .flatten() - }); - create_merge_iter_from_non_empty_iters(iters).await + let memtable = self.state.memtable().deref().as_immutable_ref(); + let imm_memtables = self.state.imm_memtables().as_slice(); + let imm_memtables = imm_memtables.iter().map(Arc::as_ref); + let tables = iter::once(memtable).chain(imm_memtables); + let iters = stream::iter(tables).filter_map(move |table| async { + table + .scan_with_ts(lower, upper) + .await + .inspect_err(|e| error!(error = ?e)) + .ok() + .flatten() + }); + create_merge_iter_from_non_empty_iters(iters).await + } } - pub async fn build_sst_iter(&self) -> anyhow::Result> { - let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); - let lower = lower.map(Key::from); - let upper = upper.map(Key::from); + pub fn build_sst_iter( + &self, + ) -> impl Future>> + Send { + async { + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); - self.state.sstables_state().scan_sst(lower, upper).await + self.state.sstables_state().scan_sst(lower, upper).await + } } } diff --git a/src/key.rs b/src/key.rs index b876934..1802135 100644 --- a/src/key.rs +++ b/src/key.rs @@ -28,6 +28,10 @@ impl Key { pub fn timestamp(&self) -> u64 { self.timestamp } + + pub fn into_tuple(self) -> (T, u64) { + (self.key, self.timestamp) + } } impl From<(T, u64)> for Key { diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 59f657e..209212b 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -65,7 +65,7 @@ impl ImmutableMemTable { } pub async fn put_with_ts(&self, key: KeyBytes, value: Bytes) -> anyhow::Result<()> { - self.0.put_with_ts(key, value) + self.0.put_with_ts(key, value).await } pub async fn scan_with_ts<'a>( @@ -73,7 +73,7 @@ impl ImmutableMemTable { lower: Bound>, upper: Bound>, ) -> anyhow::Result> { - self.0.scan_with_ts(lower, upper) + self.0.scan_with_ts(lower, upper).await } } diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 72f13e3..0da0fee 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -119,7 +119,7 @@ impl MemTable { /// In week 2, day 6, also flush the data to WAL. /// todo: remote this method pub async fn put(&self, key: Bytes, value: Bytes) -> anyhow::Result<()> { - self.put_with_ts(KeyBytes::new(key, 0), value) + self.put_with_ts(KeyBytes::new(key, 0), value).await } pub async fn sync_wal(&self) -> anyhow::Result<()> { @@ -145,6 +145,7 @@ impl MemTable { lower.map(|k| KeySlice::new(k, 0)), upper.map(|k| KeySlice::new(k, 0)), ) + .await } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index b73e4a2..b93ea78 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,24 +1,25 @@ use std::ops::Bound; use async_iter_ext::StreamTools; -use futures::{Stream, TryStreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; pub fn build_time_dedup_iter( s: S, timestamp_upper: T, -) -> impl Stream> + Unpin +) -> impl Stream> + Unpin + Send where - S: Stream> + Unpin, + S: Stream> + Unpin + Send, A: PartialEq, T: PartialOrd + Copy, { let s = s - .try_filter(|(timestamp, _)| async { timestamp.le(×tamp_upper) }) + .try_filter(|(_, timestamp)| async { timestamp.le(×tamp_upper) }) .dedup_by(|left, right| match (left, right) { (Ok((left, _)), Ok((right, _))) => left.eq(right), _ => false, - }); + }) + .map(|entry| entry.map(|pair| pair.0)); s } diff --git a/src/sst/block_meta.rs b/src/sst/block_meta.rs index 360457f..16c4a9a 100644 --- a/src/sst/block_meta.rs +++ b/src/sst/block_meta.rs @@ -1,4 +1,4 @@ -use crate::key::KeyBytes; +use crate::key::{KeyBytes, KeySlice}; use bytes::Buf; use derive_getters::Getters; @@ -7,9 +7,13 @@ use derive_getters::Getters; pub struct BlockMeta { /// Offset of this data block. pub offset: usize, + /// The first key of the data block. + #[getter(skip)] pub first_key: KeyBytes, + /// The last key of the data block. + #[getter(skip)] pub last_key: KeyBytes, } @@ -47,4 +51,12 @@ impl BlockMeta { } result } + + pub fn first_key(&self) -> KeySlice { + KeySlice::new(self.first_key.raw_ref(), self.first_key.timestamp()) + } + + pub fn last_key(&self) -> KeySlice { + KeySlice::new(self.last_key.raw_ref(), self.last_key.timestamp()) + } } diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 4d3c3b6..0a46622 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -29,21 +29,19 @@ fn build_iter<'a, File>( where File: SstHandle, { - let iter = match lower { - Bound::Included(key) => future::Either::Left(future::Either::Left(build_bounded_iter( - table, - KeySlice::from_slice(key), - upper, - |meta: &BlockMeta, key| meta.last_key.raw_ref() < key, - ))), - Bound::Excluded(key) => future::Either::Left(future::Either::Right(build_bounded_iter( - table, - KeySlice::from_slice(key), - upper, - |meta, key| meta.last_key.raw_ref() <= key, - ))), - Bound::Unbounded => future::Either::Right(build_unbounded_iter(table)), - }; + let iter = + match lower { + Bound::Included(key) => future::Either::Left(future::Either::Left(build_bounded_iter( + table, + key, + upper, + |meta: &BlockMeta, key| meta.last_key() < key, + ))), + Bound::Excluded(key) => future::Either::Left(future::Either::Right( + build_bounded_iter(table, key, upper, |meta, key| meta.last_key() <= key), + )), + Bound::Unbounded => future::Either::Right(build_unbounded_iter(table)), + }; match upper { Bound::Included(upper) => future::Either::Left(future::Either::Left(transform_stop_iter( iter, @@ -60,37 +58,33 @@ where } fn transform_stop_iter<'a>( - iter: impl Stream> + 'a, - upper: &'a [u8], - f: for<'b> fn(&'b [u8], &'b [u8]) -> bool, -) -> impl Stream> + 'a { - iter.take_while(move |entry| { - let condition = entry + iter: impl Stream> + 'a, + upper: KeySlice<'a>, + f: for<'b> fn(KeySlice<'b>, KeySlice<'b>) -> bool, +) -> impl Stream> + 'a { + iter.take_while(move |entry| async { + entry .as_ref() - .map(|entry| f(&entry.key, upper)) + .map(|entry| f(entry.key.as_key_slice(), upper)) .unwrap_or(true); - ready(condition) }) } fn build_bounded_iter<'a, File>( table: &'a SsTable, low: KeySlice<'a>, - upper: Bound<&'a [u8]>, - partition: impl for<'c> Fn(&'c BlockMeta, &'c [u8]) -> bool, -) -> impl Stream> + 'a + upper: Bound>, + partition: impl for<'c> Fn(&'c BlockMeta, KeySlice<'c>) -> bool, +) -> impl Stream> + 'a where File: SstHandle, { let index = table .block_meta .as_slice() - .partition_point(|meta| partition(meta, low.raw_ref())); + .partition_point(|meta| partition(meta, low)); - let metas = table.block_meta[index..] - .iter() - .map(BlockMeta::first_key) - .map(KeyBytes::raw_ref); + let metas = table.block_meta[index..].iter().map(BlockMeta::first_key); let metas = (index..).zip(metas); let Some(((head_index, _), tail)) = split_first(metas) else { @@ -116,7 +110,7 @@ where fn build_unbounded_iter( table: &SsTable, -) -> impl Stream> + '_ +) -> impl Stream> + '_ where File: SstHandle, { diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index dc797d4..b2daaa9 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -170,7 +170,7 @@ where &'a self, lower: Bound>, upper: Bound>, - ) -> MergeIterator> { + ) -> MergeIterator> { let iters = self.levels.iter().filter_map(move |ids| { let tables = ids.iter().map(|id| self.sstables.get(id).unwrap().as_ref()); scan_sst_concat(tables, lower, upper) diff --git a/src/state/states.rs b/src/state/states.rs index b1cfdef..bc673d6 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -142,7 +142,7 @@ where fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { let snapshot = self.inner.load(); - LockedLsmIter::new(snapshot, lower, upper) + LockedLsmIter::new(snapshot, lower, upper, 0) // todo } } From cc05cab551ea637413a3b091e1e15c2052194f44 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 10 Jun 2024 19:47:21 +0800 Subject: [PATCH 15/69] fix: cargo check --- src/block/blocks.rs | 3 ++- src/bound.rs | 12 ++++++------ src/iterators/lsm.rs | 19 ++++++++++++------- src/memtable/immutable.rs | 4 ++-- src/memtable/iterator.rs | 6 ++---- src/memtable/mutable.rs | 14 ++++++-------- src/mvcc/iterator.rs | 11 ++++++++--- src/sst/compact/common.rs | 6 +++--- src/sst/iterator/iter.rs | 5 +++-- 9 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/block/blocks.rs b/src/block/blocks.rs index af06a8d..4db502b 100644 --- a/src/block/blocks.rs +++ b/src/block/blocks.rs @@ -104,7 +104,8 @@ impl Block { let (key, value) = self.get_entry_ref(index); let key = Bytes::copy_from_slice(key); let value = Bytes::copy_from_slice(value); - Entry { key, value } + // Entry { key, value } + todo!() } pub fn first_key(&self) -> KeyBytes { diff --git a/src/bound.rs b/src/bound.rs index 3936301..405a0b2 100644 --- a/src/bound.rs +++ b/src/bound.rs @@ -3,17 +3,17 @@ use std::ops::RangeBounds; use crate::key::KeyBytes; -pub struct BytesBound<'a> { - pub start: Bound<&'a KeyBytes>, - pub end: Bound<&'a KeyBytes>, +pub struct BytesBound { + pub start: Bound, + pub end: Bound, } -impl<'a> RangeBounds for BytesBound<'a> { +impl RangeBounds for BytesBound { fn start_bound(&self) -> Bound<&KeyBytes> { - self.start + self.start.as_ref() } fn end_bound(&self) -> Bound<&KeyBytes> { - self.end + self.end.as_ref() } } diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 3d3c487..cb87998 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -79,13 +79,18 @@ where let imm_memtables = self.state.imm_memtables().as_slice(); let imm_memtables = imm_memtables.iter().map(Arc::as_ref); let tables = iter::once(memtable).chain(imm_memtables); - let iters = stream::iter(tables).filter_map(move |table| async { - table - .scan_with_ts(lower, upper) - .await - .inspect_err(|e| error!(error = ?e)) - .ok() - .flatten() + let iters = stream::iter(tables).filter_map(move |table| { + let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); + let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); + + async { + table + .scan_with_ts(lower, upper) + .await + .inspect_err(|e| error!(error = ?e)) + .ok() + .flatten() + } }); create_merge_iter_from_non_empty_iters(iters).await } diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 209212b..530ab09 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -70,8 +70,8 @@ impl ImmutableMemTable { pub async fn scan_with_ts<'a>( &'a self, - lower: Bound>, - upper: Bound>, + lower: Bound, + upper: Bound, ) -> anyhow::Result> { self.0.scan_with_ts(lower, upper).await } diff --git a/src/memtable/iterator.rs b/src/memtable/iterator.rs index 3c88e7b..a2daa5e 100644 --- a/src/memtable/iterator.rs +++ b/src/memtable/iterator.rs @@ -11,7 +11,7 @@ use crate::key::KeyBytes; pub type MemTableIterator<'a> = stream::Iter>>; type ClonedSkipMapRangeIter<'a> = - iter::Map, for<'b> fn(SkipMapRangeEntry<'b>) -> InnerEntry>; + iter::Map, for<'b> fn(map::Entry<'b, KeyBytes, Bytes>) -> InnerEntry>; pub fn new_memtable_iter(iter: SkipMapRangeIter) -> MemTableIterator { let iter = iter.map(convert_entry as for<'a> fn(map::Entry<'a, KeyBytes, Bytes>) -> _); @@ -25,9 +25,7 @@ fn convert_entry(x: map::Entry<'_, KeyBytes, Bytes>) -> InnerEntry { } } -type SkipMapRangeIter<'a> = map::Range<'a, KeyBytes, BytesBound<'a>, KeyBytes, Bytes>; - -type SkipMapRangeEntry<'a> = map::Entry<'a, Bytes, Bytes>; +type SkipMapRangeIter<'a> = map::Range<'a, KeyBytes, BytesBound, KeyBytes, Bytes>; pub type NonEmptyMemTableIterRef<'a> = NonEmptyStream>; pub type MaybeEmptyMemTableIterRef<'a> = MaybeEmptyStream>; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 0da0fee..8b64fd1 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -142,8 +142,8 @@ impl MemTable { upper: Bound<&'a [u8]>, ) -> anyhow::Result> { self.scan_with_ts( - lower.map(|k| KeySlice::new(k, 0)), - upper.map(|k| KeySlice::new(k, 0)), + lower.map(|k| KeyBytes::new(Bytes::copy_from_slice(k), 0)), + upper.map(|k| KeyBytes::new(Bytes::copy_from_slice(k), 0)), ) .await } @@ -172,16 +172,14 @@ impl MemTable { pub async fn scan_with_ts<'a>( &'a self, - lower: Bound>, - upper: Bound>, + lower: Bound, + upper: Bound, ) -> anyhow::Result> { // todo: 由于 rust 的 Borrow trait 的限制，这里只能 copy - let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); - let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); let iter = self.map.range(BytesBound { - start: lower.as_ref(), - end: upper.as_ref(), + start: lower, + end: upper, }); let iter = new_memtable_iter(iter); NonEmptyStream::try_new(iter).await diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index b93ea78..f1039ba 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,3 +1,4 @@ +use std::future::ready; use std::ops::Bound; use async_iter_ext::StreamTools; @@ -10,11 +11,15 @@ pub fn build_time_dedup_iter( ) -> impl Stream> + Unpin + Send where S: Stream> + Unpin + Send, - A: PartialEq, - T: PartialOrd + Copy, + A: PartialEq + Send, + E: Send, + T: PartialOrd + Copy + Send + Sync, { let s = s - .try_filter(|(_, timestamp)| async { timestamp.le(×tamp_upper) }) + .try_filter(move |(_, timestamp)| { + let condition = timestamp.le(×tamp_upper); + ready(condition) + }) .dedup_by(|left, right| match (left, right) { (Ok((left, _)), Ok((right, _))) => left.eq(right), _ => false, diff --git a/src/sst/compact/common.rs b/src/sst/compact/common.rs index 30e9fd2..ae7fb27 100644 --- a/src/sst/compact/common.rs +++ b/src/sst/compact/common.rs @@ -1,4 +1,4 @@ -use crate::entry::Entry; +use crate::entry::{Entry, InnerEntry}; use crate::iterators::merge::MergeIteratorInner; use crate::iterators::{ create_merge_iter_from_non_empty_iters, create_two_merge_iter, iter_fut_to_stream, @@ -122,7 +122,7 @@ async fn batch( ) -> Option>> where P: Persistent, - I: Stream> + Unpin, + I: Stream> + Unpin, { let mut builder = SsTableBuilder::new(block_size); @@ -137,7 +137,7 @@ where continue; } - let key = KeySlice::from_slice(entry.key.as_ref()); + let key = entry.key.as_key_slice(); let value = entry.value.as_ref(); builder.add(key, value); } diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 0a46622..29d8686 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -62,11 +62,12 @@ fn transform_stop_iter<'a>( upper: KeySlice<'a>, f: for<'b> fn(KeySlice<'b>, KeySlice<'b>) -> bool, ) -> impl Stream> + 'a { - iter.take_while(move |entry| async { - entry + iter.take_while(move |entry| { + let x = entry .as_ref() .map(|entry| f(entry.key.as_key_slice(), upper)) .unwrap_or(true); + ready(x) }) } From 158587c7c7b2842e393739fd3c37b69d2c5292e0 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 13 Jun 2024 13:23:06 +0800 Subject: [PATCH 16/69] fix: check --- src/block/builder.rs | 18 +++++--- src/bound.rs | 2 + src/entry.rs | 12 ++++- src/iterators/lsm.rs | 92 +++++++++++++++++--------------------- src/memtable/immutable.rs | 6 +-- src/memtable/iterator.rs | 4 +- src/memtable/mutable.rs | 12 +++-- src/mvcc/iterator.rs | 27 +++++------ src/mvcc/transaction.rs | 4 +- src/mvcc/watermark.rs | 4 +- src/sst/iterator/iter.rs | 3 +- src/state/states.rs | 12 +++-- src/test_utils/iterator.rs | 26 +++++++++++ src/test_utils/mod.rs | 1 + src/wal.rs | 22 +++++++-- 15 files changed, 150 insertions(+), 95 deletions(-) create mode 100644 src/test_utils/iterator.rs diff --git a/src/block/builder.rs b/src/block/builder.rs index 7e0c342..7da5ecf 100644 --- a/src/block/builder.rs +++ b/src/block/builder.rs @@ -1,7 +1,9 @@ -use crate::key::{KeySlice, KeyVec}; -use bytes::BufMut; use std::iter; +use bytes::BufMut; + +use crate::key::{KeySlice, KeyVec}; + use super::Block; /// Builds a block. @@ -106,11 +108,13 @@ fn compress_key(first_key: &KeyVec, key: KeySlice, buffer: &mut Vec) { #[cfg(test)] mod tests { - use crate::block::{Block, BlockBuilder, BlockIterator}; - use crate::key::{KeySlice, KeyVec}; + use std::sync::Arc; + use bytes::Bytes; use nom::AsBytes; - use std::sync::Arc; + + use crate::block::{Block, BlockBuilder, BlockIterator}; + use crate::key::{KeySlice, KeyVec}; #[test] fn test_block_build_single_key() { @@ -199,7 +203,7 @@ mod tests { for _ in 0..5 { let mut iter = BlockIterator::create_and_seek_to_first(block.clone()); for i in 0..num_of_keys() { - let entry = iter.next().unwrap().unwrap(); + let entry = iter.next().unwrap().unwrap().prune_ts(); let key = entry.key.as_bytes(); let value = entry.value.as_bytes(); assert_eq!( @@ -226,7 +230,7 @@ mod tests { let mut iter = BlockIterator::create_and_seek_to_key(block, key_of(0).as_key_slice()); for offset in 1..=5 { for i in 0..num_of_keys() { - let entry = iter.next().unwrap().unwrap(); + let entry = iter.next().unwrap().unwrap().prune_ts(); let key = entry.key.as_bytes(); let value = entry.value.as_bytes(); assert_eq!( diff --git a/src/bound.rs b/src/bound.rs index 405a0b2..8c116a6 100644 --- a/src/bound.rs +++ b/src/bound.rs @@ -3,6 +3,8 @@ use std::ops::RangeBounds; use crate::key::KeyBytes; +pub type BoundRange = (Bound, Bound); + pub struct BytesBound { pub start: Bound, pub end: Bound, diff --git a/src/entry.rs b/src/entry.rs index eafdf7a..cf58a81 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -16,7 +16,7 @@ impl Eq for Keyed {} impl PartialEq for Keyed { fn eq(&self, other: &Self) -> bool { - self.key.eq(&other.key) + PartialEq::eq(&self.key, &other.key) } } @@ -56,3 +56,13 @@ impl Keyed { } } } + +#[cfg(test)] +impl InnerEntry { + pub fn prune_ts(self) -> Entry { + Entry { + key: self.key.into_inner(), + value: self.value, + } + } +} diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index cb87998..71b6e08 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -48,63 +48,53 @@ impl<'a, P> LockedLsmIter<'a, P> where P: Persistent, { - pub fn iter(&'a self) -> impl Future>> + Send { - async { - let a = self.build_memtable_iter().await; - assert_raw_stream(&a); - let b = self.build_sst_iter().await?; - assert_raw_stream(&b); - let merge = create_two_merge_iter(a, b).await?; - assert_raw_stream(&merge); - let merge = merge.map(|entry| entry.map(Keyed::into_timed_tuple)); - assert_tuple_stream(&merge); - let time_dedup = build_time_dedup_iter(merge, self.timestamp); - assert_result_stream(&time_dedup); - // todo: dedup timestamps - let iter = new_no_deleted_iter(time_dedup); - let iter = Box::new(iter) as _; - Ok(iter) - } + pub async fn iter(&'a self) -> anyhow::Result> { + let a = self.build_memtable_iter().await; + assert_raw_stream(&a); + let b = self.build_sst_iter().await?; + assert_raw_stream(&b); + let merge = create_two_merge_iter(a, b).await?; + assert_raw_stream(&merge); + let merge = merge.map(|entry| entry.map(Keyed::into_timed_tuple)); + assert_tuple_stream(&merge); + let time_dedup = build_time_dedup_iter(merge, self.timestamp); + assert_result_stream(&time_dedup); + // todo: dedup timestamps + let iter = new_no_deleted_iter(time_dedup); + let iter = Box::new(iter) as _; + Ok(iter) } - pub fn build_memtable_iter( - &self, - ) -> impl Future> + Send { - async { - let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); - let lower = lower.map(Key::from); - let upper = upper.map(Key::from); + pub async fn build_memtable_iter(&self) -> MergeIterator { + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); - let memtable = self.state.memtable().deref().as_immutable_ref(); - let imm_memtables = self.state.imm_memtables().as_slice(); - let imm_memtables = imm_memtables.iter().map(Arc::as_ref); - let tables = iter::once(memtable).chain(imm_memtables); - let iters = stream::iter(tables).filter_map(move |table| { - let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); - let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); + let memtable = self.state.memtable().deref().as_immutable_ref(); + let imm_memtables = self.state.imm_memtables().as_slice(); + let imm_memtables = imm_memtables.iter().map(Arc::as_ref); + let tables = iter::once(memtable).chain(imm_memtables); + let iters = stream::iter(tables).filter_map(move |table| { + let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); + let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); - async { - table - .scan_with_ts(lower, upper) - .await - .inspect_err(|e| error!(error = ?e)) - .ok() - .flatten() - } - }); - create_merge_iter_from_non_empty_iters(iters).await - } + async { + table + .scan_with_ts(lower, upper) + .await + .inspect_err(|e| error!(error = ?e)) + .ok() + .flatten() + } + }); + create_merge_iter_from_non_empty_iters(iters).await } - pub fn build_sst_iter( - &self, - ) -> impl Future>> + Send { - async { - let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); - let lower = lower.map(Key::from); - let upper = upper.map(Key::from); + pub async fn build_sst_iter(&self) -> anyhow::Result> { + let (lower, upper) = transform_bound(self.lower, self.upper, self.timestamp); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); - self.state.sstables_state().scan_sst(lower, upper).await - } + self.state.sstables_state().scan_sst(lower, upper).await } } diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 530ab09..ee26ed9 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -68,11 +68,11 @@ impl ImmutableMemTable { self.0.put_with_ts(key, value).await } - pub async fn scan_with_ts<'a>( - &'a self, + pub async fn scan_with_ts( + &self, lower: Bound, upper: Bound, - ) -> anyhow::Result> { + ) -> anyhow::Result> { self.0.scan_with_ts(lower, upper).await } } diff --git a/src/memtable/iterator.rs b/src/memtable/iterator.rs index a2daa5e..95cd2fe 100644 --- a/src/memtable/iterator.rs +++ b/src/memtable/iterator.rs @@ -42,6 +42,7 @@ mod test { use crate::memtable::MemTable; use crate::persistent::interface::WalHandle; use crate::persistent::wal_handle::WalFile; + use crate::test_utils::iterator::unwrap_ts_stream; #[tokio::test] async fn test_task1_memtable_iter() { @@ -139,6 +140,7 @@ mod test { ) -> impl Stream> + Send + 'a { let iter = memtable.for_testing_scan_slice(lower, upper).await.unwrap(); - create_merge_iter_from_non_empty_iters(stream::iter(iter.into_iter())).await + let x = create_merge_iter_from_non_empty_iters(stream::iter(iter.into_iter())).await; + unwrap_ts_stream(x) } } diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 8b64fd1..30c242d 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -170,11 +170,11 @@ impl MemTable { Ok(()) } - pub async fn scan_with_ts<'a>( - &'a self, + pub async fn scan_with_ts( + &self, lower: Bound, upper: Bound, - ) -> anyhow::Result> { + ) -> anyhow::Result> { // todo: 由于 rust 的 Borrow trait 的限制，这里只能 copy let iter = self.map.range(BytesBound { @@ -206,7 +206,11 @@ impl MemTable { lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, ) -> anyhow::Result> { - self.scan(lower, upper).await + self.scan_with_ts( + lower.map(|k| KeyBytes::new(Bytes::copy_from_slice(k), 0)), + upper.map(|k| KeyBytes::new(Bytes::copy_from_slice(k), 0)), + ) + .await } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index f1039ba..d3e8cfe 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,6 +1,7 @@ use std::future::ready; use std::ops::Bound; +use crate::bound::BoundRange; use async_iter_ext::StreamTools; use futures::{Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; @@ -15,24 +16,18 @@ where E: Send, T: PartialOrd + Copy + Send + Sync, { - let s = s - .try_filter(move |(_, timestamp)| { - let condition = timestamp.le(×tamp_upper); - ready(condition) - }) - .dedup_by(|left, right| match (left, right) { - (Ok((left, _)), Ok((right, _))) => left.eq(right), - _ => false, - }) - .map(|entry| entry.map(|pair| pair.0)); - s + s.try_filter(move |(_, timestamp)| { + let condition = timestamp.le(×tamp_upper); + ready(condition) + }) + .dedup_by(|left, right| match (left, right) { + (Ok((left, _)), Ok((right, _))) => left.eq(right), + _ => false, + }) + .map(|entry| entry.map(|pair| pair.0)) } -pub fn transform_bound( - lower: Bound, - upper: Bound, - timestamp: T, -) -> (Bound<(A, T)>, Bound<(A, T)>) +pub fn transform_bound(lower: Bound, upper: Bound, timestamp: T) -> BoundRange<(A, T)> where T: Bounded + Clone, { diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 12deec0..7f603f7 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -50,8 +50,8 @@ impl Transaction

{ Ok(stream::empty()) } - pub fn commit(self) -> impl Future> + Send { - async { todo!() } + pub async fn commit(self) -> anyhow::Result<()> { + todo!() } } diff --git a/src/mvcc/watermark.rs b/src/mvcc/watermark.rs index 16e4b72..c120ddf 100644 --- a/src/mvcc/watermark.rs +++ b/src/mvcc/watermark.rs @@ -14,7 +14,7 @@ impl Watermark { pub fn add_reader(&mut self, ts: u64) { self.readers .entry(ts) - .and_modify(|count| *count = *count + 1) + .and_modify(|count| *count += 1) .or_insert(1); } @@ -23,7 +23,7 @@ impl Watermark { if *count == 1 { self.readers.remove(&ts); } else { - *count = *count - 1; + *count -= 1; } } diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 29d8686..eef95d8 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -205,7 +205,8 @@ mod tests { .next() .await .unwrap() - .unwrap_or_else(|_| panic!("panic on {}", i)); + .unwrap_or_else(|_| panic!("panic on {}", i)) + .prune_ts(); let key = entry.key.as_bytes(); let value = entry.value.as_bytes(); assert_eq!( diff --git a/src/state/states.rs b/src/state/states.rs index bc673d6..0b9c98c 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -339,7 +339,7 @@ mod test { use futures::StreamExt; use tempfile::{tempdir, TempDir}; - use crate::entry::Entry; + use crate::entry::{Entry, InnerEntry}; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::two_merge::create_inner; use crate::iterators::utils::{assert_stream_eq, build_stream, build_tuple_stream}; @@ -349,6 +349,7 @@ mod test { use crate::sst::SstOptions; use crate::state::mut_op::Op; use crate::state::states::LsmStorageState; + use crate::test_utils::iterator::unwrap_ts_stream; #[tokio::test] async fn test_task2_storage_integration() { @@ -679,7 +680,8 @@ mod test { ); { let guard = storage.scan(Included(b"00"), Included(b"00")); - let mut iter = guard.build_memtable_iter().await; + let iter = guard.build_memtable_iter().await; + let iter = unwrap_ts_stream(iter); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), build_tuple_stream([("00", "2333")]), @@ -688,7 +690,8 @@ mod test { } { let guard = storage.scan(Included(b"00"), Included(b"00")); - let mut iter = guard.build_sst_iter().await.unwrap(); + let iter = guard.build_sst_iter().await.unwrap(); + let iter = unwrap_ts_stream(iter); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), build_tuple_stream([("00", "2333333")]), @@ -700,6 +703,7 @@ mod test { let a = guard.build_memtable_iter().await; let b = guard.build_sst_iter().await.unwrap(); let iter = create_inner(a, b).await.unwrap(); + let iter = unwrap_ts_stream(iter); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), build_tuple_stream([("00", "2333"), ("00", "2333333")]), @@ -711,6 +715,7 @@ mod test { let a = guard.build_memtable_iter().await; let b = guard.build_sst_iter().await.unwrap(); let iter = create_two_merge_iter(a, b).await.unwrap(); + let iter = unwrap_ts_stream(iter); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), build_tuple_stream([("00", "2333")]), @@ -722,6 +727,7 @@ mod test { let a = guard.build_memtable_iter().await; let b = guard.build_sst_iter().await.unwrap(); let iter = create_two_merge_iter(a, b).await.unwrap(); + let iter = unwrap_ts_stream(iter); let iter = new_no_deleted_iter(iter); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), diff --git a/src/test_utils/iterator.rs b/src/test_utils/iterator.rs new file mode 100644 index 0000000..cf6b7fc --- /dev/null +++ b/src/test_utils/iterator.rs @@ -0,0 +1,26 @@ +use crate::entry::{Entry, InnerEntry}; +use futures::{Stream, StreamExt}; + +pub fn unwrap_ts_stream(s: S) -> impl Stream> +where + S: Stream>, +{ + s.map(|item| { + item.map(|entry| Entry { + key: entry.key.into_inner(), + value: entry.value, + }) + }) +} + +pub fn unwrap_ts_iterator(s: S) -> impl Iterator> +where + S: Iterator>, +{ + s.map(|item| { + item.map(|entry| Entry { + key: entry.key.into_inner(), + value: entry.value, + }) + }) +} diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index a119e18..af57c7c 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -5,6 +5,7 @@ use crate::persistent::Persistent; use crate::state::Map; mod command; +pub mod iterator; mod map; pub async fn insert_sst>( diff --git a/src/wal.rs b/src/wal.rs index 3621923..b17d74e 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -103,6 +103,12 @@ async fn get_file(path: impl AsRef) -> anyhow::Result { Ok(file) } +impl Wal { + pub async fn put_for_test<'a>(&'a self, key: &'a [u8], value: &'a [u8]) -> anyhow::Result<()> { + self.put(KeySlice::from_slice(key), value).await + } +} + #[cfg(test)] mod tests { use crate::key::KeyBytes; @@ -120,10 +126,18 @@ mod tests { { let wal = Wal::create(id, &persistent).await.unwrap(); - wal.put("111".as_bytes(), "a".as_bytes()).await.unwrap(); - wal.put("222".as_bytes(), "bb".as_bytes()).await.unwrap(); - wal.put("333".as_bytes(), "ccc".as_bytes()).await.unwrap(); - wal.put("4".as_bytes(), "".as_bytes()).await.unwrap(); + wal.put_for_test("111".as_bytes(), "a".as_bytes()) + .await + .unwrap(); + wal.put_for_test("222".as_bytes(), "bb".as_bytes()) + .await + .unwrap(); + wal.put_for_test("333".as_bytes(), "ccc".as_bytes()) + .await + .unwrap(); + wal.put_for_test("4".as_bytes(), "".as_bytes()) + .await + .unwrap(); wal.sync().await.unwrap(); } From 722359a3d2b7f2c25f13f9add9611722148dccb9 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 13 Jun 2024 13:27:30 +0800 Subject: [PATCH 17/69] wip --- src/sst/builder.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sst/builder.rs b/src/sst/builder.rs index e34cd0e..b9cd1ee 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -102,6 +102,7 @@ impl SsTableBuilder { } // first/last key + // todo: unwrap 能不能去掉？ let first_key = meta.first().unwrap().first_key.clone(); let last_key = meta.last().unwrap().last_key.clone(); @@ -132,7 +133,7 @@ impl SsTableBuilder { .first_key(first_key) .last_key(last_key) .bloom(Some(bloom)) - .max_ts(0) + .max_ts(0) // todo .build(); Ok(table) From 05f088c97ee4699c9ca4a4870455d46460a04e6c Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 14 Jun 2024 22:35:18 +0800 Subject: [PATCH 18/69] wip --- src/block/blocks.rs | 12 ++++++++++-- src/block/builder.rs | 11 +++++++++-- src/key.rs | 3 +-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/block/blocks.rs b/src/block/blocks.rs index 4db502b..e25f42b 100644 --- a/src/block/blocks.rs +++ b/src/block/blocks.rs @@ -1,5 +1,5 @@ use crate::key::KeyBytes; -use bytes::Bytes; +use bytes::{Buf, Bytes}; use crate::entry::{Entry, InnerEntry}; @@ -79,7 +79,9 @@ impl Block { } fn get_uncompressed_key_ref(data: &[u8]) -> (&[u8], &[u8]) { - get_value(data) + let (raw_key, data) = get_value(data); + let (data, timestamp) = get_u64(data); + } fn get_compressed_key_ref<'b>(first_key: &[u8], data: &'b [u8]) -> (&'b [u8], &'b [u8]) { @@ -136,6 +138,12 @@ fn get_u16(data: &[u8]) -> (&[u8], usize) { (new_data, value) } +fn get_u64(data: &[u8]) -> (&[u8], u64) { + let new_data = &data[8..]; + let value = (&data[..8]).get_u64(); + (new_data, value) +} + fn get_data_by_len(data: &[u8], len: usize) -> (&[u8], &[u8]) { (&data[len..], &data[..len]) } diff --git a/src/block/builder.rs b/src/block/builder.rs index 7da5ecf..c8ea7ac 100644 --- a/src/block/builder.rs +++ b/src/block/builder.rs @@ -76,8 +76,7 @@ impl BlockBuilder { compress_key(first_key, key, &mut self.data); } else { // first key - self.data.extend((key.len() as u16).to_be_bytes()); - self.data.extend(key.raw_ref()); + encode_key(key, &mut self.data); } self.data.extend((value.len() as u16).to_be_bytes()); @@ -93,6 +92,7 @@ impl BlockBuilder { fn compress_key(first_key: &KeyVec, key: KeySlice, buffer: &mut Vec) { let first_key = first_key.raw_ref(); + let timestamp = key.timestamp(); let key = key.raw_ref(); let common_prefix = iter::zip(first_key.iter(), key.iter()) @@ -104,6 +104,13 @@ fn compress_key(first_key: &KeyVec, key: KeySlice, buffer: &mut Vec) { if postfix > 0 { buffer.extend_from_slice(&key[common_prefix..]); } + buffer.put_u64(timestamp); +} + +fn encode_key(key: KeySlice, buffer: &mut Vec) { + buffer.put_u16(key.len() as u16); + buffer.extend(key.raw_ref()); + buffer.put_u64(key.timestamp()); } #[cfg(test)] diff --git a/src/key.rs b/src/key.rs index 1802135..007d5bc 100644 --- a/src/key.rs +++ b/src/key.rs @@ -121,8 +121,7 @@ impl Key { impl<'a> Key<&'a [u8]> { pub fn to_key_vec(self) -> KeyVec { - todo!() - // Key(self.0.to_vec()) + self.map(|key| key.to_vec()) } /// Create a key slice from a slice. Will be removed in week 3. From ab97b9386847212f5b3749596f6761b70cca4f06 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 20:53:46 +0800 Subject: [PATCH 19/69] wip --- src/block/blocks.rs | 36 ++++++++++++++++++++---------------- src/block/iterator.rs | 1 - src/key.rs | 4 ++++ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/block/blocks.rs b/src/block/blocks.rs index e25f42b..8e8f5cb 100644 --- a/src/block/blocks.rs +++ b/src/block/blocks.rs @@ -1,7 +1,7 @@ -use crate::key::KeyBytes; use bytes::{Buf, Bytes}; -use crate::entry::{Entry, InnerEntry}; +use crate::entry::InnerEntry; +use crate::key::{KeyBytes, KeySlice}; /// A block is the smallest unit of read and caching in LSM tree. It is a collection of sorted key-value pairs. #[derive(Debug)] @@ -51,7 +51,7 @@ impl Block { self.offsets.len() } - pub fn get_entry_ref(&self, index: usize) -> (&[u8], &[u8]) { + pub fn get_entry_ref(&self, index: usize) -> (KeySlice, &[u8]) { // get key let (data, key) = self.parse_key_ref(index); @@ -67,30 +67,34 @@ impl Block { (data, key, value) } - fn parse_key_ref(&self, index: usize) -> (&[u8], &[u8]) { + fn parse_key_ref(&self, index: usize) -> (&[u8], KeySlice) { let data = &self.data[self.offsets[index] as usize..]; if index == 0 { Self::get_uncompressed_key_ref(data) } else { let first_key = self.first_key_ref(); - Self::get_compressed_key_ref(first_key, data) + Self::get_compressed_key_ref(first_key.raw_ref(), data) } } - fn get_uncompressed_key_ref(data: &[u8]) -> (&[u8], &[u8]) { - let (raw_key, data) = get_value(data); + fn get_uncompressed_key_ref(data: &[u8]) -> (&[u8], KeySlice) { + let (data, raw_key) = get_value(data); let (data, timestamp) = get_u64(data); - + let output = KeySlice::new(raw_key, timestamp); + (data, output) } - fn get_compressed_key_ref<'b>(first_key: &[u8], data: &'b [u8]) -> (&'b [u8], &'b [u8]) { + // structure: [common_prefix_len, postfix_len, postfix, timestamp] + fn get_compressed_key_ref<'b>(first_key: &[u8], data: &'b [u8]) -> (&'b [u8], KeySlice<'b>) { let (data, common_prefix_len) = get_u16(data); let prefix = &first_key[..common_prefix_len]; let (data, postfix_len) = get_u16(data); let (data, postfix) = get_data_by_len(data, postfix_len); + let (data, timestamp) = get_u64(data); + // todo: 这里需要能把 (prefix: &[u8], postfix: &[u8]) 当作 &[u8] 的相关数据结构 (tuple of slices) let key = prefix .iter() @@ -99,30 +103,30 @@ impl Block { .collect::>() .leak(); + let key = KeySlice::new(key, timestamp); + (data, key) } pub fn get_entry(&self, index: usize) -> InnerEntry { let (key, value) = self.get_entry_ref(index); - let key = Bytes::copy_from_slice(key); + let key = key.to_key_bytes(); let value = Bytes::copy_from_slice(value); - // Entry { key, value } - todo!() + InnerEntry { key, value } } pub fn first_key(&self) -> KeyBytes { - let key = self.first_key_ref(); - KeyBytes::from_bytes(Bytes::copy_from_slice(key)) + self.first_key_ref().to_key_bytes() } - fn first_key_ref(&self) -> &[u8] { + fn first_key_ref(&self) -> KeySlice { let (_, key) = self.parse_key_ref(0); key } pub fn last_key(&self) -> KeyBytes { let (_, key) = self.parse_key_ref(self.offsets.len() - 1); - KeyBytes::from_bytes(Bytes::copy_from_slice(key)) + key.to_key_bytes() } } diff --git a/src/block/iterator.rs b/src/block/iterator.rs index 50553e2..574e535 100644 --- a/src/block/iterator.rs +++ b/src/block/iterator.rs @@ -36,7 +36,6 @@ impl BlockIterator { let mut current = self.block.len(); for index in 0..self.block.len() { let (this_key, _) = self.block.get_entry_ref(index); - let this_key = Key::from_slice(this_key); if this_key >= key { current = index; break; diff --git a/src/key.rs b/src/key.rs index 007d5bc..88f5a2f 100644 --- a/src/key.rs +++ b/src/key.rs @@ -66,6 +66,10 @@ impl> Key { pub fn for_testing_ts(self) -> u64 { 0 } + + pub fn to_key_bytes(self) -> KeyBytes { + self.map(|slice| Bytes::copy_from_slice(slice.as_ref())) + } } impl Key> { From 03cf677d2ac99c81f53237db7dcd5cfa4bc68826 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 21:00:21 +0800 Subject: [PATCH 20/69] fix: cargo check --tests --- Cargo.toml | 2 +- src/block/iterator.rs | 6 +++--- src/iterators/lsm.rs | 7 +++---- src/iterators/maybe_empty.rs | 2 +- src/iterators/merge.rs | 6 +++--- src/iterators/mod.rs | 4 ++-- src/iterators/two_merge.rs | 2 +- src/iterators/utils.rs | 7 +++++-- src/key.rs | 4 ++-- src/lsm/core.rs | 14 ++++++------- src/manifest.rs | 2 +- src/memtable/immutable.rs | 10 +++++----- src/memtable/mutable.rs | 6 +++--- src/mvcc/transaction.rs | 6 +++--- src/persistent/file_object.rs | 10 +++++----- src/persistent/wal_handle.rs | 2 +- src/sst/bloom.rs | 2 +- src/sst/builder.rs | 5 +++-- src/sst/compact/common.rs | 11 +++++------ src/sst/compact/leveled.rs | 12 +++++------ src/sst/compact/simple_leveled.rs | 4 ++-- src/sst/iterator/concat.rs | 4 ++-- src/sst/iterator/iter.rs | 6 +++--- src/sst/iterator/merged.rs | 2 +- src/sst/sstables.rs | 33 +++++++++++++++---------------- src/state/inner.rs | 4 ++-- src/state/states.rs | 25 +++++++++++------------ src/test_utils/mod.rs | 4 ++-- src/wal.rs | 6 +++--- 29 files changed, 103 insertions(+), 105 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 377d7d2..06e4ec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ criterion = { version = "0.5.1", features = ["async_tokio"] } maplit = "1.0.2" [lints.rust] -unused = "allow" +#unused = "allow" unsafe_code = "forbid" [[bench]] diff --git a/src/block/iterator.rs b/src/block/iterator.rs index 574e535..f6d0755 100644 --- a/src/block/iterator.rs +++ b/src/block/iterator.rs @@ -1,8 +1,8 @@ use crate::block::blocks::Block; -use crate::entry::{Entry, InnerEntry}; -use crate::key::{Key, KeySlice}; +use crate::entry::{InnerEntry}; +use crate::key::{KeySlice}; use std::sync::Arc; -use tracing::info; + // Iterates on a block. pub struct BlockIterator { diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 71b6e08..40ff09d 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use std::collections::Bound; -use std::future::Future; + use std::iter; use std::ops::Deref; use std::sync::Arc; @@ -12,10 +12,9 @@ use tracing::error; use crate::entry::{Entry, InnerEntry, Keyed}; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{ - create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, - NoDeletedIterator, TwoMergeIterator, + create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, TwoMergeIterator, }; -use crate::key::{Key, KeySlice}; +use crate::key::{Key}; use crate::memtable::MemTableIterator; use crate::mvcc::iterator::{build_time_dedup_iter, transform_bound}; use crate::persistent::Persistent; diff --git a/src/iterators/maybe_empty.rs b/src/iterators/maybe_empty.rs index d32bc46..7a0aeb5 100644 --- a/src/iterators/maybe_empty.rs +++ b/src/iterators/maybe_empty.rs @@ -1,5 +1,5 @@ use futures::{Stream, StreamExt}; -use std::future::Future; + use tracing::Instrument; pub type MaybeEmptyStream = Option>; diff --git a/src/iterators/merge.rs b/src/iterators/merge.rs index 36d0250..c4b1e43 100644 --- a/src/iterators/merge.rs +++ b/src/iterators/merge.rs @@ -1,15 +1,15 @@ use std::collections::BinaryHeap; use std::fmt::Debug; use std::future::ready; -use std::future::Future; -use std::marker::PhantomData; + + use std::pin::Pin; use std::task::{Context, Poll}; use futures::stream::unfold; use futures::{pin_mut, FutureExt, Stream, StreamExt}; use pin_project::pin_project; -use tracing::{error, info, Instrument}; +use tracing::{error}; use crate::iterators::maybe_empty::NonEmptyStream; use crate::iterators::merge::heap::HeapWrapper; diff --git a/src/iterators/mod.rs b/src/iterators/mod.rs index 1a0976d..3348501 100644 --- a/src/iterators/mod.rs +++ b/src/iterators/mod.rs @@ -10,9 +10,9 @@ pub mod utils; pub use lsm::LockedLsmIter; pub use maybe_empty::{MaybeEmptyStream, NonEmptyStream}; pub use merge::{create_merge_iter, create_merge_iter_from_non_empty_iters, MergeIterator}; -pub use no_deleted::NoDeletedIterator; + pub use ok_iter::OkIter; pub use two_merge::{create_two_merge_iter, TwoMergeIterator}; pub use utils::iter_fut_iter_to_stream; pub use utils::split_first; -pub use utils::{eq, iter_fut_to_stream, transpose_try_iter}; +pub use utils::{transpose_try_iter}; diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index 1c23007..1c40cbc 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -1,5 +1,5 @@ use std::fmt::Debug; -use std::future::Future; + use futures::stream::unfold; use futures::{Stream, StreamExt}; diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index 24692e8..013b070 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -1,16 +1,19 @@ -use bytes::Bytes; + use std::fmt::Debug; use std::future::Future; use std::iter::Map; use std::iter::Once; use std::pin::pin; use std::{iter, vec}; +use bytes::Bytes; + -use crate::entry::Entry; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; use futures::{stream, FutureExt, Stream, StreamExt}; +use crate::entry::Entry; + pub fn transpose_try_iter(iterator: Result) -> Either>> where diff --git a/src/key.rs b/src/key.rs index 88f5a2f..e0aa41c 100644 --- a/src/key.rs +++ b/src/key.rs @@ -1,10 +1,10 @@ use bytes::Bytes; use derive_new::new; use nom::AsBytes; -use std::borrow::Borrow; + use std::cmp::Ordering; use std::fmt::Debug; -use std::ops::Bound; + #[derive(PartialEq, Eq, Debug, new, Default, Clone, Copy)] pub struct Key { diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 830d176..48e407b 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -1,15 +1,15 @@ use std::future::{ready, Future}; use std::sync::Arc; -use std::thread; -use std::thread::sleep; + + use std::time::Duration; use bytes::Bytes; -use futures::executor::block_on; -use futures::{ready, FutureExt, StreamExt}; + +use futures::{FutureExt, StreamExt}; use futures_concurrency::stream::Merge; -use tokio::sync::MutexGuard; -use tokio::task::{block_in_place, JoinHandle}; + +use tokio::task::{JoinHandle}; use tokio::time::interval; use tokio_stream::wrappers::IntervalStream; use tokio_util::sync::CancellationToken; @@ -18,7 +18,7 @@ use tracing::error; use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::{LsmStorageState, Map}; -use crate::utils::func::do_nothing; + pub struct Lsm { state: Arc>, diff --git a/src/manifest.rs b/src/manifest.rs index 8ca1ecc..eb4f767 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -8,7 +8,7 @@ use derive_more::From; use serde::{Deserialize, Serialize}; use serde_json::Deserializer; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{Mutex, MutexGuard}; +use tokio::sync::{Mutex}; use crate::persistent::Persistent; use crate::sst::compact::common::CompactionTask; diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index ee26ed9..f0bb7cd 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -1,19 +1,19 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; -use std::sync::atomic::Ordering; -use crate::bound::BytesBound; -use crate::iterators::NonEmptyStream; + + + use crate::key::{KeyBytes, KeySlice}; use bytemuck::TransparentWrapper; use bytes::Bytes; use crossbeam_skiplist::map; use deref_ext::DerefExt; use derive_new::new; -use nom::AsBytes; + use ref_cast::RefCast; -use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; +use crate::memtable::iterator::{MaybeEmptyMemTableIterRef}; use crate::memtable::mutable::MemTable; use crate::persistent::interface::WalHandle; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 30c242d..29fe0da 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,6 +1,6 @@ use std::fmt::{Debug, Formatter}; -use std::future::Future; -use std::ops::{Bound, RangeBounds}; + +use std::ops::{Bound}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -19,7 +19,7 @@ use crate::key::{KeyBytes, KeySlice}; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::immutable::ImmutableMemTable; use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; -use crate::persistent::interface::{ManifestHandle, WalHandle}; +use crate::persistent::interface::{WalHandle}; use crate::persistent::Persistent; use crate::state::Map; use crate::wal::Wal; diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 7f603f7..cf8b37b 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,12 +1,12 @@ use crate::entry::Entry; -use crate::iterators::LockedLsmIter; + use crate::persistent::Persistent; -use crate::state::{LsmStorageState, LsmStorageStateInner, Map}; +use crate::state::{LsmStorageStateInner, Map}; use bytes::Bytes; use crossbeam_skiplist::SkipMap; use futures::{stream, Stream}; use std::collections::{Bound, HashSet}; -use std::future::Future; + use std::sync::atomic::AtomicBool; use std::sync::Arc; use tokio::sync::Mutex; diff --git a/src/persistent/file_object.rs b/src/persistent/file_object.rs index b3dfd64..4352533 100644 --- a/src/persistent/file_object.rs +++ b/src/persistent/file_object.rs @@ -1,16 +1,16 @@ use anyhow::Context; -use bytes::Bytes; -use std::fs::{File, OpenOptions}; -use std::future::Future; + +use std::fs::{File}; + use std::io::Write; use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::sync::Arc; use derive_new::new; -use nom::AsBytes; + use tokio::io::BufWriter; -use tokio::spawn; + use tokio::task::spawn_blocking; use tracing::Instrument; diff --git a/src/persistent/wal_handle.rs b/src/persistent/wal_handle.rs index f05f171..59d03d1 100644 --- a/src/persistent/wal_handle.rs +++ b/src/persistent/wal_handle.rs @@ -1,5 +1,5 @@ use derive_new::new; -use std::future::Future; + use std::io::Error; use std::pin::{pin, Pin}; use std::task::{Context, Poll}; diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index 3f814a8..0c95110 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use crate::key::KeySlice; + use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; diff --git a/src/sst/builder.rs b/src/sst/builder.rs index b9cd1ee..87f9069 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -1,5 +1,5 @@ use std::mem; -use std::ops::Bound::Unbounded; + use std::sync::Arc; use anyhow::Result; @@ -11,9 +11,10 @@ use tempfile::TempDir; use crate::block::{BlockBuilder, BlockCache}; use crate::key::{KeySlice, KeyVec}; use crate::memtable::ImmutableMemTable; -use crate::persistent::file_object::FileObject; + use crate::persistent::interface::WalHandle; use crate::persistent::{LocalFs, Persistent}; +use crate::persistent::file_object::FileObject; use crate::sst::bloom::Bloom; use crate::sst::{BlockMeta, SsTable}; diff --git a/src/sst/compact/common.rs b/src/sst/compact/common.rs index ae7fb27..5d07d71 100644 --- a/src/sst/compact/common.rs +++ b/src/sst/compact/common.rs @@ -1,16 +1,15 @@ -use crate::entry::{Entry, InnerEntry}; +use crate::entry::{InnerEntry}; use crate::iterators::merge::MergeIteratorInner; use crate::iterators::{ - create_merge_iter_from_non_empty_iters, create_two_merge_iter, iter_fut_to_stream, - MergeIterator, NonEmptyStream, + create_two_merge_iter, }; -use crate::key::KeySlice; + use derive_new::new; use futures::{stream, Stream, StreamExt}; use getset::CopyGetters; use serde::{Deserialize, Serialize}; -use std::future::{ready, Future}; -use std::ops::{Range, RangeBounds}; + +use std::ops::{Range}; use std::sync::Arc; use tracing::error; diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 42b53b6..3fb794a 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -1,20 +1,20 @@ use std::cmp::max; -use std::future::{ready, Future}; + use std::iter; -use std::sync::Arc; + use crate::manifest::{Compaction, Manifest, ManifestRecord}; use derive_new::new; use getset::CopyGetters; -use ordered_float::NotNan; -use tokio::sync::MutexGuard; -use tracing::{info, trace}; + + +use tracing::{trace}; use typed_builder::TypedBuilder; use crate::persistent::{Persistent, SstHandle}; use crate::sst::compact::common::{apply_compaction, compact_generate_new_sst, CompactionTask}; use crate::sst::compact::CompactionOptions::Leveled; -use crate::sst::{SsTable, SstOptions, Sstables}; +use crate::sst::{SstOptions, Sstables}; use crate::utils::num::power_of_2; #[derive(Debug, Clone, new, TypedBuilder, CopyGetters)] diff --git a/src/sst/compact/simple_leveled.rs b/src/sst/compact/simple_leveled.rs index d8efd87..007e20c 100644 --- a/src/sst/compact/simple_leveled.rs +++ b/src/sst/compact/simple_leveled.rs @@ -1,5 +1,5 @@ -use crate::sst::Sstables; -use itertools::Itertools; + + #[derive(Debug, Clone)] pub struct SimpleLeveledCompactionOptions { diff --git a/src/sst/iterator/concat.rs b/src/sst/iterator/concat.rs index 77098a9..4253130 100644 --- a/src/sst/iterator/concat.rs +++ b/src/sst/iterator/concat.rs @@ -1,10 +1,10 @@ use std::ops::Bound; use anyhow::Result; -use bytes::Bytes; + use futures::{stream, Stream, StreamExt}; -use crate::entry::{Entry, InnerEntry}; +use crate::entry::{InnerEntry}; use crate::key::KeySlice; use crate::persistent::SstHandle; use crate::sst::iterator::iter::SsTableIterator; diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index eef95d8..9a497a4 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -7,16 +7,16 @@ use std::task::{Context, Poll}; use futures::{future, FutureExt}; use futures::{stream, Stream, StreamExt}; use pin_project::pin_project; -use tracing::info; + use crate::block::BlockIterator; use crate::entry::{Entry, InnerEntry}; use crate::iterators::{iter_fut_iter_to_stream, split_first, MergeIterator, TwoMergeIterator}; -use crate::key::{KeyBytes, KeySlice}; +use crate::key::{KeySlice}; use crate::persistent::SstHandle; use crate::sst::bloom::Bloom; use crate::sst::iterator::concat::SstConcatIterator; -use crate::sst::{bloom, BlockMeta, SsTable}; +use crate::sst::{BlockMeta, SsTable}; // 暂时用 box，目前 rust 不能够方便地在 struct 中存 closure type InnerIter<'a> = Pin> + Send + 'a>>; diff --git a/src/sst/iterator/merged.rs b/src/sst/iterator/merged.rs index eff3f1b..c34c966 100644 --- a/src/sst/iterator/merged.rs +++ b/src/sst/iterator/merged.rs @@ -1,4 +1,4 @@ -use crate::entry::{Entry, InnerEntry}; +use crate::entry::{InnerEntry}; use crate::iterators::{MergeIterator, TwoMergeIterator}; use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::iterator::iter::SsTableIterator; diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index b2daaa9..28e2666 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -1,42 +1,41 @@ use anyhow::anyhow; use deref_ext::DerefExt; -use std::cmp::max; + use std::collections::{Bound, HashMap}; use std::fmt::{Debug, Formatter}; -use std::future::ready; + use std::iter::repeat; -use std::pin::Pin; + use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; -use std::{iter, mem}; +use std::{mem}; -use futures::{pin_mut, stream, FutureExt, Stream, StreamExt}; +use futures::{stream, FutureExt, StreamExt}; use itertools::Itertools; -use ordered_float::NotNan; -use tokio::sync::RwLock; + + use tracing::error; -use crate::entry::{Entry, InnerEntry}; -use crate::iterators::merge::MergeIteratorInner; +use crate::entry::{InnerEntry}; + use crate::iterators::{ - create_merge_iter, create_merge_iter_from_non_empty_iters, create_two_merge_iter, - iter_fut_to_stream, MergeIterator, NonEmptyStream, + create_merge_iter, create_two_merge_iter, MergeIterator, }; use crate::key::KeySlice; -use crate::manifest::{Compaction, Flush, ManifestRecord}; +use crate::manifest::{Compaction, Flush}; use crate::memtable::ImmutableMemTable; -use crate::persistent::{Persistent, SstHandle}; +use crate::persistent::{SstHandle}; use crate::sst::compact::{ - CompactionOptions, LeveledCompactionOptions, SimpleLeveledCompactionOptions, + CompactionOptions, }; use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::iterator::{ - create_sst_concat_and_seek_to_first, scan_sst_concat, MergedSstIterator, SsTableIterator, + scan_sst_concat, MergedSstIterator, SsTableIterator, }; use crate::sst::option::SstOptions; -use crate::sst::{bloom, SsTable, SsTableBuilder}; -use crate::state::LsmStorageStateInner; +use crate::sst::{SsTable}; + #[derive(Default)] pub struct Sstables { diff --git a/src/state/inner.rs b/src/state/inner.rs index 08838b1..395b0f8 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -1,4 +1,4 @@ -use crossbeam_skiplist::SkipMap; + use std::cmp::max; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -14,7 +14,7 @@ use crate::memtable::{ImmutableMemTable, MemTable}; use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; use crate::sst::{SsTable, SstOptions, Sstables}; -use crate::wal::Wal; + #[derive(Getters, TypedBuilder)] pub struct LsmStorageStateInner { diff --git a/src/state/states.rs b/src/state/states.rs index 0b9c98c..bfb1b73 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1,9 +1,8 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; -use std::future::Future; use std::ops::Deref; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arc_swap::ArcSwap; use bytes::Bytes; @@ -20,10 +19,9 @@ use crate::memtable::MemTable; use crate::mvcc::core::LsmMvccInner; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; -use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; +use crate::sst::compact::leveled::force_compact; use crate::state::inner::LsmStorageStateInner; -use crate::state::mut_op::Op; use crate::state::Map; use crate::utils::vec::pop; @@ -322,12 +320,12 @@ where self.delete(Bytes::copy_from_slice(key)).await } - async fn write_batch_for_test( - &self, - records: impl IntoIterator>, - ) -> anyhow::Result<()> { - todo!() - } + // async fn write_batch_for_test( + // &self, + // records: impl IntoIterator>, + // ) -> anyhow::Result<()> { + // todo!() + // } } #[cfg(test)] @@ -339,15 +337,14 @@ mod test { use futures::StreamExt; use tempfile::{tempdir, TempDir}; - use crate::entry::{Entry, InnerEntry}; + use crate::entry::Entry; + use crate::iterators::create_two_merge_iter; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::two_merge::create_inner; - use crate::iterators::utils::{assert_stream_eq, build_stream, build_tuple_stream}; - use crate::iterators::{create_two_merge_iter, eq}; + use crate::iterators::utils::{assert_stream_eq, build_stream, build_tuple_stream, eq}; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; use crate::sst::SstOptions; - use crate::state::mut_op::Op; use crate::state::states::LsmStorageState; use crate::test_utils::iterator::unwrap_ts_stream; diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index af57c7c..1f0f5ff 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,7 +1,7 @@ -use std::error::Error; + use std::ops::Range; -use crate::persistent::Persistent; + use crate::state::Map; mod command; diff --git a/src/wal.rs b/src/wal.rs index b17d74e..6dfd424 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,6 +1,6 @@ -use std::future::Future; + use std::io::Cursor; -use std::ops::DerefMut; + use std::path::Path; use std::sync::Arc; @@ -88,7 +88,7 @@ impl Wal { } pub async fn sync(&self) -> anyhow::Result<()> { - let mut guard = self.file.lock().await; + let guard = self.file.lock().await; guard.sync_all().await?; Ok(()) } From e003bf395d6413624b0b9a98feed653dd9a73a98 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 21:00:46 +0800 Subject: [PATCH 21/69] fix: cargo fmt --- src/block/iterator.rs | 5 ++--- src/iterators/lsm.rs | 2 +- src/iterators/merge.rs | 3 +-- src/iterators/mod.rs | 2 +- src/iterators/two_merge.rs | 1 - src/iterators/utils.rs | 7 ++----- src/key.rs | 1 - src/lsm/core.rs | 4 +--- src/manifest.rs | 2 +- src/memtable/immutable.rs | 5 +---- src/memtable/mutable.rs | 4 ++-- src/persistent/file_object.rs | 2 +- src/sst/bloom.rs | 1 - src/sst/builder.rs | 2 +- src/sst/compact/common.rs | 8 +++----- src/sst/compact/leveled.rs | 4 +--- src/sst/compact/simple_leveled.rs | 3 --- src/sst/iterator/concat.rs | 2 +- src/sst/iterator/iter.rs | 3 +-- src/sst/iterator/merged.rs | 2 +- src/sst/sstables.rs | 22 +++++++--------------- src/state/inner.rs | 2 -- src/state/states.rs | 4 ++-- src/test_utils/mod.rs | 2 -- src/wal.rs | 1 - 25 files changed, 30 insertions(+), 64 deletions(-) diff --git a/src/block/iterator.rs b/src/block/iterator.rs index f6d0755..f69400e 100644 --- a/src/block/iterator.rs +++ b/src/block/iterator.rs @@ -1,9 +1,8 @@ use crate::block::blocks::Block; -use crate::entry::{InnerEntry}; -use crate::key::{KeySlice}; +use crate::entry::InnerEntry; +use crate::key::KeySlice; use std::sync::Arc; - // Iterates on a block. pub struct BlockIterator { /// The internal `Block`, wrapped by an `Arc` diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 40ff09d..28bd222 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -14,7 +14,7 @@ use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{ create_merge_iter_from_non_empty_iters, create_two_merge_iter, MergeIterator, TwoMergeIterator, }; -use crate::key::{Key}; +use crate::key::Key; use crate::memtable::MemTableIterator; use crate::mvcc::iterator::{build_time_dedup_iter, transform_bound}; use crate::persistent::Persistent; diff --git a/src/iterators/merge.rs b/src/iterators/merge.rs index c4b1e43..f56e4c6 100644 --- a/src/iterators/merge.rs +++ b/src/iterators/merge.rs @@ -2,14 +2,13 @@ use std::collections::BinaryHeap; use std::fmt::Debug; use std::future::ready; - use std::pin::Pin; use std::task::{Context, Poll}; use futures::stream::unfold; use futures::{pin_mut, FutureExt, Stream, StreamExt}; use pin_project::pin_project; -use tracing::{error}; +use tracing::error; use crate::iterators::maybe_empty::NonEmptyStream; use crate::iterators::merge::heap::HeapWrapper; diff --git a/src/iterators/mod.rs b/src/iterators/mod.rs index 3348501..501f93c 100644 --- a/src/iterators/mod.rs +++ b/src/iterators/mod.rs @@ -15,4 +15,4 @@ pub use ok_iter::OkIter; pub use two_merge::{create_two_merge_iter, TwoMergeIterator}; pub use utils::iter_fut_iter_to_stream; pub use utils::split_first; -pub use utils::{transpose_try_iter}; +pub use utils::transpose_try_iter; diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index 1c40cbc..96ed310 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -1,6 +1,5 @@ use std::fmt::Debug; - use futures::stream::unfold; use futures::{Stream, StreamExt}; diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index 013b070..24692e8 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -1,19 +1,16 @@ - +use bytes::Bytes; use std::fmt::Debug; use std::future::Future; use std::iter::Map; use std::iter::Once; use std::pin::pin; use std::{iter, vec}; -use bytes::Bytes; - +use crate::entry::Entry; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; use futures::{stream, FutureExt, Stream, StreamExt}; -use crate::entry::Entry; - pub fn transpose_try_iter(iterator: Result) -> Either>> where diff --git a/src/key.rs b/src/key.rs index e0aa41c..61f75fc 100644 --- a/src/key.rs +++ b/src/key.rs @@ -5,7 +5,6 @@ use nom::AsBytes; use std::cmp::Ordering; use std::fmt::Debug; - #[derive(PartialEq, Eq, Debug, new, Default, Clone, Copy)] pub struct Key { key: T, diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 48e407b..4c62422 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -1,7 +1,6 @@ use std::future::{ready, Future}; use std::sync::Arc; - use std::time::Duration; use bytes::Bytes; @@ -9,7 +8,7 @@ use bytes::Bytes; use futures::{FutureExt, StreamExt}; use futures_concurrency::stream::Merge; -use tokio::task::{JoinHandle}; +use tokio::task::JoinHandle; use tokio::time::interval; use tokio_stream::wrappers::IntervalStream; use tokio_util::sync::CancellationToken; @@ -19,7 +18,6 @@ use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::{LsmStorageState, Map}; - pub struct Lsm { state: Arc>, cancel_token: CancellationToken, diff --git a/src/manifest.rs b/src/manifest.rs index eb4f767..365cd15 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -8,7 +8,7 @@ use derive_more::From; use serde::{Deserialize, Serialize}; use serde_json::Deserializer; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{Mutex}; +use tokio::sync::Mutex; use crate::persistent::Persistent; use crate::sst::compact::common::CompactionTask; diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index f0bb7cd..6b936b4 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -1,9 +1,6 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; - - - use crate::key::{KeyBytes, KeySlice}; use bytemuck::TransparentWrapper; use bytes::Bytes; @@ -13,7 +10,7 @@ use derive_new::new; use ref_cast::RefCast; -use crate::memtable::iterator::{MaybeEmptyMemTableIterRef}; +use crate::memtable::iterator::MaybeEmptyMemTableIterRef; use crate::memtable::mutable::MemTable; use crate::persistent::interface::WalHandle; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 29fe0da..a3b0a1b 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,6 +1,6 @@ use std::fmt::{Debug, Formatter}; -use std::ops::{Bound}; +use std::ops::Bound; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -19,7 +19,7 @@ use crate::key::{KeyBytes, KeySlice}; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::immutable::ImmutableMemTable; use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; -use crate::persistent::interface::{WalHandle}; +use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; use crate::state::Map; use crate::wal::Wal; diff --git a/src/persistent/file_object.rs b/src/persistent/file_object.rs index 4352533..589fe56 100644 --- a/src/persistent/file_object.rs +++ b/src/persistent/file_object.rs @@ -1,6 +1,6 @@ use anyhow::Context; -use std::fs::{File}; +use std::fs::File; use std::io::Write; use std::os::unix::fs::FileExt; diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index 0c95110..322740a 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -1,6 +1,5 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. - use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 87f9069..1c4ba20 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -12,9 +12,9 @@ use crate::block::{BlockBuilder, BlockCache}; use crate::key::{KeySlice, KeyVec}; use crate::memtable::ImmutableMemTable; +use crate::persistent::file_object::FileObject; use crate::persistent::interface::WalHandle; use crate::persistent::{LocalFs, Persistent}; -use crate::persistent::file_object::FileObject; use crate::sst::bloom::Bloom; use crate::sst::{BlockMeta, SsTable}; diff --git a/src/sst/compact/common.rs b/src/sst/compact/common.rs index 5d07d71..c97c84f 100644 --- a/src/sst/compact/common.rs +++ b/src/sst/compact/common.rs @@ -1,15 +1,13 @@ -use crate::entry::{InnerEntry}; +use crate::entry::InnerEntry; +use crate::iterators::create_two_merge_iter; use crate::iterators::merge::MergeIteratorInner; -use crate::iterators::{ - create_two_merge_iter, -}; use derive_new::new; use futures::{stream, Stream, StreamExt}; use getset::CopyGetters; use serde::{Deserialize, Serialize}; -use std::ops::{Range}; +use std::ops::Range; use std::sync::Arc; use tracing::error; diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 3fb794a..21fc2a2 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -2,13 +2,11 @@ use std::cmp::max; use std::iter; - use crate::manifest::{Compaction, Manifest, ManifestRecord}; use derive_new::new; use getset::CopyGetters; - -use tracing::{trace}; +use tracing::trace; use typed_builder::TypedBuilder; use crate::persistent::{Persistent, SstHandle}; diff --git a/src/sst/compact/simple_leveled.rs b/src/sst/compact/simple_leveled.rs index 007e20c..3686034 100644 --- a/src/sst/compact/simple_leveled.rs +++ b/src/sst/compact/simple_leveled.rs @@ -1,6 +1,3 @@ - - - #[derive(Debug, Clone)] pub struct SimpleLeveledCompactionOptions { pub size_ratio_percent: usize, diff --git a/src/sst/iterator/concat.rs b/src/sst/iterator/concat.rs index 4253130..4e7b0dd 100644 --- a/src/sst/iterator/concat.rs +++ b/src/sst/iterator/concat.rs @@ -4,7 +4,7 @@ use anyhow::Result; use futures::{stream, Stream, StreamExt}; -use crate::entry::{InnerEntry}; +use crate::entry::InnerEntry; use crate::key::KeySlice; use crate::persistent::SstHandle; use crate::sst::iterator::iter::SsTableIterator; diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 9a497a4..734b117 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -8,11 +8,10 @@ use futures::{future, FutureExt}; use futures::{stream, Stream, StreamExt}; use pin_project::pin_project; - use crate::block::BlockIterator; use crate::entry::{Entry, InnerEntry}; use crate::iterators::{iter_fut_iter_to_stream, split_first, MergeIterator, TwoMergeIterator}; -use crate::key::{KeySlice}; +use crate::key::KeySlice; use crate::persistent::SstHandle; use crate::sst::bloom::Bloom; use crate::sst::iterator::concat::SstConcatIterator; diff --git a/src/sst/iterator/merged.rs b/src/sst/iterator/merged.rs index c34c966..337fd6a 100644 --- a/src/sst/iterator/merged.rs +++ b/src/sst/iterator/merged.rs @@ -1,4 +1,4 @@ -use crate::entry::{InnerEntry}; +use crate::entry::InnerEntry; use crate::iterators::{MergeIterator, TwoMergeIterator}; use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::iterator::iter::SsTableIterator; diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index 28e2666..eb7f336 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -6,36 +6,28 @@ use std::fmt::{Debug, Formatter}; use std::iter::repeat; +use std::mem; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; -use std::{mem}; use futures::{stream, FutureExt, StreamExt}; use itertools::Itertools; - use tracing::error; -use crate::entry::{InnerEntry}; +use crate::entry::InnerEntry; -use crate::iterators::{ - create_merge_iter, create_two_merge_iter, MergeIterator, -}; +use crate::iterators::{create_merge_iter, create_two_merge_iter, MergeIterator}; use crate::key::KeySlice; use crate::manifest::{Compaction, Flush}; use crate::memtable::ImmutableMemTable; -use crate::persistent::{SstHandle}; -use crate::sst::compact::{ - CompactionOptions, -}; +use crate::persistent::SstHandle; +use crate::sst::compact::CompactionOptions; use crate::sst::iterator::concat::SstConcatIterator; -use crate::sst::iterator::{ - scan_sst_concat, MergedSstIterator, SsTableIterator, -}; +use crate::sst::iterator::{scan_sst_concat, MergedSstIterator, SsTableIterator}; use crate::sst::option::SstOptions; -use crate::sst::{SsTable}; - +use crate::sst::SsTable; #[derive(Default)] pub struct Sstables { diff --git a/src/state/inner.rs b/src/state/inner.rs index 395b0f8..48bba79 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -1,4 +1,3 @@ - use std::cmp::max; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -15,7 +14,6 @@ use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; use crate::sst::{SsTable, SstOptions, Sstables}; - #[derive(Getters, TypedBuilder)] pub struct LsmStorageStateInner { pub(crate) memtable: Arc>, diff --git a/src/state/states.rs b/src/state/states.rs index bfb1b73..9ad4434 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1,8 +1,8 @@ use std::collections::Bound; use std::fmt::{Debug, Formatter}; use std::ops::Deref; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use arc_swap::ArcSwap; use bytes::Bytes; @@ -19,8 +19,8 @@ use crate::memtable::MemTable; use crate::mvcc::core::LsmMvccInner; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; -use crate::sst::{SsTableBuilder, SstOptions}; use crate::sst::compact::leveled::force_compact; +use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::LsmStorageStateInner; use crate::state::Map; use crate::utils::vec::pop; diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index 1f0f5ff..5130866 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,7 +1,5 @@ - use std::ops::Range; - use crate::state::Map; mod command; diff --git a/src/wal.rs b/src/wal.rs index 6dfd424..04a5953 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,4 +1,3 @@ - use std::io::Cursor; use std::path::Path; From 08d6ea9fa6445f244a8610f804c2baab846e82f0 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 21:06:50 +0800 Subject: [PATCH 22/69] refactor test utils --- src/iterators/merge.rs | 3 +- src/iterators/utils.rs | 58 +++++++++++++++++-------------- src/sst/bloom.rs | 2 +- src/sst/builder.rs | 75 ++++++++++++++++++++-------------------- src/sst/iterator/iter.rs | 2 +- src/state/states.rs | 3 +- 6 files changed, 75 insertions(+), 68 deletions(-) diff --git a/src/iterators/merge.rs b/src/iterators/merge.rs index f56e4c6..0c8ba6d 100644 --- a/src/iterators/merge.rs +++ b/src/iterators/merge.rs @@ -132,7 +132,8 @@ mod test { use crate::entry::Entry; use crate::iterators::create_merge_iter; use crate::iterators::merge::MergeIteratorInner; - use crate::iterators::utils::{assert_stream_eq, build_stream, build_tuple_stream}; + use crate::iterators::utils::test_utils::{build_stream, build_tuple_stream}; + use crate::iterators::utils::{assert_stream_eq, }; #[tokio::test] async fn test_empty() { diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index 24692e8..ea14645 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -1,12 +1,12 @@ -use bytes::Bytes; + use std::fmt::Debug; use std::future::Future; use std::iter::Map; use std::iter::Once; use std::pin::pin; -use std::{iter, vec}; +use std::{iter}; + -use crate::entry::Entry; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; @@ -94,31 +94,37 @@ where } #[cfg(test)] -pub type EntryStream = Iter>; +pub mod test_utils { + use std::vec; + use bytes::Bytes; + use futures::{stream, Stream}; + use futures::stream::Iter; + use crate::entry::Entry; -#[cfg(test)] -pub fn build_stream<'a>(source: impl IntoIterator) -> EntryStream { - let s: Vec<_> = source - .into_iter() - .map(|(key, value)| Entry::from_slice(key.as_bytes(), value.as_bytes())) - .collect(); - stream::iter(s) -} + pub type EntryStream = Iter>; -#[cfg(test)] -pub fn build_tuple_stream<'a>( - source: impl IntoIterator, -) -> impl Stream { - let s: Vec<_> = source - .into_iter() - .map(|(key, value)| { - ( - Bytes::copy_from_slice(key.as_bytes()), - Bytes::copy_from_slice(value.as_bytes()), - ) - }) - .collect(); - stream::iter(s) + pub fn build_stream<'a>(source: impl IntoIterator) -> EntryStream { + let s: Vec<_> = source + .into_iter() + .map(|(key, value)| Entry::from_slice(key.as_bytes(), value.as_bytes())) + .collect(); + stream::iter(s) + } + + pub fn build_tuple_stream<'a>( + source: impl IntoIterator, + ) -> impl Stream { + let s: Vec<_> = source + .into_iter() + .map(|(key, value)| { + ( + Bytes::copy_from_slice(key.as_bytes()), + Bytes::copy_from_slice(value.as_bytes()), + ) + }) + .collect(); + stream::iter(s) + } } #[cfg(test)] diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index 322740a..ab9e16c 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -122,7 +122,7 @@ pub fn may_contain(bloom: Option<&Bloom>, key: &[u8]) -> bool { #[cfg(test)] mod tests { use crate::sst::bloom::Bloom; - use crate::sst::builder::{key_of, num_of_keys}; + use crate::sst::builder::test_util::{key_of, num_of_keys}; #[test] fn test_task1_bloom_filter() { diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 1c4ba20..71d8f88 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -1,22 +1,17 @@ use std::mem; - use std::sync::Arc; use anyhow::Result; use bytes::BufMut; use nom::AsBytes; -#[cfg(test)] -use tempfile::TempDir; use crate::block::{BlockBuilder, BlockCache}; -use crate::key::{KeySlice, KeyVec}; +use crate::key::KeySlice; use crate::memtable::ImmutableMemTable; - -use crate::persistent::file_object::FileObject; use crate::persistent::interface::WalHandle; -use crate::persistent::{LocalFs, Persistent}; -use crate::sst::bloom::Bloom; +use crate::persistent::Persistent; use crate::sst::{BlockMeta, SsTable}; +use crate::sst::bloom::Bloom; /// Builds an SSTable from key-value pairs. pub struct SsTableBuilder { @@ -152,52 +147,56 @@ impl SsTableBuilder { } #[cfg(test)] -impl SsTableBuilder { - async fn build_for_test(self, dir: &TempDir, id: usize) -> anyhow::Result> { - let persistent = LocalFs::new(dir.path().to_path_buf()); - self.build(id, None, &persistent).await +pub mod test_util { + use tempfile::TempDir; + + use crate::key::KeyVec; + use crate::persistent::file_object::FileObject; + use crate::persistent::LocalFs; + use crate::sst::{SsTable, SsTableBuilder}; + + impl SsTableBuilder { + pub(crate) async fn build_for_test(self, dir: &TempDir, id: usize) -> anyhow::Result> { + let persistent = LocalFs::new(dir.path().to_path_buf()); + self.build(id, None, &persistent).await + } } -} -#[cfg(test)] -pub fn key_of(idx: usize) -> KeyVec { - KeyVec::for_testing_from_vec_no_ts(format!("key_{:03}", idx * 5).into_bytes()) -} + pub fn key_of(idx: usize) -> KeyVec { + KeyVec::for_testing_from_vec_no_ts(format!("key_{:03}", idx * 5).into_bytes()) + } -#[cfg(test)] -pub fn value_of(idx: usize) -> Vec { - format!("value_{:010}", idx).into_bytes() -} + pub fn value_of(idx: usize) -> Vec { + format!("value_{:010}", idx).into_bytes() + } -#[cfg(test)] -pub fn num_of_keys() -> usize { - 100 -} + pub fn num_of_keys() -> usize { + 100 + } -#[cfg(test)] -pub async fn generate_sst(dir: &TempDir) -> SsTable { - let mut builder = SsTableBuilder::new(128); - for idx in 0..num_of_keys() { - let key = key_of(idx); - let value = value_of(idx); - builder.add(key.as_key_slice(), &value[..]); + pub async fn generate_sst(dir: &TempDir) -> SsTable { + let mut builder = SsTableBuilder::new(128); + for idx in 0..num_of_keys() { + let key = key_of(idx); + let value = value_of(idx); + builder.add(key.as_key_slice(), &value[..]); + } + builder.build_for_test(dir, 1).await.unwrap() } - builder.build_for_test(dir, 1).await.unwrap() } #[cfg(test)] mod tests { - use crate::block::BlockCache; - use bytes::Bytes; use std::sync::Arc; + + use bytes::Bytes; use tempfile::tempdir; + use crate::block::BlockCache; use crate::key::KeySlice; use crate::persistent::{LocalFs, Persistent}; - use crate::sst::builder::{key_of, num_of_keys, value_of}; - use crate::sst::iterator::SsTableIterator; use crate::sst::{SsTable, SsTableBuilder}; - use futures::stream::StreamExt; + use crate::sst::builder::test_util::{key_of, num_of_keys, value_of}; #[tokio::test] async fn test_sst_build_single_key() { diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index 734b117..d95f658 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -191,7 +191,7 @@ mod tests { use nom::AsBytes; use tempfile::tempdir; - use crate::sst::builder::{generate_sst, key_of, num_of_keys, value_of}; + use crate::sst::builder::test_util::{generate_sst, key_of, num_of_keys, value_of}; use crate::sst::iterator::SsTableIterator; #[tokio::test] diff --git a/src/state/states.rs b/src/state/states.rs index 9ad4434..2e71be9 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -341,7 +341,8 @@ mod test { use crate::iterators::create_two_merge_iter; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::two_merge::create_inner; - use crate::iterators::utils::{assert_stream_eq, build_stream, build_tuple_stream, eq}; + use crate::iterators::utils::{assert_stream_eq, eq}; + use crate::iterators::utils::test_utils::{build_stream, build_tuple_stream}; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; use crate::sst::SstOptions; From 1af0539cc3ea23f737c921bcb90866a9b3fffe54 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 21:13:25 +0800 Subject: [PATCH 23/69] fix: clippy --- src/iterators/lsm.rs | 6 +++--- src/iterators/two_merge.rs | 2 +- src/key.rs | 5 ++--- src/lsm/core.rs | 4 ++-- src/manifest.rs | 2 +- src/memtable/immutable.rs | 1 - src/memtable/mutable.rs | 1 - src/mvcc/transaction.rs | 14 +++++++------- src/persistent/file_object.rs | 2 -- src/sst/compact/leveled.rs | 4 ++-- src/sst/sstables.rs | 24 +++++++++++------------- src/sst/tables.rs | 1 - src/state/inner.rs | 3 --- src/state/states.rs | 1 - src/wal.rs | 2 +- 15 files changed, 30 insertions(+), 42 deletions(-) diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 28bd222..5574fca 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -37,11 +37,11 @@ pub struct LockedLsmIter<'a, P: Persistent> { timestamp: u64, } -fn assert_raw_stream(s: &impl Stream>) {} +fn assert_raw_stream(_s: &impl Stream>) {} -fn assert_tuple_stream(s: &impl Stream, u64)>>) {} +fn assert_tuple_stream(_s: &impl Stream, u64)>>) {} -fn assert_result_stream(s: &impl Stream>>) {} +fn assert_result_stream(_s: &impl Stream>>) {} impl<'a, P> LockedLsmIter<'a, P> where diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index 96ed310..4c1043a 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use futures::stream::unfold; -use futures::{Stream, StreamExt}; +use futures::{Stream}; use crate::iterators::no_duplication::{new_no_duplication, NoDuplication}; use crate::iterators::{MaybeEmptyStream, NonEmptyStream}; diff --git a/src/key.rs b/src/key.rs index 61f75fc..8f6c58c 100644 --- a/src/key.rs +++ b/src/key.rs @@ -100,9 +100,8 @@ impl Key { } /// Create a `KeyBytes` from a `Bytes`. Will be removed in week 3. - pub fn from_bytes(bytes: Bytes) -> KeyBytes { + pub fn from_bytes(_bytes: Bytes) -> KeyBytes { todo!() - // Key(bytes) } /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. @@ -111,7 +110,7 @@ impl Key { // self.0.as_ref() } - pub fn for_testing_from_bytes_no_ts(bytes: Bytes) -> KeyBytes { + pub fn for_testing_from_bytes_no_ts(_bytes: Bytes) -> KeyBytes { todo!() // Key(bytes) } diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 4c62422..020447d 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -148,8 +148,8 @@ mod tests { use nom::AsBytes; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; - use tracing_futures::Instrument; - use tracing_subscriber::fmt::format::FmtSpan; + + use crate::lsm::core::Lsm; use crate::persistent::{LocalFs, Persistent}; diff --git a/src/manifest.rs b/src/manifest.rs index 365cd15..b44e44a 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,5 +1,5 @@ use std::future::Future; -use std::io::{Read, Write}; +use std::io::{Read}; use std::sync::Arc; use crate::persistent::interface::ManifestHandle; diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 6b936b4..37aa595 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -5,7 +5,6 @@ use crate::key::{KeyBytes, KeySlice}; use bytemuck::TransparentWrapper; use bytes::Bytes; use crossbeam_skiplist::map; -use deref_ext::DerefExt; use derive_new::new; use ref_cast::RefCast; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index a3b0a1b..764a6f3 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -21,7 +21,6 @@ use crate::memtable::immutable::ImmutableMemTable; use crate::memtable::iterator::{new_memtable_iter, MaybeEmptyMemTableIterRef}; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; -use crate::state::Map; use crate::wal::Wal; /// A basic mem-table based on crossbeam-skiplist. diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index cf8b37b..f4040ba 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -23,19 +23,19 @@ pub struct Transaction { impl Map for Transaction

{ type Error = anyhow::Error; - async fn get(&self, key: &[u8]) -> Result, Self::Error> { + async fn get(&self, _key: &[u8]) -> Result, Self::Error> { todo!() } async fn put( &self, - key: impl Into + Send, - value: impl Into + Send, + _key: impl Into + Send, + _value: impl Into + Send, ) -> Result<(), Self::Error> { todo!() } - async fn delete(&self, key: impl Into + Send) -> Result<(), Self::Error> { + async fn delete(&self, _key: impl Into + Send) -> Result<(), Self::Error> { todo!() } } @@ -43,10 +43,10 @@ impl Map for Transaction

{ impl Transaction

{ pub fn scan<'a>( &self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + _lower: Bound<&'a [u8]>, + _upper: Bound<&'a [u8]>, ) -> anyhow::Result>> { - todo!(); + // todo Ok(stream::empty()) } diff --git a/src/persistent/file_object.rs b/src/persistent/file_object.rs index 589fe56..bcfd3a2 100644 --- a/src/persistent/file_object.rs +++ b/src/persistent/file_object.rs @@ -2,7 +2,6 @@ use anyhow::Context; use std::fs::File; -use std::io::Write; use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::sync::Arc; @@ -14,7 +13,6 @@ use tokio::io::BufWriter; use tokio::task::spawn_blocking; use tracing::Instrument; -use crate::persistent::interface::WalHandle; use crate::persistent::manifest_handle::ManifestFile; use crate::persistent::wal_handle::WalFile; use crate::persistent::{Persistent, SstHandle}; diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 21fc2a2..6c69f54 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -364,8 +364,8 @@ mod tests { .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) .build(); - let mut state = LsmStorageState::new(options, persistent).await.unwrap(); - let next_sst_id = AtomicUsize::default(); + let state = LsmStorageState::new(options, persistent).await.unwrap(); + let _next_sst_id = AtomicUsize::default(); let state_lock = Mutex::default(); for i in 0..5 { diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index eb7f336..ab0793c 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -1,5 +1,4 @@ use anyhow::anyhow; -use deref_ext::DerefExt; use std::collections::{Bound, HashMap}; use std::fmt::{Debug, Formatter}; @@ -11,8 +10,7 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; -use futures::{stream, FutureExt, StreamExt}; -use itertools::Itertools; +use futures::{stream}; use tracing::error; @@ -273,22 +271,22 @@ struct DebugLevel { #[cfg(test)] mod tests { use std::ops::Bound::Unbounded; - use std::path::PathBuf; + use std::sync::Arc; - use std::time::Duration; + - use crate::iterators::{create_two_merge_iter, NonEmptyStream}; + use crate::key::KeySlice; - use futures::StreamExt; - use tempfile::{tempdir, TempDir}; - use tokio::time::timeout; - use tracing::{info, Instrument}; - use tracing_subscriber::fmt::format::FmtSpan; + + use tempfile::{TempDir}; + + + use crate::persistent::file_object::FileObject; use crate::persistent::LocalFs; - use crate::sst::iterator::SsTableIterator; - use crate::sst::{SsTable, SsTableBuilder, SstOptions, Sstables}; + + use crate::sst::{SsTableBuilder, SstOptions, Sstables}; #[tokio::test] async fn test() { diff --git a/src/sst/tables.rs b/src/sst/tables.rs index a091aee..4d585ea 100644 --- a/src/sst/tables.rs +++ b/src/sst/tables.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use bytes::Buf; use derive_getters::Getters; -use futures::StreamExt; use typed_builder::TypedBuilder; use crate::block::{Block, BlockCache, BlockIterator}; diff --git a/src/state/inner.rs b/src/state/inner.rs index 48bba79..35c73f5 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -119,17 +119,14 @@ async fn build_state( |(mut imm_memtables, mut sstables), manifest| async { match manifest { ManifestRecord::Flush(record) => { - let flush = &record; fold_flush_manifest(&mut imm_memtables, &mut sstables, record)?; Ok((imm_memtables, sstables)) } ManifestRecord::NewMemtable(record) => { - let new_mem = &record; fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; Ok((imm_memtables, sstables)) } ManifestRecord::Compaction(record) => { - let compact = &record; sstables.fold_compaction_manifest(record); Ok((imm_memtables, sstables)) } diff --git a/src/state/states.rs b/src/state/states.rs index 2e71be9..9e37eae 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use arc_swap::ArcSwap; use bytes::Bytes; -use deref_ext::DerefExt; use derive_getters::Getters; use futures::StreamExt; use tokio::sync::{Mutex, MutexGuard}; diff --git a/src/wal.rs b/src/wal.rs index 04a5953..9ca4bee 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -111,7 +111,7 @@ impl Wal { #[cfg(test)] mod tests { use crate::key::KeyBytes; - use bytes::Bytes; + use tempfile::tempdir; use crate::persistent::LocalFs; From 47253e5ebb18683d4bcdf1a5654182d1d7e823c3 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 23 Jun 2024 21:25:13 +0800 Subject: [PATCH 24/69] fix: clippy & fmt --- src/iterators/lsm.rs | 1 + src/iterators/merge.rs | 3 +- src/iterators/two_merge.rs | 2 +- src/iterators/utils.rs | 87 +++++++++++++++++++------------------- src/key.rs | 11 ++--- src/lsm/core.rs | 2 - src/manifest.rs | 3 +- src/memtable/iterator.rs | 1 + src/memtable/mutable.rs | 6 +-- src/mvcc/core.rs | 4 +- src/sst/builder.rs | 12 ++++-- src/sst/compact/mod.rs | 1 - src/sst/compact/tiered.rs | 7 --- src/sst/iterator/iter.rs | 2 +- src/sst/sstables.rs | 48 ++++----------------- src/state/mut_op.rs | 1 + src/state/states.rs | 5 ++- src/wal.rs | 4 +- 18 files changed, 78 insertions(+), 122 deletions(-) delete mode 100644 src/sst/compact/tiered.rs diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 5574fca..94760db 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -23,6 +23,7 @@ use crate::state::LsmStorageStateInner; pub type LsmIterator<'a> = Box> + Unpin + Send + 'a>; +#[allow(dead_code)] type LsmIteratorInner<'a, File> = TwoMergeIterator< Entry, MergeIterator>, diff --git a/src/iterators/merge.rs b/src/iterators/merge.rs index 0c8ba6d..d8758db 100644 --- a/src/iterators/merge.rs +++ b/src/iterators/merge.rs @@ -132,8 +132,7 @@ mod test { use crate::entry::Entry; use crate::iterators::create_merge_iter; use crate::iterators::merge::MergeIteratorInner; - use crate::iterators::utils::test_utils::{build_stream, build_tuple_stream}; - use crate::iterators::utils::{assert_stream_eq, }; + use crate::iterators::utils::test_utils::{assert_stream_eq, build_stream, build_tuple_stream}; #[tokio::test] async fn test_empty() { diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index 4c1043a..f886e3e 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use futures::stream::unfold; -use futures::{Stream}; +use futures::Stream; use crate::iterators::no_duplication::{new_no_duplication, NoDuplication}; use crate::iterators::{MaybeEmptyStream, NonEmptyStream}; diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index ea14645..a362d45 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -1,16 +1,13 @@ - -use std::fmt::Debug; use std::future::Future; use std::iter::Map; use std::iter::Once; -use std::pin::pin; -use std::{iter}; +use std::iter; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; -use futures::{stream, FutureExt, Stream, StreamExt}; +use futures::{stream, FutureExt, StreamExt}; pub fn transpose_try_iter(iterator: Result) -> Either>> where @@ -58,48 +55,15 @@ where stream::iter(iterator.map(FutureExt::into_stream as fn(_) -> _)).flatten() } -pub async fn assert_stream_eq(s1: S1, s2: S2) -where - S1: Stream, - S2: Stream, - S1::Item: PartialEq + Debug, - S2::Item: Debug, -{ - let s1: Vec<_> = s1.collect().await; - let s2: Vec<_> = s2.collect().await; - assert_eq!(s1, s2); -} - -pub async fn eq(s1: S1, s2: S2) -> bool -where - S1: Stream, - S2: Stream, - S1::Item: PartialEq + Debug, - S2::Item: Debug, -{ - let mut s1 = pin!(s1); - let mut s2 = pin!(s2); - loop { - match (s1.next().await, s2.next().await) { - (Some(x1), Some(x2)) => { - if x1 != x2 { - dbg!((x1, x2)); - return false; - } - } - (Some(_), None) | (None, Some(_)) => return false, - (None, None) => return true, - } - } -} - #[cfg(test)] pub mod test_utils { - use std::vec; + use crate::entry::Entry; use bytes::Bytes; - use futures::{stream, Stream}; use futures::stream::Iter; - use crate::entry::Entry; + use futures::{stream, Stream, StreamExt}; + use std::fmt::Debug; + use std::pin::pin; + use std::vec; pub type EntryStream = Iter>; @@ -125,11 +89,46 @@ pub mod test_utils { .collect(); stream::iter(s) } + + pub async fn assert_stream_eq(s1: S1, s2: S2) + where + S1: Stream, + S2: Stream, + S1::Item: PartialEq + Debug, + S2::Item: Debug, + { + let s1: Vec<_> = s1.collect().await; + let s2: Vec<_> = s2.collect().await; + assert_eq!(s1, s2); + } + + pub async fn eq(s1: S1, s2: S2) -> bool + where + S1: Stream, + S2: Stream, + S1::Item: PartialEq + Debug, + S2::Item: Debug, + { + let mut s1 = pin!(s1); + let mut s2 = pin!(s2); + loop { + match (s1.next().await, s2.next().await) { + (Some(x1), Some(x2)) => { + if x1 != x2 { + dbg!((x1, x2)); + return false; + } + } + (Some(_), None) | (None, Some(_)) => return false, + (None, None) => return true, + } + } + } } #[cfg(test)] mod tests { - use super::eq; + use crate::iterators::utils::test_utils::eq; use futures::stream; use std::fmt::Debug; diff --git a/src/key.rs b/src/key.rs index 8f6c58c..cf3fb2e 100644 --- a/src/key.rs +++ b/src/key.rs @@ -127,30 +127,25 @@ impl<'a> Key<&'a [u8]> { } /// Create a key slice from a slice. Will be removed in week 3. - pub fn from_slice(slice: &'a [u8]) -> Self { + pub fn from_slice(_slice: &'a [u8]) -> Self { todo!() - // Self(slice) } /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(self) -> &'a [u8] { todo!() - // self.0 } pub fn for_testing_key_ref(self) -> &'a [u8] { todo!() - // self.0 } - pub fn for_testing_from_slice_no_ts(slice: &'a [u8]) -> Self { + pub fn for_testing_from_slice_no_ts(_slice: &'a [u8]) -> Self { todo!() - // Self(slice) } - pub fn for_testing_from_slice_with_ts(slice: &'a [u8], _ts: u64) -> Self { + pub fn for_testing_from_slice_with_ts(_slice: &'a [u8], _ts: u64) -> Self { todo!() - // Self(slice) } } diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 020447d..6ba3a16 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -148,8 +148,6 @@ mod tests { use nom::AsBytes; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; - - use crate::lsm::core::Lsm; use crate::persistent::{LocalFs, Persistent}; diff --git a/src/manifest.rs b/src/manifest.rs index b44e44a..4d6574c 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -1,5 +1,4 @@ use std::future::Future; -use std::io::{Read}; use std::sync::Arc; use crate::persistent::interface::ManifestHandle; @@ -109,7 +108,7 @@ mod tests { } { - let (manifest, records) = Manifest::recover(&persistent).await.unwrap(); + let (_manifest, records) = Manifest::recover(&persistent).await.unwrap(); let record = Compaction(CompactionTask::new(1, 2, 3), vec![1, 2, 3]); diff --git a/src/memtable/iterator.rs b/src/memtable/iterator.rs index 95cd2fe..d664e32 100644 --- a/src/memtable/iterator.rs +++ b/src/memtable/iterator.rs @@ -27,6 +27,7 @@ fn convert_entry(x: map::Entry<'_, KeyBytes, Bytes>) -> InnerEntry { type SkipMapRangeIter<'a> = map::Range<'a, KeyBytes, BytesBound, KeyBytes, Bytes>; +#[allow(dead_code)] pub type NonEmptyMemTableIterRef<'a> = NonEmptyStream>; pub type MaybeEmptyMemTableIterRef<'a> = MaybeEmptyStream>; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 764a6f3..f33b113 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,7 +1,7 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; -use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -185,10 +185,6 @@ impl MemTable { } } -fn build_path(dir: impl AsRef, id: usize) -> PathBuf { - dir.as_ref().join(format!("{}.wal", id)) -} - #[cfg(test)] impl MemTable { pub async fn for_testing_put_slice(&self, key: &[u8], value: &[u8]) -> anyhow::Result<()> { diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index fc99196..eb72f73 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -47,8 +47,8 @@ impl LsmMvccInner { pub fn new_txn( &self, - inner: Arc>, - serializable: bool, + _inner: Arc>, + _serializable: bool, ) -> Arc> { unimplemented!() } diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 71d8f88..9f22b95 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -10,8 +10,8 @@ use crate::key::KeySlice; use crate::memtable::ImmutableMemTable; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; -use crate::sst::{BlockMeta, SsTable}; use crate::sst::bloom::Bloom; +use crate::sst::{BlockMeta, SsTable}; /// Builds an SSTable from key-value pairs. pub struct SsTableBuilder { @@ -156,7 +156,11 @@ pub mod test_util { use crate::sst::{SsTable, SsTableBuilder}; impl SsTableBuilder { - pub(crate) async fn build_for_test(self, dir: &TempDir, id: usize) -> anyhow::Result> { + pub(crate) async fn build_for_test( + self, + dir: &TempDir, + id: usize, + ) -> anyhow::Result> { let persistent = LocalFs::new(dir.path().to_path_buf()); self.build(id, None, &persistent).await } @@ -195,8 +199,8 @@ mod tests { use crate::block::BlockCache; use crate::key::KeySlice; use crate::persistent::{LocalFs, Persistent}; - use crate::sst::{SsTable, SsTableBuilder}; use crate::sst::builder::test_util::{key_of, num_of_keys, value_of}; + use crate::sst::{SsTable, SsTableBuilder}; #[tokio::test] async fn test_sst_build_single_key() { @@ -292,6 +296,7 @@ mod tests { assert_eq!(*sst.max_ts(), 6); } + #[allow(dead_code)] pub async fn generate_sst_with_ts( id: usize, persistent: &P, @@ -308,6 +313,7 @@ mod tests { builder.build(id, block_cache, persistent).await.unwrap() } + #[allow(dead_code)] fn generate_test_data() -> Vec<((Bytes, u64), Bytes)> { (0..100) .map(|id| { diff --git a/src/sst/compact/mod.rs b/src/sst/compact/mod.rs index bef8f31..3ef28e6 100644 --- a/src/sst/compact/mod.rs +++ b/src/sst/compact/mod.rs @@ -2,7 +2,6 @@ pub mod common; pub mod leveled; mod option; mod simple_leveled; -mod tiered; pub use leveled::LeveledCompactionOptions; pub use option::CompactionOptions; diff --git a/src/sst/compact/tiered.rs b/src/sst/compact/tiered.rs deleted file mode 100644 index faf3595..0000000 --- a/src/sst/compact/tiered.rs +++ /dev/null @@ -1,7 +0,0 @@ -#[derive(Debug, Clone)] -pub struct TieredCompactionOptions { - pub num_tiers: usize, - pub max_size_amplification_percent: usize, - pub size_ratio: usize, - pub min_merge_width: usize, -} diff --git a/src/sst/iterator/iter.rs b/src/sst/iterator/iter.rs index d95f658..1a387b7 100644 --- a/src/sst/iterator/iter.rs +++ b/src/sst/iterator/iter.rs @@ -127,7 +127,7 @@ pub struct SsTableIterator<'a, File> { } impl<'a, File> SsTableIterator<'a, File> { - pub fn may_contain(&self, key: &[u8]) -> bool { + pub fn may_contain(&self, _key: &[u8]) -> bool { true // todo // bloom::may_contain(self.bloom, key) diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index ab0793c..d2172d2 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -10,7 +10,7 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; -use futures::{stream}; +use futures::stream; use tracing::error; @@ -170,15 +170,6 @@ where create_merge_iter(iters).await } - fn level_size(&self, level: usize) -> usize { - if level == 0 { - self.l0_sstables.len() - } else { - let ids = self.levels.get(level - 1).unwrap(); - ids.len() - } - } - pub(super) fn table_ids_mut(&mut self, level: usize) -> &mut Vec { if level == 0 { &mut self.l0_sstables @@ -207,19 +198,6 @@ where todo!() } - fn debug_level(&self, level: usize) -> DebugLevel { - let tables = self.tables(level); - let (size, count) = tables.fold((0, 0), |(size, count), table| { - (size + table.table_size(), count + 1) - }); - let ids = self.table_ids(level); - DebugLevel { - ids: ids.clone(), - size, - count, - } - } - pub fn fold_compaction_manifest(&mut self, Compaction(task, result_ids): Compaction) { let source = self.table_ids_mut(task.source()); source.remove(task.source_index()); @@ -229,7 +207,7 @@ where } fn filter_sst_by_bloom( - table: &SsTable, + _table: &SsTable, lower: Bound, upper: Bound, ) -> bool { @@ -262,30 +240,19 @@ pub fn fold_flush_manifest( Ok(()) } -struct DebugLevel { - ids: Vec, - size: u64, - count: usize, -} - #[cfg(test)] mod tests { use std::ops::Bound::Unbounded; - + use std::sync::Arc; - - use crate::key::KeySlice; - - use tempfile::{TempDir}; - - - + + use tempfile::TempDir; use crate::persistent::file_object::FileObject; use crate::persistent::LocalFs; - + use crate::sst::{SsTableBuilder, SstOptions, Sstables}; #[tokio::test] @@ -318,7 +285,8 @@ mod tests { }; sst.insert_sst(Arc::new(table)); - let iter = sst.scan_sst(Unbounded, Unbounded).await.unwrap(); + let _iter = sst.scan_sst(Unbounded, Unbounded).await.unwrap(); + todo!() // assert_eq!() } } diff --git a/src/state/mut_op.rs b/src/state/mut_op.rs index fd0c610..1b41e41 100644 --- a/src/state/mut_op.rs +++ b/src/state/mut_op.rs @@ -1,3 +1,4 @@ +#[allow(dead_code)] #[derive(Debug)] pub enum Op { Put { key: T, value: T }, diff --git a/src/state/states.rs b/src/state/states.rs index 9e37eae..1806d1e 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -340,8 +340,9 @@ mod test { use crate::iterators::create_two_merge_iter; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::two_merge::create_inner; - use crate::iterators::utils::{assert_stream_eq, eq}; - use crate::iterators::utils::test_utils::{build_stream, build_tuple_stream}; + use crate::iterators::utils::test_utils::{ + assert_stream_eq, build_stream, build_tuple_stream, eq, + }; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; use crate::sst::SstOptions; diff --git a/src/wal.rs b/src/wal.rs index 9ca4bee..36b05bf 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -111,7 +111,7 @@ impl Wal { #[cfg(test)] mod tests { use crate::key::KeyBytes; - + use tempfile::tempdir; use crate::persistent::LocalFs; @@ -141,7 +141,7 @@ mod tests { } { - let (wal, map) = Wal::recover(id, &persistent).await.unwrap(); + let (_wal, map) = Wal::recover(id, &persistent).await.unwrap(); assert_eq!(map.get(&KeyBytes::new_no_ts(b"111")).unwrap().value(), "a"); assert_eq!(map.get(&KeyBytes::new_no_ts(b"222")).unwrap().value(), "bb"); From 328d67ee7e34ccc86e444cc9c7216e79be4c549c Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 24 Jun 2024 13:20:55 +0800 Subject: [PATCH 25/69] fix: clippy & fmt --- Cargo.toml | 2 +- src/lsm/core.rs | 12 ++++-------- src/wal.rs | 12 +----------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 06e4ec9..377d7d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ criterion = { version = "0.5.1", features = ["async_tokio"] } maplit = "1.0.2" [lints.rust] -#unused = "allow" +unused = "allow" unsafe_code = "forbid" [[bench]] diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 6ba3a16..b8c29e1 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -21,28 +21,23 @@ use crate::state::{LsmStorageState, Map}; pub struct Lsm { state: Arc>, cancel_token: CancellationToken, - flush_handle: Option>, - compaction_handle: Option>, } impl Lsm

{ pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { let state = Arc::new(LsmStorageState::new(options, persistent).await?); let cancel_token = CancellationToken::new(); - let flush_handle = Self::spawn_flush(state.clone(), cancel_token.clone()); - let compaction_handle = Self::spawn_compaction(state.clone(), cancel_token.clone()); + let _ = Self::spawn_flush(state.clone(), cancel_token.clone()); + let _ = Self::spawn_compaction(state.clone(), cancel_token.clone()); let this = Self { state, cancel_token, - flush_handle: Some(flush_handle), - compaction_handle: Some(compaction_handle), }; Ok(this) } pub async fn sync(&self) -> anyhow::Result<()> { - // todo - Ok(()) + todo!() } fn spawn_flush( @@ -187,6 +182,7 @@ mod tests { // .is_empty()); // } + #[allow(dead_code)] async fn build_lsm(dir: &TempDir) -> anyhow::Result> { let persistent = LocalFs::new(dir.path().to_path_buf()); let options = SstOptions::builder() diff --git a/src/wal.rs b/src/wal.rs index 36b05bf..c53accf 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,12 +1,11 @@ use std::io::Cursor; -use std::path::Path; use std::sync::Arc; use crate::key::{KeyBytes, KeySlice}; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; -use tokio::fs::{File, OpenOptions}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Mutex; use tracing_futures::Instrument; @@ -93,15 +92,6 @@ impl Wal { } } -async fn get_file(path: impl AsRef) -> anyhow::Result { - let file = OpenOptions::new() - .create(true) - .append(true) - .open(path) - .await?; - Ok(file) -} - impl Wal { pub async fn put_for_test<'a>(&'a self, key: &'a [u8], value: &'a [u8]) -> anyhow::Result<()> { self.put(KeySlice::from_slice(key), value).await From 21945d67a6df3288697ef7dcb56046b68043a472 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 18 Jul 2024 13:39:45 +0800 Subject: [PATCH 26/69] fix: block meta encoding/decoding --- src/key.rs | 18 +++++++++++++++++- src/sst/block_meta.rs | 20 ++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/key.rs b/src/key.rs index cf3fb2e..0e9a9d8 100644 --- a/src/key.rs +++ b/src/key.rs @@ -1,4 +1,4 @@ -use bytes::Bytes; +use bytes::{Buf, Bytes}; use derive_new::new; use nom::AsBytes; @@ -69,6 +69,15 @@ impl> Key { pub fn to_key_bytes(self) -> KeyBytes { self.map(|slice| Bytes::copy_from_slice(slice.as_ref())) } + + pub fn to_byte_iter(&self) -> impl Iterator + '_ { + let key = self.key.as_ref(); + let key_len = (key.len() as u16).to_be_bytes().into_iter(); + let key = key.iter().copied(); + let timestamp = self.timestamp.to_be_bytes().into_iter(); + + key_len.chain(key).chain(timestamp) + } } impl Key> { @@ -119,6 +128,13 @@ impl Key { todo!() // self.0.as_ref() } + + pub fn decode(buf: &mut impl Buf) -> KeyBytes { + let key_len = buf.get_u16() as usize; + let key = buf.copy_to_bytes(key_len); + let timestamp = buf.get_u64(); + Key::new(key, timestamp) + } } impl<'a> Key<&'a [u8]> { diff --git a/src/sst/block_meta.rs b/src/sst/block_meta.rs index 16c4a9a..2ee14bd 100644 --- a/src/sst/block_meta.rs +++ b/src/sst/block_meta.rs @@ -20,27 +20,19 @@ pub struct BlockMeta { impl BlockMeta { pub fn encode(&self) -> impl Iterator + '_ { let offset = (self.offset as u16).to_be_bytes().into_iter(); - let first_key_len = (self.first_key.len() as u16).to_be_bytes().into_iter(); - let first_key = self.first_key.raw_ref().iter().copied(); - let last_key_len = (self.last_key.len() as u16).to_be_bytes().into_iter(); - let last_key = self.last_key.raw_ref().iter().copied(); offset - .chain(first_key_len) - .chain(first_key) - .chain(last_key_len) - .chain(last_key) + .chain(self.first_key.to_byte_iter()) + .chain(self.last_key.to_byte_iter()) } pub fn decode(mut data: impl Buf) -> Self { let offset = data.get_u16() as usize; - let first_key_len = data.get_u16() as usize; - let first_key = data.copy_to_bytes(first_key_len); - let last_key_len = data.get_u16() as usize; - let last_key = data.copy_to_bytes(last_key_len); + let first_key = KeyBytes::decode(&mut data); + let last_key = KeyBytes::decode(&mut data); Self { offset, - first_key: KeyBytes::from_bytes(first_key), - last_key: KeyBytes::from_bytes(last_key), + first_key, + last_key, } } From f588be2dd6566625d42836b75c8049bc0f3a8db5 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 18 Jul 2024 13:48:17 +0800 Subject: [PATCH 27/69] fix: block builder test --- src/block/builder.rs | 1 + src/key.rs | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/block/builder.rs b/src/block/builder.rs index c8ea7ac..4086ddc 100644 --- a/src/block/builder.rs +++ b/src/block/builder.rs @@ -107,6 +107,7 @@ fn compress_key(first_key: &KeyVec, key: KeySlice, buffer: &mut Vec) { buffer.put_u64(timestamp); } +// todo: 太多的 encoding 方法了，需要统一 fn encode_key(key: KeySlice, buffer: &mut Vec) { buffer.put_u16(key.len() as u16); buffer.extend(key.raw_ref()); diff --git a/src/key.rs b/src/key.rs index 0e9a9d8..6fd16f3 100644 --- a/src/key.rs +++ b/src/key.rs @@ -149,19 +149,19 @@ impl<'a> Key<&'a [u8]> { /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(self) -> &'a [u8] { - todo!() + self.key } pub fn for_testing_key_ref(self) -> &'a [u8] { todo!() } - pub fn for_testing_from_slice_no_ts(_slice: &'a [u8]) -> Self { - todo!() + pub fn for_testing_from_slice_no_ts(slice: &'a [u8]) -> Self { + Self::new(slice, 123) } - pub fn for_testing_from_slice_with_ts(_slice: &'a [u8], _ts: u64) -> Self { - todo!() + pub fn for_testing_from_slice_with_ts(slice: &'a [u8], ts: u64) -> Self { + Self::new(slice, ts) } } From 696b7b765d24b92cff2b8a6deeceeadd71c1c31e Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 18 Jul 2024 22:20:01 +0800 Subject: [PATCH 28/69] fix: test --- src/sst/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 9f22b95..b55beb3 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -244,7 +244,7 @@ mod tests { let dir = tempdir().unwrap(); let sst = builder.build_for_test(&dir, 1).await.unwrap(); assert!( - sst.block_meta.len() <= 25, + sst.block_meta.len() <= 40, "you have {} blocks, expect 25", sst.block_meta.len() ); From cdfbd4de0eb6898cc51d1b4ba19bed67bc6c0b6e Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 19 Jul 2024 21:18:36 +0800 Subject: [PATCH 29/69] fix: tests --- src/key.rs | 12 +++--------- src/sst/builder.rs | 6 +++++- src/sst/tables.rs | 2 +- src/state/states.rs | 1 + src/wal.rs | 44 ++++++++++++++++++++++++++++++++------------ 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/key.rs b/src/key.rs index 6fd16f3..d8924e4 100644 --- a/src/key.rs +++ b/src/key.rs @@ -44,8 +44,8 @@ impl From<(T, u64)> for Key { #[cfg(test)] impl Key { - pub fn new_no_ts(key: &[u8]) -> Self { - Self::new(Bytes::copy_from_slice(key), 0) + pub fn new_for_test(key: &[u8], ts: u64) -> Self { + Self::new(Bytes::copy_from_slice(key), ts) } } @@ -115,8 +115,7 @@ impl Key { /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(&self) -> &[u8] { - todo!() - // self.0.as_ref() + self.key.as_ref() } pub fn for_testing_from_bytes_no_ts(_bytes: Bytes) -> KeyBytes { @@ -142,11 +141,6 @@ impl<'a> Key<&'a [u8]> { self.map(|key| key.to_vec()) } - /// Create a key slice from a slice. Will be removed in week 3. - pub fn from_slice(_slice: &'a [u8]) -> Self { - todo!() - } - /// Always use `raw_ref` to access the key in week 1 + 2. This function will be removed in week 3. pub fn raw_ref(self) -> &'a [u8] { self.key diff --git a/src/sst/builder.rs b/src/sst/builder.rs index b55beb3..2ee6f38 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -20,6 +20,7 @@ pub struct SsTableBuilder { pub(crate) meta: Vec, block_size: usize, key_hashes: Vec, + max_ts: u64, // todo: use Option } impl SsTableBuilder { @@ -31,6 +32,7 @@ impl SsTableBuilder { meta: Vec::new(), block_size, key_hashes: Vec::new(), + max_ts: 0, } } @@ -45,6 +47,7 @@ impl SsTableBuilder { self.key_hashes.push(farmhash::fingerprint32(key.raw_ref())); return; } + self.max_ts = self.max_ts.max(key.timestamp()); let old_builder = mem::replace(&mut self.builder, BlockBuilder::new(self.block_size)); Self::add_block(&mut self.data, &mut self.meta, old_builder); self.add(key, value); @@ -90,6 +93,7 @@ impl SsTableBuilder { mut meta, block_size: _, key_hashes, + max_ts, } = self; // add last block @@ -129,7 +133,7 @@ impl SsTableBuilder { .first_key(first_key) .last_key(last_key) .bloom(Some(bloom)) - .max_ts(0) // todo + .max_ts(max_ts) // todo .build(); Ok(table) diff --git a/src/sst/tables.rs b/src/sst/tables.rs index 4d585ea..31400d6 100644 --- a/src/sst/tables.rs +++ b/src/sst/tables.rs @@ -29,7 +29,7 @@ pub struct SsTable { last_key: KeyBytes, pub(crate) bloom: Option, /// The maximum timestamp stored in this SST, implemented in week 3. - max_ts: u64, + max_ts: u64, // todo: use Option? } impl Debug for SsTable { diff --git a/src/state/states.rs b/src/state/states.rs index 1806d1e..624bfcb 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -162,6 +162,7 @@ where memtable.deref().approximate_size() > *self.options.target_sst_size() } + // todo: 这个函数用到的 snapshot 不用 load？直接从 caller 传过来？ pub(crate) async fn force_freeze_memtable( &self, _guard: &MutexGuard<'_, ()>, diff --git a/src/wal.rs b/src/wal.rs index c53accf..650b933 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -60,6 +60,7 @@ impl Wal { pub async fn put<'a>(&'a self, key: KeySlice<'a>, value: &'a [u8]) -> anyhow::Result<()> { let mut guard = self.file.lock().await; + // todo: 这里 write 的操作和 encode key/value 重复，需要合并 guard .write_u32(key.len() as u32) .instrument(tracing::info_span!("wal_put_write_key_len")) @@ -93,8 +94,13 @@ impl Wal { } impl Wal { - pub async fn put_for_test<'a>(&'a self, key: &'a [u8], value: &'a [u8]) -> anyhow::Result<()> { - self.put(KeySlice::from_slice(key), value).await + pub async fn put_for_test<'a>( + &'a self, + key: &'a [u8], + ts: u64, + value: &'a [u8], + ) -> anyhow::Result<()> { + self.put(KeySlice::new(key, ts), value).await } } @@ -115,16 +121,16 @@ mod tests { { let wal = Wal::create(id, &persistent).await.unwrap(); - wal.put_for_test("111".as_bytes(), "a".as_bytes()) + wal.put_for_test("111".as_bytes(), 123, "a".as_bytes()) .await .unwrap(); - wal.put_for_test("222".as_bytes(), "bb".as_bytes()) + wal.put_for_test("222".as_bytes(), 234, "bb".as_bytes()) .await .unwrap(); - wal.put_for_test("333".as_bytes(), "ccc".as_bytes()) + wal.put_for_test("333".as_bytes(), 345, "ccc".as_bytes()) .await .unwrap(); - wal.put_for_test("4".as_bytes(), "".as_bytes()) + wal.put_for_test("4".as_bytes(), 456, "".as_bytes()) .await .unwrap(); wal.sync().await.unwrap(); @@ -132,15 +138,29 @@ mod tests { { let (_wal, map) = Wal::recover(id, &persistent).await.unwrap(); - - assert_eq!(map.get(&KeyBytes::new_no_ts(b"111")).unwrap().value(), "a"); - assert_eq!(map.get(&KeyBytes::new_no_ts(b"222")).unwrap().value(), "bb"); assert_eq!( - map.get(&KeyBytes::new_no_ts(b"333")).unwrap().value(), + map.get(&KeyBytes::new_for_test(b"111", 123)) + .unwrap() + .value(), + "a" + ); + assert_eq!( + map.get(&KeyBytes::new_for_test(b"222", 234)) + .unwrap() + .value(), + "bb" + ); + assert_eq!( + map.get(&KeyBytes::new_for_test(b"333", 345)) + .unwrap() + .value(), "ccc" ); - assert_eq!(map.get(&KeyBytes::new_no_ts(b"4")).unwrap().value(), ""); - assert!(map.get(&KeyBytes::new_no_ts(b"555")).is_none()); + assert_eq!( + map.get(&KeyBytes::new_for_test(b"4", 456)).unwrap().value(), + "" + ); + assert!(map.get(&KeyBytes::new_for_test(b"555", 0)).is_none()); } } } From 220d703dd7345fd96e3804ade451d481b4f3c520 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 20 Jul 2024 21:49:39 +0800 Subject: [PATCH 30/69] fix: test --- src/block/blocks.rs | 6 ++--- src/key.rs | 6 +---- src/sst/compact/leveled.rs | 8 +++--- src/sst/sstables.rs | 50 +------------------------------------- 4 files changed, 9 insertions(+), 61 deletions(-) diff --git a/src/block/blocks.rs b/src/block/blocks.rs index 8e8f5cb..f1fdc8a 100644 --- a/src/block/blocks.rs +++ b/src/block/blocks.rs @@ -110,13 +110,13 @@ impl Block { pub fn get_entry(&self, index: usize) -> InnerEntry { let (key, value) = self.get_entry_ref(index); - let key = key.to_key_bytes(); + let key = key.copy_to_key_bytes(); let value = Bytes::copy_from_slice(value); InnerEntry { key, value } } pub fn first_key(&self) -> KeyBytes { - self.first_key_ref().to_key_bytes() + self.first_key_ref().copy_to_key_bytes() } fn first_key_ref(&self) -> KeySlice { @@ -126,7 +126,7 @@ impl Block { pub fn last_key(&self) -> KeyBytes { let (_, key) = self.parse_key_ref(self.offsets.len() - 1); - key.to_key_bytes() + key.copy_to_key_bytes() } } diff --git a/src/key.rs b/src/key.rs index d8924e4..f6e9ad6 100644 --- a/src/key.rs +++ b/src/key.rs @@ -62,11 +62,7 @@ impl> Key { self.key.as_ref().is_empty() } - pub fn for_testing_ts(self) -> u64 { - 0 - } - - pub fn to_key_bytes(self) -> KeyBytes { + pub fn copy_to_key_bytes(self) -> KeyBytes { self.map(|slice| Bytes::copy_from_slice(slice.as_ref())) } diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 6c69f54..2761720 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -298,8 +298,8 @@ mod tests { { assert_eq!(sstables.l0_sstables, [4, 3, 2]); - assert_eq!(sstables.levels, vec![vec![12, 13, 14], vec![], vec![]]); - assert_eq!(sstables.sstables.len(), 6); + assert_eq!(sstables.levels, vec![vec![12, 13, 14, 15], vec![], vec![]]); + assert_eq!(sstables.sstables.len(), 7); } compact_with_task( @@ -314,8 +314,8 @@ mod tests { { assert_eq!(sstables.l0_sstables, [4, 3, 2]); - assert_eq!(sstables.levels, vec![vec![13, 14], vec![16], vec![]]); - assert_eq!(sstables.sstables.len(), 6); + assert_eq!(sstables.levels, vec![vec![13, 14, 15], vec![17], vec![]]); + assert_eq!(sstables.sstables.len(), 7); } } diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index d2172d2..ebfadbe 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -241,52 +241,4 @@ pub fn fold_flush_manifest( } #[cfg(test)] -mod tests { - use std::ops::Bound::Unbounded; - - use std::sync::Arc; - - use crate::key::KeySlice; - - use tempfile::TempDir; - - use crate::persistent::file_object::FileObject; - use crate::persistent::LocalFs; - - use crate::sst::{SsTableBuilder, SstOptions, Sstables}; - - #[tokio::test] - async fn test() { - // tracing_subscriber::fmt::fmt() - // .with_span_events(FmtSpan::EXIT | FmtSpan::ENTER | FmtSpan::CLOSE) - // .with_target(false) - // .with_level(false) - // .init(); - let options = SstOptions::builder() - .target_sst_size(1024) - .block_size(4096) - .num_memtable_limit(1000) - .compaction_option(Default::default()) - .enable_wal(false) - .build(); - let dir = TempDir::new().unwrap(); - let path = dir.as_ref(); - let persistent = LocalFs::new(path.to_path_buf()); - let mut sst = Sstables::::new(&options); - let table = { - let mut builder = SsTableBuilder::new(16); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"11"), b"11"); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"22"), b"22"); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"33"), b"11"); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"44"), b"22"); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"55"), b"11"); - builder.add(KeySlice::for_testing_from_slice_no_ts(b"66"), b"22"); - builder.build(0, None, &persistent).await.unwrap() - }; - sst.insert_sst(Arc::new(table)); - - let _iter = sst.scan_sst(Unbounded, Unbounded).await.unwrap(); - todo!() - // assert_eq!() - } -} +mod tests {} From d3d7bbedc20e436da87a2f85ceb08531eee056bd Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 20 Jul 2024 22:45:54 +0800 Subject: [PATCH 31/69] fix: time dedup iter --- src/entry.rs | 1 + src/mvcc/iterator.rs | 46 ++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index cf58a81..271ac1f 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -12,6 +12,7 @@ pub struct Keyed { pub value: V, } +// todo: use Derivative for auto deriving impl Eq for Keyed {} impl PartialEq for Keyed { diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index d3e8cfe..0d8fa60 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,11 +1,13 @@ +use std::collections::Bound::{Excluded, Included, Unbounded}; use std::future::ready; use std::ops::Bound; -use crate::bound::BoundRange; use async_iter_ext::StreamTools; use futures::{Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; +use crate::bound::BoundRange; + pub fn build_time_dedup_iter( s: S, timestamp_upper: T, @@ -17,6 +19,7 @@ where T: PartialOrd + Copy + Send + Sync, { s.try_filter(move |(_, timestamp)| { + // todo: use binary search? let condition = timestamp.le(×tamp_upper); ready(condition) }) @@ -29,21 +32,40 @@ where pub fn transform_bound(lower: Bound, upper: Bound, timestamp: T) -> BoundRange<(A, T)> where - T: Bounded + Clone, + T: Bounded, { - use Bound::{Excluded, Included, Unbounded}; - - let lower = match lower { - Included(a) => Included((a, timestamp.clone())), - Excluded(a) => Excluded((a, T::min_value())), - Unbounded => Unbounded, - }; + ( + transform_lower_bound(lower), + transform_upper_bound(upper, timestamp), + ) +} - let upper = match upper { +fn transform_lower_bound(lower: Bound) -> Bound<(A, T)> +where + T: Bounded, +{ + use Bound::{Excluded, Included, Unbounded}; + match lower { Included(a) => Included((a, T::min_value())), Excluded(a) => Excluded((a, T::max_value())), Unbounded => Unbounded, - }; + } +} + +fn transform_upper_bound(upper: Bound, timestamp: T) -> Bound<(A, T)> +where + T: Bounded, +{ + use Bound::{Excluded, Included, Unbounded}; + match upper { + Included(a) => Included((a, timestamp)), + Excluded(a) => Excluded((a, T::min_value())), + Unbounded => Unbounded, + } +} - (lower, upper) +#[cfg(test)] +mod tests { + #[tokio::test] + async fn test_build_time_dedup_iter() {} } From 9c356c18d553c626e346e2ac4b78a2ad6f66a9c5 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 21 Jul 2024 15:24:19 +0800 Subject: [PATCH 32/69] add test --- src/entry.rs | 3 ++- src/mvcc/iterator.rs | 50 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/entry.rs b/src/entry.rs index 271ac1f..bc4750c 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -2,11 +2,12 @@ use std::cmp::Ordering; use crate::key::{Key, KeyBytes}; use bytes::Bytes; +use derive_new::new; pub type Entry = Keyed; pub type InnerEntry = Keyed; -#[derive(Debug)] +#[derive(Debug, new)] pub struct Keyed { pub key: K, pub value: V, diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 0d8fa60..d51e4fc 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -66,6 +66,54 @@ where #[cfg(test)] mod tests { + use crate::entry::Keyed; + use crate::mvcc::iterator::build_time_dedup_iter; + use futures::{stream, StreamExt, TryStreamExt}; + #[tokio::test] - async fn test_build_time_dedup_iter() {} + async fn test_build_time_dedup_iter() { + test_time_dedup_iter_helper([(Keyed::new("a", "a1"), 1)], 3, [Keyed::new("a", "a1")]).await; + test_time_dedup_iter_helper( + [ + (Keyed::new("a", "a1"), 1), + (Keyed::new("a", "a2"), 2), + (Keyed::new("b", "b3"), 3), + ], + 2, + [Keyed::new("a", "a2")], + ) + .await; + test_time_dedup_iter_helper( + [ + (Keyed::new("a", "a1"), 1), + (Keyed::new("a", "a2"), 2), + (Keyed::new("a", "a3"), 3), + (Keyed::new("b", "b2"), 2), + (Keyed::new("c", "c1"), 1), + (Keyed::new("c", "c3"), 3), + ], + 2, + [ + Keyed::new("a", "a2"), + Keyed::new("b", "b2"), + Keyed::new("c", "c1"), + ], + ) + .await; + } + + async fn test_time_dedup_iter_helper(s: I, timestamp_upper: u64, expected: S) + where + I: IntoIterator, u64)>, + I::IntoIter: Send, + S: IntoIterator>, + { + let s = stream::iter(s).map(|pair| Ok::<_, ()>(pair)); + let result: Vec<_> = build_time_dedup_iter(s, timestamp_upper) + .try_collect() + .await + .unwrap(); + let expected: Vec<_> = expected.into_iter().collect(); + assert_eq!(result, expected); + } } From 1f8cb7f608c352c6f9ee825fdeaa3eabd7b11443 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 22 Jul 2024 13:54:27 +0800 Subject: [PATCH 33/69] wip: txn iterator --- src/iterators/lsm.rs | 18 ++++++++++----- src/iterators/mod.rs | 2 +- src/iterators/utils.rs | 2 +- src/mvcc/core.rs | 20 +++++++++++------ src/mvcc/iterator.rs | 49 +++++++++++++++++++++++++++++++++++++---- src/mvcc/transaction.rs | 32 +++++++++++++++++++++++---- src/state/states.rs | 14 +++++++----- 7 files changed, 108 insertions(+), 29 deletions(-) diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 94760db..bf7aacb 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -33,8 +33,8 @@ type LsmIteratorInner<'a, File> = TwoMergeIterator< #[derive(new)] pub struct LockedLsmIter<'a, P: Persistent> { state: arc_swap::Guard>>, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, + pub(crate) lower: Bound<&'a [u8]>, + pub(crate) upper: Bound<&'a [u8]>, timestamp: u64, } @@ -49,6 +49,15 @@ where P: Persistent, { pub async fn iter(&'a self) -> anyhow::Result> { + let time_dedup = self.iter_with_delete().await?; + let iter = new_no_deleted_iter(time_dedup); + let iter = Box::new(iter) as _; + Ok(iter) + } + + pub async fn iter_with_delete( + &self, + ) -> anyhow::Result> + Unpin + Send + '_> { let a = self.build_memtable_iter().await; assert_raw_stream(&a); let b = self.build_sst_iter().await?; @@ -59,10 +68,7 @@ where assert_tuple_stream(&merge); let time_dedup = build_time_dedup_iter(merge, self.timestamp); assert_result_stream(&time_dedup); - // todo: dedup timestamps - let iter = new_no_deleted_iter(time_dedup); - let iter = Box::new(iter) as _; - Ok(iter) + Ok(time_dedup) } pub async fn build_memtable_iter(&self) -> MergeIterator { diff --git a/src/iterators/mod.rs b/src/iterators/mod.rs index 501f93c..7a9839d 100644 --- a/src/iterators/mod.rs +++ b/src/iterators/mod.rs @@ -1,4 +1,4 @@ -mod lsm; +pub mod lsm; mod maybe_empty; pub mod merge; pub mod no_deleted; diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index a362d45..cf66919 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -7,7 +7,7 @@ use std::iter; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; -use futures::{stream, FutureExt, StreamExt}; +use futures::{stream, FutureExt, Stream, StreamExt}; pub fn transpose_try_iter(iterator: Result) -> Either>> where diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index eb72f73..0ea7a71 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -1,16 +1,15 @@ use crate::mvcc::transaction::Transaction; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; -use crate::state::LsmStorageState; +use crate::state::{LsmStorageState, LsmStorageStateInner}; use parking_lot::Mutex; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; pub(crate) struct CommittedTxnData { pub(crate) key_hashes: HashSet, - #[allow(dead_code)] pub(crate) read_ts: u64, - #[allow(dead_code)] pub(crate) commit_ts: u64, } @@ -47,9 +46,16 @@ impl LsmMvccInner { pub fn new_txn( &self, - _inner: Arc>, - _serializable: bool, - ) -> Arc> { - unimplemented!() + inner: arc_swap::Guard>>, + serializable: bool, + ) -> Transaction

{ + // todo: use external dependency to get time + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let key_hashes = serializable.then(|| Mutex::default()); + let tx = Transaction::new(ts, inner, key_hashes); + tx } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index d51e4fc..991fdb3 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,12 +1,53 @@ +use async_iter_ext::StreamTools; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use futures::{stream, Stream, StreamExt, TryStreamExt}; +use num_traits::Bounded; use std::collections::Bound::{Excluded, Included, Unbounded}; use std::future::ready; use std::ops::Bound; - -use async_iter_ext::StreamTools; -use futures::{Stream, StreamExt, TryStreamExt}; -use num_traits::Bounded; +use std::sync::Arc; use crate::bound::BoundRange; +use crate::entry::{Entry, Keyed}; +use crate::iterators::lsm::LsmIterator; +use crate::iterators::no_deleted::new_no_deleted_iter; +use crate::iterators::{create_two_merge_iter, LockedLsmIter}; +use crate::persistent::Persistent; + +pub struct LockedTxnIter<'a, P: Persistent> { + local_storage: arc_swap::Guard>>, + lsm_iter: LockedLsmIter<'a, P>, +} + +impl<'a, P: Persistent> LockedTxnIter<'a, P> { + pub async fn iter(&'a self) -> anyhow::Result> { + let lsm_iter = self.lsm_iter.iter_with_delete().await?; + let local_iter = txn_local_iterator( + self.local_storage.as_ref(), + self.lsm_iter.lower.map(Bytes::copy_from_slice), + self.lsm_iter.upper.map(Bytes::copy_from_slice), + ); + let merged = create_two_merge_iter(local_iter, lsm_iter).await?; + let iter = new_no_deleted_iter(merged); + let iter = Box::new(iter) as _; + Ok(iter) + } +} + +pub fn txn_local_iterator<'a>( + map: &'a SkipMap, + lower: Bound, + upper: Bound, +) -> impl Stream> + Unpin + Send + 'a { + let it = map.range((lower, upper)).map(|entry| { + let key = entry.key().clone(); + let value = entry.value().clone(); + let pair = Keyed::new(key, value); + Ok::<_, anyhow::Error>(pair) + }); + stream::iter(it) +} pub fn build_time_dedup_iter( s: S, diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index f4040ba..bfdc408 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -7,16 +7,20 @@ use crossbeam_skiplist::SkipMap; use futures::{stream, Stream}; use std::collections::{Bound, HashSet}; +use crate::mvcc::iterator::txn_local_iterator; +use parking_lot::Mutex; use std::sync::atomic::AtomicBool; use std::sync::Arc; -use tokio::sync::Mutex; pub struct Transaction { pub(crate) read_ts: u64, - pub(crate) inner: Arc>, + pub(crate) inner: arc_swap::Guard>>, + + // todo: need Arc<...> ? pub(crate) local_storage: Arc>, pub(crate) committed: Arc, /// Write set and read set + /// todo: check deadlock? pub(crate) key_hashes: Option, HashSet)>>, } @@ -41,11 +45,31 @@ impl Map for Transaction

{ } impl Transaction

{ + pub fn new( + read_ts: u64, + inner: arc_swap::Guard>>, + key_hashes: Option, HashSet)>>, + ) -> Self { + Self { + read_ts, + inner, + local_storage: Arc::default(), + committed: Arc::default(), + key_hashes, + } + } + pub fn scan<'a>( &self, - _lower: Bound<&'a [u8]>, - _upper: Bound<&'a [u8]>, + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, ) -> anyhow::Result>> { + let local_iterator = txn_local_iterator( + &self.local_storage, + lower.map(Bytes::copy_from_slice), + upper.map(Bytes::copy_from_slice), + ); + // todo Ok(stream::empty()) } diff --git a/src/state/states.rs b/src/state/states.rs index 624bfcb..13765b7 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1,13 +1,13 @@ +use anyhow::anyhow; +use arc_swap::ArcSwap; +use bytes::Bytes; +use derive_getters::Getters; +use futures::StreamExt; use std::collections::Bound; use std::fmt::{Debug, Formatter}; use std::ops::Deref; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; - -use arc_swap::ArcSwap; -use bytes::Bytes; -use derive_getters::Getters; -use futures::StreamExt; use tokio::sync::{Mutex, MutexGuard}; use tracing_futures::Instrument; @@ -81,7 +81,9 @@ where } pub fn new_txn(&self) -> anyhow::Result> { - todo!() + let mvcc = self.mvcc.as_ref().ok_or(anyhow!("no mvcc"))?; + let tx = mvcc.new_txn(self.inner.load(), false); + Ok(tx) } } From bcbadeaf391ac1d1436c908a5699ab788abd2be7 Mon Sep 17 00:00:00 2001 From: Kermit Date: Tue, 23 Jul 2024 22:29:27 +0800 Subject: [PATCH 34/69] fix: txn iterator --- src/iterators/lsm.rs | 2 +- src/mvcc/core.rs | 8 +- src/mvcc/iterator.rs | 27 ++--- src/mvcc/transaction.rs | 24 ++--- src/state/states.rs | 220 ++++++++++++---------------------------- 5 files changed, 96 insertions(+), 185 deletions(-) diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index bf7aacb..0533d2b 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -32,7 +32,7 @@ type LsmIteratorInner<'a, File> = TwoMergeIterator< #[derive(new)] pub struct LockedLsmIter<'a, P: Persistent> { - state: arc_swap::Guard>>, + state: Arc>, pub(crate) lower: Bound<&'a [u8]>, pub(crate) upper: Bound<&'a [u8]>, timestamp: u64, diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 0ea7a71..d95b111 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -46,7 +46,7 @@ impl LsmMvccInner { pub fn new_txn( &self, - inner: arc_swap::Guard>>, + inner: Arc>, serializable: bool, ) -> Transaction

{ // todo: use external dependency to get time @@ -54,8 +54,8 @@ impl LsmMvccInner { .duration_since(UNIX_EPOCH) .unwrap() .as_secs(); - let key_hashes = serializable.then(|| Mutex::default()); - let tx = Transaction::new(ts, inner, key_hashes); - tx + let key_hashes = serializable.then(Mutex::default); + + Transaction::new(ts, inner, key_hashes) } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 991fdb3..0de4150 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,6 +1,13 @@ +use crate::bound::BoundRange; +use crate::entry::{Entry, Keyed}; +use crate::iterators::lsm::LsmIterator; +use crate::iterators::no_deleted::new_no_deleted_iter; +use crate::iterators::{create_two_merge_iter, LockedLsmIter}; +use crate::persistent::Persistent; use async_iter_ext::StreamTools; use bytes::Bytes; use crossbeam_skiplist::SkipMap; +use derive_new::new; use futures::{stream, Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; use std::collections::Bound::{Excluded, Included, Unbounded}; @@ -8,15 +15,9 @@ use std::future::ready; use std::ops::Bound; use std::sync::Arc; -use crate::bound::BoundRange; -use crate::entry::{Entry, Keyed}; -use crate::iterators::lsm::LsmIterator; -use crate::iterators::no_deleted::new_no_deleted_iter; -use crate::iterators::{create_two_merge_iter, LockedLsmIter}; -use crate::persistent::Persistent; - +#[derive(new)] pub struct LockedTxnIter<'a, P: Persistent> { - local_storage: arc_swap::Guard>>, + local_storage: &'a SkipMap, lsm_iter: LockedLsmIter<'a, P>, } @@ -24,7 +25,7 @@ impl<'a, P: Persistent> LockedTxnIter<'a, P> { pub async fn iter(&'a self) -> anyhow::Result> { let lsm_iter = self.lsm_iter.iter_with_delete().await?; let local_iter = txn_local_iterator( - self.local_storage.as_ref(), + self.local_storage, self.lsm_iter.lower.map(Bytes::copy_from_slice), self.lsm_iter.upper.map(Bytes::copy_from_slice), ); @@ -35,11 +36,11 @@ impl<'a, P: Persistent> LockedTxnIter<'a, P> { } } -pub fn txn_local_iterator<'a>( - map: &'a SkipMap, +pub fn txn_local_iterator( + map: &SkipMap, lower: Bound, upper: Bound, -) -> impl Stream> + Unpin + Send + 'a { +) -> impl Stream> + Unpin + Send + '_ { let it = map.range((lower, upper)).map(|entry| { let key = entry.key().clone(); let value = entry.value().clone(); @@ -149,7 +150,7 @@ mod tests { I::IntoIter: Send, S: IntoIterator>, { - let s = stream::iter(s).map(|pair| Ok::<_, ()>(pair)); + let s = stream::iter(s).map(Ok::<_, ()>); let result: Vec<_> = build_time_dedup_iter(s, timestamp_upper) .try_collect() .await diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index bfdc408..cf21bad 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -7,14 +7,16 @@ use crossbeam_skiplist::SkipMap; use futures::{stream, Stream}; use std::collections::{Bound, HashSet}; -use crate::mvcc::iterator::txn_local_iterator; +use crate::iterators::LockedLsmIter; +use crate::mvcc::iterator::{txn_local_iterator, LockedTxnIter}; +use arc_swap::access::Access; use parking_lot::Mutex; use std::sync::atomic::AtomicBool; use std::sync::Arc; pub struct Transaction { pub(crate) read_ts: u64, - pub(crate) inner: arc_swap::Guard>>, + pub(crate) inner: Arc>, // todo: need Arc<...> ? pub(crate) local_storage: Arc>, @@ -47,7 +49,7 @@ impl Map for Transaction

{ impl Transaction

{ pub fn new( read_ts: u64, - inner: arc_swap::Guard>>, + inner: Arc>, key_hashes: Option, HashSet)>>, ) -> Self { Self { @@ -59,19 +61,15 @@ impl Transaction

{ } } + // todo: no need for Result? pub fn scan<'a>( - &self, + &'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, - ) -> anyhow::Result>> { - let local_iterator = txn_local_iterator( - &self.local_storage, - lower.map(Bytes::copy_from_slice), - upper.map(Bytes::copy_from_slice), - ); - - // todo - Ok(stream::empty()) + ) -> anyhow::Result> { + let inner_iter = LockedLsmIter::new(self.inner.clone(), lower, upper, self.read_ts); // todo + let guard = LockedTxnIter::new(&self.local_storage, inner_iter); + Ok(guard) } pub async fn commit(self) -> anyhow::Result<()> { diff --git a/src/state/states.rs b/src/state/states.rs index 13765b7..b051762 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -82,7 +82,7 @@ where pub fn new_txn(&self) -> anyhow::Result> { let mvcc = self.mvcc.as_ref().ok_or(anyhow!("no mvcc"))?; - let tx = mvcc.new_txn(self.inner.load(), false); + let tx = mvcc.new_txn(self.inner.load_full(), false); Ok(tx) } } @@ -140,7 +140,7 @@ where } fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { - let snapshot = self.inner.load(); + let snapshot = self.inner.load_full(); LockedLsmIter::new(snapshot, lower, upper, 0) // todo } } @@ -346,6 +346,7 @@ mod test { use crate::iterators::utils::test_utils::{ assert_stream_eq, build_stream, build_tuple_stream, eq, }; + use crate::mvcc::transaction::Transaction; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; use crate::sst::SstOptions; @@ -814,14 +815,7 @@ mod test { ); assert_eq!(snapshot1.get_for_test(b"c").await.unwrap(), None); - { - let iter = snapshot1.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "1"), ("b", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot1, Unbounded, Unbounded, [("a", "1"), ("b", "1")]).await; assert_eq!( snapshot2.get_for_test(b"a").await.unwrap(), @@ -833,14 +827,7 @@ mod test { ); assert_eq!(snapshot2.get_for_test(b"c").await.unwrap(), None); - { - let iter = snapshot2.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "2"), ("b", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot2, Unbounded, Unbounded, [("a", "2"), ("b", "1")]).await; assert_eq!( snapshot3.get_for_test(b"a").await.unwrap(), @@ -852,14 +839,7 @@ mod test { Some(Bytes::from_static(b"1")) ); - { - let iter = snapshot3.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "2"), ("c", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot3, Unbounded, Unbounded, [("a", "2"), ("c", "1")]).await; if !flush { let guard = storage.state_lock.lock().await; @@ -890,14 +870,7 @@ mod test { ); assert_eq!(snapshot1.get_for_test(b"c").await.unwrap(), None); - { - let iter = snapshot1.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "1"), ("b", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot1, Unbounded, Unbounded, [("a", "1"), ("b", "1")]).await; assert_eq!( snapshot2.get_for_test(b"a").await.unwrap(), @@ -909,14 +882,7 @@ mod test { ); assert_eq!(snapshot2.get_for_test(b"c").await.unwrap(), None); - { - let iter = snapshot2.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "2"), ("b", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot2, Unbounded, Unbounded, [("a", "2"), ("b", "1")]).await; assert_eq!( snapshot3.get_for_test(b"a").await.unwrap(), @@ -928,14 +894,7 @@ mod test { Some(Bytes::from_static(b"1")) ); - { - let iter = snapshot3.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "2"), ("c", "1")]), - ) - .await; - } + assert_scan_iter(&snapshot3, Unbounded, Unbounded, [("a", "2"), ("c", "1")]).await; assert_eq!( snapshot4.get_for_test(b"a").await.unwrap(), @@ -950,14 +909,13 @@ mod test { Some(Bytes::from_static(b"1")) ); - { - let iter = snapshot4.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "3"), ("b", "3"), ("c", "1")]), - ) - .await; - } + assert_scan_iter( + &snapshot4, + Unbounded, + Unbounded, + [("a", "3"), ("b", "3"), ("c", "1")], + ) + .await; assert_eq!( snapshot5.get_for_test(b"a").await.unwrap(), @@ -972,14 +930,13 @@ mod test { Some(Bytes::from_static(b"1")) ); - { - let iter = snapshot4.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "4"), ("b", "3"), ("c", "1")]), - ) - .await; - } + assert_scan_iter( + &snapshot4, + Unbounded, + Unbounded, + [("a", "4"), ("b", "3"), ("c", "1")], + ) + .await; assert_eq!( snapshot6.get_for_test(b"a").await.unwrap(), @@ -991,33 +948,11 @@ mod test { Some(Bytes::from_static(b"5")) ); - { - let iter = snapshot6.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "4"), ("c", "5")]), - ) - .await; - } + assert_scan_iter(&snapshot6, Unbounded, Unbounded, [("a", "4"), ("c", "5")]).await; if flush { - { - let iter = snapshot6.scan(Included(b"a"), Included(b"a")).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("a", "4")]), - ) - .await; - } - - { - let iter = snapshot6.scan(Excluded(b"a"), Excluded(b"c")).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([]), - ) - .await; - } + assert_scan_iter(&snapshot6, Included(b"a"), Included(b"a"), [("a", "4")]).await; + assert_scan_iter(&snapshot6, Excluded(b"a"), Excluded(b"c"), []).await; } } @@ -1192,59 +1127,30 @@ mod test { txn1.put_for_test(b"test1", b"233").await.unwrap(); txn2.put_for_test(b"test2", b"233").await.unwrap(); - { - let iter = txn1.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("test1", "233")]), - ) - .await; - } - - { - let iter = txn2.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("test2", "233")]), - ) - .await; - } + assert_scan_iter(&txn1, Unbounded, Unbounded, [("test1", "233")]).await; + assert_scan_iter(&txn2, Unbounded, Unbounded, [("test2", "233")]).await; let txn3 = storage.new_txn().unwrap(); - { - let iter = txn3.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([]), - ) - .await; - } + assert_scan_iter(&txn3, Unbounded, Unbounded, []).await; txn1.commit().await.unwrap(); txn2.commit().await.unwrap(); + assert_scan_iter(&txn3, Unbounded, Unbounded, []).await; + + drop(txn3); + { - let iter = txn3.scan(Unbounded, Unbounded).unwrap(); + let guard = storage.scan(Unbounded, Unbounded); + let iter = guard.iter().await.unwrap(); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([]), + build_tuple_stream([("test1", "233"), ("test2", "233")]), ) .await; } - drop(txn3); - - // todo: add this test - // { - // let iter = storage.scan(Unbounded, Unbounded); - // assert_stream_eq( - // iter.iter().await.unwrap().map(Result::unwrap).map(Entry::into_tuple), - // build_tuple_stream([("test1", "233"), ("test2", "233")]), - // ) - // .await; - // } - let txn4 = storage.new_txn().unwrap(); assert_eq!( @@ -1256,14 +1162,13 @@ mod test { Some(Bytes::from("233")) ); - { - let iter = txn4.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("test1", "233"), ("test2", "233")]), - ) - .await; - } + assert_scan_iter( + &txn4, + Unbounded, + Unbounded, + [("test1", "233"), ("test2", "233")], + ) + .await; txn4.put_for_test(b"test2", b"2333").await.unwrap(); assert_eq!( @@ -1275,14 +1180,13 @@ mod test { Some(Bytes::from("2333")) ); - { - let iter = txn4.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("test1", "233"), ("test2", "2333")]), - ) - .await; - } + assert_scan_iter( + &txn4, + Unbounded, + Unbounded, + [("test1", "233"), ("test2", "2333")], + ) + .await; txn4.delete_for_test(b"test2").await.unwrap(); @@ -1292,13 +1196,21 @@ mod test { ); assert_eq!(txn4.get_for_test(b"test2").await.unwrap(), None); - { - let iter = txn4.scan(Unbounded, Unbounded).unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("test1", "233")]), - ) - .await; - } + assert_scan_iter(&txn4, Unbounded, Unbounded, [("test1", "233")]).await; + } + + async fn assert_scan_iter( + snapshot: &Transaction

, + lower: Bound<&[u8]>, + upper: Bound<&[u8]>, + expected: impl IntoIterator, + ) { + let guard = snapshot.scan(lower, upper).unwrap(); + let iter = guard.iter().await.unwrap(); + assert_stream_eq( + iter.map(Result::unwrap).map(Entry::into_tuple), + build_tuple_stream(expected), + ) + .await; } } From a91ee88812b65c7ff841986792b2d1879a269292 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 25 Jul 2024 22:58:46 +0800 Subject: [PATCH 35/69] wip --- src/mvcc/core.rs | 6 ++---- src/sst/option.rs | 3 +++ src/state/states.rs | 11 ++++++++++- src/utils/mod.rs | 1 + src/utils/time.rs | 6 ++++++ 5 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 src/utils/time.rs diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index d95b111..9d42a8d 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -2,6 +2,7 @@ use crate::mvcc::transaction::Transaction; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; use crate::state::{LsmStorageState, LsmStorageStateInner}; +use crate::utils::time::now_unix; use parking_lot::Mutex; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -50,10 +51,7 @@ impl LsmMvccInner { serializable: bool, ) -> Transaction

{ // todo: use external dependency to get time - let ts = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); + let ts = now_unix().unwrap(); let key_hashes = serializable.then(Mutex::default); Transaction::new(ts, inner, key_hashes) diff --git a/src/sst/option.rs b/src/sst/option.rs index 7b7c664..dee50a0 100644 --- a/src/sst/option.rs +++ b/src/sst/option.rs @@ -12,4 +12,7 @@ pub struct SstOptions { num_memtable_limit: usize, compaction_option: CompactionOptions, enable_wal: bool, + + #[builder(default)] + enable_mvcc: bool, } diff --git a/src/state/states.rs b/src/state/states.rs index b051762..67c8aed 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -22,6 +22,7 @@ use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::LsmStorageStateInner; use crate::state::Map; +use crate::utils::time::now_unix; use crate::utils::vec::pop; #[derive(Getters)] @@ -67,6 +68,13 @@ where .await?; let sst_id = AtomicUsize::new(next_sst_id); + let mvcc = if *options.enable_mvcc() { + // todo: use external dependency to get time + Some(LsmMvccInner::new(now_unix()?)) + } else { + None + }; + let this = Self { inner: ArcSwap::new(Arc::new(inner)), block_cache: Arc::new(BlockCache::new(1024)), @@ -75,7 +83,7 @@ where persistent, options, sst_id, - mvcc: None, // todo + mvcc, }; Ok(this) } @@ -794,6 +802,7 @@ mod test { .num_memtable_limit(1000) .compaction_option(Default::default()) .enable_wal(true) + .enable_mvcc(true) .build(); let storage = LsmStorageState::new(options, persistent).await.unwrap(); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index ae9e6a4..2c82e73 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod func; pub mod num; +pub mod time; pub mod vec; diff --git a/src/utils/time.rs b/src/utils/time.rs new file mode 100644 index 0000000..8c567ce --- /dev/null +++ b/src/utils/time.rs @@ -0,0 +1,6 @@ +use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH}; + +pub fn now_unix() -> Result { + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + Ok(now) +} From 14c1a54afcce0d6e1e3cb31c40e53fa35bcaf9bb Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 25 Jul 2024 23:05:00 +0800 Subject: [PATCH 36/69] wip --- src/mvcc/transaction.rs | 28 +++++++++++++++++++--------- src/state/states.rs | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index cf21bad..de45cf5 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -13,6 +13,7 @@ use arc_swap::access::Access; use parking_lot::Mutex; use std::sync::atomic::AtomicBool; use std::sync::Arc; +use tokio_stream::StreamExt; pub struct Transaction { pub(crate) read_ts: u64, @@ -29,20 +30,29 @@ pub struct Transaction { impl Map for Transaction

{ type Error = anyhow::Error; - async fn get(&self, _key: &[u8]) -> Result, Self::Error> { - todo!() + async fn get(&self, key: &[u8]) -> Result, Self::Error> { + let guard = self.scan(Bound::Included(key), Bound::Included(key)); + let output = guard + .iter() + .await? + .next() + .await + .transpose()? + .map(|entry| entry.value); + Ok(output) } async fn put( &self, - _key: impl Into + Send, - _value: impl Into + Send, + key: impl Into + Send, + value: impl Into + Send, ) -> Result<(), Self::Error> { - todo!() + self.local_storage.insert(key.into(), value.into()); + Ok(()) } - async fn delete(&self, _key: impl Into + Send) -> Result<(), Self::Error> { - todo!() + async fn delete(&self, key: impl Into + Send) -> Result<(), Self::Error> { + self.put(key, Bytes::new()).await } } @@ -66,10 +76,10 @@ impl Transaction

{ &'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, - ) -> anyhow::Result> { + ) -> LockedTxnIter<'a, P> { let inner_iter = LockedLsmIter::new(self.inner.clone(), lower, upper, self.read_ts); // todo let guard = LockedTxnIter::new(&self.local_storage, inner_iter); - Ok(guard) + guard } pub async fn commit(self) -> anyhow::Result<()> { diff --git a/src/state/states.rs b/src/state/states.rs index 67c8aed..81d7b8b 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1214,7 +1214,7 @@ mod test { upper: Bound<&[u8]>, expected: impl IntoIterator, ) { - let guard = snapshot.scan(lower, upper).unwrap(); + let guard = snapshot.scan(lower, upper); let iter = guard.iter().await.unwrap(); assert_stream_eq( iter.map(Result::unwrap).map(Entry::into_tuple), From 90ef246d965889a4a0930dcbca8e0150e32360eb Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 25 Jul 2024 23:14:05 +0800 Subject: [PATCH 37/69] wip: time provider --- src/lib.rs | 1 + src/state/states.rs | 6 ++++++ src/time/mod.rs | 13 +++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 src/time/mod.rs diff --git a/src/lib.rs b/src/lib.rs index 93c02f2..60b721b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,4 +15,5 @@ mod lsm; mod manifest; pub mod mvcc; mod test_utils; +pub mod time; mod utils; diff --git a/src/state/states.rs b/src/state/states.rs index 81d7b8b..2102b55 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -808,6 +808,12 @@ mod test { storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); + + assert_eq!( + storage.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + let snapshot1 = storage.new_txn().unwrap(); storage.put_for_test(b"a", b"2").await.unwrap(); let snapshot2 = storage.new_txn().unwrap(); diff --git a/src/time/mod.rs b/src/time/mod.rs new file mode 100644 index 0000000..a4075f9 --- /dev/null +++ b/src/time/mod.rs @@ -0,0 +1,13 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +pub trait TimeProvider: Send + Sync { + fn now(&self) -> u64; +} + +pub struct TimeIncrement(AtomicU64); + +impl TimeProvider for TimeIncrement { + fn now(&self) -> u64 { + self.0.fetch_add(1, Ordering::Relaxed) + } +} From 9b2c4233e16faf72f6a25440da78a87b8d0b9ef9 Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 2 Aug 2024 13:48:57 +0800 Subject: [PATCH 38/69] feat: use external time provider --- benches/ycsb.rs | 8 ++++++-- src/lsm/core.rs | 32 +++++++++++++++++++++----------- src/mvcc/core.rs | 22 ++++++++++++++-------- src/sst/compact/leveled.rs | 5 ++++- src/state/states.rs | 27 +++++++++++++++++++-------- src/time/mod.rs | 17 ++++++++++++++++- src/utils/mod.rs | 1 - src/utils/time.rs | 6 ------ 8 files changed, 80 insertions(+), 38 deletions(-) delete mode 100644 src/utils/time.rs diff --git a/benches/ycsb.rs b/benches/ycsb.rs index 604f80f..ff73d3c 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -12,6 +12,7 @@ use ycsb::workload::CoreWorkload; use better_mini_lsm::persistent::LocalFs; use better_mini_lsm::sst::SstOptions; use better_mini_lsm::state::{LsmStorageState, Map}; +use better_mini_lsm::time::SystemTime; #[derive(Clone)] struct LsmStorageStateBench(Arc>); @@ -63,8 +64,11 @@ fn ycsb_bench(c: &mut Criterion) { .enable_wal(false) .build(); let runtime = tokio::runtime::Runtime::new().unwrap(); - let state = - runtime.block_on(async { LsmStorageState::new(options, persistent).await.unwrap() }); + let state = runtime.block_on(async { + LsmStorageState::new(options, persistent, Box::new(SystemTime)) + .await + .unwrap() + }); let database = LsmStorageStateBench(Arc::new(state)); let props = Properties { operation_count: 1000, diff --git a/src/lsm/core.rs b/src/lsm/core.rs index b8c29e1..07e8f16 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -8,24 +8,28 @@ use bytes::Bytes; use futures::{FutureExt, StreamExt}; use futures_concurrency::stream::Merge; +use crate::mvcc::core::TimeProviderWrapper; +use crate::persistent::Persistent; +use crate::sst::SstOptions; +use crate::state::{LsmStorageState, Map}; use tokio::task::JoinHandle; use tokio::time::interval; use tokio_stream::wrappers::IntervalStream; use tokio_util::sync::CancellationToken; use tracing::error; -use crate::persistent::Persistent; -use crate::sst::SstOptions; -use crate::state::{LsmStorageState, Map}; - pub struct Lsm { state: Arc>, cancel_token: CancellationToken, } impl Lsm

{ - pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { - let state = Arc::new(LsmStorageState::new(options, persistent).await?); + pub async fn new( + options: SstOptions, + persistent: P, + time_provider: TimeProviderWrapper, + ) -> anyhow::Result { + let state = Arc::new(LsmStorageState::new(options, persistent, time_provider).await?); let cancel_token = CancellationToken::new(); let _ = Self::spawn_flush(state.clone(), cancel_token.clone()); let _ = Self::spawn_compaction(state.clone(), cancel_token.clone()); @@ -150,7 +154,7 @@ mod tests { use crate::sst::SstOptions; use crate::state::Map; use crate::test_utils::insert_sst; - + use crate::time::TimeIncrement; // todo: WAL causes the "too many open files" error // #[tokio::test] // async fn test_task2_auto_flush() { @@ -192,7 +196,7 @@ mod tests { .compaction_option(Default::default()) .enable_wal(false) .build(); - Lsm::new(options, persistent).await + Lsm::new(options, persistent, Box::::default()).await } #[tokio::test] @@ -211,7 +215,9 @@ mod tests { .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) .build(); - let lsm = Lsm::new(options, persistent).await.unwrap(); + let lsm = Lsm::new(options, persistent, Box::::default()) + .await + .unwrap(); for i in 0..10 { let begin = i * 100; insert_sst(&lsm, begin..begin + 100).await.unwrap(); @@ -247,7 +253,9 @@ mod tests { .build(); let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options.clone(), persistent).await.unwrap(); + let lsm = Lsm::new(options.clone(), persistent, Box::::default()) + .await + .unwrap(); add_data(&lsm).await.unwrap(); sleep(Duration::from_secs(2)).await; @@ -260,7 +268,9 @@ mod tests { { let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options, persistent).await.unwrap(); + let lsm = Lsm::new(options, persistent, Box::::default()) + .await + .unwrap(); assert_eq!( &lsm.get(b"key-0").await.unwrap().unwrap()[..], b"value-1024".as_slice() diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 9d42a8d..fcb8fd1 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -1,12 +1,13 @@ +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; + +use parking_lot::Mutex; + use crate::mvcc::transaction::Transaction; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; -use crate::state::{LsmStorageState, LsmStorageStateInner}; -use crate::utils::time::now_unix; -use parking_lot::Mutex; -use std::collections::{BTreeMap, HashSet}; -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use crate::state::LsmStorageStateInner; +use crate::time::TimeProvider; pub(crate) struct CommittedTxnData { pub(crate) key_hashes: HashSet, @@ -14,20 +15,25 @@ pub(crate) struct CommittedTxnData { pub(crate) commit_ts: u64, } +pub type TimeProviderWrapper = Box; + pub(crate) struct LsmMvccInner { pub(crate) write_lock: Mutex<()>, pub(crate) commit_lock: Mutex<()>, pub(crate) ts: Arc>, pub(crate) committed_txns: Arc>>, + time_provider: TimeProviderWrapper, } impl LsmMvccInner { - pub fn new(initial_ts: u64) -> Self { + pub fn new(time_provider: TimeProviderWrapper) -> Self { + let initial_ts = time_provider.now(); Self { write_lock: Mutex::new(()), commit_lock: Mutex::new(()), ts: Arc::new(Mutex::new((initial_ts, Watermark::new()))), committed_txns: Arc::new(Mutex::new(BTreeMap::new())), + time_provider, } } @@ -51,7 +57,7 @@ impl LsmMvccInner { serializable: bool, ) -> Transaction

{ // todo: use external dependency to get time - let ts = now_unix().unwrap(); + let ts = self.time_provider.now(); let key_hashes = serializable.then(Mutex::default); Transaction::new(ts, inner, key_hashes) diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 2761720..ba16b15 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -226,6 +226,7 @@ mod tests { use crate::sst::{SstOptions, Sstables}; use crate::state::{LsmStorageState, Map}; use crate::test_utils::insert_sst; + use crate::time::TimeIncrement; #[test] fn test_select_level_source() { @@ -364,7 +365,9 @@ mod tests { .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) .build(); - let state = LsmStorageState::new(options, persistent).await.unwrap(); + let state = LsmStorageState::new(options, persistent, Box::::default()) + .await + .unwrap(); let _next_sst_id = AtomicUsize::default(); let state_lock = Mutex::default(); diff --git a/src/state/states.rs b/src/state/states.rs index 2102b55..f538d59 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -15,14 +15,14 @@ use crate::block::BlockCache; use crate::iterators::LockedLsmIter; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; -use crate::mvcc::core::LsmMvccInner; +use crate::mvcc::core::{LsmMvccInner, TimeProviderWrapper}; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::LsmStorageStateInner; use crate::state::Map; -use crate::utils::time::now_unix; +use crate::time::TimeProvider; use crate::utils::vec::pop; #[derive(Getters)] @@ -55,7 +55,11 @@ impl

LsmStorageState

where P: Persistent, { - pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { + pub async fn new( + options: SstOptions, + persistent: P, + time_provider: TimeProviderWrapper, + ) -> anyhow::Result { let (manifest, manifest_records) = Manifest::recover(&persistent).await?; let block_cache = Arc::new(BlockCache::new(1024)); let (inner, next_sst_id) = LsmStorageStateInner::recover( @@ -70,7 +74,7 @@ where let mvcc = if *options.enable_mvcc() { // todo: use external dependency to get time - Some(LsmMvccInner::new(now_unix()?)) + Some(LsmMvccInner::new(time_provider)) } else { None }; @@ -360,6 +364,7 @@ mod test { use crate::sst::SstOptions; use crate::state::states::LsmStorageState; use crate::test_utils::iterator::unwrap_ts_stream; + use crate::time::TimeIncrement; #[tokio::test] async fn test_task2_storage_integration() { @@ -780,7 +785,7 @@ mod test { .compaction_option(Default::default()) .enable_wal(false) .build(); - LsmStorageState::new(options, persistent).await + LsmStorageState::new(options, persistent, Box::::default()).await } #[tokio::test] @@ -804,7 +809,9 @@ mod test { .enable_wal(true) .enable_mvcc(true) .build(); - let storage = LsmStorageState::new(options, persistent).await.unwrap(); + let storage = LsmStorageState::new(options, persistent, Box::::default()) + .await + .unwrap(); storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); @@ -982,7 +989,9 @@ mod test { .compaction_option(Default::default()) .enable_wal(true) .build(); - let storage = LsmStorageState::new(options, persistent).await.unwrap(); + let storage = LsmStorageState::new(options, persistent, Box::::default()) + .await + .unwrap(); let txn1 = storage.new_txn().unwrap(); let txn2 = storage.new_txn().unwrap(); @@ -1135,7 +1144,9 @@ mod test { .compaction_option(Default::default()) .enable_wal(true) .build(); - let storage = LsmStorageState::new(options, persistent).await.unwrap(); + let storage = LsmStorageState::new(options, persistent, Box::::default()) + .await + .unwrap(); let txn1 = storage.new_txn().unwrap(); let txn2 = storage.new_txn().unwrap(); diff --git a/src/time/mod.rs b/src/time/mod.rs index a4075f9..8130392 100644 --- a/src/time/mod.rs +++ b/src/time/mod.rs @@ -1,9 +1,11 @@ use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::UNIX_EPOCH; -pub trait TimeProvider: Send + Sync { +pub trait TimeProvider: Send + Sync + 'static { fn now(&self) -> u64; } +#[derive(Default)] pub struct TimeIncrement(AtomicU64); impl TimeProvider for TimeIncrement { @@ -11,3 +13,16 @@ impl TimeProvider for TimeIncrement { self.0.fetch_add(1, Ordering::Relaxed) } } + +#[derive(Default)] +pub struct SystemTime; + +impl TimeProvider for SystemTime { + fn now(&self) -> u64 { + // todo: remove unwrap? + std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 2c82e73..ae9e6a4 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,4 +1,3 @@ pub mod func; pub mod num; -pub mod time; pub mod vec; diff --git a/src/utils/time.rs b/src/utils/time.rs deleted file mode 100644 index 8c567ce..0000000 --- a/src/utils/time.rs +++ /dev/null @@ -1,6 +0,0 @@ -use std::time::{SystemTime, SystemTimeError, UNIX_EPOCH}; - -pub fn now_unix() -> Result { - let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); - Ok(now) -} From 41130b7c78cd3e658fa2fef5f7b54cfe16e5208a Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 2 Aug 2024 14:16:49 +0800 Subject: [PATCH 39/69] fix: scan with ts --- src/mvcc/core.rs | 8 ++------ src/mvcc/transaction.rs | 2 +- src/state/states.rs | 20 +++++++++++++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index fcb8fd1..7545764 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -22,18 +22,15 @@ pub(crate) struct LsmMvccInner { pub(crate) commit_lock: Mutex<()>, pub(crate) ts: Arc>, pub(crate) committed_txns: Arc>>, - time_provider: TimeProviderWrapper, } impl LsmMvccInner { - pub fn new(time_provider: TimeProviderWrapper) -> Self { - let initial_ts = time_provider.now(); + pub fn new(initial_ts: u64) -> Self { Self { write_lock: Mutex::new(()), commit_lock: Mutex::new(()), ts: Arc::new(Mutex::new((initial_ts, Watermark::new()))), committed_txns: Arc::new(Mutex::new(BTreeMap::new())), - time_provider, } } @@ -55,9 +52,8 @@ impl LsmMvccInner { &self, inner: Arc>, serializable: bool, + ts: u64, ) -> Transaction

{ - // todo: use external dependency to get time - let ts = self.time_provider.now(); let key_hashes = serializable.then(Mutex::default); Transaction::new(ts, inner, key_hashes) diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index de45cf5..0bd63c3 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -77,7 +77,7 @@ impl Transaction

{ lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>, ) -> LockedTxnIter<'a, P> { - let inner_iter = LockedLsmIter::new(self.inner.clone(), lower, upper, self.read_ts); // todo + let inner_iter = LockedLsmIter::new(self.inner.clone(), lower, upper, self.read_ts); let guard = LockedTxnIter::new(&self.local_storage, inner_iter); guard } diff --git a/src/state/states.rs b/src/state/states.rs index f538d59..6703d39 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -35,6 +35,7 @@ pub struct LsmStorageState { pub(crate) options: SstOptions, pub(crate) sst_id: AtomicUsize, mvcc: Option, + time_provider: TimeProviderWrapper, } impl

Debug for LsmStorageState

@@ -74,7 +75,7 @@ where let mvcc = if *options.enable_mvcc() { // todo: use external dependency to get time - Some(LsmMvccInner::new(time_provider)) + Some(LsmMvccInner::new(time_provider.now())) } else { None }; @@ -88,13 +89,14 @@ where options, sst_id, mvcc, + time_provider, }; Ok(this) } pub fn new_txn(&self) -> anyhow::Result> { let mvcc = self.mvcc.as_ref().ok_or(anyhow!("no mvcc"))?; - let tx = mvcc.new_txn(self.inner.load_full(), false); + let tx = mvcc.new_txn(self.inner.load_full(), false, self.time_provider.now()); Ok(tx) } } @@ -153,7 +155,7 @@ where fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { let snapshot = self.inner.load_full(); - LockedLsmIter::new(snapshot, lower, upper, 0) // todo + LockedLsmIter::new(snapshot, lower, upper, self.time_provider.now()) } } @@ -822,6 +824,18 @@ mod test { ); let snapshot1 = storage.new_txn().unwrap(); + { + let guard = snapshot1.scan(Included(b"a"), Included(b"a")); + let mut iter = guard.iter().await.unwrap(); + while let Some(elem) = iter.next().await { + let elem = elem.unwrap(); + dbg!(elem); + } + } + assert_eq!( + snapshot1.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); storage.put_for_test(b"a", b"2").await.unwrap(); let snapshot2 = storage.new_txn().unwrap(); storage.delete_for_test(b"b").await.unwrap(); From 7e500e012439dd8ce3af114ea9def1ea5b254d7e Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 2 Aug 2024 14:43:35 +0800 Subject: [PATCH 40/69] wip --- src/iterators/lsm.rs | 1 + src/memtable/immutable.rs | 4 --- src/memtable/mutable.rs | 69 ++++++++++++++++++++++++++++++++------- src/state/states.rs | 22 ++++++++----- 4 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/iterators/lsm.rs b/src/iterators/lsm.rs index 0533d2b..1cade1c 100644 --- a/src/iterators/lsm.rs +++ b/src/iterators/lsm.rs @@ -81,6 +81,7 @@ where let imm_memtables = imm_memtables.iter().map(Arc::as_ref); let tables = iter::once(memtable).chain(imm_memtables); let iters = stream::iter(tables).filter_map(move |table| { + // todo: 这里不用每个 loop 都 copy，可以放在外面？ let lower = lower.map(|ks| ks.map(Bytes::copy_from_slice)); let upper = upper.map(|ks| ks.map(Bytes::copy_from_slice)); diff --git a/src/memtable/immutable.rs b/src/memtable/immutable.rs index 37aa595..dd3a789 100644 --- a/src/memtable/immutable.rs +++ b/src/memtable/immutable.rs @@ -60,10 +60,6 @@ impl ImmutableMemTable { self.0.get_with_ts(key) } - pub async fn put_with_ts(&self, key: KeyBytes, value: Bytes) -> anyhow::Result<()> { - self.0.put_with_ts(key, value).await - } - pub async fn scan_with_ts( &self, lower: Bound, diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index f33b113..faa8f1c 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -112,15 +112,6 @@ impl MemTable { self.get_with_ts(KeySlice::new(key, 0)) } - /// Put a key-value pair into the mem-table. - /// - /// In week 1, day 1, simply put the key-value pair into the skipmap. - /// In week 2, day 6, also flush the data to WAL. - /// todo: remote this method - pub async fn put(&self, key: Bytes, value: Bytes) -> anyhow::Result<()> { - self.put_with_ts(KeyBytes::new(key, 0), value).await - } - pub async fn sync_wal(&self) -> anyhow::Result<()> { if let Some(ref wal) = self.wal { wal.sync().await?; @@ -207,6 +198,15 @@ impl MemTable { ) .await } + + /// Put a key-value pair into the mem-table. + /// + /// In week 1, day 1, simply put the key-value pair into the skipmap. + /// In week 2, day 6, also flush the data to WAL. + /// todo: remote this method + pub async fn put(&self, key: Bytes, value: Bytes) -> anyhow::Result<()> { + self.put_with_ts(KeyBytes::new(key, 0), value).await + } } impl MemTable { @@ -225,11 +225,16 @@ impl MemTable { #[cfg(test)] mod test { - use tempfile::tempdir; - + use std::ops::Bound::Included; + use crate::key::Key; use crate::manifest::Manifest; use crate::memtable::mutable::MemTable; use crate::persistent::LocalFs; + use crate::time::{TimeIncrement, TimeProvider}; + use bytes::Bytes; + use futures::StreamExt; + use tempfile::tempdir; + use crate::mvcc::iterator::transform_bound; #[tokio::test] async fn test_task1_memtable_get_wal() { @@ -330,4 +335,46 @@ mod test { ); } } + + #[tokio::test] + async fn test_memtable_mvcc() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let manifest = Manifest::create(&persistent).await.unwrap(); + let id = 123; + + let memtable = MemTable::create_with_wal(id, &persistent, &manifest) + .await + .unwrap(); + let time_provider = Box::::default(); + + memtable + .put_with_ts( + Key::new(Bytes::copy_from_slice(b"key1"), time_provider.now()), + Bytes::copy_from_slice(b"value1"), + ) + .await + .unwrap(); + + { + let (lower, upper) = transform_bound(Included(b"key1"), Included(b"key1"), time_provider.now()); + let lower = lower.map(Key::from); + let upper = upper.map(Key::from); + let lower = lower.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); + let upper = upper.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); + let mut iter = memtable.scan_with_ts(lower, upper).await.unwrap(); + + let (new_iter, elem) = iter.unwrap().next().await; + dbg!(elem); + let new_iter = new_iter.unwrap(); + // iter = new_iter; + assert!(new_iter.is_none()); + + + // let (new_iter, elem) = iter.unwrap().next().await; + // dbg!(elem); + // let new_iter = new_iter.unwrap(); + // iter = new_iter; + } + } } diff --git a/src/state/states.rs b/src/state/states.rs index 6703d39..7f0460e 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -13,6 +13,7 @@ use tracing_futures::Instrument; use crate::block::BlockCache; use crate::iterators::LockedLsmIter; +use crate::key::KeyBytes; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; use crate::mvcc::core::{LsmMvccInner, TimeProviderWrapper}; @@ -126,9 +127,10 @@ where value: impl Into + Send, ) -> anyhow::Result<()> { let snapshot = self.inner.load(); + let key = KeyBytes::new(key.into(), self.time_provider.now()); snapshot .memtable() - .put(key.into(), value.into()) + .put_with_ts(key, value.into()) .instrument(tracing::info_span!("memtable_put")) .await?; self.try_freeze_memtable(&snapshot) @@ -139,7 +141,8 @@ where async fn delete(&self, key: impl Into + Send) -> anyhow::Result<()> { let snapshot = self.inner.load(); - snapshot.memtable().put(key.into(), Bytes::new()).await?; + let key = KeyBytes::new(key.into(), self.time_provider.now()); + snapshot.memtable().put_with_ts(key, Bytes::new()).await?; self.try_freeze_memtable(&snapshot).await?; Ok(()) } @@ -818,20 +821,21 @@ mod test { storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); - assert_eq!( - storage.get_for_test(b"a").await.unwrap(), - Some(Bytes::from_static(b"1")) - ); - - let snapshot1 = storage.new_txn().unwrap(); { - let guard = snapshot1.scan(Included(b"a"), Included(b"a")); + let guard = storage.scan(Included(b"a"), Included(b"a")); let mut iter = guard.iter().await.unwrap(); while let Some(elem) = iter.next().await { let elem = elem.unwrap(); dbg!(elem); } } + assert_eq!( + storage.get_for_test(b"a").await.unwrap(), + Some(Bytes::from_static(b"1")) + ); + + let snapshot1 = storage.new_txn().unwrap(); + assert_eq!( snapshot1.get_for_test(b"a").await.unwrap(), Some(Bytes::from_static(b"1")) From 7d0e16ef82b6e579626b82ae25b794812bd7064a Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 4 Aug 2024 13:22:36 +0800 Subject: [PATCH 41/69] fix: test_task2_memtable_mvcc --- src/memtable/mutable.rs | 15 ++++----------- src/mvcc/iterator.rs | 16 ++++++++-------- src/state/states.rs | 10 +--------- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index faa8f1c..5880f17 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -225,16 +225,16 @@ impl MemTable { #[cfg(test)] mod test { - use std::ops::Bound::Included; use crate::key::Key; use crate::manifest::Manifest; use crate::memtable::mutable::MemTable; + use crate::mvcc::iterator::transform_bound; use crate::persistent::LocalFs; use crate::time::{TimeIncrement, TimeProvider}; use bytes::Bytes; use futures::StreamExt; + use std::ops::Bound::Included; use tempfile::tempdir; - use crate::mvcc::iterator::transform_bound; #[tokio::test] async fn test_task1_memtable_get_wal() { @@ -357,7 +357,8 @@ mod test { .unwrap(); { - let (lower, upper) = transform_bound(Included(b"key1"), Included(b"key1"), time_provider.now()); + let now = time_provider.now(); + let (lower, upper) = transform_bound(Included(b"key1"), Included(b"key1"), now); let lower = lower.map(Key::from); let upper = upper.map(Key::from); let lower = lower.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); @@ -365,16 +366,8 @@ mod test { let mut iter = memtable.scan_with_ts(lower, upper).await.unwrap(); let (new_iter, elem) = iter.unwrap().next().await; - dbg!(elem); let new_iter = new_iter.unwrap(); - // iter = new_iter; assert!(new_iter.is_none()); - - - // let (new_iter, elem) = iter.unwrap().next().await; - // dbg!(elem); - // let new_iter = new_iter.unwrap(); - // iter = new_iter; } } } diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 0de4150..0b811ba 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -77,31 +77,31 @@ where T: Bounded, { ( - transform_lower_bound(lower), - transform_upper_bound(upper, timestamp), + transform_lower_bound(lower, timestamp), + transform_upper_bound(upper), ) } -fn transform_lower_bound(lower: Bound) -> Bound<(A, T)> +fn transform_lower_bound(lower: Bound, timestamp: T) -> Bound<(A, T)> where T: Bounded, { use Bound::{Excluded, Included, Unbounded}; match lower { - Included(a) => Included((a, T::min_value())), - Excluded(a) => Excluded((a, T::max_value())), + Included(a) => Included((a, timestamp)), + Excluded(a) => Excluded((a, T::min_value())), Unbounded => Unbounded, } } -fn transform_upper_bound(upper: Bound, timestamp: T) -> Bound<(A, T)> +fn transform_upper_bound(upper: Bound) -> Bound<(A, T)> where T: Bounded, { use Bound::{Excluded, Included, Unbounded}; match upper { - Included(a) => Included((a, timestamp)), - Excluded(a) => Excluded((a, T::min_value())), + Included(a) => Included((a, T::min_value())), + Excluded(a) => Excluded((a, T::max_value())), Unbounded => Unbounded, } } diff --git a/src/state/states.rs b/src/state/states.rs index 7f0460e..e20f5c5 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -821,14 +821,6 @@ mod test { storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); - { - let guard = storage.scan(Included(b"a"), Included(b"a")); - let mut iter = guard.iter().await.unwrap(); - while let Some(elem) = iter.next().await { - let elem = elem.unwrap(); - dbg!(elem); - } - } assert_eq!( storage.get_for_test(b"a").await.unwrap(), Some(Bytes::from_static(b"1")) @@ -971,7 +963,7 @@ mod test { ); assert_scan_iter( - &snapshot4, + &snapshot5, Unbounded, Unbounded, [("a", "4"), ("b", "3"), ("c", "1")], From 45a70bd6f19de1152055ebab3272da1b924011eb Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 4 Aug 2024 13:32:18 +0800 Subject: [PATCH 42/69] fix: test_task1_storage_get --- src/state/states.rs | 66 --------------------------------------------- 1 file changed, 66 deletions(-) diff --git a/src/state/states.rs b/src/state/states.rs index e20f5c5..6cb1534 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -698,72 +698,6 @@ mod test { storage.get_for_test(b"0").await.unwrap(), Some(Bytes::from_static(b"2333333")) ); - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let iter = guard.build_memtable_iter().await; - let iter = unwrap_ts_stream(iter); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333")]), - ) - .await; - } - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let iter = guard.build_sst_iter().await.unwrap(); - let iter = unwrap_ts_stream(iter); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333333")]), - ) - .await; - } - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let a = guard.build_memtable_iter().await; - let b = guard.build_sst_iter().await.unwrap(); - let iter = create_inner(a, b).await.unwrap(); - let iter = unwrap_ts_stream(iter); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333"), ("00", "2333333")]), - ) - .await; - } - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let a = guard.build_memtable_iter().await; - let b = guard.build_sst_iter().await.unwrap(); - let iter = create_two_merge_iter(a, b).await.unwrap(); - let iter = unwrap_ts_stream(iter); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333")]), - ) - .await; - } - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let a = guard.build_memtable_iter().await; - let b = guard.build_sst_iter().await.unwrap(); - let iter = create_two_merge_iter(a, b).await.unwrap(); - let iter = unwrap_ts_stream(iter); - let iter = new_no_deleted_iter(iter); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333")]), - ) - .await; - } - { - let guard = storage.scan(Included(b"00"), Included(b"00")); - let iter = guard.iter().await.unwrap(); - assert_stream_eq( - iter.map(Result::unwrap).map(Entry::into_tuple), - build_tuple_stream([("00", "2333")]), - ) - .await; - } assert_eq!( storage.get_for_test(b"00").await.unwrap(), Some(Bytes::from_static(b"2333")) From 5ee394a26ca24aeb75561a5ed835aa15654ba104 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 4 Aug 2024 14:01:22 +0800 Subject: [PATCH 43/69] fix: test_wal_integration --- src/lsm/core.rs | 22 ++++++++++++---------- src/state/inner.rs | 4 ++++ src/state/states.rs | 7 ++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 07e8f16..6ad97c4 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -41,7 +41,7 @@ impl Lsm

{ } pub async fn sync(&self) -> anyhow::Result<()> { - todo!() + self.state.sync_wal().await } fn spawn_flush( @@ -142,9 +142,11 @@ enum Signal { #[cfg(test)] mod tests { - use std::time::Duration; - + use arc_swap::access::DynAccess; + use futures::StreamExt; use nom::AsBytes; + use std::ops::Bound::{Included, Unbounded}; + use std::time::Duration; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; @@ -154,7 +156,7 @@ mod tests { use crate::sst::SstOptions; use crate::state::Map; use crate::test_utils::insert_sst; - use crate::time::TimeIncrement; + use crate::time::{SystemTime, TimeIncrement}; // todo: WAL causes the "too many open files" error // #[tokio::test] // async fn test_task2_auto_flush() { @@ -253,7 +255,7 @@ mod tests { .build(); let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options.clone(), persistent, Box::::default()) + let lsm = Lsm::new(options.clone(), persistent, Box::::default()) .await .unwrap(); add_data(&lsm).await.unwrap(); @@ -268,16 +270,16 @@ mod tests { { let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options, persistent, Box::::default()) + let lsm = Lsm::new(options, persistent, Box::::default()) .await .unwrap(); assert_eq!( - &lsm.get(b"key-0").await.unwrap().unwrap()[..], - b"value-1024".as_slice() + std::str::from_utf8(&lsm.get(b"key-0").await.unwrap().unwrap()[..]).unwrap(), + "value-1024", ); assert_eq!( - &lsm.get(b"key-1").await.unwrap().unwrap()[..], - b"value-1024".as_slice() + std::str::from_utf8(&lsm.get(b"key-1").await.unwrap().unwrap()[..]).unwrap(), + "value-1024", ); assert_eq!(lsm.get(b"key-2").await.unwrap(), None); } diff --git a/src/state/inner.rs b/src/state/inner.rs index 35c73f5..e8da32b 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -71,6 +71,10 @@ impl LsmStorageStateInner

{ }; Ok((this, next_sst_id + 1)) } + + pub async fn sync_wal(&self) -> anyhow::Result<()> { + self.memtable.sync_wal().await + } } impl Clone for LsmStorageStateInner

{ diff --git a/src/state/states.rs b/src/state/states.rs index 6cb1534..98d28b3 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; +use arc_swap::access::Access; use arc_swap::ArcSwap; use bytes::Bytes; use derive_getters::Getters; @@ -100,6 +101,10 @@ where let tx = mvcc.new_txn(self.inner.load_full(), false, self.time_provider.now()); Ok(tx) } + + pub async fn sync_wal(&self) -> anyhow::Result<()> { + self.inner.load().sync_wal().await + } } // KV store @@ -156,7 +161,7 @@ where self.sst_id().fetch_add(1, Ordering::Relaxed) } - fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { + pub fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { let snapshot = self.inner.load_full(); LockedLsmIter::new(snapshot, lower, upper, self.time_provider.now()) } From ed5ac31134908df977e340e33f0c74646f3a0bb8 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 4 Aug 2024 15:06:51 +0800 Subject: [PATCH 44/69] wip --- benches/ycsb.rs | 7 +--- src/lsm/core.rs | 22 +++-------- src/memtable/mutable.rs | 2 +- src/mvcc/core.rs | 6 ++- src/sst/builder.rs | 5 ++- src/sst/compact/leveled.rs | 4 +- src/sst/option.rs | 3 ++ src/sst/sstables.rs | 6 +-- src/sst/tables.rs | 11 +++++- src/state/inner.rs | 28 +++++++++++-- src/state/mod.rs | 2 +- src/state/states.rs | 80 ++++++++++++++------------------------ src/wal.rs | 1 + 13 files changed, 90 insertions(+), 87 deletions(-) diff --git a/benches/ycsb.rs b/benches/ycsb.rs index ff73d3c..84f8469 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -64,11 +64,8 @@ fn ycsb_bench(c: &mut Criterion) { .enable_wal(false) .build(); let runtime = tokio::runtime::Runtime::new().unwrap(); - let state = runtime.block_on(async { - LsmStorageState::new(options, persistent, Box::new(SystemTime)) - .await - .unwrap() - }); + let state = + runtime.block_on(async { LsmStorageState::new(options, persistent).await.unwrap() }); let database = LsmStorageStateBench(Arc::new(state)); let props = Properties { operation_count: 1000, diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 6ad97c4..0a29a4a 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -24,12 +24,8 @@ pub struct Lsm { } impl Lsm

{ - pub async fn new( - options: SstOptions, - persistent: P, - time_provider: TimeProviderWrapper, - ) -> anyhow::Result { - let state = Arc::new(LsmStorageState::new(options, persistent, time_provider).await?); + pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { + let state = Arc::new(LsmStorageState::new(options, persistent).await?); let cancel_token = CancellationToken::new(); let _ = Self::spawn_flush(state.clone(), cancel_token.clone()); let _ = Self::spawn_compaction(state.clone(), cancel_token.clone()); @@ -198,7 +194,7 @@ mod tests { .compaction_option(Default::default()) .enable_wal(false) .build(); - Lsm::new(options, persistent, Box::::default()).await + Lsm::new(options, persistent).await } #[tokio::test] @@ -217,9 +213,7 @@ mod tests { .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) .build(); - let lsm = Lsm::new(options, persistent, Box::::default()) - .await - .unwrap(); + let lsm = Lsm::new(options, persistent).await.unwrap(); for i in 0..10 { let begin = i * 100; insert_sst(&lsm, begin..begin + 100).await.unwrap(); @@ -255,9 +249,7 @@ mod tests { .build(); let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options.clone(), persistent, Box::::default()) - .await - .unwrap(); + let lsm = Lsm::new(options.clone(), persistent).await.unwrap(); add_data(&lsm).await.unwrap(); sleep(Duration::from_secs(2)).await; @@ -270,9 +262,7 @@ mod tests { { let persistent = LocalFs::new(dir.path().to_path_buf()); - let lsm = Lsm::new(options, persistent, Box::::default()) - .await - .unwrap(); + let lsm = Lsm::new(options, persistent).await.unwrap(); assert_eq!( std::str::from_utf8(&lsm.get(b"key-0").await.unwrap().unwrap()[..]).unwrap(), "value-1024", diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 5880f17..d853b95 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -2,7 +2,7 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use bytemuck::TransparentWrapperAlloc; diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 7545764..940e34b 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -52,10 +52,12 @@ impl LsmMvccInner { &self, inner: Arc>, serializable: bool, - ts: u64, ) -> Transaction

{ let key_hashes = serializable.then(Mutex::default); - + let ts = { + let guard = self.ts.lock(); + guard.0 + }; Transaction::new(ts, inner, key_hashes) } } diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 2ee6f38..6f3cd85 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -122,6 +122,9 @@ impl SsTableBuilder { bloom.encode(&mut data); data.put_u32(bloom_offset); + // encode max_ts + data.put_u64(max_ts); + let file = persistent.create_sst(id, data).await?; let table = SsTable::builder() @@ -133,7 +136,7 @@ impl SsTableBuilder { .first_key(first_key) .last_key(last_key) .bloom(Some(bloom)) - .max_ts(max_ts) // todo + .max_ts(max_ts) .build(); Ok(table) diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index ba16b15..9ca635c 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -365,9 +365,7 @@ mod tests { .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) .build(); - let state = LsmStorageState::new(options, persistent, Box::::default()) - .await - .unwrap(); + let state = LsmStorageState::new(options, persistent).await.unwrap(); let _next_sst_id = AtomicUsize::default(); let state_lock = Mutex::default(); diff --git a/src/sst/option.rs b/src/sst/option.rs index dee50a0..101fa0f 100644 --- a/src/sst/option.rs +++ b/src/sst/option.rs @@ -15,4 +15,7 @@ pub struct SstOptions { #[builder(default)] enable_mvcc: bool, + + #[builder(default)] + serializable: bool, } diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index ebfadbe..9b2caf2 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -78,14 +78,14 @@ impl Sstables { pub fn levels(&self) -> &[Vec] { &self.levels } +} +// only for test +impl Sstables { pub fn sstables(&self) -> &HashMap>> { &self.sstables } -} -// only for test -impl Sstables { // todo: delete it pub fn sstables_mut(&mut self) -> &mut HashMap>> { &mut self.sstables diff --git a/src/sst/tables.rs b/src/sst/tables.rs index 31400d6..a028df6 100644 --- a/src/sst/tables.rs +++ b/src/sst/tables.rs @@ -29,7 +29,7 @@ pub struct SsTable { last_key: KeyBytes, pub(crate) bloom: Option, /// The maximum timestamp stored in this SST, implemented in week 3. - max_ts: u64, // todo: use Option? + pub max_ts: u64, // todo: use Option? } impl Debug for SsTable { @@ -54,6 +54,13 @@ impl SsTable { ) -> Result { let file = persistent.open_sst(id).await?; let mut end = file.size(); + + let max_ts = { + let data = file.read(end - 8, 8).await?; + end = end - 8; + u64::from_be_bytes(data.as_slice().try_into()?) + }; + let bloom = { let bloom_offset_begin = end - 4; let bloom_offset = file.read(bloom_offset_begin, 4).await?.as_slice().get_u32() as u64; @@ -92,7 +99,7 @@ impl SsTable { first_key, last_key, bloom: Some(bloom), - max_ts: 0, + max_ts, }; Ok(table) } diff --git a/src/state/inner.rs b/src/state/inner.rs index e8da32b..1864793 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -14,6 +14,12 @@ use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; use crate::sst::{SsTable, SstOptions, Sstables}; +pub struct RecoveredState { + pub state: LsmStorageStateInner

, + pub next_sst_id: usize, + pub initial_ts: u64, +} + #[derive(Getters, TypedBuilder)] pub struct LsmStorageStateInner { pub(crate) memtable: Arc>, @@ -28,7 +34,7 @@ impl LsmStorageStateInner

{ manifest_records: Vec, persistent: &P, block_cache: Option>, - ) -> anyhow::Result<(Self, usize)> { + ) -> anyhow::Result> { let (imm_memtables, mut sstables_state) = build_state(options, manifest_records, persistent).await?; // todo: split sst_ids & sst hashmap @@ -64,12 +70,27 @@ impl LsmStorageStateInner

{ let memtable = MemTable::create_with_wal(next_sst_id, persistent, manifest).await?; + let max_ts = { + let memtable_max_ts = imm_memtables + .iter() + .flat_map(|table| table.iter().map(|entry| entry.key().timestamp())); + let sst_max_ts = sstables_state.sstables().values().map(|sst| sst.max_ts); + memtable_max_ts.chain(sst_max_ts).reduce(max).unwrap_or(0) + }; + let this = Self { memtable: Arc::new(memtable), imm_memtables, sstables_state: Arc::new(sstables_state), }; - Ok((this, next_sst_id + 1)) + + let recovered = RecoveredState { + state: this, + next_sst_id: next_sst_id + 1, + initial_ts: max_ts, + }; + + Ok(recovered) } pub async fn sync_wal(&self) -> anyhow::Result<()> { @@ -127,7 +148,8 @@ async fn build_state( Ok((imm_memtables, sstables)) } ManifestRecord::NewMemtable(record) => { - fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; + let max_ts = + fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; Ok((imm_memtables, sstables)) } ManifestRecord::Compaction(record) => { diff --git a/src/state/mod.rs b/src/state/mod.rs index af330bc..8a39c1f 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -1,4 +1,4 @@ -mod inner; +pub mod inner; mod map; mod mut_op; mod states; diff --git a/src/state/states.rs b/src/state/states.rs index 98d28b3..a501bf1 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -18,11 +18,12 @@ use crate::key::KeyBytes; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; use crate::mvcc::core::{LsmMvccInner, TimeProviderWrapper}; +use crate::mvcc::iterator::LockedTxnIter; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; -use crate::state::inner::LsmStorageStateInner; +use crate::state::inner::{LsmStorageStateInner, RecoveredState}; use crate::state::Map; use crate::time::TimeProvider; use crate::utils::vec::pop; @@ -37,7 +38,6 @@ pub struct LsmStorageState { pub(crate) options: SstOptions, pub(crate) sst_id: AtomicUsize, mvcc: Option, - time_provider: TimeProviderWrapper, } impl

Debug for LsmStorageState

@@ -58,14 +58,14 @@ impl

LsmStorageState

where P: Persistent, { - pub async fn new( - options: SstOptions, - persistent: P, - time_provider: TimeProviderWrapper, - ) -> anyhow::Result { + pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { let (manifest, manifest_records) = Manifest::recover(&persistent).await?; let block_cache = Arc::new(BlockCache::new(1024)); - let (inner, next_sst_id) = LsmStorageStateInner::recover( + let RecoveredState { + state: inner, + next_sst_id, + initial_ts, + } = LsmStorageStateInner::recover( &options, &manifest, manifest_records, @@ -76,8 +76,7 @@ where let sst_id = AtomicUsize::new(next_sst_id); let mvcc = if *options.enable_mvcc() { - // todo: use external dependency to get time - Some(LsmMvccInner::new(time_provider.now())) + Some(LsmMvccInner::new(initial_ts)) } else { None }; @@ -91,14 +90,13 @@ where options, sst_id, mvcc, - time_provider, }; Ok(this) } pub fn new_txn(&self) -> anyhow::Result> { let mvcc = self.mvcc.as_ref().ok_or(anyhow!("no mvcc"))?; - let tx = mvcc.new_txn(self.inner.load_full(), false, self.time_provider.now()); + let tx = mvcc.new_txn(self.inner.load_full(), *self.options.serializable()); Ok(tx) } @@ -115,15 +113,8 @@ where type Error = anyhow::Error; async fn get(&self, key: &[u8]) -> anyhow::Result> { - let guard = self.scan(Bound::Included(key), Bound::Included(key)); - let value = guard - .iter() - .await? - .next() - .await - .transpose()? - .map(|entry| entry.value); - Ok(value) + let txn = self.new_txn()?; + txn.get(key).await } async fn put( @@ -131,25 +122,16 @@ where key: impl Into + Send, value: impl Into + Send, ) -> anyhow::Result<()> { - let snapshot = self.inner.load(); - let key = KeyBytes::new(key.into(), self.time_provider.now()); - snapshot - .memtable() - .put_with_ts(key, value.into()) - .instrument(tracing::info_span!("memtable_put")) - .await?; - self.try_freeze_memtable(&snapshot) - .instrument(tracing::info_span!("try_freeze_memtable")) - .await?; - Ok(()) + // todo: check options.serializable + let txn = self.new_txn()?; + txn.put(key, value).await?; + txn.commit().await } async fn delete(&self, key: impl Into + Send) -> anyhow::Result<()> { - let snapshot = self.inner.load(); - let key = KeyBytes::new(key.into(), self.time_provider.now()); - snapshot.memtable().put_with_ts(key, Bytes::new()).await?; - self.try_freeze_memtable(&snapshot).await?; - Ok(()) + let txn = self.new_txn()?; + txn.delete(key).await?; + txn.commit().await } } @@ -161,9 +143,12 @@ where self.sst_id().fetch_add(1, Ordering::Relaxed) } - pub fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedLsmIter<'a, P> { - let snapshot = self.inner.load_full(); - LockedLsmIter::new(snapshot, lower, upper, self.time_provider.now()) + pub fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedTxnIter<'a, P> { + todo!() + // let txn = self.new_txn()?; + // txn.scan(lower, upper).await + // let snapshot = self.inner.load_full(); + // LockedLsmIter::new(snapshot, lower, upper, self.time_provider.now()) } } @@ -729,7 +714,7 @@ mod test { .compaction_option(Default::default()) .enable_wal(false) .build(); - LsmStorageState::new(options, persistent, Box::::default()).await + LsmStorageState::new(options, persistent).await } #[tokio::test] @@ -753,9 +738,7 @@ mod test { .enable_wal(true) .enable_mvcc(true) .build(); - let storage = LsmStorageState::new(options, persistent, Box::::default()) - .await - .unwrap(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); @@ -938,9 +921,7 @@ mod test { .compaction_option(Default::default()) .enable_wal(true) .build(); - let storage = LsmStorageState::new(options, persistent, Box::::default()) - .await - .unwrap(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); let txn1 = storage.new_txn().unwrap(); let txn2 = storage.new_txn().unwrap(); @@ -1092,10 +1073,9 @@ mod test { .num_memtable_limit(1000) .compaction_option(Default::default()) .enable_wal(true) + .enable_mvcc(true) .build(); - let storage = LsmStorageState::new(options, persistent, Box::::default()) - .await - .unwrap(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); let txn1 = storage.new_txn().unwrap(); let txn2 = storage.new_txn().unwrap(); diff --git a/src/wal.rs b/src/wal.rs index 650b933..6a33f6e 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,3 +1,4 @@ +use std::cmp::max; use std::io::Cursor; use std::sync::Arc; From f66da10e3c0631710fe0b8c9d3a7c47015e5611c Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 4 Aug 2024 15:35:03 +0800 Subject: [PATCH 45/69] add tests --- src/mvcc/transaction.rs | 5 + src/state/inner.rs | 19 +++- src/state/states.rs | 219 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 4 deletions(-) diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 0bd63c3..30d40d4 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -83,6 +83,11 @@ impl Transaction

{ } pub async fn commit(self) -> anyhow::Result<()> { + // todo: use write batch + for entry in self.local_storage.iter() { + // self.inner + // todo + } todo!() } } diff --git a/src/state/inner.rs b/src/state/inner.rs index 1864793..d9a5cfc 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -1,18 +1,21 @@ -use std::cmp::max; -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; - +use bytes::Bytes; use derive_getters::Getters; use futures::stream; use futures::stream::{StreamExt, TryStreamExt}; +use std::cmp::max; +use std::fmt::{Debug, Formatter}; +use std::future::Future; +use std::sync::Arc; use typed_builder::TypedBuilder; use crate::block::BlockCache; +use crate::key::KeyBytes; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::{ImmutableMemTable, MemTable}; use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; use crate::sst::{SsTable, SstOptions, Sstables}; +use crate::state::{LsmStorageState, Map}; pub struct RecoveredState { pub state: LsmStorageStateInner

, @@ -28,6 +31,14 @@ pub struct LsmStorageStateInner { } impl LsmStorageStateInner

{ + pub async fn put(&self, key: KeyBytes, value: impl Into + Send) -> anyhow::Result<()> { + self.memtable().put_with_ts(key, value.into()).await?; + // self.try_freeze_memtable(&snapshot) + // .await?; + // todo + Ok(()) + } + pub async fn recover( options: &SstOptions, manifest: &Manifest, diff --git a/src/state/states.rs b/src/state/states.rs index a501bf1..68b91a4 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -358,6 +358,7 @@ mod test { use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::states::LsmStorageState; + use crate::state::Map; use crate::test_utils::iterator::unwrap_ts_stream; use crate::time::TimeIncrement; @@ -1168,4 +1169,222 @@ mod test { ) .await; } + + #[tokio::test] + async fn test_serializable_1() { + let dir = tempdir().unwrap(); + let storage = build_serializable_lsm(&dir).await; + + storage.put_for_test(b"key1", b"1").await.unwrap(); + storage.put_for_test(b"key2", b"2").await.unwrap(); + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn1.put_for_test(b"key1", &txn1.get_for_test(b"key2").await.unwrap().unwrap()) + .await + .unwrap(); + txn2.put_for_test(b"key2", &txn2.get_for_test(b"key1").await.unwrap().unwrap()) + .await + .unwrap(); + txn1.commit().await.unwrap(); + assert!(txn2.commit().await.is_err()); + assert_eq!( + storage.get_for_test(b"key1").await.unwrap(), + Some(Bytes::from("2")) + ); + assert_eq!( + storage.get_for_test(b"key2").await.unwrap(), + Some(Bytes::from("2")) + ); + } + + #[tokio::test] + async fn test_serializable_2() { + let dir = tempdir().unwrap(); + let storage = build_serializable_lsm(&dir).await; + + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn1.put_for_test(b"key1", b"1").await.unwrap(); + txn2.put_for_test(b"key1", b"2").await.unwrap(); + txn1.commit().await.unwrap(); + txn2.commit().await.unwrap(); + assert_eq!( + storage.get_for_test(b"key1").await.unwrap(), + Some(Bytes::from("2")) + ); + } + + #[tokio::test] + async fn test_serializable_3_ts_range() { + let dir = tempdir().unwrap(); + let storage = build_serializable_lsm(&dir).await; + + storage.put_for_test(b"key1", b"1").await.unwrap(); + storage.put_for_test(b"key2", b"2").await.unwrap(); + let txn1 = storage.new_txn().unwrap(); + txn1.put_for_test(b"key1", &txn1.get_for_test(b"key2").await.unwrap().unwrap()) + .await + .unwrap(); + txn1.commit().await.unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn2.put_for_test(b"key2", &txn2.get_for_test(b"key1").await.unwrap().unwrap()) + .await + .unwrap(); + txn2.commit().await.unwrap(); + assert_eq!( + storage.get_for_test(b"key1").await.unwrap(), + Some(Bytes::from("2")) + ); + assert_eq!( + storage.get_for_test(b"key2").await.unwrap(), + Some(Bytes::from("2")) + ); + } + + #[tokio::test] + async fn test_serializable_4_scan() { + let dir = tempdir().unwrap(); + let storage = build_serializable_lsm(&dir).await; + + storage.put_for_test(b"key1", b"1").await.unwrap(); + storage.put_for_test(b"key2", b"2").await.unwrap(); + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn1.put_for_test(b"key1", &txn1.get_for_test(b"key2").await.unwrap().unwrap()) + .await + .unwrap(); + txn1.commit().await.unwrap(); + + { + let guard = txn2.scan(Unbounded, Unbounded); + let mut iter = guard.iter().await.unwrap(); + while let Some(entry) = iter.next().await { + // todo: check entry + let entry = entry.unwrap(); + } + } + + txn2.put_for_test(b"key2", b"1").await.unwrap(); + assert!(txn2.commit().await.is_err()); + assert_eq!( + storage.get_for_test(b"key1").await.unwrap(), + Some(Bytes::from("2")) + ); + assert_eq!( + storage.get_for_test(b"key2").await.unwrap(), + Some(Bytes::from("2")) + ); + } + + #[tokio::test] + async fn test_serializable_5_read_only() { + let dir = tempdir().unwrap(); + let storage = build_serializable_lsm(&dir).await; + + storage.put_for_test(b"key1", b"1").await.unwrap(); + storage.put_for_test(b"key2", b"2").await.unwrap(); + let txn1 = storage.new_txn().unwrap(); + txn1.put_for_test(b"key1", &txn1.get_for_test(b"key2").await.unwrap().unwrap()) + .await + .unwrap(); + txn1.commit().await.unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn2.get_for_test(b"key1").await.unwrap().unwrap(); + + { + let guard = txn2.scan(Unbounded, Unbounded); + let mut iter = guard.iter().await.unwrap(); + while let Some(entry) = iter.next().await { + // todo: check entry + let entry = entry.unwrap(); + } + } + + txn2.commit().await.unwrap(); + assert_eq!( + storage.get_for_test(b"key1").await.unwrap(), + Some(Bytes::from("2")) + ); + assert_eq!( + storage.get_for_test(b"key2").await.unwrap(), + Some(Bytes::from("2")) + ); + } + + async fn build_serializable_lsm(dir: &TempDir) -> LsmStorageState { + let persistent = LocalFs::new(dir.path().to_path_buf()); + let options = SstOptions::builder() + .target_sst_size(1024) + .block_size(4096) + .num_memtable_limit(1000) + .compaction_option(Default::default()) + .enable_wal(true) + .enable_mvcc(true) + .serializable(true) + .build(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); + storage + } + + // todo: week 3, day 7 test + // #[test] + // fn test_task3_mvcc_compaction() { + // let dir = tempdir().unwrap(); + // let options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); + // let storage = MiniLsm::open(&dir, options.clone()).unwrap(); + // storage + // .write_batch(&[ + // WriteBatchRecord::Put("table1_a", "1"), + // WriteBatchRecord::Put("table1_b", "1"), + // WriteBatchRecord::Put("table1_c", "1"), + // WriteBatchRecord::Put("table2_a", "1"), + // WriteBatchRecord::Put("table2_b", "1"), + // WriteBatchRecord::Put("table2_c", "1"), + // ]) + // .unwrap(); + // storage.force_flush().unwrap(); + // let snapshot0 = storage.new_txn().unwrap(); + // storage + // .write_batch(&[ + // WriteBatchRecord::Put("table1_a", "2"), + // WriteBatchRecord::Del("table1_b"), + // WriteBatchRecord::Put("table1_c", "2"), + // WriteBatchRecord::Put("table2_a", "2"), + // WriteBatchRecord::Del("table2_b"), + // WriteBatchRecord::Put("table2_c", "2"), + // ]) + // .unwrap(); + // storage.force_flush().unwrap(); + // storage.add_compaction_filter(CompactionFilter::Prefix(Bytes::from("table2_"))); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("table1_a"), Bytes::from("2")), + // (Bytes::from("table1_a"), Bytes::from("1")), + // (Bytes::from("table1_b"), Bytes::new()), + // (Bytes::from("table1_b"), Bytes::from("1")), + // (Bytes::from("table1_c"), Bytes::from("2")), + // (Bytes::from("table1_c"), Bytes::from("1")), + // (Bytes::from("table2_a"), Bytes::from("2")), + // (Bytes::from("table2_b"), Bytes::new()), + // (Bytes::from("table2_c"), Bytes::from("2")), + // ], + // ); + // + // drop(snapshot0); + // + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("table1_a"), Bytes::from("2")), + // (Bytes::from("table1_c"), Bytes::from("2")), + // ], + // ); + // } } From 36496535d2b218da3b622ef0cc679f1461fb8ee7 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 5 Aug 2024 13:16:18 +0800 Subject: [PATCH 46/69] wip --- src/key.rs | 4 ++-- src/memtable/mutable.rs | 23 ++++++++++++++----- src/mvcc/core.rs | 2 +- src/mvcc/transaction.rs | 40 ++++++++++++++++++++++++--------- src/wal.rs | 50 +++++++++++++++++------------------------ 5 files changed, 71 insertions(+), 48 deletions(-) diff --git a/src/key.rs b/src/key.rs index f6e9ad6..bbe28db 100644 --- a/src/key.rs +++ b/src/key.rs @@ -7,8 +7,8 @@ use std::fmt::Debug; #[derive(PartialEq, Eq, Debug, new, Default, Clone, Copy)] pub struct Key { - key: T, - timestamp: u64, + pub key: T, + pub timestamp: u64, } pub type KeySlice<'a> = Key<&'a [u8]>; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index d853b95..cf7add8 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,7 +1,7 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; - +use std::slice; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; @@ -14,6 +14,7 @@ use ref_cast::RefCast; use tracing_futures::Instrument; use crate::bound::BytesBound; +use crate::entry::Entry; use crate::iterators::NonEmptyStream; use crate::key::{KeyBytes, KeySlice}; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; @@ -148,13 +149,25 @@ impl MemTable { } pub async fn put_with_ts(&self, key: KeyBytes, value: Bytes) -> anyhow::Result<()> { - let size = key.len() + value.len(); + let KeyBytes { key, timestamp } = key; + let entry = Entry::new(key, value); + self.put_batch(slice::from_ref(&entry), timestamp).await + } + + pub async fn put_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { if let Some(wal) = self.wal.as_ref() { - wal.put(key.as_key_slice(), value.as_bytes()) + wal.put_batch(entries, timestamp) .instrument(tracing::info_span!("wal_put")) - .await? + .await?; + } + let mut size = 0; + for entry in entries { + let key = KeyBytes::new(entry.key.clone(), timestamp); + let value = entry.value.clone(); + + size += key.len() + value.len(); + self.map.insert(key, value); } - self.map.insert(key, value); self.approximate_size.fetch_add(size, Ordering::Release); Ok(()) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 940e34b..1606e61 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -58,6 +58,6 @@ impl LsmMvccInner { let guard = self.ts.lock(); guard.0 }; - Transaction::new(ts, inner, key_hashes) + Transaction::new(ts, inner, key_hashes, self.ts.clone()) } } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 30d40d4..5ac4eb0 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,30 +1,35 @@ -use crate::entry::Entry; +use std::collections::{Bound, HashSet}; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; -use crate::persistent::Persistent; -use crate::state::{LsmStorageStateInner, Map}; use bytes::Bytes; use crossbeam_skiplist::SkipMap; -use futures::{stream, Stream}; -use std::collections::{Bound, HashSet}; - -use crate::iterators::LockedLsmIter; -use crate::mvcc::iterator::{txn_local_iterator, LockedTxnIter}; -use arc_swap::access::Access; use parking_lot::Mutex; -use std::sync::atomic::AtomicBool; -use std::sync::Arc; use tokio_stream::StreamExt; +use crate::iterators::LockedLsmIter; +use crate::mvcc::iterator::LockedTxnIter; +use crate::mvcc::watermark::Watermark; +use crate::persistent::Persistent; +use crate::state::{LsmStorageStateInner, Map}; + pub struct Transaction { pub(crate) read_ts: u64, pub(crate) inner: Arc>, // todo: need Arc<...> ? pub(crate) local_storage: Arc>, + + // todo: delete it? pub(crate) committed: Arc, /// Write set and read set /// todo: check deadlock? pub(crate) key_hashes: Option, HashSet)>>, + + // todo: remove u64? + // todo: remove mutex? + // todo: 理论上存储一个 callback: fn(read_ts: u64) 即可 + watermark: Arc>, } impl Map for Transaction

{ @@ -61,13 +66,19 @@ impl Transaction

{ read_ts: u64, inner: Arc>, key_hashes: Option, HashSet)>>, + watermark: Arc>, ) -> Self { + { + let mut guard = watermark.lock(); + guard.1.add_reader(read_ts); + } Self { read_ts, inner, local_storage: Arc::default(), committed: Arc::default(), key_hashes, + watermark, } } @@ -92,6 +103,13 @@ impl Transaction

{ } } +impl Drop for Transaction

{ + fn drop(&mut self) { + let mut guard = self.watermark.lock(); + guard.1.remove_reader(self.read_ts); + } +} + #[cfg(test)] impl Transaction

{ pub async fn get_for_test(&self, key: &[u8]) -> anyhow::Result> { diff --git a/src/wal.rs b/src/wal.rs index 6a33f6e..ed7639f 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,19 +1,19 @@ use std::cmp::max; use std::io::Cursor; - +use std::slice; use std::sync::Arc; use crate::key::{KeyBytes, KeySlice}; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; +use crate::entry::Entry; +use crate::persistent::interface::WalHandle; +use crate::persistent::Persistent; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Mutex; use tracing_futures::Instrument; -use crate::persistent::interface::WalHandle; -use crate::persistent::Persistent; - pub struct Wal { file: Arc>, } @@ -59,31 +59,22 @@ impl Wal { Ok((wal, map)) } - pub async fn put<'a>(&'a self, key: KeySlice<'a>, value: &'a [u8]) -> anyhow::Result<()> { + pub async fn put_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { let mut guard = self.file.lock().await; - // todo: 这里 write 的操作和 encode key/value 重复，需要合并 - guard - .write_u32(key.len() as u32) - .instrument(tracing::info_span!("wal_put_write_key_len")) - .await?; - guard - .write_all(key.raw_ref()) - .instrument(tracing::info_span!("wal_put_write_all_key")) - .await?; - guard.write_u64(key.timestamp()).await?; - - guard - .write_u32(value.len() as u32) - .instrument(tracing::info_span!("wal_put_write_value_len")) - .await?; - guard - .write_all(value) - .instrument(tracing::info_span!("wal_put_write_all_value")) - .await?; - guard - .flush() - .instrument(tracing::info_span!("wal_put_flush")) - .await?; + for entry in entries { + let key = entry.key.as_ref(); + guard.write_u32(key.len() as u32).await?; + guard.write_all(key).await?; + guard.write_u64(timestamp).await?; + + let value = entry.value.as_ref(); + + guard.write_u32(value.len() as u32).await?; + guard.write_all(value).await?; + } + guard.flush().await?; + guard.sync_all().await?; + Ok(()) } @@ -101,7 +92,8 @@ impl Wal { ts: u64, value: &'a [u8], ) -> anyhow::Result<()> { - self.put(KeySlice::new(key, ts), value).await + let entry = Entry::new(Bytes::copy_from_slice(key), Bytes::copy_from_slice(value)); + self.put_batch(slice::from_ref(&entry), ts).await } } From df8f7b75e0df51b31033df779357437d5293a263 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 5 Aug 2024 22:30:12 +0800 Subject: [PATCH 47/69] wip --- src/iterators/utils.rs | 22 ++++++++++++++++++++++ src/memtable/mutable.rs | 13 +++++++++---- src/mvcc/transaction.rs | 29 ++++++++++++++++++----------- src/wal.rs | 28 ++++++++++++++++------------ 4 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index cf66919..4aba831 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -55,6 +55,28 @@ where stream::iter(iterator.map(FutureExt::into_stream as fn(_) -> _)).flatten() } +pub struct Ref { + data: T, + finished: bool, +} + +impl Ref { + pub fn new(data: T) -> Self { + Self { + data, + finished: false, + } + } +} + +impl<'a, T> Iterator for &'a Ref { + type Item = &'a T; + + fn next(&mut self) -> Option { + todo!() + } +} + #[cfg(test)] pub mod test_utils { use crate::entry::Entry; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index cf7add8..b655101 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,9 +1,9 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; -use std::slice; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; +use std::{iter, slice}; use bytemuck::TransparentWrapperAlloc; use bytes::Bytes; @@ -14,7 +14,7 @@ use ref_cast::RefCast; use tracing_futures::Instrument; use crate::bound::BytesBound; -use crate::entry::Entry; +use crate::entry::{Entry, Keyed}; use crate::iterators::NonEmptyStream; use crate::key::{KeyBytes, KeySlice}; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; @@ -155,15 +155,20 @@ impl MemTable { } pub async fn put_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { + // todo: entries 可以改成 iterator if let Some(wal) = self.wal.as_ref() { + let entries = entries + .iter() + .map(|e| Keyed::new(e.key.as_ref(), e.value.as_ref())); wal.put_batch(entries, timestamp) .instrument(tracing::info_span!("wal_put")) .await?; } let mut size = 0; for entry in entries { - let key = KeyBytes::new(entry.key.clone(), timestamp); - let value = entry.value.clone(); + let Entry { key, value } = entry; + let key = KeyBytes::new(key.clone(), timestamp); + let value = value.clone(); size += key.len() + value.len(); self.map.insert(key, value); diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 5ac4eb0..52ec8ad 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,17 +1,18 @@ use std::collections::{Bound, HashSet}; +use std::future::Future; use std::sync::atomic::AtomicBool; use std::sync::Arc; -use bytes::Bytes; -use crossbeam_skiplist::SkipMap; -use parking_lot::Mutex; -use tokio_stream::StreamExt; - +use crate::entry::Entry; use crate::iterators::LockedLsmIter; use crate::mvcc::iterator::LockedTxnIter; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; use crate::state::{LsmStorageStateInner, Map}; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use parking_lot::Mutex; +use tokio_stream::StreamExt; pub struct Transaction { pub(crate) read_ts: u64, @@ -94,12 +95,18 @@ impl Transaction

{ } pub async fn commit(self) -> anyhow::Result<()> { - // todo: use write batch - for entry in self.local_storage.iter() { - // self.inner - // todo - } - todo!() + let commit_ts = { + let mut guard = self.watermark.lock(); + guard.0 += 1; + guard.0 + }; + let entries: Vec<_> = self + .local_storage + .iter() + .map(|e| Entry::new(e.key().clone(), e.value().clone())) + .collect(); + self.inner.memtable.put_batch(&entries, commit_ts).await?; + Ok(()) } } diff --git a/src/wal.rs b/src/wal.rs index ed7639f..254fbe7 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,13 +1,13 @@ -use std::cmp::max; -use std::io::Cursor; -use std::slice; -use std::sync::Arc; - use crate::key::{KeyBytes, KeySlice}; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; +use std::cmp::max; +use std::future::Future; +use std::io::Cursor; +use std::sync::Arc; +use std::{iter, slice}; -use crate::entry::Entry; +use crate::entry::{Entry, Keyed}; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -59,16 +59,20 @@ impl Wal { Ok((wal, map)) } - pub async fn put_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { + pub async fn put_batch( + &self, + entries: impl Iterator> + Send, + timestamp: u64, + ) -> anyhow::Result<()> { + // todo: atomic wal let mut guard = self.file.lock().await; for entry in entries { - let key = entry.key.as_ref(); + let key = entry.key; guard.write_u32(key.len() as u32).await?; guard.write_all(key).await?; guard.write_u64(timestamp).await?; - let value = entry.value.as_ref(); - + let value = entry.value; guard.write_u32(value.len() as u32).await?; guard.write_all(value).await?; } @@ -92,8 +96,8 @@ impl Wal { ts: u64, value: &'a [u8], ) -> anyhow::Result<()> { - let entry = Entry::new(Bytes::copy_from_slice(key), Bytes::copy_from_slice(value)); - self.put_batch(slice::from_ref(&entry), ts).await + let entry = Keyed::new(key, value); + self.put_batch(iter::once(entry), ts).await } } From 47940806ad0c16a71eec9416418f59b749e41ee4 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 5 Aug 2024 22:53:57 +0800 Subject: [PATCH 48/69] wip --- src/mvcc/core.rs | 4 +-- src/mvcc/transaction.rs | 70 ++++++++++++++++++++++++++--------------- src/state/states.rs | 4 +-- 3 files changed, 48 insertions(+), 30 deletions(-) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 1606e61..4030793 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -49,7 +49,7 @@ impl LsmMvccInner { } pub fn new_txn( - &self, + self: &Arc, inner: Arc>, serializable: bool, ) -> Transaction

{ @@ -58,6 +58,6 @@ impl LsmMvccInner { let guard = self.ts.lock(); guard.0 }; - Transaction::new(ts, inner, key_hashes, self.ts.clone()) + Transaction::new(ts, inner, key_hashes, self.clone()) } } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 52ec8ad..0269396 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,18 +1,25 @@ use std::collections::{Bound, HashSet}; -use std::future::Future; +use std::ops::Bound::Excluded; use std::sync::atomic::AtomicBool; use std::sync::Arc; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use parking_lot::Mutex; +use tokio_stream::StreamExt; + use crate::entry::Entry; use crate::iterators::LockedLsmIter; +use crate::mvcc::core::LsmMvccInner; use crate::mvcc::iterator::LockedTxnIter; -use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; use crate::state::{LsmStorageStateInner, Map}; -use bytes::Bytes; -use crossbeam_skiplist::SkipMap; -use parking_lot::Mutex; -use tokio_stream::StreamExt; + +#[derive(Debug, Default)] +pub struct RWSet { + read_set: HashSet, + write_set: HashSet, +} pub struct Transaction { pub(crate) read_ts: u64, @@ -25,12 +32,9 @@ pub struct Transaction { pub(crate) committed: Arc, /// Write set and read set /// todo: check deadlock? - pub(crate) key_hashes: Option, HashSet)>>, + pub(crate) key_hashes: Option>, - // todo: remove u64? - // todo: remove mutex? - // todo: 理论上存储一个 callback: fn(read_ts: u64) 即可 - watermark: Arc>, + mvcc: Arc, } impl Map for Transaction

{ @@ -66,11 +70,11 @@ impl Transaction

{ pub fn new( read_ts: u64, inner: Arc>, - key_hashes: Option, HashSet)>>, - watermark: Arc>, + key_hashes: Option>, + mvcc: Arc, ) -> Self { { - let mut guard = watermark.lock(); + let mut guard = mvcc.ts.lock(); guard.1.add_reader(read_ts); } Self { @@ -79,7 +83,7 @@ impl Transaction

{ local_storage: Arc::default(), committed: Arc::default(), key_hashes, - watermark, + mvcc, } } @@ -95,24 +99,38 @@ impl Transaction

{ } pub async fn commit(self) -> anyhow::Result<()> { - let commit_ts = { - let mut guard = self.watermark.lock(); - guard.0 += 1; - guard.0 + // todo: commit lock / write lock ? + let _commit_guard = self.mvcc.commit_lock.lock(); + let expected_commit_ts = { + // todo: 这里的锁可以去掉？ + let mut guard = self.mvcc.ts.lock(); + guard.0 + 1 }; - let entries: Vec<_> = self - .local_storage - .iter() - .map(|e| Entry::new(e.key().clone(), e.value().clone())) - .collect(); - self.inner.memtable.put_batch(&entries, commit_ts).await?; + let conflict = if let Some(key_hashes) = self.key_hashes.as_ref() { + let guard = self.mvcc.committed_txns.lock(); + let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); + let rw_set_guard = key_hashes.lock(); + let read_set = &rw_set_guard.read_set; + guard + .range(range) + .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) + } else { + false + }; + + // let entries: Vec<_> = self + // .local_storage + // .iter() + // .map(|e| Entry::new(e.key().clone(), e.value().clone())) + // .collect(); + // self.inner.memtable.put_batch(&entries, commit_ts).await?; Ok(()) } } impl Drop for Transaction

{ fn drop(&mut self) { - let mut guard = self.watermark.lock(); + let mut guard = self.mvcc.ts.lock(); guard.1.remove_reader(self.read_ts); } } diff --git a/src/state/states.rs b/src/state/states.rs index 68b91a4..a46f0af 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -37,7 +37,7 @@ pub struct LsmStorageState { pub(crate) persistent: P, pub(crate) options: SstOptions, pub(crate) sst_id: AtomicUsize, - mvcc: Option, + mvcc: Option>, } impl

Debug for LsmStorageState

@@ -76,7 +76,7 @@ where let sst_id = AtomicUsize::new(next_sst_id); let mvcc = if *options.enable_mvcc() { - Some(LsmMvccInner::new(initial_ts)) + Some(Arc::new(LsmMvccInner::new(initial_ts))) } else { None }; From 02b4a15e1869f25375323a52745449b2b82f6dca Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 7 Aug 2024 13:50:17 +0800 Subject: [PATCH 49/69] wip --- src/mvcc/core.rs | 8 +++----- src/mvcc/transaction.rs | 43 +++++++++++++++++++++++------------------ src/utils/mod.rs | 1 + src/utils/scoped.rs | 12 ++++++++++++ 4 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 src/utils/scoped.rs diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 4030793..5185888 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -18,19 +18,17 @@ pub(crate) struct CommittedTxnData { pub type TimeProviderWrapper = Box; pub(crate) struct LsmMvccInner { - pub(crate) write_lock: Mutex<()>, - pub(crate) commit_lock: Mutex<()>, + pub(crate) write_lock: Mutex<()>, // 使用 tokio channel 替代，因为所有 write 都需要 serialized pub(crate) ts: Arc>, - pub(crate) committed_txns: Arc>>, + pub(crate) committed_txns: Arc>>, } impl LsmMvccInner { pub fn new(initial_ts: u64) -> Self { Self { write_lock: Mutex::new(()), - commit_lock: Mutex::new(()), ts: Arc::new(Mutex::new((initial_ts, Watermark::new()))), - committed_txns: Arc::new(Mutex::new(BTreeMap::new())), + committed_txns: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), } } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 0269396..62bff1b 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -14,6 +14,7 @@ use crate::mvcc::core::LsmMvccInner; use crate::mvcc::iterator::LockedTxnIter; use crate::persistent::Persistent; use crate::state::{LsmStorageStateInner, Map}; +use crate::utils::scoped::Scoped; #[derive(Debug, Default)] pub struct RWSet { @@ -32,7 +33,7 @@ pub struct Transaction { pub(crate) committed: Arc, /// Write set and read set /// todo: check deadlock? - pub(crate) key_hashes: Option>, + pub(crate) key_hashes: Option>>, mvcc: Arc, } @@ -82,7 +83,7 @@ impl Transaction

{ inner, local_storage: Arc::default(), committed: Arc::default(), - key_hashes, + key_hashes: key_hashes.map(Scoped::new), mvcc, } } @@ -99,24 +100,28 @@ impl Transaction

{ } pub async fn commit(self) -> anyhow::Result<()> { + let guard = self.mvcc.committed_txns.lock().await; + // todo: commit lock / write lock ? - let _commit_guard = self.mvcc.commit_lock.lock(); - let expected_commit_ts = { - // todo: 这里的锁可以去掉？ - let mut guard = self.mvcc.ts.lock(); - guard.0 + 1 - }; - let conflict = if let Some(key_hashes) = self.key_hashes.as_ref() { - let guard = self.mvcc.committed_txns.lock(); - let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); - let rw_set_guard = key_hashes.lock(); - let read_set = &rw_set_guard.read_set; - guard - .range(range) - .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) - } else { - false - }; + // let _commit_guard = self.mvcc.commit_lock.lock().await; + // let expected_commit_ts = { + // // todo: 这里的锁可以去掉？ + // let mut guard = self.mvcc.ts.lock(); + // guard.0 + 1 + // }; + // let conflict = if let Some(key_hashes) = self.key_hashes.as_ref() { + // key_hashes.with_ref(|key_hashes| { + // let guard = self.mvcc.committed_txns.lock(); + // let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); + // let rw_set_guard = key_hashes.lock(); + // let read_set = &rw_set_guard.read_set; + // guard + // .range(range) + // .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) + // }) + // } else { + // false + // }; // let entries: Vec<_> = self // .local_storage diff --git a/src/utils/mod.rs b/src/utils/mod.rs index ae9e6a4..c290d3f 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod func; pub mod num; pub mod vec; +pub mod scoped; diff --git a/src/utils/scoped.rs b/src/utils/scoped.rs new file mode 100644 index 0000000..0deef90 --- /dev/null +++ b/src/utils/scoped.rs @@ -0,0 +1,12 @@ +use derive_new::new; + +#[derive(Debug, Default, new)] +pub struct Scoped { + inner: T +} + +impl Scoped { + pub fn with_ref(&self, f: F) -> B where F: FnOnce(&T) -> B { + f(&self.inner) + } +} From b2bd12bd1e4a6fce73af27c97ccf9e50979790bd Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 7 Aug 2024 22:30:18 +0800 Subject: [PATCH 50/69] wip --- src/mvcc/core.rs | 5 +-- src/mvcc/transaction.rs | 89 +++++++++++++++++++++++------------------ src/state/inner.rs | 6 +++ src/state/states.rs | 3 ++ src/utils/mod.rs | 2 +- src/utils/scoped.rs | 38 +++++++++++++++++- 6 files changed, 97 insertions(+), 46 deletions(-) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 5185888..f8b74a5 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -18,7 +18,6 @@ pub(crate) struct CommittedTxnData { pub type TimeProviderWrapper = Box; pub(crate) struct LsmMvccInner { - pub(crate) write_lock: Mutex<()>, // 使用 tokio channel 替代，因为所有 write 都需要 serialized pub(crate) ts: Arc>, pub(crate) committed_txns: Arc>>, } @@ -26,7 +25,6 @@ pub(crate) struct LsmMvccInner { impl LsmMvccInner { pub fn new(initial_ts: u64) -> Self { Self { - write_lock: Mutex::new(()), ts: Arc::new(Mutex::new((initial_ts, Watermark::new()))), committed_txns: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), } @@ -51,11 +49,10 @@ impl LsmMvccInner { inner: Arc>, serializable: bool, ) -> Transaction

{ - let key_hashes = serializable.then(Mutex::default); let ts = { let guard = self.ts.lock(); guard.0 }; - Transaction::new(ts, inner, key_hashes, self.clone()) + Transaction::new(ts, inner, serializable, self.clone()) } } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 62bff1b..befd10e 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,20 +1,20 @@ +use anyhow::anyhow; +use bytes::Bytes; +use crossbeam_skiplist::SkipMap; +use parking_lot::Mutex; use std::collections::{Bound, HashSet}; use std::ops::Bound::Excluded; use std::sync::atomic::AtomicBool; use std::sync::Arc; - -use bytes::Bytes; -use crossbeam_skiplist::SkipMap; -use parking_lot::Mutex; use tokio_stream::StreamExt; use crate::entry::Entry; use crate::iterators::LockedLsmIter; -use crate::mvcc::core::LsmMvccInner; +use crate::mvcc::core::{CommittedTxnData, LsmMvccInner}; use crate::mvcc::iterator::LockedTxnIter; use crate::persistent::Persistent; use crate::state::{LsmStorageStateInner, Map}; -use crate::utils::scoped::Scoped; +use crate::utils::scoped::{Scoped, ScopedMutex}; #[derive(Debug, Default)] pub struct RWSet { @@ -33,7 +33,7 @@ pub struct Transaction { pub(crate) committed: Arc, /// Write set and read set /// todo: check deadlock? - pub(crate) key_hashes: Option>>, + pub(crate) key_hashes: Option>, mvcc: Arc, } @@ -71,7 +71,7 @@ impl Transaction

{ pub fn new( read_ts: u64, inner: Arc>, - key_hashes: Option>, + serializable: bool, mvcc: Arc, ) -> Self { { @@ -83,7 +83,7 @@ impl Transaction

{ inner, local_storage: Arc::default(), committed: Arc::default(), - key_hashes: key_hashes.map(Scoped::new), + key_hashes: serializable.then(|| ScopedMutex::default()), mvcc, } } @@ -99,36 +99,47 @@ impl Transaction

{ guard } - pub async fn commit(self) -> anyhow::Result<()> { - let guard = self.mvcc.committed_txns.lock().await; - - // todo: commit lock / write lock ? - // let _commit_guard = self.mvcc.commit_lock.lock().await; - // let expected_commit_ts = { - // // todo: 这里的锁可以去掉？ - // let mut guard = self.mvcc.ts.lock(); - // guard.0 + 1 - // }; - // let conflict = if let Some(key_hashes) = self.key_hashes.as_ref() { - // key_hashes.with_ref(|key_hashes| { - // let guard = self.mvcc.committed_txns.lock(); - // let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); - // let rw_set_guard = key_hashes.lock(); - // let read_set = &rw_set_guard.read_set; - // guard - // .range(range) - // .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) - // }) - // } else { - // false - // }; - - // let entries: Vec<_> = self - // .local_storage - // .iter() - // .map(|e| Entry::new(e.key().clone(), e.value().clone())) - // .collect(); - // self.inner.memtable.put_batch(&entries, commit_ts).await?; + // todo: 区分 snapshot isolation vs serializable isolation + pub async fn commit(mut self) -> anyhow::Result<()> { + let mut commit_guard = self.mvcc.committed_txns.lock().await; + + let expected_commit_ts = { + // todo: 这里的锁可以去掉？ + let mut guard = self.mvcc.ts.lock(); + guard.0 + 1 + }; + let key_hashes = self.key_hashes.take().map(ScopedMutex::into_inner); + let conflict = if let Some(key_hashes) = key_hashes.as_ref() { + let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); + let read_set = &key_hashes.read_set; + commit_guard + .range(range) + .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) + } else { + false + }; + if conflict { + return Err(anyhow!("commit conflict")); + } + + let entries: Vec<_> = self + .local_storage + .iter() + .map(|e| Entry::new(e.key().clone(), e.value().clone())) + .collect(); + // todo: 如果 write_batch 失败怎么保证 atomicity + self.inner.write_batch(&entries, expected_commit_ts).await?; + self.mvcc.update_commit_ts(expected_commit_ts); + + if let Some(key_hashes) = key_hashes { + let committed_data = CommittedTxnData { + key_hashes: key_hashes.write_set, + read_ts: self.read_ts, + commit_ts: expected_commit_ts, + }; + commit_guard.insert(expected_commit_ts, committed_data); + } + Ok(()) } } diff --git a/src/state/inner.rs b/src/state/inner.rs index d9a5cfc..1286a8d 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use typed_builder::TypedBuilder; use crate::block::BlockCache; +use crate::entry::Entry; use crate::key::KeyBytes; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::{ImmutableMemTable, MemTable}; @@ -31,6 +32,11 @@ pub struct LsmStorageStateInner { } impl LsmStorageStateInner

{ + pub async fn write_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { + // todo: 使用 type system 保证单线程调用 + todo!() + } + pub async fn put(&self, key: KeyBytes, value: impl Into + Send) -> anyhow::Result<()> { self.memtable().put_with_ts(key, value.into()).await?; // self.try_freeze_memtable(&snapshot) diff --git a/src/state/states.rs b/src/state/states.rs index a46f0af..9308d82 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -34,6 +34,7 @@ pub struct LsmStorageState { block_cache: Arc, manifest: Manifest, pub(crate) state_lock: Mutex<()>, + write_lock: Mutex<()>, pub(crate) persistent: P, pub(crate) options: SstOptions, pub(crate) sst_id: AtomicUsize, @@ -85,6 +86,7 @@ where inner: ArcSwap::new(Arc::new(inner)), block_cache: Arc::new(BlockCache::new(1024)), manifest, + write_lock: Mutex::default(), state_lock: Mutex::default(), persistent, options, @@ -122,6 +124,7 @@ where key: impl Into + Send, value: impl Into + Send, ) -> anyhow::Result<()> { + // let _guard = // todo: check options.serializable let txn = self.new_txn()?; txn.put(key, value).await?; diff --git a/src/utils/mod.rs b/src/utils/mod.rs index c290d3f..ff2aeb1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,4 +1,4 @@ pub mod func; pub mod num; -pub mod vec; pub mod scoped; +pub mod vec; diff --git a/src/utils/scoped.rs b/src/utils/scoped.rs index 0deef90..d5e63ad 100644 --- a/src/utils/scoped.rs +++ b/src/utils/scoped.rs @@ -1,12 +1,46 @@ use derive_new::new; +use parking_lot::{Mutex, MutexGuard}; #[derive(Debug, Default, new)] pub struct Scoped { - inner: T + inner: T, } impl Scoped { - pub fn with_ref(&self, f: F) -> B where F: FnOnce(&T) -> B { + pub fn with_ref(&self, f: F) -> B + where + F: FnOnce(&T) -> B, + { f(&self.inner) } + + // todo: into_inner 不应该存在？ + pub fn into_inner(self) -> T { + self.inner + } +} + +#[derive(Debug, Default)] +pub struct ScopedMutex { + inner: Mutex, +} + +impl ScopedMutex { + pub fn new(t: T) -> Self { + Self { + inner: Mutex::new(t), + } + } + + pub fn lock_with(&self, f: F) -> B + where + F: FnOnce(MutexGuard) -> B, + { + let guard = self.inner.lock(); + f(guard) + } + + pub fn into_inner(self) -> T { + self.inner.into_inner() + } } From 65b164b30800d725512f8ad86e9d26ca42d764fc Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 8 Aug 2024 13:28:39 +0800 Subject: [PATCH 51/69] wip --- src/mvcc/core.rs | 6 +++--- src/mvcc/transaction.rs | 30 ++++++++++++++---------------- src/state/inner.rs | 5 ----- src/state/states.rs | 21 +++++++++++++++------ 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index f8b74a5..c638a91 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -6,7 +6,7 @@ use parking_lot::Mutex; use crate::mvcc::transaction::Transaction; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; -use crate::state::LsmStorageStateInner; +use crate::state::{LsmStorageState, LsmStorageStateInner}; use crate::time::TimeProvider; pub(crate) struct CommittedTxnData { @@ -46,13 +46,13 @@ impl LsmMvccInner { pub fn new_txn( self: &Arc, - inner: Arc>, + state: &LsmStorageState

, serializable: bool, ) -> Transaction

{ let ts = { let guard = self.ts.lock(); guard.0 }; - Transaction::new(ts, inner, serializable, self.clone()) + Transaction::new(ts, state, serializable, self.clone()) } } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index befd10e..0f75f42 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -13,7 +13,7 @@ use crate::iterators::LockedLsmIter; use crate::mvcc::core::{CommittedTxnData, LsmMvccInner}; use crate::mvcc::iterator::LockedTxnIter; use crate::persistent::Persistent; -use crate::state::{LsmStorageStateInner, Map}; +use crate::state::{LsmStorageState, LsmStorageStateInner, Map}; use crate::utils::scoped::{Scoped, ScopedMutex}; #[derive(Debug, Default)] @@ -22,9 +22,9 @@ pub struct RWSet { write_set: HashSet, } -pub struct Transaction { +pub struct Transaction<'a, P: Persistent> { pub(crate) read_ts: u64, - pub(crate) inner: Arc>, + pub(crate) state: &'a LsmStorageState

, // todo: need Arc<...> ? pub(crate) local_storage: Arc>, @@ -38,7 +38,7 @@ pub struct Transaction { mvcc: Arc, } -impl Map for Transaction

{ +impl<'a, P: Persistent> Map for Transaction<'a, P> { type Error = anyhow::Error; async fn get(&self, key: &[u8]) -> Result, Self::Error> { @@ -67,10 +67,10 @@ impl Map for Transaction

{ } } -impl Transaction

{ +impl<'a, P: Persistent> Transaction<'a, P> { pub fn new( read_ts: u64, - inner: Arc>, + state: &'a LsmStorageState

, serializable: bool, mvcc: Arc, ) -> Self { @@ -80,7 +80,7 @@ impl Transaction

{ } Self { read_ts, - inner, + state, local_storage: Arc::default(), committed: Arc::default(), key_hashes: serializable.then(|| ScopedMutex::default()), @@ -89,12 +89,9 @@ impl Transaction

{ } // todo: no need for Result? - pub fn scan<'a>( - &'a self, - lower: Bound<&'a [u8]>, - upper: Bound<&'a [u8]>, - ) -> LockedTxnIter<'a, P> { - let inner_iter = LockedLsmIter::new(self.inner.clone(), lower, upper, self.read_ts); + pub fn scan(&'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedTxnIter<'a, P> { + let inner = self.state.inner.load_full(); + let inner_iter = LockedLsmIter::new(inner, lower, upper, self.read_ts); let guard = LockedTxnIter::new(&self.local_storage, inner_iter); guard } @@ -122,13 +119,14 @@ impl Transaction

{ return Err(anyhow!("commit conflict")); } + // todo: avoid collecting let entries: Vec<_> = self .local_storage .iter() .map(|e| Entry::new(e.key().clone(), e.value().clone())) .collect(); // todo: 如果 write_batch 失败怎么保证 atomicity - self.inner.write_batch(&entries, expected_commit_ts).await?; + self.state.write_batch(&entries, expected_commit_ts).await?; self.mvcc.update_commit_ts(expected_commit_ts); if let Some(key_hashes) = key_hashes { @@ -144,7 +142,7 @@ impl Transaction

{ } } -impl Drop for Transaction

{ +impl<'a, P: Persistent> Drop for Transaction<'a, P> { fn drop(&mut self) { let mut guard = self.mvcc.ts.lock(); guard.1.remove_reader(self.read_ts); @@ -152,7 +150,7 @@ impl Drop for Transaction

{ } #[cfg(test)] -impl Transaction

{ +impl<'a, P: Persistent> Transaction<'a, P> { pub async fn get_for_test(&self, key: &[u8]) -> anyhow::Result> { self.get(key).await } diff --git a/src/state/inner.rs b/src/state/inner.rs index 1286a8d..6f60ce4 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -32,11 +32,6 @@ pub struct LsmStorageStateInner { } impl LsmStorageStateInner

{ - pub async fn write_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { - // todo: 使用 type system 保证单线程调用 - todo!() - } - pub async fn put(&self, key: KeyBytes, value: impl Into + Send) -> anyhow::Result<()> { self.memtable().put_with_ts(key, value.into()).await?; // self.try_freeze_memtable(&snapshot) diff --git a/src/state/states.rs b/src/state/states.rs index 9308d82..37f0f53 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -13,6 +13,7 @@ use tokio::sync::{Mutex, MutexGuard}; use tracing_futures::Instrument; use crate::block::BlockCache; +use crate::entry::Entry; use crate::iterators::LockedLsmIter; use crate::key::KeyBytes; use crate::manifest::{Flush, Manifest, ManifestRecord}; @@ -97,8 +98,9 @@ where } pub fn new_txn(&self) -> anyhow::Result> { + // todo: avoid clone? let mvcc = self.mvcc.as_ref().ok_or(anyhow!("no mvcc"))?; - let tx = mvcc.new_txn(self.inner.load_full(), *self.options.serializable()); + let tx = mvcc.new_txn(self, *self.options.serializable()); Ok(tx) } @@ -142,6 +144,13 @@ impl

LsmStorageState

where P: Persistent, { + pub async fn write_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { + let guard = self.inner.load(); + guard.memtable.put_batch(entries, timestamp).await?; + self.try_freeze_memtable(guard.as_ref()).await?; + Ok(()) + } + pub(crate) fn next_sst_id(&self) -> usize { self.sst_id().fetch_add(1, Ordering::Relaxed) } @@ -342,11 +351,11 @@ where #[cfg(test)] mod test { - use std::collections::Bound; - use std::ops::Bound::{Excluded, Included, Unbounded}; - use bytes::Bytes; use futures::StreamExt; + use std::collections::Bound; + use std::ops::Bound::{Excluded, Included, Unbounded}; + use std::sync::Arc; use tempfile::{tempdir, TempDir}; use crate::entry::Entry; @@ -1158,8 +1167,8 @@ mod test { assert_scan_iter(&txn4, Unbounded, Unbounded, [("test1", "233")]).await; } - async fn assert_scan_iter( - snapshot: &Transaction

, + async fn assert_scan_iter<'a, P: Persistent>( + snapshot: &'a Transaction<'a, P>, lower: Bound<&[u8]>, upper: Bound<&[u8]>, expected: impl IntoIterator, From 964da58357fe37b7a9ef14483f97449ddb8f4d97 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 8 Aug 2024 22:49:31 +0800 Subject: [PATCH 52/69] wip --- src/mvcc/iterator.rs | 30 +++++++++++++++++++++++++++++- src/mvcc/transaction.rs | 2 +- src/sst/tables.rs | 2 +- src/state/inner.rs | 3 +-- src/state/states.rs | 19 +++++++++++-------- 5 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 0b811ba..bd6b756 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -3,6 +3,7 @@ use crate::entry::{Entry, Keyed}; use crate::iterators::lsm::LsmIterator; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{create_two_merge_iter, LockedLsmIter}; +use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use async_iter_ext::StreamTools; use bytes::Bytes; @@ -10,10 +11,37 @@ use crossbeam_skiplist::SkipMap; use derive_new::new; use futures::{stream, Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; +use ouroboros::self_referencing; use std::collections::Bound::{Excluded, Included, Unbounded}; use std::future::ready; use std::ops::Bound; -use std::sync::Arc; + +struct TxnWithBound<'a, P: Persistent> { + txn: Transaction<'a, P>, + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, +} + +#[self_referencing] +pub struct LockedTxnIterWithTxn<'a, P: Persistent> { + txn: TxnWithBound<'a, P>, + + #[borrows(txn)] + #[covariant] + iter: LockedTxnIter<'this, P>, +} + +impl<'a, P: Persistent> LockedTxnIterWithTxn<'a, P> { + pub fn new_(txn: Transaction<'a, P>, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> Self { + Self::new(TxnWithBound { txn, lower, upper }, |txn| { + txn.txn.scan(txn.lower, txn.upper) + }) + } + + pub async fn iter(&'a self) -> anyhow::Result> { + self.with_iter(|iter| iter.iter()).await + } +} #[derive(new)] pub struct LockedTxnIter<'a, P: Persistent> { diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 0f75f42..7860690 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -83,7 +83,7 @@ impl<'a, P: Persistent> Transaction<'a, P> { state, local_storage: Arc::default(), committed: Arc::default(), - key_hashes: serializable.then(|| ScopedMutex::default()), + key_hashes: serializable.then(ScopedMutex::default), mvcc, } } diff --git a/src/sst/tables.rs b/src/sst/tables.rs index a028df6..f4acac3 100644 --- a/src/sst/tables.rs +++ b/src/sst/tables.rs @@ -57,7 +57,7 @@ impl SsTable { let max_ts = { let data = file.read(end - 8, 8).await?; - end = end - 8; + end -= 8; u64::from_be_bytes(data.as_slice().try_into()?) }; diff --git a/src/state/inner.rs b/src/state/inner.rs index 6f60ce4..9c31b42 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -160,8 +160,7 @@ async fn build_state( Ok((imm_memtables, sstables)) } ManifestRecord::NewMemtable(record) => { - let max_ts = - fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; + fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; Ok((imm_memtables, sstables)) } ManifestRecord::Compaction(record) => { diff --git a/src/state/states.rs b/src/state/states.rs index 37f0f53..410bb82 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -20,6 +20,7 @@ use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; use crate::mvcc::core::{LsmMvccInner, TimeProviderWrapper}; use crate::mvcc::iterator::LockedTxnIter; +use crate::mvcc::iterator::LockedTxnIterWithTxn; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; @@ -155,12 +156,14 @@ where self.sst_id().fetch_add(1, Ordering::Relaxed) } - pub fn scan<'a>(&self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedTxnIter<'a, P> { - todo!() - // let txn = self.new_txn()?; - // txn.scan(lower, upper).await - // let snapshot = self.inner.load_full(); - // LockedLsmIter::new(snapshot, lower, upper, self.time_provider.now()) + pub fn scan<'a>( + &'a self, + lower: Bound<&'a [u8]>, + upper: Bound<&'a [u8]>, + ) -> LockedTxnIterWithTxn<'a, P> { + // todo: remove unwrap + let txn = self.new_txn().unwrap(); + LockedTxnIterWithTxn::new_(txn, lower, upper) } } @@ -1334,8 +1337,8 @@ mod test { .enable_mvcc(true) .serializable(true) .build(); - let storage = LsmStorageState::new(options, persistent).await.unwrap(); - storage + + LsmStorageState::new(options, persistent).await.unwrap() } // todo: week 3, day 7 test From b136bc994629cb3399c191c4d35bbcc475d049c2 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sun, 11 Aug 2024 23:27:19 +0800 Subject: [PATCH 53/69] use ATPIT instead of TAIT --- src/iterators/merge.rs | 50 +++++++++++++++++++++++------ src/iterators/two_merge.rs | 64 +++++++++++++++++++++++++++----------- src/lib.rs | 1 + src/state/states.rs | 14 ++------- 4 files changed, 89 insertions(+), 40 deletions(-) diff --git a/src/iterators/merge.rs b/src/iterators/merge.rs index d8758db..ad8eb35 100644 --- a/src/iterators/merge.rs +++ b/src/iterators/merge.rs @@ -44,7 +44,7 @@ where Item: Ord + Debug, I: Stream> + Unpin, { - iters: Pin>>, + iters: Pin::HeapStream>>, } impl MergeIteratorInner @@ -67,7 +67,7 @@ where .collect() .await; Self { - iters: Box::pin(build_heap_stream(iters)), + iters: Box::pin(HeapStreamBuilderImpl::build_heap_stream(iters)), } } } @@ -87,17 +87,47 @@ where } } -type HeapStream> + Unpin> = - impl Stream>; +pub trait HeapStreamBuilder { + type HeapStream> + Unpin>: Stream< + Item = anyhow::Result, + >; -fn build_heap_stream(heap: BinaryHeap>) -> HeapStream -where - I: Stream> + Unpin, - Item: Ord + Debug, -{ - unfold(heap, unfold_fn) + fn build_heap_stream( + heap: BinaryHeap>, + ) -> Self::HeapStream + where + I: Stream> + Unpin, + Item: Ord + Debug; +} + +pub struct HeapStreamBuilderImpl; + +impl HeapStreamBuilder for HeapStreamBuilderImpl { + type HeapStream> + Unpin> = + impl Stream>; + + fn build_heap_stream( + heap: BinaryHeap>, + ) -> Self::HeapStream + where + I: Stream> + Unpin, + Item: Ord + Debug, + { + unfold(heap, unfold_fn) + } } +// type HeapStream> + Unpin> = +// impl Stream>; +// +// fn build_heap_stream(heap: BinaryHeap>) -> HeapStream +// where +// I: Stream> + Unpin, +// Item: Ord + Debug, +// { +// unfold(heap, unfold_fn) +// } + async fn unfold_fn( mut heap: BinaryHeap>, ) -> Option<(anyhow::Result, BinaryHeap>)> diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index f886e3e..e32413f 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -8,36 +8,64 @@ use crate::iterators::{MaybeEmptyStream, NonEmptyStream}; // Merges two iterators of different types into one. If the two iterators have the same key, only /// produce the key once and prefer the entry from A. -pub type TwoMergeIterator = NoDuplication>; +pub type TwoMergeIterator = + NoDuplication<::MyTwoMergeIterInner>; pub async fn create_two_merge_iter( a: A, b: B, -) -> anyhow::Result>> +) -> anyhow::Result< + NoDuplication<::MyTwoMergeIterInner>, +> where Item: Ord + Debug + Unpin, A: Stream> + Unpin, B: Stream> + Unpin, { - let inner = create_inner(a, b).await?; + let inner = TwoMergeIterImpl::create_inner(a, b).await?; Ok(new_no_duplication(inner)) } -pub type TwoMergeIterInner< - Item: Ord + Debug + Unpin, - A: Stream> + Unpin, - B: Stream> + Unpin, -> = impl Stream> + Unpin; -pub async fn create_inner(a: A, b: B) -> anyhow::Result> -where - Item: Ord + Debug + Unpin, - A: Stream> + Unpin, - B: Stream> + Unpin, -{ - let a = NonEmptyStream::try_new(a).await?; - let b = NonEmptyStream::try_new(b).await?; - let x = unfold((a, b), unfold_fn); - Ok(Box::pin(x)) +pub trait TwoMergeIter { + type MyTwoMergeIterInner< + Item: Ord + Debug + Unpin, + A: Stream> + Unpin, + B: Stream> + Unpin, + >: Stream> + Unpin; + + async fn create_inner( + a: A, + b: B, + ) -> anyhow::Result> + where + Item: Ord + Debug + Unpin, + A: Stream> + Unpin, + B: Stream> + Unpin; +} + +pub struct TwoMergeIterImpl; + +impl TwoMergeIter for TwoMergeIterImpl { + type MyTwoMergeIterInner< + Item: Ord + Debug + Unpin, + A: Stream> + Unpin, + B: Stream> + Unpin, + > = impl Stream> + Unpin; + + async fn create_inner( + a: A, + b: B, + ) -> anyhow::Result> + where + Item: Ord + Debug + Unpin, + A: Stream> + Unpin, + B: Stream> + Unpin, + { + let a = NonEmptyStream::try_new(a).await?; + let b = NonEmptyStream::try_new(b).await?; + let x = unfold((a, b), unfold_fn); + Ok(Box::pin(x)) + } } async fn unfold_fn( diff --git a/src/lib.rs b/src/lib.rs index 60b721b..587ee79 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![feature(type_alias_impl_trait)] +#![feature(impl_trait_in_assoc_type)] mod block; mod bound; diff --git a/src/state/states.rs b/src/state/states.rs index 410bb82..72a3dbb 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -14,12 +14,9 @@ use tracing_futures::Instrument; use crate::block::BlockCache; use crate::entry::Entry; -use crate::iterators::LockedLsmIter; -use crate::key::KeyBytes; use crate::manifest::{Flush, Manifest, ManifestRecord}; use crate::memtable::MemTable; -use crate::mvcc::core::{LsmMvccInner, TimeProviderWrapper}; -use crate::mvcc::iterator::LockedTxnIter; +use crate::mvcc::core::LsmMvccInner; use crate::mvcc::iterator::LockedTxnIterWithTxn; use crate::mvcc::transaction::Transaction; use crate::persistent::Persistent; @@ -358,13 +355,9 @@ mod test { use futures::StreamExt; use std::collections::Bound; use std::ops::Bound::{Excluded, Included, Unbounded}; - use std::sync::Arc; use tempfile::{tempdir, TempDir}; use crate::entry::Entry; - use crate::iterators::create_two_merge_iter; - use crate::iterators::no_deleted::new_no_deleted_iter; - use crate::iterators::two_merge::create_inner; use crate::iterators::utils::test_utils::{ assert_stream_eq, build_stream, build_tuple_stream, eq, }; @@ -373,9 +366,6 @@ mod test { use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::states::LsmStorageState; - use crate::state::Map; - use crate::test_utils::iterator::unwrap_ts_stream; - use crate::time::TimeIncrement; #[tokio::test] async fn test_task2_storage_integration() { @@ -1337,7 +1327,7 @@ mod test { .enable_mvcc(true) .serializable(true) .build(); - + LsmStorageState::new(options, persistent).await.unwrap() } From 325bfd80896b140b6f0b0bc3cbf439989208ea65 Mon Sep 17 00:00:00 2001 From: Kermit Date: Mon, 12 Aug 2024 00:00:24 +0800 Subject: [PATCH 54/69] upgrade toolchain --- rust-toolchain.toml | 2 +- src/lib.rs | 1 - src/mvcc/core.rs | 6 +++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eb84621..eee1312 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-02-09" +channel = "nightly-2024-08-11" diff --git a/src/lib.rs b/src/lib.rs index 587ee79..107a7f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(type_alias_impl_trait)] #![feature(impl_trait_in_assoc_type)] mod block; diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index c638a91..bbfd4fa 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -44,11 +44,11 @@ impl LsmMvccInner { ts.1.watermark().unwrap_or(ts.0) } - pub fn new_txn( + pub fn new_txn<'a, P: Persistent>( self: &Arc, - state: &LsmStorageState

, + state: &'a LsmStorageState

, serializable: bool, - ) -> Transaction

{ + ) -> Transaction<'a, P> { let ts = { let guard = self.ts.lock(); guard.0 From 7795bd39ade45d78a10137aae1f47e92bfefb3de Mon Sep 17 00:00:00 2001 From: Kermit Date: Tue, 13 Aug 2024 21:51:44 +0800 Subject: [PATCH 55/69] fix: test --- src/iterators/merge/heap.rs | 2 +- src/lsm/core.rs | 2 ++ src/mvcc/core.rs | 6 +++--- src/sst/compact/leveled.rs | 1 + src/state/states.rs | 2 ++ 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/iterators/merge/heap.rs b/src/iterators/merge/heap.rs index 492ce9d..a016844 100644 --- a/src/iterators/merge/heap.rs +++ b/src/iterators/merge/heap.rs @@ -1,7 +1,7 @@ use crate::iterators::NonEmptyStream; use std::cmp; -pub(super) struct HeapWrapper { +pub struct HeapWrapper { pub index: usize, pub iter: NonEmptyStream, } diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 0a29a4a..fd6a660 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -212,6 +212,7 @@ mod tests { .num_memtable_limit(1000) .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) + .enable_mvcc(true) .build(); let lsm = Lsm::new(options, persistent).await.unwrap(); for i in 0..10 { @@ -246,6 +247,7 @@ mod tests { .num_memtable_limit(10) .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(true) + .enable_mvcc(true) .build(); let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index bbfd4fa..f1c5f01 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -17,9 +17,9 @@ pub(crate) struct CommittedTxnData { pub type TimeProviderWrapper = Box; -pub(crate) struct LsmMvccInner { - pub(crate) ts: Arc>, - pub(crate) committed_txns: Arc>>, +pub struct LsmMvccInner { + pub ts: Arc>, + pub committed_txns: Arc>>, } impl LsmMvccInner { diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 9ca635c..806ca7d 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -364,6 +364,7 @@ mod tests { .num_memtable_limit(1000) .compaction_option(CompactionOptions::Leveled(compaction_options)) .enable_wal(false) + .enable_mvcc(true) .build(); let state = LsmStorageState::new(options, persistent).await.unwrap(); let _next_sst_id = AtomicUsize::default(); diff --git a/src/state/states.rs b/src/state/states.rs index 72a3dbb..c73ab11 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -719,6 +719,7 @@ mod test { .num_memtable_limit(1000) .compaction_option(Default::default()) .enable_wal(false) + .enable_mvcc(true) .build(); LsmStorageState::new(options, persistent).await } @@ -926,6 +927,7 @@ mod test { .num_memtable_limit(1000) .compaction_option(Default::default()) .enable_wal(true) + .enable_mvcc(true) .build(); let storage = LsmStorageState::new(options, persistent).await.unwrap(); From 40105d6dddb8e93965d25e65459f76809a9f2536 Mon Sep 17 00:00:00 2001 From: Kermit Date: Tue, 13 Aug 2024 21:53:24 +0800 Subject: [PATCH 56/69] fix: warning --- src/iterators/two_merge.rs | 8 ++++---- src/mvcc/core.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/iterators/two_merge.rs b/src/iterators/two_merge.rs index e32413f..a0c6768 100644 --- a/src/iterators/two_merge.rs +++ b/src/iterators/two_merge.rs @@ -1,7 +1,7 @@ -use std::fmt::Debug; - use futures::stream::unfold; use futures::Stream; +use std::fmt::Debug; +use std::future::Future; use crate::iterators::no_duplication::{new_no_duplication, NoDuplication}; use crate::iterators::{MaybeEmptyStream, NonEmptyStream}; @@ -33,10 +33,10 @@ pub trait TwoMergeIter { B: Stream> + Unpin, >: Stream> + Unpin; - async fn create_inner( + fn create_inner( a: A, b: B, - ) -> anyhow::Result> + ) -> impl Future>> where Item: Ord + Debug + Unpin, A: Stream> + Unpin, diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index f1c5f01..cb71b5f 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -9,10 +9,10 @@ use crate::persistent::Persistent; use crate::state::{LsmStorageState, LsmStorageStateInner}; use crate::time::TimeProvider; -pub(crate) struct CommittedTxnData { - pub(crate) key_hashes: HashSet, - pub(crate) read_ts: u64, - pub(crate) commit_ts: u64, +pub struct CommittedTxnData { + pub key_hashes: HashSet, + pub read_ts: u64, + pub commit_ts: u64, } pub type TimeProviderWrapper = Box; From 619781cd607eda504d46802ccc4535a81ebf5438 Mon Sep 17 00:00:00 2001 From: Kermit Date: Tue, 13 Aug 2024 22:05:25 +0800 Subject: [PATCH 57/69] fix: test --- src/mvcc/core.rs | 1 + src/mvcc/transaction.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index cb71b5f..9800e30 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -9,6 +9,7 @@ use crate::persistent::Persistent; use crate::state::{LsmStorageState, LsmStorageStateInner}; use crate::time::TimeProvider; +#[derive(Debug)] pub struct CommittedTxnData { pub key_hashes: HashSet, pub read_ts: u64, diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 7860690..7da739c 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -111,7 +111,7 @@ impl<'a, P: Persistent> Transaction<'a, P> { let read_set = &key_hashes.read_set; commit_guard .range(range) - .any(|(_, data)| data.key_hashes.is_disjoint(read_set)) + .any(|(_, data)| !data.key_hashes.is_disjoint(read_set)) } else { false }; From 372ff764ff1bec43fb36948ad01e5d2af7f68659 Mon Sep 17 00:00:00 2001 From: Kermit Date: Tue, 13 Aug 2024 22:47:01 +0800 Subject: [PATCH 58/69] add inspect iterator this commit will result in linking error --- src/iterators/inspect.rs | 24 ++++++++++++++++++++++++ src/iterators/mod.rs | 1 + src/mvcc/iterator.rs | 15 ++++++++++++++- src/mvcc/transaction.rs | 29 +++++++++++++++++++++++++++-- 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 src/iterators/inspect.rs diff --git a/src/iterators/inspect.rs b/src/iterators/inspect.rs new file mode 100644 index 0000000..d2647a4 --- /dev/null +++ b/src/iterators/inspect.rs @@ -0,0 +1,24 @@ +use futures::{Stream, StreamExt}; + +pub struct InspectIterImpl; + +pub trait InspectIter { + type Stream, F: FnMut(&Item)>: Stream; + + fn inspect_stream(s: S, f: F) -> Self::Stream + where + S: Stream, + F: FnMut(&Item); +} + +impl InspectIter for InspectIterImpl { + type Stream, F: FnMut(&Item)> = impl Stream; + + fn inspect_stream(s: S, mut f: F) -> Self::Stream + where + S: Stream, + F: FnMut(&Item), + { + s.inspect(move |item| f(item)) + } +} diff --git a/src/iterators/mod.rs b/src/iterators/mod.rs index 7a9839d..5224c4b 100644 --- a/src/iterators/mod.rs +++ b/src/iterators/mod.rs @@ -4,6 +4,7 @@ pub mod merge; pub mod no_deleted; mod no_duplication; mod ok_iter; +pub mod inspect; pub mod two_merge; pub mod utils; diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index bd6b756..762b92d 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -3,7 +3,7 @@ use crate::entry::{Entry, Keyed}; use crate::iterators::lsm::LsmIterator; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{create_two_merge_iter, LockedLsmIter}; -use crate::mvcc::transaction::Transaction; +use crate::mvcc::transaction::{RWSet, Transaction}; use crate::persistent::Persistent; use async_iter_ext::StreamTools; use bytes::Bytes; @@ -15,6 +15,8 @@ use ouroboros::self_referencing; use std::collections::Bound::{Excluded, Included, Unbounded}; use std::future::ready; use std::ops::Bound; +use crate::iterators::inspect::{InspectIter, InspectIterImpl}; +use crate::utils::scoped::ScopedMutex; struct TxnWithBound<'a, P: Persistent> { txn: Transaction<'a, P>, @@ -47,6 +49,7 @@ impl<'a, P: Persistent> LockedTxnIterWithTxn<'a, P> { pub struct LockedTxnIter<'a, P: Persistent> { local_storage: &'a SkipMap, lsm_iter: LockedLsmIter<'a, P>, + key_hashes: Option<&'a ScopedMutex>, } impl<'a, P: Persistent> LockedTxnIter<'a, P> { @@ -59,6 +62,16 @@ impl<'a, P: Persistent> LockedTxnIter<'a, P> { ); let merged = create_two_merge_iter(local_iter, lsm_iter).await?; let iter = new_no_deleted_iter(merged); + let iter = InspectIterImpl::inspect_stream( + iter, + |entry| { + let Ok(entry) = entry else { return }; + let Some(key_hashes) = self.key_hashes else { return }; + let key = entry.key.as_ref(); + key_hashes.lock_with(|mut set| set.add_read_key(key)); + + } + ); let iter = Box::new(iter) as _; Ok(iter) } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 7da739c..6c64074 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -22,6 +22,16 @@ pub struct RWSet { write_set: HashSet, } +impl RWSet { + pub fn add_read_key(&mut self, key: &[u8]) { + self.read_set.insert(farmhash::hash32(key)); + } + + pub fn add_write_key(&mut self, key: &[u8]) { + self.write_set.insert(farmhash::hash32(key)); + } +} + pub struct Transaction<'a, P: Persistent> { pub(crate) read_ts: u64, pub(crate) state: &'a LsmStorageState

, @@ -50,6 +60,9 @@ impl<'a, P: Persistent> Map for Transaction<'a, P> { .await .transpose()? .map(|entry| entry.value); + if let Some(key_hashes) = self.key_hashes.as_ref() { + key_hashes.lock_with(|mut set| set.add_read_key(key)); + } Ok(output) } @@ -58,11 +71,19 @@ impl<'a, P: Persistent> Map for Transaction<'a, P> { key: impl Into + Send, value: impl Into + Send, ) -> Result<(), Self::Error> { - self.local_storage.insert(key.into(), value.into()); + let key = key.into(); + self.local_storage.insert(key.clone(), value.into()); + if let Some(key_hashes) = self.key_hashes.as_ref() { + key_hashes.lock_with(|mut set| set.add_write_key(key.as_ref())); + } Ok(()) } async fn delete(&self, key: impl Into + Send) -> Result<(), Self::Error> { + let key = key.into(); + if let Some(key_hashes) = self.key_hashes.as_ref() { + key_hashes.lock_with(|mut set| set.add_write_key(key.as_ref())); + } self.put(key, Bytes::new()).await } } @@ -92,7 +113,7 @@ impl<'a, P: Persistent> Transaction<'a, P> { pub fn scan(&'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedTxnIter<'a, P> { let inner = self.state.inner.load_full(); let inner_iter = LockedLsmIter::new(inner, lower, upper, self.read_ts); - let guard = LockedTxnIter::new(&self.local_storage, inner_iter); + let guard = LockedTxnIter::new(&self.local_storage, inner_iter, self.key_hashes.as_ref()); guard } @@ -109,6 +130,10 @@ impl<'a, P: Persistent> Transaction<'a, P> { let conflict = if let Some(key_hashes) = key_hashes.as_ref() { let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); let read_set = &key_hashes.read_set; + println!("===="); + dbg!(key_hashes); + dbg!(&commit_guard); + println!("===="); commit_guard .range(range) .any(|(_, data)| !data.key_hashes.is_disjoint(read_set)) From 22d9b25e9810e0330e35d9f874bd9adacc927617 Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 14 Aug 2024 12:52:49 +0800 Subject: [PATCH 59/69] fix: Cargo.toml --- Cargo.lock | 35 ++++++++++++++++++++++++++--------- Cargo.toml | 2 +- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a24d2d0..05f9a13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,6 +415,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "criterion" version = "0.5.1" @@ -559,9 +568,11 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ + "convert_case", "proc-macro2", "quote", "syn 2.0.69", + "unicode-xid", ] [[package]] @@ -2071,6 +2082,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2248,6 +2265,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -2263,15 +2289,6 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-targets" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 377d7d2..1b4980f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ tokio-util = "0.7.11" futures-concurrency = "7.6.1" ordered-float = "4.2.2" getset = "0.1.2" -derive_more = "1.0.0" +derive_more = { version = "1.0.0", features = ["full"] } tracing-futures = { version = "0.2.5", features = ["futures-03"] } num-traits = "0.2.19" From 8080fc4a6a44bd3a0186aec7c8901a33c6527936 Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 14 Aug 2024 13:26:01 +0800 Subject: [PATCH 60/69] add test & fix clippy --- Cargo.toml | 2 +- benches/ycsb.rs | 1 - src/iterators/mod.rs | 2 +- src/iterators/utils.rs | 2 +- src/lsm/core.rs | 7 +++---- src/memtable/mutable.rs | 8 +++----- src/mvcc/core.rs | 2 +- src/mvcc/iterator.rs | 23 ++++++++++------------- src/mvcc/transaction.rs | 7 +++---- src/mvcc/watermark.rs | 6 ++++++ src/sst/bloom.rs | 4 ++-- src/sst/builder.rs | 38 ++++++++++++++++++++++---------------- src/sst/compact/leveled.rs | 2 +- src/state/inner.rs | 3 --- src/state/states.rs | 3 --- src/wal.rs | 9 +++------ 16 files changed, 57 insertions(+), 62 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1b4980f..f9f3436 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ criterion = { version = "0.5.1", features = ["async_tokio"] } maplit = "1.0.2" [lints.rust] -unused = "allow" +#unused = "allow" unsafe_code = "forbid" [[bench]] diff --git a/benches/ycsb.rs b/benches/ycsb.rs index 84f8469..604f80f 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -12,7 +12,6 @@ use ycsb::workload::CoreWorkload; use better_mini_lsm::persistent::LocalFs; use better_mini_lsm::sst::SstOptions; use better_mini_lsm::state::{LsmStorageState, Map}; -use better_mini_lsm::time::SystemTime; #[derive(Clone)] struct LsmStorageStateBench(Arc>); diff --git a/src/iterators/mod.rs b/src/iterators/mod.rs index 5224c4b..d25dd8a 100644 --- a/src/iterators/mod.rs +++ b/src/iterators/mod.rs @@ -1,10 +1,10 @@ +pub mod inspect; pub mod lsm; mod maybe_empty; pub mod merge; pub mod no_deleted; mod no_duplication; mod ok_iter; -pub mod inspect; pub mod two_merge; pub mod utils; diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index 4aba831..db2ffb1 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -7,7 +7,7 @@ use std::iter; use either::Either; use futures::future::IntoStream; use futures::stream::{FlatMap, Flatten, Iter}; -use futures::{stream, FutureExt, Stream, StreamExt}; +use futures::{stream, FutureExt, StreamExt}; pub fn transpose_try_iter(iterator: Result) -> Either>> where diff --git a/src/lsm/core.rs b/src/lsm/core.rs index fd6a660..c3772f7 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -8,7 +8,6 @@ use bytes::Bytes; use futures::{FutureExt, StreamExt}; use futures_concurrency::stream::Merge; -use crate::mvcc::core::TimeProviderWrapper; use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::{LsmStorageState, Map}; @@ -139,9 +138,9 @@ enum Signal { #[cfg(test)] mod tests { use arc_swap::access::DynAccess; - use futures::StreamExt; + use nom::AsBytes; - use std::ops::Bound::{Included, Unbounded}; + use std::time::Duration; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; @@ -152,7 +151,7 @@ mod tests { use crate::sst::SstOptions; use crate::state::Map; use crate::test_utils::insert_sst; - use crate::time::{SystemTime, TimeIncrement}; + // todo: WAL causes the "too many open files" error // #[tokio::test] // async fn test_task2_auto_flush() { diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index b655101..5963550 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,15 +1,14 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::{iter, slice}; +use std::slice; use bytemuck::TransparentWrapperAlloc; use bytes::Bytes; use crossbeam_skiplist::SkipMap; use derive_getters::Getters; -use nom::AsBytes; use ref_cast::RefCast; use tracing_futures::Instrument; @@ -250,7 +249,6 @@ mod test { use crate::persistent::LocalFs; use crate::time::{TimeIncrement, TimeProvider}; use bytes::Bytes; - use futures::StreamExt; use std::ops::Bound::Included; use tempfile::tempdir; @@ -381,7 +379,7 @@ mod test { let upper = upper.map(Key::from); let lower = lower.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); let upper = upper.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); - let mut iter = memtable.scan_with_ts(lower, upper).await.unwrap(); + let iter = memtable.scan_with_ts(lower, upper).await.unwrap(); let (new_iter, elem) = iter.unwrap().next().await; let new_iter = new_iter.unwrap(); diff --git a/src/mvcc/core.rs b/src/mvcc/core.rs index 9800e30..646bf1e 100644 --- a/src/mvcc/core.rs +++ b/src/mvcc/core.rs @@ -6,7 +6,7 @@ use parking_lot::Mutex; use crate::mvcc::transaction::Transaction; use crate::mvcc::watermark::Watermark; use crate::persistent::Persistent; -use crate::state::{LsmStorageState, LsmStorageStateInner}; +use crate::state::LsmStorageState; use crate::time::TimeProvider; #[derive(Debug)] diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 762b92d..1fd1a5a 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,10 +1,12 @@ use crate::bound::BoundRange; use crate::entry::{Entry, Keyed}; +use crate::iterators::inspect::{InspectIter, InspectIterImpl}; use crate::iterators::lsm::LsmIterator; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{create_two_merge_iter, LockedLsmIter}; use crate::mvcc::transaction::{RWSet, Transaction}; use crate::persistent::Persistent; +use crate::utils::scoped::ScopedMutex; use async_iter_ext::StreamTools; use bytes::Bytes; use crossbeam_skiplist::SkipMap; @@ -12,11 +14,8 @@ use derive_new::new; use futures::{stream, Stream, StreamExt, TryStreamExt}; use num_traits::Bounded; use ouroboros::self_referencing; -use std::collections::Bound::{Excluded, Included, Unbounded}; use std::future::ready; use std::ops::Bound; -use crate::iterators::inspect::{InspectIter, InspectIterImpl}; -use crate::utils::scoped::ScopedMutex; struct TxnWithBound<'a, P: Persistent> { txn: Transaction<'a, P>, @@ -62,16 +61,14 @@ impl<'a, P: Persistent> LockedTxnIter<'a, P> { ); let merged = create_two_merge_iter(local_iter, lsm_iter).await?; let iter = new_no_deleted_iter(merged); - let iter = InspectIterImpl::inspect_stream( - iter, - |entry| { - let Ok(entry) = entry else { return }; - let Some(key_hashes) = self.key_hashes else { return }; - let key = entry.key.as_ref(); - key_hashes.lock_with(|mut set| set.add_read_key(key)); - - } - ); + let iter = InspectIterImpl::inspect_stream(iter, |entry| { + let Ok(entry) = entry else { return }; + let Some(key_hashes) = self.key_hashes else { + return; + }; + let key = entry.key.as_ref(); + key_hashes.lock_with(|mut set| set.add_read_key(key)); + }); let iter = Box::new(iter) as _; Ok(iter) } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 6c64074..4bf8b24 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -1,7 +1,6 @@ use anyhow::anyhow; use bytes::Bytes; use crossbeam_skiplist::SkipMap; -use parking_lot::Mutex; use std::collections::{Bound, HashSet}; use std::ops::Bound::Excluded; use std::sync::atomic::AtomicBool; @@ -13,8 +12,8 @@ use crate::iterators::LockedLsmIter; use crate::mvcc::core::{CommittedTxnData, LsmMvccInner}; use crate::mvcc::iterator::LockedTxnIter; use crate::persistent::Persistent; -use crate::state::{LsmStorageState, LsmStorageStateInner, Map}; -use crate::utils::scoped::{Scoped, ScopedMutex}; +use crate::state::{LsmStorageState, Map}; +use crate::utils::scoped::ScopedMutex; #[derive(Debug, Default)] pub struct RWSet { @@ -123,7 +122,7 @@ impl<'a, P: Persistent> Transaction<'a, P> { let expected_commit_ts = { // todo: 这里的锁可以去掉？ - let mut guard = self.mvcc.ts.lock(); + let guard = self.mvcc.ts.lock(); guard.0 + 1 }; let key_hashes = self.key_hashes.take().map(ScopedMutex::into_inner); diff --git a/src/mvcc/watermark.rs b/src/mvcc/watermark.rs index c120ddf..ea1690c 100644 --- a/src/mvcc/watermark.rs +++ b/src/mvcc/watermark.rs @@ -4,6 +4,12 @@ pub struct Watermark { readers: BTreeMap, } +impl Default for Watermark { + fn default() -> Self { + Self::new() + } +} + impl Watermark { pub fn new() -> Self { Self { diff --git a/src/sst/bloom.rs b/src/sst/bloom.rs index ab9e16c..fe45976 100644 --- a/src/sst/bloom.rs +++ b/src/sst/bloom.rs @@ -71,7 +71,7 @@ impl Bloom { /// Build bloom filter from key hashes pub fn build_from_key_hashes(keys: &[u32], bits_per_key: usize) -> Self { let k = (bits_per_key as f64 * 0.69) as u32; - let k = k.min(30).max(1); + let k = k.clamp(1, 30); let nbits = (keys.len() * bits_per_key).max(64); let nbytes = (nbits + 7) / 8; let nbits = nbytes * 8; @@ -103,7 +103,7 @@ impl Bloom { } fn compute_index(value: u32, num_hash: u8, nbits: usize) -> impl Iterator { - let delta = (value >> 17) | (value << 15); + let delta = value.rotate_left(15); (0..num_hash).scan(value, move |h, _| { let new_h = (*h).wrapping_add(delta); diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 6f3cd85..29e4ca5 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -204,10 +204,13 @@ mod tests { use tempfile::tempdir; use crate::block::BlockCache; - use crate::key::KeySlice; + + use crate::key::{Key, KeySlice}; use crate::persistent::{LocalFs, Persistent}; use crate::sst::builder::test_util::{key_of, num_of_keys, value_of}; + use crate::sst::iterator::SsTableIterator; use crate::sst::{SsTable, SsTableBuilder}; + use futures::StreamExt; #[tokio::test] async fn test_sst_build_single_key() { @@ -273,21 +276,24 @@ mod tests { } // todo: add test - // #[tokio::test] - // async fn test_sst_build_multi_version_hard() { - // let dir = tempdir().unwrap(); - // let persistent = LocalFs::new(dir.path().to_path_buf()); - // let data = generate_test_data(); - // let _ = generate_sst_with_ts(1, &persistent, data.clone(), None).await; - // let sst = SsTable::open(1, None, &persistent).await.unwrap(); - // let sst_iter = SsTableIterator::create_and_seek_to_first(&sst) - // .map(|entry| { - // let entry = entry.unwrap(); - // let key = entry.key; - // todo!() - // }); - // - // } + #[tokio::test] + async fn test_sst_build_multi_version_hard() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let data = generate_test_data(); + let _ = generate_sst_with_ts(1, &persistent, data.clone(), None).await; + let sst = SsTable::open(1, None, &persistent).await.unwrap(); + let result: Vec<_> = SsTableIterator::create_and_seek_to_first(&sst) + .map(|entry| { + let entry = entry.unwrap(); + let Key { key, timestamp } = entry.key; + let value = entry.value; + ((key, timestamp), value) + }) + .collect() + .await; + assert_eq!(data, result); + } #[tokio::test] async fn test_task3_sst_ts() { diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 806ca7d..510415d 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -226,7 +226,7 @@ mod tests { use crate::sst::{SstOptions, Sstables}; use crate::state::{LsmStorageState, Map}; use crate::test_utils::insert_sst; - use crate::time::TimeIncrement; + #[test] fn test_select_level_source() { diff --git a/src/state/inner.rs b/src/state/inner.rs index 9c31b42..c01550a 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -4,19 +4,16 @@ use futures::stream; use futures::stream::{StreamExt, TryStreamExt}; use std::cmp::max; use std::fmt::{Debug, Formatter}; -use std::future::Future; use std::sync::Arc; use typed_builder::TypedBuilder; use crate::block::BlockCache; -use crate::entry::Entry; use crate::key::KeyBytes; use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; use crate::memtable::{ImmutableMemTable, MemTable}; use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; use crate::sst::{SsTable, SstOptions, Sstables}; -use crate::state::{LsmStorageState, Map}; pub struct RecoveredState { pub state: LsmStorageStateInner

, diff --git a/src/state/states.rs b/src/state/states.rs index c73ab11..94acc71 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -3,14 +3,12 @@ use arc_swap::access::Access; use arc_swap::ArcSwap; use bytes::Bytes; use derive_getters::Getters; -use futures::StreamExt; use std::collections::Bound; use std::fmt::{Debug, Formatter}; use std::ops::Deref; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::{Mutex, MutexGuard}; -use tracing_futures::Instrument; use crate::block::BlockCache; use crate::entry::Entry; @@ -24,7 +22,6 @@ use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::{LsmStorageStateInner, RecoveredState}; use crate::state::Map; -use crate::time::TimeProvider; use crate::utils::vec::pop; #[derive(Getters)] diff --git a/src/wal.rs b/src/wal.rs index 254fbe7..67ec53f 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -1,18 +1,15 @@ -use crate::key::{KeyBytes, KeySlice}; +use crate::key::KeyBytes; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; -use std::cmp::max; -use std::future::Future; use std::io::Cursor; use std::sync::Arc; -use std::{iter, slice}; +use std::iter; -use crate::entry::{Entry, Keyed}; +use crate::entry::Keyed; use crate::persistent::interface::WalHandle; use crate::persistent::Persistent; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Mutex; -use tracing_futures::Instrument; pub struct Wal { file: Arc>, From 67b0977cacf119c8b3f4d1406b2a25c585118b41 Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 14 Aug 2024 22:42:14 +0800 Subject: [PATCH 61/69] add test --- src/lsm/core.rs | 6 +- src/memtable/mutable.rs | 2 +- src/sst/builder.rs | 2 +- src/sst/compact/leveled.rs | 1 - src/sst/sstables.rs | 1 + src/state/states.rs | 359 ++++++++++++++++++------------------- src/wal.rs | 2 +- 7 files changed, 180 insertions(+), 193 deletions(-) diff --git a/src/lsm/core.rs b/src/lsm/core.rs index c3772f7..e74bcb7 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -138,9 +138,9 @@ enum Signal { #[cfg(test)] mod tests { use arc_swap::access::DynAccess; - + use nom::AsBytes; - + use std::time::Duration; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; @@ -151,7 +151,7 @@ mod tests { use crate::sst::SstOptions; use crate::state::Map; use crate::test_utils::insert_sst; - + // todo: WAL causes the "too many open files" error // #[tokio::test] // async fn test_task2_auto_flush() { diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 5963550..239c7c3 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -1,9 +1,9 @@ use std::fmt::{Debug, Formatter}; use std::ops::Bound; +use std::slice; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::slice; use bytemuck::TransparentWrapperAlloc; use bytes::Bytes; diff --git a/src/sst/builder.rs b/src/sst/builder.rs index 29e4ca5..460f3a7 100644 --- a/src/sst/builder.rs +++ b/src/sst/builder.rs @@ -204,7 +204,7 @@ mod tests { use tempfile::tempdir; use crate::block::BlockCache; - + use crate::key::{Key, KeySlice}; use crate::persistent::{LocalFs, Persistent}; use crate::sst::builder::test_util::{key_of, num_of_keys, value_of}; diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 510415d..9a7e065 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -226,7 +226,6 @@ mod tests { use crate::sst::{SstOptions, Sstables}; use crate::state::{LsmStorageState, Map}; use crate::test_utils::insert_sst; - #[test] fn test_select_level_source() { diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index 9b2caf2..72e2464 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -146,6 +146,7 @@ where .iter() .map(|id| self.sstables.get(id).unwrap()) .filter_map(move |table| { + // todo: scan not use bloom? if !filter_sst_by_bloom(table, lower, upper) { None } else { diff --git a/src/state/states.rs b/src/state/states.rs index 94acc71..714ed30 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -349,20 +349,23 @@ where #[cfg(test)] mod test { use bytes::Bytes; - use futures::StreamExt; + use futures::{stream, Stream, StreamExt}; use std::collections::Bound; use std::ops::Bound::{Excluded, Included, Unbounded}; use tempfile::{tempdir, TempDir}; - use crate::entry::Entry; + use crate::entry::{Entry, InnerEntry, Keyed}; use crate::iterators::utils::test_utils::{ assert_stream_eq, build_stream, build_tuple_stream, eq, }; + use crate::iterators::{create_merge_iter, MergeIterator}; use crate::mvcc::transaction::Transaction; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; + use crate::sst::iterator::SsTableIterator; use crate::sst::SstOptions; use crate::state::states::LsmStorageState; + use crate::state::LsmStorageStateInner; #[tokio::test] async fn test_task2_storage_integration() { @@ -945,128 +948,160 @@ mod test { } // todo: add test - // #[tokio::test] - // async fn test_task3_mvcc_compaction() { - // use Op::{Put, Del}; - // let dir = tempdir().unwrap(); - // let persistent = LocalFs::new(dir.path().to_path_buf()); - // let options = SstOptions::builder() - // .target_sst_size(1024) - // .block_size(4096) - // .num_memtable_limit(1000) - // .compaction_option(Default::default()) - // .enable_wal(true) - // .build(); - // let storage = LsmStorageState::new(options, persistent).await.unwrap(); - // - // let snapshot0 = storage.new_txn().unwrap(); - // storage - // .write_batch_for_test(&[ - // Put {key: b"a", value: b"1"}, - // Put{key: b"b", value: b"1"}, - // ]).await - // .unwrap(); - // let snapshot1 = storage.new_txn().unwrap(); - // storage - // .write_batch_for_test(&[ - // Put{key: b"a", value: b"2"}, - // Put{key: b"d", value: b"2"}, - // ]).await - // .unwrap(); - // let snapshot2 = storage.new_txn().unwrap(); - // storage - // .write_batch_for_test(&[ - // Put{key: b"a", value: b"3"}, - // Del(b"d"), - // ]).await - // .unwrap(); - // let snapshot3 = storage.new_txn().unwrap(); - // storage - // .write_batch_for_test(&[ - // Put{key: b"c", value: b"4"}, - // Del(b"a"), - // ]).await - // .unwrap(); - // - // storage.force_flush().unwrap(); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("a"), Bytes::new()), - // (Bytes::from("a"), Bytes::from("3")), - // (Bytes::from("a"), Bytes::from("2")), - // (Bytes::from("a"), Bytes::from("1")), - // (Bytes::from("b"), Bytes::from("1")), - // (Bytes::from("c"), Bytes::from("4")), - // (Bytes::from("d"), Bytes::new()), - // (Bytes::from("d"), Bytes::from("2")), - // ], - // ); - // - // drop(snapshot0); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("a"), Bytes::new()), - // (Bytes::from("a"), Bytes::from("3")), - // (Bytes::from("a"), Bytes::from("2")), - // (Bytes::from("a"), Bytes::from("1")), - // (Bytes::from("b"), Bytes::from("1")), - // (Bytes::from("c"), Bytes::from("4")), - // (Bytes::from("d"), Bytes::new()), - // (Bytes::from("d"), Bytes::from("2")), - // ], - // ); - // - // drop(snapshot1); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("a"), Bytes::new()), - // (Bytes::from("a"), Bytes::from("3")), - // (Bytes::from("a"), Bytes::from("2")), - // (Bytes::from("b"), Bytes::from("1")), - // (Bytes::from("c"), Bytes::from("4")), - // (Bytes::from("d"), Bytes::new()), - // (Bytes::from("d"), Bytes::from("2")), - // ], - // ); - // - // drop(snapshot2); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("a"), Bytes::new()), - // (Bytes::from("a"), Bytes::from("3")), - // (Bytes::from("b"), Bytes::from("1")), - // (Bytes::from("c"), Bytes::from("4")), - // ], - // ); - // - // drop(snapshot3); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("b"), Bytes::from("1")), - // (Bytes::from("c"), Bytes::from("4")), - // ], - // ); - // } + #[tokio::test] + async fn test_task3_mvcc_compaction() { + let dir = tempdir().unwrap(); + let persistent = LocalFs::new(dir.path().to_path_buf()); + let options = SstOptions::builder() + .target_sst_size(1024) + .block_size(4096) + .num_memtable_limit(1000) + .compaction_option(Default::default()) + .enable_wal(true) + .enable_mvcc(true) + .build(); + let storage = LsmStorageState::new(options, persistent).await.unwrap(); + + /* + a b c d + ----------- 0 + 1 1 + ----------- 1 + 2 2 + ----------- 2 + 3 - + ----------- 3 + - 4 + + */ + let snapshot0 = storage.new_txn().unwrap(); + storage.put_for_test(b"a", b"1").await.unwrap(); + storage.put_for_test(b"b", b"1").await.unwrap(); + + let snapshot1 = storage.new_txn().unwrap(); + storage.put_for_test(b"a", b"2").await.unwrap(); + storage.put_for_test(b"d", b"2").await.unwrap(); + + let snapshot2 = storage.new_txn().unwrap(); + storage.put_for_test(b"a", b"3").await.unwrap(); + storage.delete_for_test(b"d").await.unwrap(); + + let snapshot3 = storage.new_txn().unwrap(); + storage.put_for_test(b"c", b"4").await.unwrap(); + storage.delete_for_test(b"a").await.unwrap(); + + { + let guard = storage.state_lock.lock().await; + storage.force_flush_imm_memtable(&guard).await.unwrap(); + storage.force_compact(&guard).await.unwrap(); + } + + { + let inner = storage.inner.load(); + let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; + assert_mvcc_compaction_iter( + iter, + [ + ("a", ""), + ("a", "3"), + ("a", "2"), + ("a", "1"), + ("b", "1"), + ("c", "4"), + ("d", ""), + ("d", "2"), + ], + ) + .await; + } + + drop(snapshot0); + { + let guard = storage.state_lock.lock().await; + storage.force_compact(&guard).await.unwrap(); + } + + { + let inner = storage.inner.load(); + let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; + assert_mvcc_compaction_iter( + iter, + [ + ("a", ""), + ("a", "3"), + ("a", "2"), + ("a", "1"), + ("b", "1"), + ("c", "4"), + ("d", ""), + ("d", "2"), + ], + ) + .await; + } + + drop(snapshot1); + { + let guard = storage.state_lock.lock().await; + storage.force_compact(&guard).await.unwrap(); + } + + { + let inner = storage.inner.load(); + let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; + assert_mvcc_compaction_iter( + iter, + [ + ("a", ""), + ("a", "3"), + ("a", "2"), + ("b", "1"), + ("c", "4"), + ("d", ""), + ("d", "2"), + ], + ) + .await; + } + + drop(snapshot2); + { + let guard = storage.state_lock.lock().await; + storage.force_compact(&guard).await.unwrap(); + } + + { + let inner = storage.inner.load(); + let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; + assert_mvcc_compaction_iter(iter, [("a", ""), ("a", "3"), ("b", "1"), ("c", "4")]) + .await; + } + + drop(snapshot3); + { + let guard = storage.state_lock.lock().await; + storage.force_compact(&guard).await.unwrap(); + } + + { + let inner = storage.inner.load(); + let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; + assert_mvcc_compaction_iter(iter, [("b", "1"), ("c", "4")]).await; + } + } + + async fn construct_test_mvcc_compaction_iter<'a, P: Persistent>( + storage: &'a LsmStorageStateInner

, + ) -> impl Stream> + 'a { + let iter = storage + .sstables_state + .sstables() + .values() + .map(|sst| SsTableIterator::scan(sst.as_ref(), Unbounded, Unbounded)); + let iter = stream::iter(iter); + let result = create_merge_iter(iter).await; + result + } #[tokio::test] async fn test_txn_integration() { @@ -1174,6 +1209,20 @@ mod test { .await; } + async fn assert_mvcc_compaction_iter( + iter: impl Stream>, + expected: impl IntoIterator, + ) { + let iter = iter + .map(Result::unwrap) + .map(|entry| (entry.key.key, entry.value)); + let expected = expected + .into_iter() + .map(|(key, value)| (Bytes::from(key), Bytes::from(value))); + let expected = stream::iter(expected); + assert_stream_eq(iter, expected).await; + } + #[tokio::test] async fn test_serializable_1() { let dir = tempdir().unwrap(); @@ -1329,66 +1378,4 @@ mod test { LsmStorageState::new(options, persistent).await.unwrap() } - - // todo: week 3, day 7 test - // #[test] - // fn test_task3_mvcc_compaction() { - // let dir = tempdir().unwrap(); - // let options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); - // let storage = MiniLsm::open(&dir, options.clone()).unwrap(); - // storage - // .write_batch(&[ - // WriteBatchRecord::Put("table1_a", "1"), - // WriteBatchRecord::Put("table1_b", "1"), - // WriteBatchRecord::Put("table1_c", "1"), - // WriteBatchRecord::Put("table2_a", "1"), - // WriteBatchRecord::Put("table2_b", "1"), - // WriteBatchRecord::Put("table2_c", "1"), - // ]) - // .unwrap(); - // storage.force_flush().unwrap(); - // let snapshot0 = storage.new_txn().unwrap(); - // storage - // .write_batch(&[ - // WriteBatchRecord::Put("table1_a", "2"), - // WriteBatchRecord::Del("table1_b"), - // WriteBatchRecord::Put("table1_c", "2"), - // WriteBatchRecord::Put("table2_a", "2"), - // WriteBatchRecord::Del("table2_b"), - // WriteBatchRecord::Put("table2_c", "2"), - // ]) - // .unwrap(); - // storage.force_flush().unwrap(); - // storage.add_compaction_filter(CompactionFilter::Prefix(Bytes::from("table2_"))); - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("table1_a"), Bytes::from("2")), - // (Bytes::from("table1_a"), Bytes::from("1")), - // (Bytes::from("table1_b"), Bytes::new()), - // (Bytes::from("table1_b"), Bytes::from("1")), - // (Bytes::from("table1_c"), Bytes::from("2")), - // (Bytes::from("table1_c"), Bytes::from("1")), - // (Bytes::from("table2_a"), Bytes::from("2")), - // (Bytes::from("table2_b"), Bytes::new()), - // (Bytes::from("table2_c"), Bytes::from("2")), - // ], - // ); - // - // drop(snapshot0); - // - // storage.force_full_compaction().unwrap(); - // - // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); - // check_iter_result_by_key( - // &mut iter, - // vec![ - // (Bytes::from("table1_a"), Bytes::from("2")), - // (Bytes::from("table1_c"), Bytes::from("2")), - // ], - // ); - // } } diff --git a/src/wal.rs b/src/wal.rs index 67ec53f..8205d6a 100644 --- a/src/wal.rs +++ b/src/wal.rs @@ -2,8 +2,8 @@ use crate::key::KeyBytes; use bytes::{Buf, Bytes}; use crossbeam_skiplist::SkipMap; use std::io::Cursor; -use std::sync::Arc; use std::iter; +use std::sync::Arc; use crate::entry::Keyed; use crate::persistent::interface::WalHandle; From 6f7ff34168064ea844c8a6a9c0abcd2736435dc6 Mon Sep 17 00:00:00 2001 From: Kermit Date: Wed, 14 Aug 2024 22:52:22 +0800 Subject: [PATCH 62/69] fix: test --- src/state/states.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/src/state/states.rs b/src/state/states.rs index 714ed30..4bd6a42 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -972,8 +972,8 @@ mod test { 3 - ----------- 3 - 4 + */ - */ let snapshot0 = storage.new_txn().unwrap(); storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); @@ -992,6 +992,7 @@ mod test { { let guard = storage.state_lock.lock().await; + storage.force_freeze_memtable(&guard).await.unwrap(); storage.force_flush_imm_memtable(&guard).await.unwrap(); storage.force_compact(&guard).await.unwrap(); } @@ -1378,4 +1379,66 @@ mod test { LsmStorageState::new(options, persistent).await.unwrap() } + + // todo: week 3, day 7 test + // #[test] + // fn test_task3_mvcc_compaction() { + // let dir = tempdir().unwrap(); + // let options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); + // let storage = MiniLsm::open(&dir, options.clone()).unwrap(); + // storage + // .write_batch(&[ + // WriteBatchRecord::Put("table1_a", "1"), + // WriteBatchRecord::Put("table1_b", "1"), + // WriteBatchRecord::Put("table1_c", "1"), + // WriteBatchRecord::Put("table2_a", "1"), + // WriteBatchRecord::Put("table2_b", "1"), + // WriteBatchRecord::Put("table2_c", "1"), + // ]) + // .unwrap(); + // storage.force_flush().unwrap(); + // let snapshot0 = storage.new_txn().unwrap(); + // storage + // .write_batch(&[ + // WriteBatchRecord::Put("table1_a", "2"), + // WriteBatchRecord::Del("table1_b"), + // WriteBatchRecord::Put("table1_c", "2"), + // WriteBatchRecord::Put("table2_a", "2"), + // WriteBatchRecord::Del("table2_b"), + // WriteBatchRecord::Put("table2_c", "2"), + // ]) + // .unwrap(); + // storage.force_flush().unwrap(); + // storage.add_compaction_filter(CompactionFilter::Prefix(Bytes::from("table2_"))); + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("table1_a"), Bytes::from("2")), + // (Bytes::from("table1_a"), Bytes::from("1")), + // (Bytes::from("table1_b"), Bytes::new()), + // (Bytes::from("table1_b"), Bytes::from("1")), + // (Bytes::from("table1_c"), Bytes::from("2")), + // (Bytes::from("table1_c"), Bytes::from("1")), + // (Bytes::from("table2_a"), Bytes::from("2")), + // (Bytes::from("table2_b"), Bytes::new()), + // (Bytes::from("table2_c"), Bytes::from("2")), + // ], + // ); + // + // drop(snapshot0); + // + // storage.force_full_compaction().unwrap(); + // + // let mut iter = construct_merge_iterator_over_storage(&storage.inner.state.read()); + // check_iter_result_by_key( + // &mut iter, + // vec![ + // (Bytes::from("table1_a"), Bytes::from("2")), + // (Bytes::from("table1_c"), Bytes::from("2")), + // ], + // ); + // } } From 8b5a715298503a67fc7b521630dffb0a2aecb663 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 15 Aug 2024 22:27:06 +0800 Subject: [PATCH 63/69] feat: add WatermarkGcIter --- src/mvcc/iterator.rs | 91 ++++++++++++++++++++++++++++++++++++++++++-- src/state/states.rs | 8 ++-- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 1fd1a5a..662e803 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -1,9 +1,10 @@ use crate::bound::BoundRange; -use crate::entry::{Entry, Keyed}; +use crate::entry::{Entry, InnerEntry, Keyed}; use crate::iterators::inspect::{InspectIter, InspectIterImpl}; use crate::iterators::lsm::LsmIterator; use crate::iterators::no_deleted::new_no_deleted_iter; use crate::iterators::{create_two_merge_iter, LockedLsmIter}; +use crate::key::KeyBytes; use crate::mvcc::transaction::{RWSet, Transaction}; use crate::persistent::Persistent; use crate::utils::scoped::ScopedMutex; @@ -110,6 +111,53 @@ where .map(|entry| entry.map(|pair| pair.0)) } +pub trait WatermarkGcIter { + type Stream>>: Stream< + Item = anyhow::Result, + >; + + fn build_watermark_gc_iter(s: S, watermark: u64) -> Self::Stream + where + S: Stream>; +} + +struct WatermarkGcIterImpl; + +impl WatermarkGcIter for WatermarkGcIterImpl { + type Stream>> = + impl Stream>; + + fn build_watermark_gc_iter(s: S, watermark: u64) -> Self::Stream + where + S: Stream>, + { + s.scan(None, move |state: &mut Option, entry| { + let item = match entry { + Err(e) => Some(Some(Err(e))), + Ok(entry) => { + if let Some(prev_key) = state { + if prev_key.key == entry.key.key { + if entry.key.timestamp <= watermark { + Some(None) + } else { + Some(Some(Ok(entry))) + } + } else { + *state = Some(entry.key.clone()); + Some(Some(Ok(entry))) + } + } else { + *state = Some(entry.key.clone()); + Some(Some(Ok(entry))) + } + } + }; + ready(item) + }) + .filter_map(|entry| async { entry }) + } +} + pub fn transform_bound(lower: Bound, upper: Bound, timestamp: T) -> BoundRange<(A, T)> where T: Bounded, @@ -146,9 +194,12 @@ where #[cfg(test)] mod tests { - use crate::entry::Keyed; - use crate::mvcc::iterator::build_time_dedup_iter; - use futures::{stream, StreamExt, TryStreamExt}; + use crate::entry::{InnerEntry, Keyed}; + use crate::iterators::utils::test_utils::assert_stream_eq; + use crate::key::KeyBytes; + use crate::mvcc::iterator::{build_time_dedup_iter, WatermarkGcIter, WatermarkGcIterImpl}; + use bytes::Bytes; + use futures::{stream, Stream, StreamExt, TryStreamExt}; #[tokio::test] async fn test_build_time_dedup_iter() { @@ -196,4 +247,36 @@ mod tests { let expected: Vec<_> = expected.into_iter().collect(); assert_eq!(result, expected); } + + #[tokio::test] + async fn test_watermark_gc() { + test_watermark_gc_helper([("a", 0), ("b", 0)], 5, [("a", 0), ("b", 0)]).await; + test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 0)], 2, [("a", 3), ("b", 0)]).await; + test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 5)], 2, [("a", 3), ("b", 5)]).await; + } + + async fn test_watermark_gc_helper( + input: impl IntoIterator, + watermark: u64, + expected_output: impl IntoIterator, + ) { + fn transform( + iter: impl IntoIterator, + ) -> impl Stream> { + let s = iter.into_iter().map(|(key, timestamp)| { + Ok(InnerEntry::new( + KeyBytes::new(Bytes::from(key), timestamp), + Bytes::new(), + )) + }); + stream::iter(s) + } + + assert_stream_eq( + WatermarkGcIterImpl::build_watermark_gc_iter(transform(input), watermark) + .map(Result::unwrap), + transform(expected_output).map(Result::unwrap), + ) + .await; + } } diff --git a/src/state/states.rs b/src/state/states.rs index 4bd6a42..1f27673 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -354,11 +354,11 @@ mod test { use std::ops::Bound::{Excluded, Included, Unbounded}; use tempfile::{tempdir, TempDir}; - use crate::entry::{Entry, InnerEntry, Keyed}; + use crate::entry::{Entry, InnerEntry}; + use crate::iterators::create_merge_iter; use crate::iterators::utils::test_utils::{ assert_stream_eq, build_stream, build_tuple_stream, eq, }; - use crate::iterators::{create_merge_iter, MergeIterator}; use crate::mvcc::transaction::Transaction; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; @@ -1100,8 +1100,8 @@ mod test { .values() .map(|sst| SsTableIterator::scan(sst.as_ref(), Unbounded, Unbounded)); let iter = stream::iter(iter); - let result = create_merge_iter(iter).await; - result + + create_merge_iter(iter).await } #[tokio::test] From bfdb22f69d8d92d959740387853b5ddac8233308 Mon Sep 17 00:00:00 2001 From: Kermit Date: Thu, 15 Aug 2024 22:47:46 +0800 Subject: [PATCH 64/69] fix: clippy --- Cargo.toml | 1 - benches/ycsb.rs | 3 +++ src/iterators/utils.rs | 22 ---------------------- src/lib.rs | 2 +- src/lsm/core.rs | 26 +++++++++++++++++++------- src/lsm/mod.rs | 2 +- src/memtable/mutable.rs | 2 +- src/mvcc/transaction.rs | 4 ---- src/mvcc/watermark.rs | 3 +++ src/state/states.rs | 11 +++++------ src/test_utils/command.rs | 5 ----- src/test_utils/iterator.rs | 12 ------------ src/test_utils/map.rs | 1 - src/test_utils/mod.rs | 6 ++++-- src/utils/func.rs | 1 - src/utils/mod.rs | 1 - src/utils/scoped.rs | 20 -------------------- 17 files changed, 37 insertions(+), 85 deletions(-) delete mode 100644 src/test_utils/command.rs delete mode 100644 src/test_utils/map.rs delete mode 100644 src/utils/func.rs diff --git a/Cargo.toml b/Cargo.toml index f9f3436..b8347fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,6 @@ criterion = { version = "0.5.1", features = ["async_tokio"] } maplit = "1.0.2" [lints.rust] -#unused = "allow" unsafe_code = "forbid" [[bench]] diff --git a/benches/ycsb.rs b/benches/ycsb.rs index 604f80f..ac370dd 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -19,7 +19,10 @@ struct LsmStorageStateBench(Arc>); impl IsSend for LsmStorageStateBench {} impl IsSync for LsmStorageStateBench {} +#[allow(dead_code)] trait IsSend: Send {} + +#[allow(dead_code)] trait IsSync: Sync {} impl DB for LsmStorageStateBench { diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index db2ffb1..a362d45 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -55,28 +55,6 @@ where stream::iter(iterator.map(FutureExt::into_stream as fn(_) -> _)).flatten() } -pub struct Ref { - data: T, - finished: bool, -} - -impl Ref { - pub fn new(data: T) -> Self { - Self { - data, - finished: false, - } - } -} - -impl<'a, T> Iterator for &'a Ref { - type Item = &'a T; - - fn next(&mut self) -> Option { - todo!() - } -} - #[cfg(test)] pub mod test_utils { use crate::entry::Entry; diff --git a/src/lib.rs b/src/lib.rs index 107a7f7..763f64e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ pub mod sst; pub mod state; mod wal; -mod lsm; +pub mod lsm; mod manifest; pub mod mvcc; mod test_utils; diff --git a/src/lsm/core.rs b/src/lsm/core.rs index e74bcb7..8cf6ac7 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -5,12 +5,12 @@ use std::time::Duration; use bytes::Bytes; -use futures::{FutureExt, StreamExt}; -use futures_concurrency::stream::Merge; - use crate::persistent::Persistent; use crate::sst::SstOptions; use crate::state::{LsmStorageState, Map}; +use futures::{FutureExt, StreamExt}; +use futures_concurrency::stream::Merge; +use tokio::runtime::Handle; use tokio::task::JoinHandle; use tokio::time::interval; use tokio_stream::wrappers::IntervalStream; @@ -20,17 +20,21 @@ use tracing::error; pub struct Lsm { state: Arc>, cancel_token: CancellationToken, + flush_handle: Option>, + spawn_handle: Option>, } impl Lsm

{ pub async fn new(options: SstOptions, persistent: P) -> anyhow::Result { let state = Arc::new(LsmStorageState::new(options, persistent).await?); let cancel_token = CancellationToken::new(); - let _ = Self::spawn_flush(state.clone(), cancel_token.clone()); - let _ = Self::spawn_compaction(state.clone(), cancel_token.clone()); + let flush_handle = Self::spawn_flush(state.clone(), cancel_token.clone()); + let spawn_handle = Self::spawn_compaction(state.clone(), cancel_token.clone()); let this = Self { state, cancel_token, + flush_handle: Some(flush_handle), + spawn_handle: Some(spawn_handle), }; Ok(this) } @@ -114,6 +118,16 @@ where impl Drop for Lsm

{ fn drop(&mut self) { self.cancel_token.cancel(); + let flush_handle = self.flush_handle.take(); + let spawn_handle = self.spawn_handle.take(); + Handle::current().block_on(async { + if let Some(flush_handle) = flush_handle { + flush_handle.await.inspect_err(|e| error!(error = ?e)).ok(); + } + if let Some(spawn_handle) = spawn_handle { + spawn_handle.await.inspect_err(|e| error!(error = ?e)).ok(); + } + }) } } @@ -137,8 +151,6 @@ enum Signal { #[cfg(test)] mod tests { - use arc_swap::access::DynAccess; - use nom::AsBytes; use std::time::Duration; diff --git a/src/lsm/mod.rs b/src/lsm/mod.rs index 9816037..5a7ca06 100644 --- a/src/lsm/mod.rs +++ b/src/lsm/mod.rs @@ -1 +1 @@ -mod core; +pub mod core; diff --git a/src/memtable/mutable.rs b/src/memtable/mutable.rs index 239c7c3..75aa3e1 100644 --- a/src/memtable/mutable.rs +++ b/src/memtable/mutable.rs @@ -381,7 +381,7 @@ mod test { let upper = upper.map(|ks| ks.map(|b| Bytes::copy_from_slice(b))); let iter = memtable.scan_with_ts(lower, upper).await.unwrap(); - let (new_iter, elem) = iter.unwrap().next().await; + let (new_iter, _) = iter.unwrap().next().await; let new_iter = new_iter.unwrap(); assert!(new_iter.is_none()); } diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 4bf8b24..9e58147 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -3,7 +3,6 @@ use bytes::Bytes; use crossbeam_skiplist::SkipMap; use std::collections::{Bound, HashSet}; use std::ops::Bound::Excluded; -use std::sync::atomic::AtomicBool; use std::sync::Arc; use tokio_stream::StreamExt; @@ -38,8 +37,6 @@ pub struct Transaction<'a, P: Persistent> { // todo: need Arc<...> ? pub(crate) local_storage: Arc>, - // todo: delete it? - pub(crate) committed: Arc, /// Write set and read set /// todo: check deadlock? pub(crate) key_hashes: Option>, @@ -102,7 +99,6 @@ impl<'a, P: Persistent> Transaction<'a, P> { read_ts, state, local_storage: Arc::default(), - committed: Arc::default(), key_hashes: serializable.then(ScopedMutex::default), mvcc, } diff --git a/src/mvcc/watermark.rs b/src/mvcc/watermark.rs index ea1690c..fa6d491 100644 --- a/src/mvcc/watermark.rs +++ b/src/mvcc/watermark.rs @@ -36,7 +36,10 @@ impl Watermark { pub fn watermark(&self) -> Option { self.readers.keys().copied().next() } +} +#[cfg(test)] +impl Watermark { fn num_retained_snapshots(&self) -> usize { self.readers.len() } diff --git a/src/state/states.rs b/src/state/states.rs index 1f27673..2d38779 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -1,5 +1,4 @@ use anyhow::anyhow; -use arc_swap::access::Access; use arc_swap::ArcSwap; use bytes::Bytes; use derive_getters::Getters; @@ -1091,9 +1090,9 @@ mod test { } } - async fn construct_test_mvcc_compaction_iter<'a, P: Persistent>( - storage: &'a LsmStorageStateInner

, - ) -> impl Stream> + 'a { + async fn construct_test_mvcc_compaction_iter( + storage: &LsmStorageStateInner

, + ) -> impl Stream> + '_ { let iter = storage .sstables_state .sstables() @@ -1314,7 +1313,7 @@ mod test { let mut iter = guard.iter().await.unwrap(); while let Some(entry) = iter.next().await { // todo: check entry - let entry = entry.unwrap(); + let _entry = entry.unwrap(); } } @@ -1350,7 +1349,7 @@ mod test { let mut iter = guard.iter().await.unwrap(); while let Some(entry) = iter.next().await { // todo: check entry - let entry = entry.unwrap(); + let _entry = entry.unwrap(); } } diff --git a/src/test_utils/command.rs b/src/test_utils/command.rs deleted file mode 100644 index 96fcb12..0000000 --- a/src/test_utils/command.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub enum Command { - Get { key: Vec }, - Put { key: Vec, value: Vec }, - Delete { key: Vec }, -} diff --git a/src/test_utils/iterator.rs b/src/test_utils/iterator.rs index cf6b7fc..a92dd49 100644 --- a/src/test_utils/iterator.rs +++ b/src/test_utils/iterator.rs @@ -12,15 +12,3 @@ where }) }) } - -pub fn unwrap_ts_iterator(s: S) -> impl Iterator> -where - S: Iterator>, -{ - s.map(|item| { - item.map(|entry| Entry { - key: entry.key.into_inner(), - value: entry.value, - }) - }) -} diff --git a/src/test_utils/map.rs b/src/test_utils/map.rs deleted file mode 100644 index 014c2fa..0000000 --- a/src/test_utils/map.rs +++ /dev/null @@ -1 +0,0 @@ -pub struct Map {} diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index 5130866..da5dd00 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,11 +1,13 @@ +#[cfg(test)] use std::ops::Range; +#[cfg(test)] use crate::state::Map; -mod command; +#[cfg(test)] pub mod iterator; -mod map; +#[cfg(test)] pub async fn insert_sst>( state: &M, range: Range, diff --git a/src/utils/func.rs b/src/utils/func.rs deleted file mode 100644 index 10a9f6a..0000000 --- a/src/utils/func.rs +++ /dev/null @@ -1 +0,0 @@ -pub fn do_nothing(_: T) {} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index ff2aeb1..4a2578f 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,4 +1,3 @@ -pub mod func; pub mod num; pub mod scoped; pub mod vec; diff --git a/src/utils/scoped.rs b/src/utils/scoped.rs index d5e63ad..d8e8516 100644 --- a/src/utils/scoped.rs +++ b/src/utils/scoped.rs @@ -1,25 +1,5 @@ -use derive_new::new; use parking_lot::{Mutex, MutexGuard}; -#[derive(Debug, Default, new)] -pub struct Scoped { - inner: T, -} - -impl Scoped { - pub fn with_ref(&self, f: F) -> B - where - F: FnOnce(&T) -> B, - { - f(&self.inner) - } - - // todo: into_inner 不应该存在？ - pub fn into_inner(self) -> T { - self.inner - } -} - #[derive(Debug, Default)] pub struct ScopedMutex { inner: Mutex, From e568af40333dd63fd869681376ee6eb92be83160 Mon Sep 17 00:00:00 2001 From: Kermit Date: Fri, 16 Aug 2024 13:55:31 +0800 Subject: [PATCH 65/69] try fix mvcc gc --- src/mvcc/iterator.rs | 64 +++++++++++++++++++++----------------- src/sst/compact/common.rs | 19 ++++++++--- src/sst/compact/leveled.rs | 10 +++++- src/state/states.rs | 2 ++ 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 662e803..5c94351 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -112,49 +112,53 @@ where } pub trait WatermarkGcIter { - type Stream>>: Stream< - Item = anyhow::Result, - >; + type Stream> + Send + Unpin>: Stream> + + Send + + Unpin; fn build_watermark_gc_iter(s: S, watermark: u64) -> Self::Stream where - S: Stream>; + S: Stream> + Send + Unpin; } -struct WatermarkGcIterImpl; +pub struct WatermarkGcIterImpl; impl WatermarkGcIter for WatermarkGcIterImpl { - type Stream>> = - impl Stream>; + type Stream> + Send + Unpin> = + impl Stream> + Send + Unpin; fn build_watermark_gc_iter(s: S, watermark: u64) -> Self::Stream where - S: Stream>, + S: Stream> + Send + Unpin, { - s.scan(None, move |state: &mut Option, entry| { - let item = match entry { - Err(e) => Some(Some(Err(e))), - Ok(entry) => { - if let Some(prev_key) = state { - if prev_key.key == entry.key.key { - if entry.key.timestamp <= watermark { - Some(None) + let result = s + .scan(None, move |state: &mut Option, entry| { + let item = match entry { + Err(e) => Some(Some(Err(e))), + Ok(entry) => { + if let Some(prev_key) = state { + if prev_key.key == entry.key.key { + if entry.key.timestamp <= watermark { + Some(None) + } else { + Some(Some(Ok(entry))) + } } else { + *state = Some(entry.key.clone()); Some(Some(Ok(entry))) } } else { *state = Some(entry.key.clone()); Some(Some(Ok(entry))) } - } else { - *state = Some(entry.key.clone()); - Some(Some(Ok(entry))) } - } - }; - ready(item) - }) - .filter_map(|entry| async { entry }) + }; + ready(item) + }) + .filter_map(|entry| async { entry }); + + // todo: remove Box::pin? + Box::pin(result) } } @@ -255,11 +259,13 @@ mod tests { test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 5)], 2, [("a", 3), ("b", 5)]).await; } - async fn test_watermark_gc_helper( - input: impl IntoIterator, - watermark: u64, - expected_output: impl IntoIterator, - ) { + async fn test_watermark_gc_helper(input: S1, watermark: u64, expected_output: S2) + where + S1: IntoIterator, + S1::IntoIter: Send, + S2: IntoIterator, + S2::IntoIter: Send, + { fn transform( iter: impl IntoIterator, ) -> impl Stream> { diff --git a/src/sst/compact/common.rs b/src/sst/compact/common.rs index c97c84f..3e18dbc 100644 --- a/src/sst/compact/common.rs +++ b/src/sst/compact/common.rs @@ -7,13 +7,14 @@ use futures::{stream, Stream, StreamExt}; use getset::CopyGetters; use serde::{Deserialize, Serialize}; -use std::ops::Range; -use std::sync::Arc; -use tracing::error; - +use crate::mvcc::iterator::{WatermarkGcIter, WatermarkGcIterImpl}; use crate::persistent::{Persistent, SstHandle}; use crate::sst::iterator::{create_sst_concat_and_seek_to_first, SsTableIterator}; use crate::sst::{SsTable, SsTableBuilder, SstOptions, Sstables}; +use futures::future::Either; +use std::ops::Range; +use std::sync::Arc; +use tracing::error; #[derive(Serialize, Deserialize, new, CopyGetters, PartialEq, Debug)] #[getset(get_copy = "pub")] @@ -66,6 +67,7 @@ pub async fn compact_generate_new_sst<'a, P: Persistent, U, L>( next_sst_id: impl Fn() -> usize + Send + 'a + Sync, options: &'a SstOptions, persistent: &'a P, + watermark: Option, ) -> anyhow::Result>>> where U: IntoIterator> + Send + 'a, @@ -86,7 +88,14 @@ where let tables = lower_sstables.into_iter().collect(); create_sst_concat_and_seek_to_first(tables) }?; - let iter = assert_send(create_two_merge_iter(l0, l1)).await?; + let merged_iter = assert_send(create_two_merge_iter(l0, l1)).await?; + let iter = match watermark { + Some(watermark) => Either::Left(WatermarkGcIterImpl::build_watermark_gc_iter( + merged_iter, + watermark, + )), + None => Either::Right(merged_iter), + }; let s: Vec<_> = assert_send( stream::unfold(iter, |mut iter| async { let id = next_sst_id(); diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 9a7e065..5598bcd 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -48,12 +48,14 @@ pub async fn force_compact( options: &SstOptions, persistent: &P, manifest: Option<&Manifest>, + watermark: Option, ) -> anyhow::Result<()> { let Some(task) = generate_task(sstables, options) else { return Ok(()); }; - let new_sst_ids = compact_with_task(sstables, next_sst_id, options, persistent, &task).await?; + let new_sst_ids = + compact_with_task(sstables, next_sst_id, options, persistent, &task, watermark).await?; if let Some(manifest) = manifest { let record = ManifestRecord::Compaction(Compaction(task, new_sst_ids)); @@ -69,6 +71,7 @@ pub async fn compact_with_task( options: &SstOptions, persistent: &P, task: &CompactionTask, + watermark: Option, ) -> anyhow::Result> { let source = task.source(); let source_index = task.source_index(); @@ -82,6 +85,7 @@ pub async fn compact_with_task( next_sst_id, options, persistent, + watermark, ) .await?; @@ -276,6 +280,7 @@ mod tests { &state.options, &state.persistent, &CompactionTask::new(0, 4, 1), + None, ) .await .unwrap(); @@ -292,6 +297,7 @@ mod tests { &state.options, &state.persistent, &CompactionTask::new(0, 3, 1), + None, ) .await .unwrap(); @@ -308,6 +314,7 @@ mod tests { &state.options, &state.persistent, &CompactionTask::new(1, 0, 2), + None, ) .await .unwrap(); @@ -329,6 +336,7 @@ mod tests { &state.options, &state.persistent, None, + None, ) .await .unwrap(); diff --git a/src/state/states.rs b/src/state/states.rs index 2d38779..2419643 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -233,6 +233,7 @@ where let new = { let mut new = Clone::clone(self.inner.load().as_ref()); let mut new_sstables = Clone::clone(new.sstables_state().as_ref()); + let watermark = self.mvcc.as_ref().map(|mvcc| mvcc.watermark()); force_compact( &mut new_sstables, @@ -240,6 +241,7 @@ where self.options(), self.persistent(), Some(&self.manifest), + watermark, ) .await?; From 281004d86263e30c62c92295bdcf6de16dda5c18 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 17 Aug 2024 15:28:16 +0800 Subject: [PATCH 66/69] add full compaction --- src/manifest.rs | 12 ++++-- src/mvcc/iterator.rs | 7 ++- src/sst/compact/common.rs | 28 ++++++++---- src/sst/compact/full.rs | 14 ++++++ src/sst/compact/leveled.rs | 87 ++++++++++++++++++++++++-------------- src/sst/compact/mod.rs | 1 + src/sst/compact/option.rs | 1 + src/sst/sstables.rs | 45 ++++++++++++++++---- src/state/inner.rs | 20 ++++++--- src/state/states.rs | 59 +++++++++++++++++++++++--- src/utils/mod.rs | 1 + src/utils/send.rs | 3 ++ 12 files changed, 214 insertions(+), 64 deletions(-) create mode 100644 src/sst/compact/full.rs create mode 100644 src/utils/send.rs diff --git a/src/manifest.rs b/src/manifest.rs index 4d6574c..9e8d170 100644 --- a/src/manifest.rs +++ b/src/manifest.rs @@ -79,7 +79,7 @@ impl Manifest { mod tests { use crate::manifest::{Compaction, Flush, Manifest, ManifestRecord, NewMemtable}; use crate::persistent::LocalFs; - use crate::sst::compact::common::CompactionTask; + use crate::sst::compact::common::{CompactionTask, SourceIndex}; use tempfile::tempdir; #[tokio::test] @@ -92,7 +92,10 @@ mod tests { { let manifest = Manifest::create(&persistent).await.unwrap(); - let record = Compaction(CompactionTask::new(1, 2, 3), vec![1, 2, 3]); + let record = Compaction( + CompactionTask::new(1, SourceIndex::Index { index: 2 }, 3), + vec![1, 2, 3], + ); manifest .add_record_when_init(R::Compaction(record)) .await @@ -110,7 +113,10 @@ mod tests { { let (_manifest, records) = Manifest::recover(&persistent).await.unwrap(); - let record = Compaction(CompactionTask::new(1, 2, 3), vec![1, 2, 3]); + let record = Compaction( + CompactionTask::new(1, SourceIndex::Index { index: 2 }, 3), + vec![1, 2, 3], + ); assert_eq!( records, diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 5c94351..7ddd854 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -131,6 +131,7 @@ impl WatermarkGcIter for WatermarkGcIterImpl { where S: Stream> + Send + Unpin, { + dbg!("build_watermark_gc_iter"); let result = s .scan(None, move |state: &mut Option, entry| { let item = match entry { @@ -155,7 +156,10 @@ impl WatermarkGcIter for WatermarkGcIterImpl { }; ready(item) }) - .filter_map(|entry| async { entry }); + .filter_map(|entry| async { entry }) + .inspect(|entry| { + dbg!(entry); + }); // todo: remove Box::pin? Box::pin(result) @@ -257,6 +261,7 @@ mod tests { test_watermark_gc_helper([("a", 0), ("b", 0)], 5, [("a", 0), ("b", 0)]).await; test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 0)], 2, [("a", 3), ("b", 0)]).await; test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 5)], 2, [("a", 3), ("b", 5)]).await; + test_watermark_gc_helper([("a", 2), ("a", 1), ("b", 5)], 3, [("a", 2), ("b", 5)]).await; } async fn test_watermark_gc_helper(input: S1, watermark: u64, expected_output: S2) diff --git a/src/sst/compact/common.rs b/src/sst/compact/common.rs index 3e18dbc..7332a44 100644 --- a/src/sst/compact/common.rs +++ b/src/sst/compact/common.rs @@ -11,6 +11,7 @@ use crate::mvcc::iterator::{WatermarkGcIter, WatermarkGcIterImpl}; use crate::persistent::{Persistent, SstHandle}; use crate::sst::iterator::{create_sst_concat_and_seek_to_first, SsTableIterator}; use crate::sst::{SsTable, SsTableBuilder, SstOptions, Sstables}; +use crate::utils::send::assert_send; use futures::future::Either; use std::ops::Range; use std::sync::Arc; @@ -20,10 +21,25 @@ use tracing::error; #[getset(get_copy = "pub")] pub struct CompactionTask { source: usize, - source_index: usize, + source_index: SourceIndex, destination: usize, } +#[derive(Serialize, Deserialize, PartialEq, Debug, Copy, Clone)] +pub enum SourceIndex { + Index { index: usize }, + Full { len: usize }, +} + +impl SourceIndex { + pub fn build_range(self) -> Range { + match self { + SourceIndex::Index { index } => index..index + 1, + SourceIndex::Full { len } => 0..len, + } + } +} + pub fn apply_compaction( sstables: &mut Sstables, source: Range, @@ -115,10 +131,6 @@ where Ok(s) } -fn assert_send(x: T) -> T { - x -} - async fn batch( iter: &mut I, sst_id: usize, @@ -139,9 +151,9 @@ where }; // 被删除的 entry 不再添加 - if entry.value.is_empty() { - continue; - } + // if entry.value.is_empty() { + // continue; + // } let key = entry.key.as_key_slice(); let value = entry.value.as_ref(); diff --git a/src/sst/compact/full.rs b/src/sst/compact/full.rs new file mode 100644 index 0000000..764b317 --- /dev/null +++ b/src/sst/compact/full.rs @@ -0,0 +1,14 @@ +use crate::persistent::SstHandle; +use crate::sst::compact::common::{CompactionTask, SourceIndex}; +use crate::sst::Sstables; + +#[derive(Debug, Clone, Copy)] +pub struct LeveledCompactionOptions; + +pub fn generate_full_compaction_task( + sstables: &Sstables, +) -> Option { + let len = sstables.l0_sstables.len(); + let task = CompactionTask::new(0, SourceIndex::Full { len }, 1); + Some(task) +} diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 5598bcd..22670e8 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -1,19 +1,19 @@ use std::cmp::max; - use std::iter; use crate::manifest::{Compaction, Manifest, ManifestRecord}; use derive_new::new; use getset::CopyGetters; -use tracing::trace; use typed_builder::TypedBuilder; use crate::persistent::{Persistent, SstHandle}; -use crate::sst::compact::common::{apply_compaction, compact_generate_new_sst, CompactionTask}; -use crate::sst::compact::CompactionOptions::Leveled; +use crate::sst::compact::common::{compact_generate_new_sst, CompactionTask, SourceIndex}; +use crate::sst::compact::full::generate_full_compaction_task; +use crate::sst::compact::CompactionOptions::{Full, Leveled, NoCompaction}; use crate::sst::{SstOptions, Sstables}; use crate::utils::num::power_of_2; +use crate::utils::send::assert_send; #[derive(Debug, Clone, new, TypedBuilder, CopyGetters)] #[getset(get_copy = "pub")] @@ -42,6 +42,7 @@ impl LeveledCompactionOptions { } } +// todo: move this function out of leveled.rs pub async fn force_compact( sstables: &mut Sstables, next_sst_id: impl Fn() -> usize + Send + Sync, @@ -50,12 +51,27 @@ pub async fn force_compact( manifest: Option<&Manifest>, watermark: Option, ) -> anyhow::Result<()> { - let Some(task) = generate_task(sstables, options) else { + // todo: 这个可以提到外面，就不用 clone state 了 + let Some(task) = (match options.compaction_option() { + Leveled(options) => generate_task(options, sstables), + Full => generate_full_compaction_task(sstables), + NoCompaction => None, + }) else { + println!("not compact"); return Ok(()); }; - let new_sst_ids = - compact_with_task(sstables, next_sst_id, options, persistent, &task, watermark).await?; + dbg!(&task); + + let new_sst_ids = assert_send(compact_with_task( + sstables, + next_sst_id, + options, + persistent, + &task, + watermark, + )) + .await?; if let Some(manifest) = manifest { let record = ManifestRecord::Compaction(Compaction(task, new_sst_ids)); @@ -74,30 +90,35 @@ pub async fn compact_with_task( watermark: Option, ) -> anyhow::Result> { let source = task.source(); - let source_index = task.source_index(); - let source_id = *sstables.table_ids(source).get(source_index).unwrap(); - let source_level = sstables.sstables.get(&source_id).unwrap().as_ref(); + let source_level: Vec<_> = match task.source_index() { + SourceIndex::Index { index } => { + let source_id = *sstables.table_ids(source).get(index).unwrap(); + let source_level = sstables.sstables.get(&source_id).unwrap().as_ref(); + let source = iter::once(source_level); + source.collect() + } + SourceIndex::Full { .. } => { + let source = sstables.tables(source); + source.collect() + } + }; + let destination = task.destination(); - let new_sst = compact_generate_new_sst( - iter::once(source_level), + let new_sst = assert_send(compact_generate_new_sst( + source_level, sstables.tables(destination), next_sst_id, options, persistent, watermark, - ) + )) .await?; let new_sst_ids: Vec<_> = new_sst.iter().map(|table| table.id()).copied().collect(); - apply_compaction( - sstables, - source_index..source_index + 1, - source, - destination, - new_sst, - ); + sstables.apply_compaction_sst_ids(task, new_sst_ids.clone()); + sstables.apply_compaction_sst(new_sst, task); Ok(new_sst_ids) } @@ -131,6 +152,7 @@ fn select_level_source( // todo: make it looking better... })? .0; + dbg!(source); if source == level_sizes.len() - 1 { None } else { @@ -180,16 +202,13 @@ fn compute_target_sizes(last_level_size: u64, options: &LeveledCompactionOptions } pub fn generate_task( + compact_options: &LeveledCompactionOptions, sstables: &Sstables, - options: &SstOptions, ) -> Option { - let Leveled(compact_options) = options.compaction_option() else { - trace!("skip force compaction"); - return None; - }; - let level_sizes = compute_level_sizes(sstables); let target_sizes = compute_target_sizes(*level_sizes.last().unwrap(), compact_options); + dbg!(&level_sizes); + dbg!(&target_sizes); // todo: only select one source sst let source = select_level_source( @@ -206,7 +225,13 @@ pub fn generate_task( .enumerate() .min_by(|(_, left_id), (_, right_id)| left_id.cmp(right_id))?; - let task = CompactionTask::new(source, source_index, destination); + let task = CompactionTask::new( + source, + SourceIndex::Index { + index: source_index, + }, + destination, + ); Some(task) } @@ -221,7 +246,7 @@ mod tests { use crate::persistent::file_object::FileObject; use crate::persistent::LocalFs; - use crate::sst::compact::common::CompactionTask; + use crate::sst::compact::common::{CompactionTask, SourceIndex}; use crate::sst::compact::leveled::{ compact_with_task, force_compact, select_level_destination, select_level_source, }; @@ -279,7 +304,7 @@ mod tests { build_next_sst_id(&state.sst_id), &state.options, &state.persistent, - &CompactionTask::new(0, 4, 1), + &CompactionTask::new(0, SourceIndex::Index { index: 4 }, 1), None, ) .await @@ -296,7 +321,7 @@ mod tests { build_next_sst_id(&state.sst_id), &state.options, &state.persistent, - &CompactionTask::new(0, 3, 1), + &CompactionTask::new(0, SourceIndex::Index { index: 3 }, 1), None, ) .await @@ -313,7 +338,7 @@ mod tests { build_next_sst_id(&state.sst_id), &state.options, &state.persistent, - &CompactionTask::new(1, 0, 2), + &CompactionTask::new(1, SourceIndex::Index { index: 0 }, 2), None, ) .await diff --git a/src/sst/compact/mod.rs b/src/sst/compact/mod.rs index 3ef28e6..1229046 100644 --- a/src/sst/compact/mod.rs +++ b/src/sst/compact/mod.rs @@ -1,4 +1,5 @@ pub mod common; +pub mod full; pub mod leveled; mod option; mod simple_leveled; diff --git a/src/sst/compact/option.rs b/src/sst/compact/option.rs index ecf2769..874096b 100644 --- a/src/sst/compact/option.rs +++ b/src/sst/compact/option.rs @@ -5,6 +5,7 @@ pub enum CompactionOptions { /// Leveled compaction with partial compaction + dynamic level support (= RocksDB's Leveled /// Compaction) Leveled(LeveledCompactionOptions), + Full, /// In no compaction mode (week 1), always flush to L0 #[default] NoCompaction, diff --git a/src/sst/sstables.rs b/src/sst/sstables.rs index 72e2464..99b9184 100644 --- a/src/sst/sstables.rs +++ b/src/sst/sstables.rs @@ -5,22 +5,21 @@ use std::fmt::{Debug, Formatter}; use std::iter::repeat; -use std::mem; +use futures::stream; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; -use futures::stream; - use tracing::error; use crate::entry::InnerEntry; use crate::iterators::{create_merge_iter, create_two_merge_iter, MergeIterator}; use crate::key::KeySlice; -use crate::manifest::{Compaction, Flush}; +use crate::manifest::Flush; use crate::memtable::ImmutableMemTable; use crate::persistent::SstHandle; +use crate::sst::compact::common::CompactionTask; use crate::sst::compact::CompactionOptions; use crate::sst::iterator::concat::SstConcatIterator; use crate::sst::iterator::{scan_sst_concat, MergedSstIterator, SsTableIterator}; @@ -97,6 +96,7 @@ impl Sstables { repeat(Vec::new()).take(opt.max_levels() - 1).collect() } CompactionOptions::NoCompaction => Vec::new(), + CompactionOptions::Full => repeat(Vec::new()).take(1).collect(), }; Self { l0_sstables: Vec::new(), @@ -199,11 +199,38 @@ where todo!() } - pub fn fold_compaction_manifest(&mut self, Compaction(task, result_ids): Compaction) { - let source = self.table_ids_mut(task.source()); - source.remove(task.source_index()); - let destination = self.table_ids_mut(task.destination()); - let _ = mem::replace(destination, result_ids); + pub fn apply_compaction_sst_ids(&mut self, task: &CompactionTask, new_sst_ids: Vec) { + let source_level = task.source(); + let source_range = task.source_index().build_range(); + self.table_ids_mut(source_level).splice(source_range, []); + + let destination_level = task.destination(); + self.table_ids_mut(destination_level) + .splice(.., new_sst_ids); + } + + pub fn apply_compaction_sst( + &mut self, + new_sst: Vec>>, + task: &CompactionTask, + ) { + let source_level = task.source(); + let source_range = task.source_index().build_range(); + let source_ids = self.table_ids(source_level).clone(); + let source_ids = &source_ids[source_range]; + for id in source_ids { + self.sstables.remove(id); + } + + let destination_level = task.destination(); + let destination_ids = self.table_ids(destination_level).clone(); + for id in &destination_ids { + self.sstables.remove(id); + } + + for table in new_sst { + self.sstables.insert(*table.id(), table); + } } } diff --git a/src/state/inner.rs b/src/state/inner.rs index c01550a..14a6310 100644 --- a/src/state/inner.rs +++ b/src/state/inner.rs @@ -9,7 +9,7 @@ use typed_builder::TypedBuilder; use crate::block::BlockCache; use crate::key::KeyBytes; -use crate::manifest::{Manifest, ManifestRecord, NewMemtable}; +use crate::manifest::{Compaction, Manifest, ManifestRecord, NewMemtable}; use crate::memtable::{ImmutableMemTable, MemTable}; use crate::persistent::Persistent; use crate::sst::sstables::fold_flush_manifest; @@ -160,8 +160,8 @@ async fn build_state( fold_new_imm_memtable(&mut imm_memtables, persistent, record).await?; Ok((imm_memtables, sstables)) } - ManifestRecord::Compaction(record) => { - sstables.fold_compaction_manifest(record); + ManifestRecord::Compaction(Compaction(task, new_sst_ids)) => { + sstables.apply_compaction_sst_ids(&task, new_sst_ids); Ok((imm_memtables, sstables)) } } @@ -174,7 +174,7 @@ async fn build_state( mod tests { use crate::manifest::{Compaction, Flush, NewMemtable}; use crate::persistent::LocalFs; - use crate::sst::compact::common::CompactionTask; + use crate::sst::compact::common::{CompactionTask, SourceIndex}; use crate::sst::compact::{CompactionOptions, LeveledCompactionOptions}; use crate::sst::SstOptions; use crate::state::inner::build_state; @@ -207,11 +207,19 @@ mod tests { Flush(1).into(), NewMemtable(5).into(), Flush(2).into(), - Compaction(CompactionTask::new(0, 2, 1), vec![6]).into(), + Compaction( + CompactionTask::new(0, SourceIndex::Index { index: 2 }, 1), + vec![6], + ) + .into(), NewMemtable(7).into(), NewMemtable(8).into(), Flush(3).into(), - Compaction(CompactionTask::new(0, 2, 1), vec![9, 10]).into(), + Compaction( + CompactionTask::new(0, SourceIndex::Index { index: 2 }, 1), + vec![9, 10], + ) + .into(), ]; let (imm, ssts) = build_state(&options, manifest_records, &persistent) diff --git a/src/state/states.rs b/src/state/states.rs index 2419643..175f999 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -356,13 +356,14 @@ mod test { use tempfile::{tempdir, TempDir}; use crate::entry::{Entry, InnerEntry}; - use crate::iterators::create_merge_iter; + use crate::iterators::merge::MergeIteratorInner; use crate::iterators::utils::test_utils::{ assert_stream_eq, build_stream, build_tuple_stream, eq, }; use crate::mvcc::transaction::Transaction; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; + use crate::sst::compact::{CompactionOptions, LeveledCompactionOptions}; use crate::sst::iterator::SsTableIterator; use crate::sst::SstOptions; use crate::state::states::LsmStorageState; @@ -953,11 +954,17 @@ mod test { async fn test_task3_mvcc_compaction() { let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); + let compaction_option = LeveledCompactionOptions::builder() + .level_size_multiplier_2_exponent(2) + .max_levels(2) + .max_bytes_for_level_base(1) + .build(); + let compaction_option = CompactionOptions::Leveled(compaction_option); let options = SstOptions::builder() .target_sst_size(1024) .block_size(4096) .num_memtable_limit(1000) - .compaction_option(Default::default()) + .compaction_option(compaction_option) .enable_wal(true) .enable_mvcc(true) .build(); @@ -976,18 +983,22 @@ mod test { */ let snapshot0 = storage.new_txn().unwrap(); + dbg!(snapshot0.read_ts); storage.put_for_test(b"a", b"1").await.unwrap(); storage.put_for_test(b"b", b"1").await.unwrap(); let snapshot1 = storage.new_txn().unwrap(); + dbg!(snapshot1.read_ts); storage.put_for_test(b"a", b"2").await.unwrap(); storage.put_for_test(b"d", b"2").await.unwrap(); let snapshot2 = storage.new_txn().unwrap(); + dbg!(snapshot2.read_ts); storage.put_for_test(b"a", b"3").await.unwrap(); storage.delete_for_test(b"d").await.unwrap(); let snapshot3 = storage.new_txn().unwrap(); + dbg!(snapshot3.read_ts); storage.put_for_test(b"c", b"4").await.unwrap(); storage.delete_for_test(b"a").await.unwrap(); @@ -998,6 +1009,17 @@ mod test { storage.force_compact(&guard).await.unwrap(); } + { + for sst in storage.inner.load().sstables_state.sstables().values() { + let it = SsTableIterator::scan(sst.as_ref(), Unbounded, Unbounded); + it.for_each(|x| async { + let x = x.unwrap(); + dbg!(x); + }) + .await; + } + } + { let inner = storage.inner.load(); let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; @@ -1018,6 +1040,11 @@ mod test { } drop(snapshot0); + { + let guard = storage.mvcc.as_ref().unwrap().ts.lock(); + let watermark = guard.1.watermark(); + dbg!(watermark); + } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1043,6 +1070,11 @@ mod test { } drop(snapshot1); + { + let guard = storage.mvcc.as_ref().unwrap().ts.lock(); + let watermark = guard.1.watermark(); + dbg!(watermark); + } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1067,6 +1099,11 @@ mod test { } drop(snapshot2); + { + let guard = storage.mvcc.as_ref().unwrap().ts.lock(); + let watermark = guard.1.watermark(); + dbg!(watermark); + } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1080,6 +1117,11 @@ mod test { } drop(snapshot3); + { + let guard = storage.mvcc.as_ref().unwrap().ts.lock(); + let watermark = guard.1.watermark(); + dbg!(watermark); + } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1095,14 +1137,19 @@ mod test { async fn construct_test_mvcc_compaction_iter( storage: &LsmStorageStateInner

, ) -> impl Stream> + '_ { - let iter = storage + let l0 = storage.sstables_state.l0_sstables().iter(); + let level_other = storage .sstables_state - .sstables() - .values() + .levels() + .iter() + .flat_map(|v| v.iter()); + let iter = l0 + .chain(level_other) + .map(|id| storage.sstables_state.sstables().get(id).unwrap()) .map(|sst| SsTableIterator::scan(sst.as_ref(), Unbounded, Unbounded)); let iter = stream::iter(iter); - create_merge_iter(iter).await + MergeIteratorInner::create(iter).await } #[tokio::test] diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 4a2578f..0c7cfa1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod num; pub mod scoped; +pub mod send; pub mod vec; diff --git a/src/utils/send.rs b/src/utils/send.rs new file mode 100644 index 0000000..8f7380e --- /dev/null +++ b/src/utils/send.rs @@ -0,0 +1,3 @@ +pub fn assert_send(x: T) -> T { + x +} From 9d2ae1feec1551b3fb951c3c0f0f2152e9293023 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 17 Aug 2024 16:27:31 +0800 Subject: [PATCH 67/69] fix: test_task3_mvcc_compaction --- src/iterators/utils.rs | 1 - src/lsm/core.rs | 1 - src/mvcc/iterator.rs | 94 ++++++++++++++++++++++++-------------- src/mvcc/transaction.rs | 29 ++++++++---- src/sst/compact/leveled.rs | 9 +--- src/state/mod.rs | 1 + src/state/states.rs | 89 ++++++++++++++++-------------------- src/state/write_batch.rs | 24 ++++++++++ 8 files changed, 146 insertions(+), 102 deletions(-) create mode 100644 src/state/write_batch.rs diff --git a/src/iterators/utils.rs b/src/iterators/utils.rs index a362d45..657699b 100644 --- a/src/iterators/utils.rs +++ b/src/iterators/utils.rs @@ -115,7 +115,6 @@ pub mod test_utils { match (s1.next().await, s2.next().await) { (Some(x1), Some(x2)) => { if x1 != x2 { - dbg!((x1, x2)); return false; } } diff --git a/src/lsm/core.rs b/src/lsm/core.rs index 8cf6ac7..a303755 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -231,7 +231,6 @@ mod tests { insert_sst(&lsm, begin..begin + 100).await.unwrap(); } sleep(Duration::from_secs(2)).await; - dbg!(&lsm.state); for i in 0..10 { let begin = i * 100; diff --git a/src/mvcc/iterator.rs b/src/mvcc/iterator.rs index 7ddd854..e555a45 100644 --- a/src/mvcc/iterator.rs +++ b/src/mvcc/iterator.rs @@ -131,41 +131,46 @@ impl WatermarkGcIter for WatermarkGcIterImpl { where S: Stream> + Send + Unpin, { - dbg!("build_watermark_gc_iter"); let result = s .scan(None, move |state: &mut Option, entry| { - let item = match entry { - Err(e) => Some(Some(Err(e))), - Ok(entry) => { - if let Some(prev_key) = state { - if prev_key.key == entry.key.key { - if entry.key.timestamp <= watermark { - Some(None) - } else { - Some(Some(Ok(entry))) - } - } else { - *state = Some(entry.key.clone()); - Some(Some(Ok(entry))) - } - } else { - *state = Some(entry.key.clone()); - Some(Some(Ok(entry))) - } - } - }; + let item = Some(build_item(state, entry, watermark)); ready(item) }) - .filter_map(|entry| async { entry }) - .inspect(|entry| { - dbg!(entry); - }); + .filter_map(|entry| async { entry }); // todo: remove Box::pin? Box::pin(result) } } +fn build_item( + state: &mut Option, + entry: anyhow::Result, + watermark: u64, +) -> Option> { + match entry { + Err(e) => Some(Err(e)), + Ok(entry) => { + if let Some(prev_key) = state { + if prev_key.key == entry.key.key { + return None; + } + } + if entry.key.timestamp > watermark { + Some(Ok(entry)) + } else { + *state = Some(entry.key.clone()); + if entry.value.is_empty() { + // is delete + None + } else { + Some(Ok(entry)) + } + } + } + } +} + pub fn transform_bound(lower: Bound, upper: Bound, timestamp: T) -> BoundRange<(A, T)> where T: Bounded, @@ -258,26 +263,47 @@ mod tests { #[tokio::test] async fn test_watermark_gc() { - test_watermark_gc_helper([("a", 0), ("b", 0)], 5, [("a", 0), ("b", 0)]).await; - test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 0)], 2, [("a", 3), ("b", 0)]).await; - test_watermark_gc_helper([("a", 3), ("a", 2), ("b", 5)], 2, [("a", 3), ("b", 5)]).await; - test_watermark_gc_helper([("a", 2), ("a", 1), ("b", 5)], 3, [("a", 2), ("b", 5)]).await; + test_watermark_gc_helper( + [("a", "a", 0), ("b", "b", 0)], + 5, + [("a", "a", 0), ("b", "b", 0)], + ) + .await; + test_watermark_gc_helper( + [("a", "a", 3), ("a", "a", 2), ("b", "b", 0)], + 2, + [("a", "a", 3), ("a", "a", 2), ("b", "b", 0)], + ) + .await; + test_watermark_gc_helper( + [("a", "a", 3), ("a", "a", 2), ("b", "b", 5)], + 2, + [("a", "a", 3), ("a", "a", 2), ("b", "b", 5)], + ) + .await; + test_watermark_gc_helper( + [("a", "a", 2), ("a", "a", 1), ("b", "b", 5)], + 3, + [("a", "a", 2), ("b", "b", 5)], + ) + .await; + test_watermark_gc_helper([("a", "", 2), ("a", "a", 1)], 2, []).await; } async fn test_watermark_gc_helper(input: S1, watermark: u64, expected_output: S2) where - S1: IntoIterator, + S1: IntoIterator, S1::IntoIter: Send, - S2: IntoIterator, + S2: IntoIterator, S2::IntoIter: Send, { fn transform( - iter: impl IntoIterator, + iter: impl IntoIterator, ) -> impl Stream> { - let s = iter.into_iter().map(|(key, timestamp)| { + let s = iter.into_iter().map(|(key, value, timestamp)| { Ok(InnerEntry::new( KeyBytes::new(Bytes::from(key), timestamp), - Bytes::new(), + Bytes::from(value), )) }); stream::iter(s) diff --git a/src/mvcc/transaction.rs b/src/mvcc/transaction.rs index 9e58147..407621e 100644 --- a/src/mvcc/transaction.rs +++ b/src/mvcc/transaction.rs @@ -3,6 +3,7 @@ use bytes::Bytes; use crossbeam_skiplist::SkipMap; use std::collections::{Bound, HashSet}; use std::ops::Bound::Excluded; +use std::slice; use std::sync::Arc; use tokio_stream::StreamExt; @@ -11,6 +12,7 @@ use crate::iterators::LockedLsmIter; use crate::mvcc::core::{CommittedTxnData, LsmMvccInner}; use crate::mvcc::iterator::LockedTxnIter; use crate::persistent::Persistent; +use crate::state::write_batch::WriteBatchRecord; use crate::state::{LsmStorageState, Map}; use crate::utils::scoped::ScopedMutex; @@ -44,6 +46,7 @@ pub struct Transaction<'a, P: Persistent> { mvcc: Arc, } +// todo: no need for async impl<'a, P: Persistent> Map for Transaction<'a, P> { type Error = anyhow::Error; @@ -68,10 +71,9 @@ impl<'a, P: Persistent> Map for Transaction<'a, P> { value: impl Into + Send, ) -> Result<(), Self::Error> { let key = key.into(); - self.local_storage.insert(key.clone(), value.into()); - if let Some(key_hashes) = self.key_hashes.as_ref() { - key_hashes.lock_with(|mut set| set.add_write_key(key.as_ref())); - } + let value = value.into(); + let record = WriteBatchRecord::Put(key, value); + self.write_batch(slice::from_ref(&record)); Ok(()) } @@ -104,6 +106,21 @@ impl<'a, P: Persistent> Transaction<'a, P> { } } + pub fn write_batch(&self, batch: &[WriteBatchRecord]) { + if let Some(key_hashes) = self.key_hashes.as_ref() { + key_hashes.lock_with(|mut set| { + for record in batch { + set.add_write_key(record.get_key().as_ref()); + } + }); + } + + for record in batch { + let pair = record.clone().into_keyed(); + self.local_storage.insert(pair.key, pair.value); + } + } + // todo: no need for Result? pub fn scan(&'a self, lower: Bound<&'a [u8]>, upper: Bound<&'a [u8]>) -> LockedTxnIter<'a, P> { let inner = self.state.inner.load_full(); @@ -125,10 +142,6 @@ impl<'a, P: Persistent> Transaction<'a, P> { let conflict = if let Some(key_hashes) = key_hashes.as_ref() { let range = (Excluded(self.read_ts), Excluded(expected_commit_ts)); let read_set = &key_hashes.read_set; - println!("===="); - dbg!(key_hashes); - dbg!(&commit_guard); - println!("===="); commit_guard .range(range) .any(|(_, data)| !data.key_hashes.is_disjoint(read_set)) diff --git a/src/sst/compact/leveled.rs b/src/sst/compact/leveled.rs index 22670e8..db9e02f 100644 --- a/src/sst/compact/leveled.rs +++ b/src/sst/compact/leveled.rs @@ -57,12 +57,9 @@ pub async fn force_compact( Full => generate_full_compaction_task(sstables), NoCompaction => None, }) else { - println!("not compact"); return Ok(()); }; - dbg!(&task); - let new_sst_ids = assert_send(compact_with_task( sstables, next_sst_id, @@ -117,8 +114,8 @@ pub async fn compact_with_task( let new_sst_ids: Vec<_> = new_sst.iter().map(|table| table.id()).copied().collect(); - sstables.apply_compaction_sst_ids(task, new_sst_ids.clone()); sstables.apply_compaction_sst(new_sst, task); + sstables.apply_compaction_sst_ids(task, new_sst_ids.clone()); Ok(new_sst_ids) } @@ -141,7 +138,6 @@ fn select_level_source( *level_size as f64 / denominator as f64 }) .collect(); - // println!("max_bytes_for_level_base={}, scores={:?}", max_bytes_for_level_base, scores); let source = scores .iter() .enumerate() @@ -152,7 +148,6 @@ fn select_level_source( // todo: make it looking better... })? .0; - dbg!(source); if source == level_sizes.len() - 1 { None } else { @@ -207,8 +202,6 @@ pub fn generate_task( ) -> Option { let level_sizes = compute_level_sizes(sstables); let target_sizes = compute_target_sizes(*level_sizes.last().unwrap(), compact_options); - dbg!(&level_sizes); - dbg!(&target_sizes); // todo: only select one source sst let source = select_level_source( diff --git a/src/state/mod.rs b/src/state/mod.rs index 8a39c1f..019cafd 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -2,6 +2,7 @@ pub mod inner; mod map; mod mut_op; mod states; +pub mod write_batch; pub use inner::LsmStorageStateInner; pub use map::Map; diff --git a/src/state/states.rs b/src/state/states.rs index 175f999..c314de1 100644 --- a/src/state/states.rs +++ b/src/state/states.rs @@ -20,6 +20,7 @@ use crate::persistent::Persistent; use crate::sst::compact::leveled::force_compact; use crate::sst::{SsTableBuilder, SstOptions}; use crate::state::inner::{LsmStorageStateInner, RecoveredState}; +use crate::state::write_batch::WriteBatchRecord; use crate::state::Map; use crate::utils::vec::pop; @@ -138,6 +139,12 @@ impl

LsmStorageState

where P: Persistent, { + pub async fn put_batch(&self, batch: &[WriteBatchRecord]) -> anyhow::Result<()> { + let txn = self.new_txn()?; + txn.write_batch(batch); + txn.commit().await + } + pub async fn write_batch(&self, entries: &[Entry], timestamp: u64) -> anyhow::Result<()> { let guard = self.inner.load(); guard.memtable.put_batch(entries, timestamp).await?; @@ -363,7 +370,7 @@ mod test { use crate::mvcc::transaction::Transaction; use crate::persistent::file_object::LocalFs; use crate::persistent::Persistent; - use crate::sst::compact::{CompactionOptions, LeveledCompactionOptions}; + use crate::sst::compact::CompactionOptions; use crate::sst::iterator::SsTableIterator; use crate::sst::SstOptions; use crate::state::states::LsmStorageState; @@ -952,14 +959,11 @@ mod test { // todo: add test #[tokio::test] async fn test_task3_mvcc_compaction() { + use crate::state::write_batch::WriteBatchRecord::{Del, Put}; + let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); - let compaction_option = LeveledCompactionOptions::builder() - .level_size_multiplier_2_exponent(2) - .max_levels(2) - .max_bytes_for_level_base(1) - .build(); - let compaction_option = CompactionOptions::Leveled(compaction_option); + let compaction_option = CompactionOptions::Full; let options = SstOptions::builder() .target_sst_size(1024) .block_size(4096) @@ -983,24 +987,40 @@ mod test { */ let snapshot0 = storage.new_txn().unwrap(); - dbg!(snapshot0.read_ts); - storage.put_for_test(b"a", b"1").await.unwrap(); - storage.put_for_test(b"b", b"1").await.unwrap(); + storage + .put_batch(&[ + Put(Bytes::copy_from_slice(b"a"), Bytes::copy_from_slice(b"1")), + Put(Bytes::copy_from_slice(b"b"), Bytes::copy_from_slice(b"1")), + ]) + .await + .unwrap(); let snapshot1 = storage.new_txn().unwrap(); - dbg!(snapshot1.read_ts); - storage.put_for_test(b"a", b"2").await.unwrap(); - storage.put_for_test(b"d", b"2").await.unwrap(); + storage + .put_batch(&[ + Put(Bytes::copy_from_slice(b"a"), Bytes::copy_from_slice(b"2")), + Put(Bytes::copy_from_slice(b"d"), Bytes::copy_from_slice(b"2")), + ]) + .await + .unwrap(); let snapshot2 = storage.new_txn().unwrap(); - dbg!(snapshot2.read_ts); - storage.put_for_test(b"a", b"3").await.unwrap(); - storage.delete_for_test(b"d").await.unwrap(); + storage + .put_batch(&[ + Put(Bytes::copy_from_slice(b"a"), Bytes::copy_from_slice(b"3")), + Del(Bytes::copy_from_slice(b"d")), + ]) + .await + .unwrap(); let snapshot3 = storage.new_txn().unwrap(); - dbg!(snapshot3.read_ts); - storage.put_for_test(b"c", b"4").await.unwrap(); - storage.delete_for_test(b"a").await.unwrap(); + storage + .put_batch(&[ + Put(Bytes::copy_from_slice(b"c"), Bytes::copy_from_slice(b"4")), + Del(Bytes::copy_from_slice(b"a")), + ]) + .await + .unwrap(); { let guard = storage.state_lock.lock().await; @@ -1009,17 +1029,6 @@ mod test { storage.force_compact(&guard).await.unwrap(); } - { - for sst in storage.inner.load().sstables_state.sstables().values() { - let it = SsTableIterator::scan(sst.as_ref(), Unbounded, Unbounded); - it.for_each(|x| async { - let x = x.unwrap(); - dbg!(x); - }) - .await; - } - } - { let inner = storage.inner.load(); let iter = construct_test_mvcc_compaction_iter(inner.as_ref()).await; @@ -1040,11 +1049,6 @@ mod test { } drop(snapshot0); - { - let guard = storage.mvcc.as_ref().unwrap().ts.lock(); - let watermark = guard.1.watermark(); - dbg!(watermark); - } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1070,11 +1074,6 @@ mod test { } drop(snapshot1); - { - let guard = storage.mvcc.as_ref().unwrap().ts.lock(); - let watermark = guard.1.watermark(); - dbg!(watermark); - } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1099,11 +1098,6 @@ mod test { } drop(snapshot2); - { - let guard = storage.mvcc.as_ref().unwrap().ts.lock(); - let watermark = guard.1.watermark(); - dbg!(watermark); - } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); @@ -1117,11 +1111,6 @@ mod test { } drop(snapshot3); - { - let guard = storage.mvcc.as_ref().unwrap().ts.lock(); - let watermark = guard.1.watermark(); - dbg!(watermark); - } { let guard = storage.state_lock.lock().await; storage.force_compact(&guard).await.unwrap(); diff --git a/src/state/write_batch.rs b/src/state/write_batch.rs new file mode 100644 index 0000000..ebf25e4 --- /dev/null +++ b/src/state/write_batch.rs @@ -0,0 +1,24 @@ +use crate::entry::Keyed; +use bytes::Bytes; + +#[derive(Debug, Clone)] +pub enum WriteBatchRecord { + Put(Bytes, Bytes), + Del(Bytes), +} + +impl WriteBatchRecord { + pub fn get_key(&self) -> &Bytes { + match self { + WriteBatchRecord::Put(key, _) => key, + WriteBatchRecord::Del(key) => key, + } + } + + pub fn into_keyed(self) -> Keyed { + match self { + WriteBatchRecord::Put(key, value) => Keyed::new(key, value), + WriteBatchRecord::Del(key) => Keyed::new(key, Bytes::new()), + } + } +} From 96f0d3ca5ca87e86c1db80a85991acf6850a6551 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 17 Aug 2024 16:40:44 +0800 Subject: [PATCH 68/69] fix: async runtime panic --- src/lsm/core.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/lsm/core.rs b/src/lsm/core.rs index a303755..b3afb35 100644 --- a/src/lsm/core.rs +++ b/src/lsm/core.rs @@ -120,14 +120,17 @@ impl Drop for Lsm

{ self.cancel_token.cancel(); let flush_handle = self.flush_handle.take(); let spawn_handle = self.spawn_handle.take(); - Handle::current().block_on(async { - if let Some(flush_handle) = flush_handle { - flush_handle.await.inspect_err(|e| error!(error = ?e)).ok(); - } - if let Some(spawn_handle) = spawn_handle { - spawn_handle.await.inspect_err(|e| error!(error = ?e)).ok(); - } - }) + + tokio::task::block_in_place(|| { + Handle::current().block_on(async { + if let Some(flush_handle) = flush_handle { + flush_handle.await.inspect_err(|e| error!(error = ?e)).ok(); + } + if let Some(spawn_handle) = spawn_handle { + spawn_handle.await.inspect_err(|e| error!(error = ?e)).ok(); + } + }) + }); } } @@ -208,7 +211,7 @@ mod tests { Lsm::new(options, persistent).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_auto_compaction() { let dir = tempdir().unwrap(); let persistent = LocalFs::new(dir.path().to_path_buf()); @@ -244,7 +247,7 @@ mod tests { } } - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn test_wal_integration() { let compaction_options = LeveledCompactionOptions::builder() .max_levels(3) From 89098e137e5f73c98bf539bac22da680b2312002 Mon Sep 17 00:00:00 2001 From: Kermit Date: Sat, 17 Aug 2024 16:48:25 +0800 Subject: [PATCH 69/69] fix: ycsb bench --- benches/ycsb.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/benches/ycsb.rs b/benches/ycsb.rs index ac370dd..8ace7dc 100644 --- a/benches/ycsb.rs +++ b/benches/ycsb.rs @@ -64,6 +64,7 @@ fn ycsb_bench(c: &mut Criterion) { .num_memtable_limit(1000) .compaction_option(Default::default()) .enable_wal(false) + .enable_mvcc(true) .build(); let runtime = tokio::runtime::Runtime::new().unwrap(); let state =