diff --git a/src/lib.rs b/src/lib.rs index d0a71ef..844cd64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -134,9 +134,11 @@ impl Default for Config { } pub(crate) const MAX_TOTAL_KEY_SIZE: usize = 0x3fff; // 14 bits +pub(crate) const MAX_TOTAL_VALUE_SIZE: usize = 0xffff; // 16 bits pub(crate) const NAMESPACING_RESERVED_SIZE: usize = 0xff; +pub(crate) const VALUE_RESERVED_SIZE: usize = 0xff; pub const MAX_KEY_SIZE: usize = MAX_TOTAL_KEY_SIZE - NAMESPACING_RESERVED_SIZE; -pub const MAX_VALUE_SIZE: usize = 0xffff; +pub const MAX_VALUE_SIZE: usize = MAX_TOTAL_KEY_SIZE - VALUE_RESERVED_SIZE; const _: () = assert!(MAX_KEY_SIZE <= u16::MAX as usize); const _: () = assert!(MAX_VALUE_SIZE <= u16::MAX as usize); diff --git a/src/lists.rs b/src/lists.rs index 13b1dc1..be58773 100644 --- a/src/lists.rs +++ b/src/lists.rs @@ -667,17 +667,17 @@ impl CandyStore { /// Discards the given list, removing all elements it contains and dropping the list itself. /// This is more efficient than iteration + removal of each element. - pub fn discard_list + ?Sized>(&self, list_key: &B) -> Result<()> { + pub fn discard_list + ?Sized>(&self, list_key: &B) -> Result { self.owned_discard_list(list_key.as_ref().to_owned()) } /// Owned version of [Self::discard_list] - pub fn owned_discard_list(&self, list_key: Vec) -> Result<()> { + pub fn owned_discard_list(&self, list_key: Vec) -> Result { let (list_ph, list_key) = self.make_list_key(list_key); let _guard = self.lock_list(list_ph); let Some(list_bytes) = self.get_raw(&list_key)? else { - return Ok(()); + return Ok(false); }; let list = *from_bytes::(&list_bytes); for idx in list.head_idx..list.tail_idx { @@ -693,7 +693,7 @@ impl CandyStore { } self.remove_raw(&list_key)?; - Ok(()) + Ok(true) } /// Returns the first (head) element of the list diff --git a/src/store.rs b/src/store.rs index aa2d074..8ec42ca 100644 --- a/src/store.rs +++ b/src/store.rs @@ -10,7 +10,7 @@ use crate::{ hashing::{HashSeed, PartedHash}, router::ShardRouter, shard::{CompactionThreadPool, InsertMode, InsertStatus, KVPair}, - Stats, + Stats, MAX_TOTAL_VALUE_SIZE, }; use crate::{ shard::{NUM_ROWS, ROW_WIDTH}, @@ -330,7 +330,7 @@ impl CandyStore { if full_key.len() > MAX_TOTAL_KEY_SIZE as usize { return Err(anyhow!(CandyError::KeyTooLong(full_key.len()))); } - if val.len() > MAX_VALUE_SIZE as usize { + if val.len() > MAX_TOTAL_VALUE_SIZE as usize { return Err(anyhow!(CandyError::ValueTooLong(val.len()))); } if full_key.len() + val.len() > self.config.max_shard_size as usize { @@ -499,6 +499,43 @@ impl CandyStore { pub fn merge_small_shards(&self, max_fill_level: f32) -> Result { self.root.merge_small_shards(max_fill_level) } + + /// Sets a big item, whose value is unlimited in size. Behind the scenes the value is split into chunks + /// and stored as a list. This makes this API non-atomic, i.e., crashing while writing a big value may later + /// allow you to retrieve a partial result. It is up to the caller to add a length field or a checksum to make + /// sure the value is correct. + /// + /// Returns true if the value had existed before (thus it was replaced), false otherwise + pub fn set_big(&self, key: &[u8], val: &[u8]) -> Result { + let existed = self.discard_list(key)?; + for (i, chunk) in val.chunks(MAX_VALUE_SIZE).enumerate() { + self.set_in_list(key, &i.to_le_bytes(), chunk)?; + } + Ok(existed) + } + + /// Returns a big item, collecting all the underlying chunks into a single value that's returned to the + /// caller. + pub fn get_big(&self, key: &[u8]) -> Result>> { + let mut val = vec![]; + let mut exists = false; + for res in self.iter_list(key) { + let (_, chunk) = res?; + exists = true; + val.extend_from_slice(&chunk); + } + if exists { + Ok(Some(val)) + } else { + Ok(None) + } + } + + /// Removes a big item by key. Returns true if the key had existed, false otherwise. + /// See also [Self::set_big] + pub fn remove_big(&self, key: &[u8]) -> Result { + self.discard_list(key) + } } // impl Drop for CandyStore { diff --git a/src/typed.rs b/src/typed.rs index 6643db5..00c32f2 100644 --- a/src/typed.rs +++ b/src/typed.rs @@ -188,6 +188,43 @@ where Ok(None) } } + + /// Same as [CandyStore::get_big] but serializes the key and deserializes the value + pub fn get_big(&self, key: &Q) -> Result> + where + K: Borrow, + { + let kbytes = Self::make_key(key); + if let Some(vbytes) = self.store.get_big(&kbytes)? { + Ok(Some(from_bytes::(&vbytes)?)) + } else { + Ok(None) + } + } + + /// Same as [CandyStore::set_big] but serializes the key and the value. + pub fn set_big( + &self, + key: &Q1, + val: &Q2, + ) -> Result + where + K: Borrow, + V: Borrow, + { + let kbytes = Self::make_key(key); + let vbytes = val.to_bytes::(); + self.store.set_big(&kbytes, &vbytes) + } + + /// Same as [CandyStore::remove_big] but serializes the key + pub fn remove_big(&self, k: &Q) -> Result + where + K: Borrow, + { + let kbytes = Self::make_key(k); + self.store.remove_big(&kbytes) + } } /// A wrapper around [CandyStore] that exposes the list API in a typed manner. See [CandyTypedStore] for more @@ -432,7 +469,7 @@ where } /// Same as [CandyStore::discard_list], but `list_key` is typed - pub fn discard(&self, list_key: &Q) -> Result<()> + pub fn discard(&self, list_key: &Q) -> Result where L: Borrow, { diff --git a/tests/test_bigval.rs b/tests/test_bigval.rs new file mode 100644 index 0000000..850444e --- /dev/null +++ b/tests/test_bigval.rs @@ -0,0 +1,32 @@ +mod common; + +use std::sync::Arc; + +use candystore::{CandyStore, CandyTypedStore, Config, Result}; + +use crate::common::run_in_tempdir; + +#[test] +fn test_bigval() -> Result<()> { + run_in_tempdir(|dir| { + let db = Arc::new(CandyStore::open(dir, Config::default())?); + + assert_eq!(db.set_big(b"mykey", &vec![0x99; 1_000_000])?, false); + assert_eq!(db.get_big(b"yourkey")?, None); + assert_eq!(db.get_big(b"mykey")?, Some(vec![0x99; 1_000_000])); + assert_eq!(db.remove_big(b"mykey")?, true); + assert_eq!(db.get_big(b"mykey")?, None); + assert_eq!(db.set_big(b"mykey", &vec![0x88; 100_000])?, false); + assert_eq!(db.set_big(b"mykey", &vec![0x77; 100_000])?, true); + assert_eq!(db.get_big(b"mykey")?, Some(vec![0x77; 100_000])); + + let typed = CandyTypedStore::>::new(db); + assert_eq!(typed.set_big("hello", &vec![123456789; 100_000])?, false); + assert_eq!(typed.get_big("world")?, None); + assert_eq!(typed.get_big("hello")?, Some(vec![123456789; 100_000])); + assert_eq!(typed.remove_big("hello")?, true); + assert_eq!(typed.remove_big("hello")?, false); + + Ok(()) + }) +}