Add big-values API

src/lib.rs

@@ -134,9 +134,11 @@ impl Default for Config {
 }
 
 pub(crate) const MAX_TOTAL_KEY_SIZE: usize = 0x3fff; // 14 bits
+pub(crate) const MAX_TOTAL_VALUE_SIZE: usize = 0xffff; // 16 bits
 pub(crate) const NAMESPACING_RESERVED_SIZE: usize = 0xff;
+pub(crate) const VALUE_RESERVED_SIZE: usize = 0xff;
 pub const MAX_KEY_SIZE: usize = MAX_TOTAL_KEY_SIZE - NAMESPACING_RESERVED_SIZE;
-pub const MAX_VALUE_SIZE: usize = 0xffff;
+pub const MAX_VALUE_SIZE: usize = MAX_TOTAL_KEY_SIZE - VALUE_RESERVED_SIZE;
 
 const _: () = assert!(MAX_KEY_SIZE <= u16::MAX as usize);
 const _: () = assert!(MAX_VALUE_SIZE <= u16::MAX as usize);

src/lists.rs

@@ -667,17 +667,17 @@ impl CandyStore {
 
     /// Discards the given list, removing all elements it contains and dropping the list itself.
     /// This is more efficient than iteration + removal of each element.
-    pub fn discard_list<B: AsRef<[u8]> + ?Sized>(&self, list_key: &B) -> Result<()> {
+    pub fn discard_list<B: AsRef<[u8]> + ?Sized>(&self, list_key: &B) -> Result<bool> {
         self.owned_discard_list(list_key.as_ref().to_owned())
     }
 
     /// Owned version of [Self::discard_list]
-    pub fn owned_discard_list(&self, list_key: Vec<u8>) -> Result<()> {
+    pub fn owned_discard_list(&self, list_key: Vec<u8>) -> Result<bool> {
         let (list_ph, list_key) = self.make_list_key(list_key);
         let _guard = self.lock_list(list_ph);
 
         let Some(list_bytes) = self.get_raw(&list_key)? else {
-            return Ok(());
+            return Ok(false);
         };
         let list = *from_bytes::<List>(&list_bytes);
         for idx in list.head_idx..list.tail_idx {
@@ -693,7 +693,7 @@ impl CandyStore {
         }
         self.remove_raw(&list_key)?;
 
-        Ok(())
+        Ok(true)
     }
 
     /// Returns the first (head) element of the list

src/store.rs

@@ -10,7 +10,7 @@ use crate::{
     hashing::{HashSeed, PartedHash},
     router::ShardRouter,
     shard::{CompactionThreadPool, InsertMode, InsertStatus, KVPair},
-    Stats,
+    Stats, MAX_TOTAL_VALUE_SIZE,
 };
 use crate::{
     shard::{NUM_ROWS, ROW_WIDTH},
@@ -330,7 +330,7 @@ impl CandyStore {
         if full_key.len() > MAX_TOTAL_KEY_SIZE as usize {
             return Err(anyhow!(CandyError::KeyTooLong(full_key.len())));
         }
-        if val.len() > MAX_VALUE_SIZE as usize {
+        if val.len() > MAX_TOTAL_VALUE_SIZE as usize {
             return Err(anyhow!(CandyError::ValueTooLong(val.len())));
         }
         if full_key.len() + val.len() > self.config.max_shard_size as usize {
@@ -499,6 +499,43 @@ impl CandyStore {
     pub fn merge_small_shards(&self, max_fill_level: f32) -> Result<bool> {
         self.root.merge_small_shards(max_fill_level)
     }
+
+    /// Sets a big item, whose value is unlimited in size. Behind the scenes the value is split into chunks
+    /// and stored as a list. This makes this API non-atomic, i.e., crashing while writing a big value may later
+    /// allow you to retrieve a partial result. It is up to the caller to add a length field or a checksum to make
+    /// sure the value is correct.
+    ///
+    /// Returns true if the value had existed before (thus it was replaced), false otherwise
+    pub fn set_big(&self, key: &[u8], val: &[u8]) -> Result<bool> {
+        let existed = self.discard_list(key)?;
+        for (i, chunk) in val.chunks(MAX_VALUE_SIZE).enumerate() {
+            self.set_in_list(key, &i.to_le_bytes(), chunk)?;
+        }
+        Ok(existed)
+    }
+
+    /// Returns a big item, collecting all the underlying chunks into a single value that's returned to the
+    /// caller.
+    pub fn get_big(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        let mut val = vec![];
+        let mut exists = false;
+        for res in self.iter_list(key) {
+            let (_, chunk) = res?;
+            exists = true;
+            val.extend_from_slice(&chunk);
+        }
+        if exists {
+            Ok(Some(val))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Removes a big item by key. Returns true if the key had existed, false otherwise.
+    /// See also [Self::set_big]
+    pub fn remove_big(&self, key: &[u8]) -> Result<bool> {
+        self.discard_list(key)
+    }
 }
 
 // impl Drop for CandyStore {

src/typed.rs

@@ -188,6 +188,43 @@ where
             Ok(None)
         }
     }
+
+    /// Same as [CandyStore::get_big] but serializes the key and deserializes the value
+    pub fn get_big<Q: ?Sized + Encode>(&self, key: &Q) -> Result<Option<V>>
+    where
+        K: Borrow<Q>,
+    {
+        let kbytes = Self::make_key(key);
+        if let Some(vbytes) = self.store.get_big(&kbytes)? {
+            Ok(Some(from_bytes::<V>(&vbytes)?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Same as [CandyStore::set_big] but serializes the key and the value.
+    pub fn set_big<Q1: ?Sized + Encode, Q2: ?Sized + Encode>(
+        &self,
+        key: &Q1,
+        val: &Q2,
+    ) -> Result<bool>
+    where
+        K: Borrow<Q1>,
+        V: Borrow<Q2>,
+    {
+        let kbytes = Self::make_key(key);
+        let vbytes = val.to_bytes::<LE>();
+        self.store.set_big(&kbytes, &vbytes)
+    }
+
+    /// Same as [CandyStore::remove_big] but serializes the key
+    pub fn remove_big<Q: ?Sized + Encode>(&self, k: &Q) -> Result<bool>
+    where
+        K: Borrow<Q>,
+    {
+        let kbytes = Self::make_key(k);
+        self.store.remove_big(&kbytes)
+    }
 }
 
 /// A wrapper around [CandyStore] that exposes the list API in a typed manner. See [CandyTypedStore] for more
@@ -432,7 +469,7 @@ where
     }
 
     /// Same as [CandyStore::discard_list], but `list_key` is typed
-    pub fn discard<Q: ?Sized + Encode>(&self, list_key: &Q) -> Result<()>
+    pub fn discard<Q: ?Sized + Encode>(&self, list_key: &Q) -> Result<bool>
     where
         L: Borrow<Q>,
     {

tests/test_bigval.rs

@@ -0,0 +1,32 @@
+mod common;
+
+use std::sync::Arc;
+
+use candystore::{CandyStore, CandyTypedStore, Config, Result};
+
+use crate::common::run_in_tempdir;
+
+#[test]
+fn test_bigval() -> Result<()> {
+    run_in_tempdir(|dir| {
+        let db = Arc::new(CandyStore::open(dir, Config::default())?);
+
+        assert_eq!(db.set_big(b"mykey", &vec![0x99; 1_000_000])?, false);
+        assert_eq!(db.get_big(b"yourkey")?, None);
+        assert_eq!(db.get_big(b"mykey")?, Some(vec![0x99; 1_000_000]));
+        assert_eq!(db.remove_big(b"mykey")?, true);
+        assert_eq!(db.get_big(b"mykey")?, None);
+        assert_eq!(db.set_big(b"mykey", &vec![0x88; 100_000])?, false);
+        assert_eq!(db.set_big(b"mykey", &vec![0x77; 100_000])?, true);
+        assert_eq!(db.get_big(b"mykey")?, Some(vec![0x77; 100_000]));
+
+        let typed = CandyTypedStore::<String, Vec<u32>>::new(db);
+        assert_eq!(typed.set_big("hello", &vec![123456789; 100_000])?, false);
+        assert_eq!(typed.get_big("world")?, None);
+        assert_eq!(typed.get_big("hello")?, Some(vec![123456789; 100_000]));
+        assert_eq!(typed.remove_big("hello")?, true);
+        assert_eq!(typed.remove_big("hello")?, false);
+
+        Ok(())
+    })
+}