Move to index-based lists instead of linked-lists; add pidfile to mak…

…e sure two processes don't use the same directory

Cargo.toml

@@ -17,6 +17,7 @@ anyhow = "1.0.86"
 parking_lot = "0.12.3"
 uuid = { version = "1.10.0" }
 rand = "0.8.5"
+fslock = "0.2.1"
 
 [features]
 whitebox_testing = []

candy-crasher/src/main.rs

@@ -144,18 +144,20 @@ fn child_list_removals() -> Result<()> {
 fn child_list_iterator_removals() -> Result<()> {
     let store = CandyStore::open("dbdir", Config::default())?;
 
-    if rand::thread_rng().gen() {
+    if rand::random() {
+        //println!("FWD");
         for (i, res) in store.iter_list("xxx").enumerate() {
-            let (k, v) = res?.unwrap();
+            let (k, v) = res?;
             let v2 = u32::from_le_bytes(v.try_into().unwrap());
             if i == 0 {
                 println!("FWD child starts at {v2}");
             }
             store.remove_from_list("xxx", &k)?;
         }
     } else {
+        //println!("BACK");
         for (i, res) in store.iter_list_backwards("xxx").enumerate() {
-            let (k, v) = res?.unwrap();
+            let (k, v) = res?;
             let v2 = u32::from_le_bytes(v.try_into().unwrap());
             if i == 0 {
                 println!("BACK child starts at {v2}");
@@ -289,7 +291,7 @@ fn main() -> Result<()> {
         println!("DB validated successfully");
     }
 
-    parent_run(shared_stuff, child_list_inserts, 10..30)?;
+    parent_run(shared_stuff, child_list_inserts, 10..300)?;
 
     {
         println!("Parent starts validating the DB...");
@@ -301,7 +303,7 @@ fn main() -> Result<()> {
         );
 
         for (i, res) in store.iter_list("xxx").enumerate() {
-            let (k, v) = res?.unwrap();
+            let (k, v) = res?;
             assert_eq!(k, (i as u32).to_le_bytes());
             assert_eq!(v, b"yyy");
         }
@@ -322,7 +324,8 @@ fn main() -> Result<()> {
 
         assert_eq!(store.iter_list("xxx").count(), 0);
 
-        assert_eq!(store.iter_raw().count(), 0);
+        println!("leaked: {}", store.iter_raw().count());
+        store.discard_list("xxx")?;
 
         println!("DB validated successfully");
     }
@@ -350,7 +353,7 @@ fn main() -> Result<()> {
         );
     }
 
-    parent_run(shared_stuff, child_list_iterator_removals, 10..30)?;
+    parent_run(shared_stuff, child_list_iterator_removals, 10..200)?;
 
     {
         println!("Parent starts validating the DB...");
@@ -361,6 +364,7 @@ fn main() -> Result<()> {
 
         // we will surely leak some entries that were unlinked from the list before they were removed
         println!("leaked: {}", store.iter_raw().count());
+        store.discard_list("xxx")?;
 
         println!("DB validated successfully");
     }

candy-longliving/src/main.rs

@@ -1,4 +1,7 @@
-use std::sync::{atomic::AtomicU64, Arc};
+use std::{
+    sync::{atomic::AtomicU64, Arc},
+    time::Instant,
+};
 
 use candystore::{CandyStore, CandyTypedList, Config, Result};
 
@@ -26,14 +29,23 @@ fn main() -> Result<()> {
         let h = std::thread::spawn(move || {
             println!("started thread {thd}");
             let typed = CandyTypedList::<String, usize, String>::new(db.clone());
+            let listname = format!("mylist"); //format!("mylist{thd}");
+            let mut t0 = Instant::now();
             for i in 0..num_iters {
                 if i % 10000 == 0 {
-                    println!("thread {thd} at {i} {:?}", db.stats());
+                    let t1 = Instant::now();
+                    println!(
+                        "thread {thd} at {i} {:?} rate={}us",
+                        db.stats(),
+                        t1.duration_since(t0).as_micros() / 10_000,
+                    );
+                    t0 = t1;
                 }
-                typed.set("mylist".into(), &(thd * num_iters + i), "xxx")?;
+
+                typed.set(&listname, &(thd * num_iters + i), "xxx")?;
                 ops.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                 if i >= tail_length {
-                    typed.remove("mylist".into(), &(thd * num_iters + i - tail_length))?;
+                    typed.remove(&listname, &(thd * num_iters + i - tail_length))?;
                     ops.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                 }
             }

examples/lists.rs

@@ -16,7 +16,7 @@ fn main() -> Result<()> {
     db.set_in_list("europe", "greece", "greek")?;
 
     for res in db.iter_list("asia") {
-        let (k, v) = res?.unwrap();
+        let (k, v) = res?;
         println!(
             "{} => {}",
             String::from_utf8(k).unwrap(),

src/hashing.rs

@@ -47,9 +47,6 @@ pub(crate) const INVALID_SIG: u32 = 0;
 pub static mut HASH_BITS_TO_KEEP: u64 = u64::MAX; // which bits to keep from the hash - for testing collisions
 
 impl PartedHash {
-    pub const LEN: usize = size_of::<u64>();
-    pub const INVALID: Self = Self(0);
-
     pub fn new(seed: &HashSeed, buf: &[u8]) -> Self {
         Self::from_hash(SipHasher24::new_with_key(&seed.0).hash(buf))
     }
@@ -58,10 +55,6 @@ impl PartedHash {
     pub fn is_valid(&self) -> bool {
         self.signature() != INVALID_SIG
     }
-    #[inline]
-    pub fn is_invalid(&self) -> bool {
-        self.signature() == INVALID_SIG
-    }
 
     #[inline]
     pub fn shard_selector(&self) -> u16 {

src/lib.rs

@@ -38,7 +38,7 @@
 //!     db.set_in_list("spanish", "bye", "adios")?;
 //!     db.set_in_list("spanish", "thanks", "gracias")?;
 //!
-//!     let items = db.iter_list("spanish").map(|res| res.unwrap().unwrap()).collect::<Vec<_>>();
+//!     let items = db.iter_list("spanish").map(|res| res.unwrap()).collect::<Vec<_>>();
 //!     assert_eq!(items, vec![("bye".into(), "adios".into()), ("thanks".into(), "gracias".into())]);
 //!
 //!     Ok(())
@@ -50,15 +50,14 @@
 mod encodable;
 mod hashing;
 mod insertion;
-mod list_insert;
-mod list_remove;
 mod lists;
 mod shard;
 mod store;
 mod typed;
 
 pub use hashing::HashSeed;
 pub use insertion::{GetOrCreateStatus, ReplaceStatus, SetStatus};
+pub use lists::{LinkedListIterator, ListCompactionParams};
 use std::fmt::{Display, Formatter};
 pub use store::{CandyStore, CoarseHistogram, SizeHistogram, Stats};
 pub use typed::{CandyTypedDeque, CandyTypedKey, CandyTypedList, CandyTypedStore};