Major rework of db layer + package upgrades

This is a big-bang merge of a bunch of major changes that happened on a branch. It rips out use of RocksDB entirely, replacing with custom page storage layer and the use a proper write-ahead-log (via `okaywal` for now). There's still some rough edges here, but this is much closer to the vision I had in mind.
rdaum · Jan 1, 2024 · e54023b · e54023b
1 parent 910af59
commit e54023b
Show file tree

Hide file tree

Showing 33 changed files with 2,561 additions and 1,260 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,92 +20,95 @@ license = "GPL-3"
 
 [workspace.dependencies]
 ## Command line arguments parsing.
-clap = "4.4.6"
-clap_derive = "4.4.2"
+clap = "4.4.12"
+clap_derive = "4.4.7"
 
 ## HTTP/websockets front-end
-axum = { version = "0.6.20", features = ["ws", "headers"] }
-serde_json = "1.0.107"
-serde = { version = "1.0.189", features = ["derive"] }
-serde_derive = "1.0.189"
-tower-http = { version = "0.4.4", features = ["add-extension", "auth", "compression-full", "trace"] }
+axum = { version = "0.7.3", features = ["ws"] }
+axum-extra = "0.9.1"
+serde_json = "1.0.108"
+serde = { version = "1.0.193", features = ["derive"] }
+serde_derive = "1.0.193"
+tower-http = { version = "0.5.0", features = ["add-extension", "auth", "compression-full", "trace"] }
 
 ## Asynchronous transaction processing & networking
-futures = "0.3.28"
-futures-util = { version = "0.3.28", features = ["sink", "std"] }
-async-trait = "0.1.73"
-tokio = { version = "1.33.0", features = ["full"] }
+futures = "0.3.30"
+futures-util = { version = "0.3.30", features = ["sink", "std"] }
+async-trait = "0.1.76"
+tokio = { version = "1.35.1", features = ["full"] }
 tokio-test = "0.4.3"
-tokio-util = { version = "0.7.9",  features = ["full"] }
+tokio-util = { version = "0.7.10",  features = ["full"] }
 
 # Used for RPC daemon/client
 tmq = "0.4.0"
 
 ## Logging & tracing
-tracing = "0.1.39"
+tracing = "0.1.40"
 tracing-test = "0.2.4"
-tracing-subscriber = "0.3.17"
+tracing-subscriber = "0.3.18"
 tracing-chrome = "0.7.1"
-metrics-exporter-prometheus = "0.12.1"
+metrics-exporter-prometheus = "0.13.0"
 metrics = "0.21.1"
-metrics-util = { version = "0.15.1",  default-features = false, features = ["recency", "registry", "summary"] }
-metrics-macros = "0.7.0"
+metrics-util = { version = "0.16.0",  default-features = false, features = ["recency", "registry", "summary"] }
+metrics-macros = "0.7.1"
 
 # General usefullness
 atomic-wait = "1.1.0"
 binary-layout = "3.2.0"
 bytes = "1.5.0"
 chrono = "0.4.31"
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-crossbeam-channel = "0.5.8"
+crossbeam-channel = "0.5.10"
 decorum = "0.3.1"  # For ordering & comparing our floats
-enum-primitive-derive = "0.2.2"
-inventory = "0.3.12"
-itertools = "0.11.0"
+enum-primitive-derive = "0.3.0"
+inventory = "0.3.14"
+itertools = "0.12.0"
 lazy_static = "1.4.0"
 num-traits = "0.2.17"
-rustyline = "12.0.0"
-sized-chunks = "0.6.5"
+rustyline = "13.0.0"
+sized-chunks = "0.7.0"
 strum = { version = "0.25.0", features = ["derive"] }
-uuid = { version = "1.4.1", features = ["v4"] }
-yoke = "0.7.2"
-yoke-derive = "0.7.2"
+uuid = { version = "1.6.1", features = ["v4"] }
+yoke = "0.7.3"
+yoke-derive = "0.7.3"
 
 ## Required for MOO builtins.
 pwhash = "1.0.0" # For MOO's hokey "crypt" function, which is unix's crypt(3) basically
 md5 = "0.7.0" # For MOO's "string_hash"
 rand = "0.8.5"
 onig = { version = "6.4.0", default-features = false  }
-chrono-tz = "0.8.3"
-iana-time-zone = "0.1.57"
+chrono-tz = "0.8.5"
+iana-time-zone = "0.1.59"
 
 ## Compiler grammar/parser
-pest = "2.7.4"
-pest_derive = "2.7.4"
+pest = "2.7.5"
+pest_derive = "2.7.5"
 
 ## Error declaration/ handling
-anyhow = { version = "1.0.75", features = ["backtrace"] }
-thiserror = "1.0.49"
+anyhow = { version = "1.0.78", features = ["backtrace"] }
+thiserror = "1.0.53"
 
 ## For macro-ing
 paste = "1.0.14"
 
 # For the DB & values layer.
 bincode = "2.0.0-rc.3" # For serializing/deserializing values
 im = "15.1.0" # Immutable data structures
-rocksdb = "0.21.0"
 text_io = "0.1.12" # Used for reading text dumps.
-libc = "0.2.149"
+libc = "0.2.151"
+okaywal = "0.3.1"
+io-uring = "0.6.2"
+hi_sparse_bitset = "0.3.0"
 
 # Dev dependencies
-tempfile = "3.8.0"
+tempfile = "3.9.0"
 
 # testing
 pretty_assertions = "1.4.0"
-test-case = "3.2.1"
+test-case = "3.3.1"
 unindent = "0.2.3"
 
 # Auth/Auth
 rusty_paseto = { version = "0.6.0" }
-ed25519-dalek = { version = "2.0.0", features = ["zeroize", "pkcs8", "rand_core"] }
-pem = "3.0.2"
+ed25519-dalek = { version = "2.1.0", features = ["zeroize", "pkcs8", "rand_core"] }
+pem = "3.0.3"
diff --git a/JHCore-DEV-2.db b/JHCore-DEV-2.db
@@ -1,11 +1,3 @@
-# Copyright (C) 2024 Ryan Daum <[email protected]>
-#
-# This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
 ** LambdaMOO Database, Format Version 4 **
 237
 2729

diff --git a/crates/compiler/src/moo.pest b/crates/compiler/src/moo.pest
@@ -12,14 +12,6 @@
 // this program. If not, see <https://www.gnu.org/licenses/>.
 //
 
-// Copyright (C) 2024 Ryan Daum <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
-
 program    = { SOI ~ statements ~ EOI }
 statements = { statement* }
 statement  = {

diff --git a/crates/db/Cargo.toml b/crates/db/Cargo.toml
@@ -12,6 +12,8 @@ harness = false
 [dev-dependencies]
 tempfile.workspace = true
 rand.workspace = true
+serde_json.workspace = true
+serde.workspace = true
 criterion.workspace = true
 
 [dependencies]
@@ -37,14 +39,16 @@ metrics.workspace = true
 metrics-macros.workspace = true
 
 # For the DB layer.
-rocksdb.workspace = true
 crossbeam-channel.workspace = true
 bincode.workspace = true
 im.workspace = true
 sized-chunks.workspace = true
 binary-layout.workspace = true
 libc.workspace = true
 atomic-wait.workspace = true
+okaywal.workspace = true
+io-uring.workspace = true
+hi_sparse_bitset.workspace = true
 
 # For testing & benching common bits
 serde_json.workspace = true

diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs
@@ -21,10 +21,10 @@ use crate::loader::LoaderInterface;
 use crate::tb_worldstate::TupleBoxWorldStateSource;
 
 mod db_loader_client;
-mod db_tx;
+pub mod db_tx;
 mod db_worldstate;
 pub mod loader;
-mod object_relations;
+pub mod object_relations;
 pub mod tb_worldstate;
 pub mod tuplebox;
 

diff --git a/crates/db/src/object_relations.rs b/crates/db/src/object_relations.rs
@@ -49,9 +49,9 @@ pub enum WorldStateRelation {
     ObjectPropertyValue = 8,
 }
 
-impl Into<RelationId> for WorldStateRelation {
-    fn into(self) -> RelationId {
-        RelationId(self as usize)
+impl From<WorldStateRelation> for RelationId {
+    fn from(val: WorldStateRelation) -> Self {
+        RelationId(val as usize)
     }
 }
 
@@ -190,7 +190,7 @@ async fn insert_composite_value<Codomain: Clone + Eq + PartialEq + AsByteBuffer>
 }
 
 #[allow(dead_code)]
-async fn delete_if_exists<Codomain: Clone + Eq + PartialEq + AsByteBuffer>(
+async fn delete_if_exists(
     tx: &Transaction,
     rel: WorldStateRelation,
     oid: Objid,
@@ -263,8 +263,8 @@ mod tests {
         relations[ObjectParent as usize].secondary_indexed = true;
         relations[WorldStateRelation::ObjectLocation as usize].secondary_indexed = true;
 
-        let db = TupleBox::new(1 << 24, 4096, None, &relations, WorldStateSequences::COUNT).await;
-        db
+
+        TupleBox::new(1 << 24, 32768, None, &relations, WorldStateSequences::COUNT).await
     }
 
     /// Test simple relations mapping oid->oid (with secondary index), independent of all other

diff --git a/crates/db/src/tb_worldstate.rs b/crates/db/src/tb_worldstate.rs
@@ -49,7 +49,7 @@ use crate::{object_relations, tuplebox, Database};
 
 // TODO: Totally arbitrary and needs profiling. Needs to be big enough to hold entire props and
 //   verbs.
-const PAGE_SIZE: usize = 32768;
+const PAGE_SIZE: usize = 65536;
 
 /// An implementation of `WorldState` / `WorldStateSource` that uses the TupleBox as its backing
 pub struct TupleBoxWorldStateSource {
@@ -121,7 +121,7 @@ impl DbTransaction for TupleBoxTransaction {
     async fn get_object_owner(&self, obj: Objid) -> Result<Objid, WorldStateError> {
         object_relations::get_object_value(&self.tx, WorldStateRelation::ObjectOwner, obj)
             .await
-            .ok_or_else(|| WorldStateError::ObjectNotFound(obj))
+            .ok_or(WorldStateError::ObjectNotFound(obj))
     }
 
     async fn set_object_owner(&self, obj: Objid, owner: Objid) -> Result<(), WorldStateError> {
@@ -132,7 +132,7 @@ impl DbTransaction for TupleBoxTransaction {
     async fn get_object_flags(&self, obj: Objid) -> Result<BitEnum<ObjFlag>, WorldStateError> {
         object_relations::get_object_value(&self.tx, WorldStateRelation::ObjectFlags, obj)
             .await
-            .ok_or_else(|| WorldStateError::ObjectNotFound(obj))
+            .ok_or(WorldStateError::ObjectNotFound(obj))
     }
 
     async fn set_object_flags(
@@ -147,7 +147,7 @@ impl DbTransaction for TupleBoxTransaction {
     async fn get_object_name(&self, obj: Objid) -> Result<String, WorldStateError> {
         object_relations::get_object_value(&self.tx, WorldStateRelation::ObjectName, obj)
             .await
-            .ok_or_else(|| WorldStateError::ObjectNotFound(obj))
+            .ok_or(WorldStateError::ObjectNotFound(obj))
     }
 
     async fn create_object(
@@ -228,14 +228,12 @@ impl DbTransaction for TupleBoxTransaction {
 
         // Now we can remove this object from all relevant column relations
         // First the simple ones which are keyed on the object id.
-        let oid_relations = vec![
-            WorldStateRelation::ObjectFlags,
+        let oid_relations = [WorldStateRelation::ObjectFlags,
             WorldStateRelation::ObjectName,
             WorldStateRelation::ObjectOwner,
             WorldStateRelation::ObjectParent,
             WorldStateRelation::ObjectLocation,
-            WorldStateRelation::ObjectVerbs,
-        ];
+            WorldStateRelation::ObjectVerbs];
         for rel in oid_relations.iter() {
             let relation = self.tx.relation((*rel).into()).await;
             relation
@@ -430,7 +428,7 @@ impl DbTransaction for TupleBoxTransaction {
                     c,
                 )
                 .await
-                .unwrap_or_else(|| PropDefs::empty()),
+                .unwrap_or_else(PropDefs::empty),
                 Some(props) => props,
             };
             let c_props = c_props.with_all_added(&new_props);
@@ -559,9 +557,8 @@ impl DbTransaction for TupleBoxTransaction {
                 .await
                 .ok_or_else(|| WorldStateError::VerbNotFound(obj, name.clone()))?;
         Ok(verbdefs
-            .find_named(name.as_str())
-            .get(0)
-            .ok_or_else(|| WorldStateError::VerbNotFound(obj, name))?
+            .find_named(name.as_str()).first()
+            .ok_or(WorldStateError::VerbNotFound(obj, name))?
             .clone())
     }
 
@@ -856,16 +853,16 @@ impl DbTransaction for TupleBoxTransaction {
                 Some(s) => s.as_str(),
             };
 
-            let new_prop = PropDef::new(
+
+
+            PropDef::new(
                 p.uuid(),
                 p.definer(),
                 p.location(),
-                &name,
+                name,
                 new_flags.unwrap_or_else(|| p.flags()),
                 new_owner.unwrap_or_else(|| p.owner()),
-            );
-
-            new_prop
+            )
         }) else {
             return Err(WorldStateError::PropertyNotFound(obj, format!("{}", uuid)));
         };
@@ -1072,7 +1069,7 @@ impl TupleBoxTransaction {
                     search_a,
                 )
                 .await
-                .unwrap_or_else(|| NOTHING);
+                .unwrap_or(NOTHING);
                 search_a = parent;
             }
 
@@ -1084,7 +1081,7 @@ impl TupleBoxTransaction {
                     search_b,
                 )
                 .await
-                .unwrap_or_else(|| NOTHING);
+                .unwrap_or(NOTHING);
                 search_b = parent;
             }
         }
@@ -1094,7 +1091,7 @@ impl TupleBoxTransaction {
 impl Database for TupleBoxWorldStateSource {
     fn loader_client(&mut self) -> Result<Box<dyn LoaderInterface>, WorldStateError> {
         let tx = TupleBoxTransaction::new(self.db.clone());
-        return Ok(Box::new(DbTxWorldState { tx: Box::new(tx) }));
+        Ok(Box::new(DbTxWorldState { tx: Box::new(tx) }))
     }
 
     fn world_state_source(self: Box<Self>) -> Result<Arc<dyn WorldStateSource>, WorldStateError> {
@@ -1138,8 +1135,8 @@ mod tests {
         relations[WorldStateRelation::ObjectParent as usize].secondary_indexed = true;
         relations[WorldStateRelation::ObjectLocation as usize].secondary_indexed = true;
 
-        let db = TupleBox::new(1 << 24, 4096, None, &relations, WorldStateSequences::COUNT).await;
-        db
+
+        TupleBox::new(1 << 24, 4096, None, &relations, WorldStateSequences::COUNT).await
     }
 
     #[tokio::test]