Some todo/comments/doc additions, and module scoping cleanup

crates/daemon/src/connections_tb.rs

@@ -25,9 +25,7 @@ use strum::{Display, EnumCount, EnumIter, IntoEnumIterator};
 use tracing::{debug, error, warn};
 use uuid::Uuid;
 
-use moor_db::tuplebox;
-use moor_db::tuplebox::tb::{RelationInfo, TupleBox};
-use moor_db::tuplebox::RelationId;
+use moor_db::tuplebox::{RelationId, RelationInfo, Transaction, TupleBox};
 use moor_kernel::tasks::sessions::SessionError;
 use moor_values::util::slice_ref::SliceRef;
 use moor_values::var::objid::Objid;
@@ -77,7 +75,7 @@ enum ConnectionRelation {
 
 impl ConnectionsTb {
     async fn most_recent_client_connection(
-        tx: &tuplebox::Transaction,
+        tx: &Transaction,
         connection_obj: Objid,
     ) -> Result<Vec<(SliceRef, SystemTime)>, SessionError> {
         let clients = tx

crates/db/benches/tb_single_thread.rs

@@ -17,7 +17,7 @@
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use moor_db::testing::jepsen::{History, Type, Value};
-use moor_db::tuplebox::tb::{RelationInfo, TupleBox};
+use moor_db::tuplebox::{RelationInfo, TupleBox};
 use moor_values::util::slice_ref::SliceRef;
 use std::collections::HashMap;
 use std::sync::Arc;

crates/db/src/object_relations.rs

@@ -21,7 +21,7 @@ use moor_values::util::slice_ref::SliceRef;
 use moor_values::var::objid::Objid;
 use moor_values::AsByteBuffer;
 
-use crate::tuplebox::tuples::TupleError;
+use crate::tuplebox::TupleError;
 use crate::tuplebox::{RelationId, Transaction};
 
 /// The set of binary relations that are used to represent the world state in the moor system.
@@ -248,7 +248,7 @@ mod tests {
         get_object_by_codomain, get_object_value, insert_object_value, upsert_object_value,
         WorldStateRelation, WorldStateSequences,
     };
-    use crate::tuplebox::tb::{RelationInfo, TupleBox};
+    use crate::tuplebox::{RelationInfo, TupleBox};
 
     async fn test_db() -> Arc<TupleBox> {
         let mut relations: Vec<RelationInfo> = WorldStateRelation::iter()

crates/db/src/tb_worldstate.rs

@@ -21,7 +21,7 @@ use strum::{EnumCount, IntoEnumIterator};
 use tracing::warn;
 use uuid::Uuid;
 
-use crate::tuplebox::tuples::TupleError;
+use crate::tuplebox::TupleError;
 use moor_values::model::defset::HasUuid;
 use moor_values::model::objects::{ObjAttrs, ObjFlag};
 use moor_values::model::objset::ObjSet;
@@ -43,8 +43,8 @@ use crate::loader::LoaderInterface;
 use crate::object_relations::{
     get_all_object_keys_matching, WorldStateRelation, WorldStateSequences,
 };
-use crate::tuplebox::tb::{RelationInfo, TupleBox};
 use crate::tuplebox::{CommitError, Transaction};
+use crate::tuplebox::{RelationInfo, TupleBox};
 use crate::{object_relations, Database};
 
 /// An implementation of `WorldState` / `WorldStateSource` that uses the TupleBox as its backing
@@ -1113,7 +1113,7 @@ mod tests {
     use crate::db_tx::DbTransaction;
     use crate::object_relations::{WorldStateRelation, WorldStateSequences};
     use crate::tb_worldstate::TupleBoxTransaction;
-    use crate::tuplebox::tb::{RelationInfo, TupleBox};
+    use crate::tuplebox::{RelationInfo, TupleBox};
 
     async fn test_db() -> Arc<TupleBox> {
         let mut relations: Vec<RelationInfo> = WorldStateRelation::iter()

crates/db/src/tuplebox/backing.rs

@@ -18,7 +18,7 @@
 
 use tokio::sync::mpsc::UnboundedSender;
 
-use crate::tuplebox::tx::working_set::WorkingSet;
+use crate::tuplebox::tx::WorkingSet;
 
 pub struct BackingStoreClient {
     sender: UnboundedSender<WriterMessage>,

crates/db/src/tuplebox/base_relation.rs

@@ -30,6 +30,17 @@ use crate::tuplebox::RelationId;
 ///
 // TODO: Add some kind of 'type' flag to the relation & tuple values, so that we can do
 //   type-checking on the values, though for our purposes this may be overkill at this time.
+// TODO: all the 'seek' type operations should be returning a *set* of tuples that match, not
+//   a single one. right now this is behaving like a key-value pair, not a proper binary relation.
+//   means changing the indexes here to point to sets of tuples, not single tuples. right now
+//   for moor's purposes this is irrelevant, but it will be important for proper implementation of
+//   joins and other relational operations.
+// TODO: the indexes should be paged.
+// TODO: support ordered indexes, not just hash indexes.
+//   if we're staying with in-memory, use an Adaptive Radix Tree; my implementation, but hopefully
+//   modified to support CoW/shared ownership of the tree nodes, like the im::HashMap does.
+//   if we're going to support on-disk indexes, use a CoW B+Tree, which I have implemented elsewhere,
+//   but will need to bring in here, optimize, and provide loving care to.
 #[derive(Clone)]
 pub struct BaseRelation {
     pub(crate) id: RelationId,

crates/db/src/tuplebox/coldstorage.rs

@@ -34,7 +34,7 @@ use crate::tuplebox::page_storage::{PageStore, PageStoreMutation};
 use crate::tuplebox::tb::RelationInfo;
 use crate::tuplebox::tuples::TxTuple;
 use crate::tuplebox::tuples::{PageId, SlotBox, SlotId, TupleId};
-use crate::tuplebox::tx::working_set::WorkingSet;
+use crate::tuplebox::tx::WorkingSet;
 use crate::tuplebox::RelationId;
 
 /// Uses WAL + custom page store as the persistent backing store & write-ahead-log for the tuplebox.

crates/db/src/tuplebox/mod.rs

@@ -29,12 +29,13 @@ mod coldstorage;
 mod page_storage;
 mod pool;
 
-pub mod tb;
-pub mod tuples;
+mod tb;
+mod tuples;
 mod tx;
 
+pub use tb::{RelationInfo, TupleBox};
 pub use tuples::TupleError;
-pub use tx::transaction::{CommitError, Transaction};
+pub use tx::{CommitError, Transaction};
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
 pub struct RelationId(pub usize);

crates/db/src/tuplebox/tb.rs

@@ -27,8 +27,8 @@ use crate::tuplebox::backing::BackingStoreClient;
 use crate::tuplebox::base_relation::BaseRelation;
 use crate::tuplebox::tuples::SlotBox;
 use crate::tuplebox::tuples::TxTuple;
-use crate::tuplebox::tx::transaction::{CommitError, CommitSet, Transaction};
-use crate::tuplebox::tx::working_set::WorkingSet;
+use crate::tuplebox::tx::WorkingSet;
+use crate::tuplebox::tx::{CommitError, CommitSet, Transaction};
 use crate::tuplebox::RelationId;
 
 /// Meta-data about a relation

crates/db/src/tuplebox/tx/mod.rs

@@ -12,6 +12,9 @@
 // this program. If not, see <https://www.gnu.org/licenses/>.
 //
 
-pub mod relvar;
-pub mod transaction;
-pub mod working_set;
+mod relvar;
+mod transaction;
+mod working_set;
+
+pub use transaction::{CommitError, CommitSet, Transaction};
+pub use working_set::WorkingSet;

crates/db/src/tuplebox/tx/relvar.rs

@@ -21,6 +21,7 @@ use crate::tuplebox::RelationId;
 /// A reference / handle / pointer to a relation, the actual operations are managed through the
 /// transaction.
 /// A more convenient handle tied to the lifetime of the transaction.
+// TODO: see comments on BaseRelation. changes there will require changes here.
 pub struct RelVar<'a> {
     pub(crate) tx: &'a Transaction,
     pub(crate) id: RelationId,

crates/db/src/tuplebox/tx/transaction.rs

@@ -290,7 +290,7 @@ impl Transaction {
 
 /// A set of tuples to be committed to the canonical base relations, based on a transaction's
 /// working set.
-pub(crate) struct CommitSet {
+pub struct CommitSet {
     pub(crate) ts: u64,
     relations: SparseChunk<BaseRelation, 64>,
 }

crates/db/src/tuplebox/tx/working_set.rs

@@ -26,6 +26,7 @@ use crate::tuplebox::RelationId;
 /// we will attempt to make permanent when the transaction commits.
 /// The working set is also referred to for reads/updates during the lifetime of the transaction.  
 /// It effectively "is" the transaction in regards to *base relations*.
+// TODO: see comments on BaseRelation, changes there will reqiure changes here.
 pub struct WorkingSet {
     pub(crate) ts: u64,
     pub(crate) slotbox: Arc<SlotBox>,

crates/db/tests/jepsen.rs

@@ -13,7 +13,7 @@
 //
 
 pub mod support {
-    use moor_db::tuplebox::tb::{RelationInfo, TupleBox};
+    use moor_db::tuplebox::{RelationInfo, TupleBox};
     use std::path::PathBuf;
     use std::sync::Arc;
 
@@ -40,7 +40,7 @@ mod tests {
     use tracing_test::traced_test;
 
     use moor_db::testing::jepsen::{History, Type, Value};
-    use moor_db::tuplebox::tb::TupleBox;
+    use moor_db::tuplebox::TupleBox;
     use moor_db::tuplebox::{RelationId, Transaction};
 
     use moor_values::util::slice_ref::SliceRef;

crates/db/tests/tb_restore.rs

@@ -19,8 +19,8 @@ mod test {
     use std::sync::Arc;
 
     use moor_db::testing::jepsen::{History, Type, Value};
-    use moor_db::tuplebox::tb::{RelationInfo, TupleBox};
     use moor_db::tuplebox::{RelationId, Transaction};
+    use moor_db::tuplebox::{RelationInfo, TupleBox};
     use moor_values::util::slice_ref::SliceRef;
 
     fn from_val(value: i64) -> SliceRef {

crates/db/tests/worldstate_restore.rs

@@ -17,7 +17,7 @@ mod test {
     use moor_db::db_tx::DbTransaction;
     use moor_db::object_relations::{WorldStateRelation, WorldStateSequences};
     use moor_db::tb_worldstate::TupleBoxTransaction;
-    use moor_db::tuplebox::tb::{RelationInfo, TupleBox};
+    use moor_db::tuplebox::{RelationInfo, TupleBox};
     use moor_values::model::defset::HasUuid;
     use moor_values::model::objects::ObjAttrs;
     use moor_values::model::r#match::VerbArgsSpec;