From 906f83281bcf3474ca493d2e91ce1ba7867e90d4 Mon Sep 17 00:00:00 2001 From: Ryan Daum Date: Tue, 2 Jan 2024 00:04:22 -0500 Subject: [PATCH] A little package cleanup. Moved slotbox and friends into 'tuples' module. --- crates/db/src/object_relations.rs | 3 +- crates/db/src/tb_worldstate.rs | 4 +- crates/db/src/tuplebox/base_relation.rs | 2 +- crates/db/src/tuplebox/coldstorage.rs | 2 +- crates/db/src/tuplebox/mod.rs | 4 +- crates/db/src/tuplebox/page_storage.rs | 2 +- crates/db/src/tuplebox/slots/mod.rs | 21 ----- crates/db/src/tuplebox/tb.rs | 2 +- crates/db/src/tuplebox/tuples/mod.rs | 76 ++++--------------- .../src/tuplebox/{slots => tuples}/slotbox.rs | 13 +++- .../{slots => tuples}/slotted_page.rs | 6 +- crates/db/src/tuplebox/tuples/tuple.rs | 7 +- crates/db/src/tuplebox/tuples/tuple_ref.rs | 64 ++++++++++++++++ crates/db/src/tuplebox/tuples/tx_tuple.rs | 12 +-- crates/db/src/tuplebox/tx/transaction.rs | 2 +- crates/db/src/tuplebox/tx/working_set.rs | 8 +- 16 files changed, 109 insertions(+), 119 deletions(-) delete mode 100644 crates/db/src/tuplebox/slots/mod.rs rename crates/db/src/tuplebox/{slots => tuples}/slotbox.rs (97%) rename crates/db/src/tuplebox/{slots => tuples}/slotted_page.rs (99%) create mode 100644 crates/db/src/tuplebox/tuples/tuple_ref.rs diff --git a/crates/db/src/object_relations.rs b/crates/db/src/object_relations.rs index 4358c265..214f2a16 100644 --- a/crates/db/src/object_relations.rs +++ b/crates/db/src/object_relations.rs @@ -21,7 +21,8 @@ use moor_values::util::slice_ref::SliceRef; use moor_values::var::objid::Objid; use moor_values::AsByteBuffer; -use crate::tuplebox::{RelationId, Transaction, TupleError}; +use crate::tuplebox::tuples::TupleError; +use crate::tuplebox::{RelationId, Transaction}; /// The set of binary relations that are used to represent the world state in the moor system. #[repr(usize)] diff --git a/crates/db/src/tb_worldstate.rs b/crates/db/src/tb_worldstate.rs index 287c741c..5aa57ae0 100644 --- a/crates/db/src/tb_worldstate.rs +++ b/crates/db/src/tb_worldstate.rs @@ -21,6 +21,7 @@ use strum::{EnumCount, IntoEnumIterator}; use tracing::warn; use uuid::Uuid; +use crate::tuplebox::tuples::TupleError; use moor_values::model::defset::HasUuid; use moor_values::model::objects::{ObjAttrs, ObjFlag}; use moor_values::model::objset::ObjSet; @@ -35,7 +36,6 @@ use moor_values::util::bitenum::BitEnum; use moor_values::var::objid::Objid; use moor_values::var::{v_none, Var}; use moor_values::{AsByteBuffer, NOTHING, SYSTEM_OBJECT}; -use tuplebox::TupleError; use crate::db_tx::DbTransaction; use crate::db_worldstate::DbTxWorldState; @@ -45,7 +45,7 @@ use crate::object_relations::{ }; use crate::tuplebox::tb::{RelationInfo, TupleBox}; use crate::tuplebox::{CommitError, Transaction}; -use crate::{object_relations, tuplebox, Database}; +use crate::{object_relations, Database}; /// An implementation of `WorldState` / `WorldStateSource` that uses the TupleBox as its backing pub struct TupleBoxWorldStateSource { diff --git a/crates/db/src/tuplebox/base_relation.rs b/crates/db/src/tuplebox/base_relation.rs index 1ec67875..f4e253a2 100644 --- a/crates/db/src/tuplebox/base_relation.rs +++ b/crates/db/src/tuplebox/base_relation.rs @@ -17,8 +17,8 @@ use std::sync::Arc; use moor_values::util::slice_ref::SliceRef; -use crate::tuplebox::slots::{SlotBox, TupleId}; use crate::tuplebox::tuples::TupleRef; +use crate::tuplebox::tuples::{SlotBox, TupleId}; use crate::tuplebox::RelationId; /// Represents a 'canonical' base binary relation, which is a set of tuples of domain, codomain, diff --git a/crates/db/src/tuplebox/coldstorage.rs b/crates/db/src/tuplebox/coldstorage.rs index 3cfe6d32..8ec7dd62 100644 --- a/crates/db/src/tuplebox/coldstorage.rs +++ b/crates/db/src/tuplebox/coldstorage.rs @@ -31,9 +31,9 @@ use tracing::{debug, error, info, warn}; use crate::tuplebox::backing::{BackingStoreClient, WriterMessage}; use crate::tuplebox::base_relation::BaseRelation; use crate::tuplebox::page_storage::{PageStore, PageStoreMutation}; -use crate::tuplebox::slots::{PageId, SlotBox, SlotId, TupleId}; use crate::tuplebox::tb::RelationInfo; use crate::tuplebox::tuples::TxTuple; +use crate::tuplebox::tuples::{PageId, SlotBox, SlotId, TupleId}; use crate::tuplebox::tx::working_set::WorkingSet; use crate::tuplebox::RelationId; diff --git a/crates/db/src/tuplebox/mod.rs b/crates/db/src/tuplebox/mod.rs index 8c7a4556..eefb4d55 100644 --- a/crates/db/src/tuplebox/mod.rs +++ b/crates/db/src/tuplebox/mod.rs @@ -28,9 +28,9 @@ mod base_relation; mod coldstorage; mod page_storage; mod pool; -mod slots; + pub mod tb; -mod tuples; +pub mod tuples; mod tx; pub use tuples::TupleError; diff --git a/crates/db/src/tuplebox/page_storage.rs b/crates/db/src/tuplebox/page_storage.rs index 8eeb873d..2472a68c 100644 --- a/crates/db/src/tuplebox/page_storage.rs +++ b/crates/db/src/tuplebox/page_storage.rs @@ -14,7 +14,7 @@ // TODO: there's no way this is "robust" enough to be used in production -use crate::tuplebox::slots::PageId; +use crate::tuplebox::tuples::PageId; use crate::tuplebox::RelationId; use im::{HashMap, HashSet}; use io_uring::squeue::Flags; diff --git a/crates/db/src/tuplebox/slots/mod.rs b/crates/db/src/tuplebox/slots/mod.rs deleted file mode 100644 index 2e587b36..00000000 --- a/crates/db/src/tuplebox/slots/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) 2024 Ryan Daum -// -// This program is free software: you can redistribute it and/or modify it under -// the terms of the GNU General Public License as published by the Free Software -// Foundation, version 3. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -// FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along with -// this program. If not, see . -// - -pub use crate::tuplebox::slots::slotbox::{SlotId, TupleId}; -// Export TupleId & SlotBox -pub use crate::tuplebox::slots::slotbox::PageId; -pub use crate::tuplebox::slots::slotbox::SlotBox; - -mod slotbox; -mod slotted_page; diff --git a/crates/db/src/tuplebox/tb.rs b/crates/db/src/tuplebox/tb.rs index b37e8e91..6fadc668 100644 --- a/crates/db/src/tuplebox/tb.rs +++ b/crates/db/src/tuplebox/tb.rs @@ -25,7 +25,7 @@ use tracing::info; use crate::tuplebox::backing::BackingStoreClient; use crate::tuplebox::base_relation::BaseRelation; -use crate::tuplebox::slots::SlotBox; +use crate::tuplebox::tuples::SlotBox; use crate::tuplebox::tuples::TxTuple; use crate::tuplebox::tx::transaction::{CommitError, CommitSet, Transaction}; use crate::tuplebox::tx::working_set::WorkingSet; diff --git a/crates/db/src/tuplebox/tuples/mod.rs b/crates/db/src/tuplebox/tuples/mod.rs index 4bda5569..58355581 100644 --- a/crates/db/src/tuplebox/tuples/mod.rs +++ b/crates/db/src/tuplebox/tuples/mod.rs @@ -12,72 +12,22 @@ // this program. If not, see . // -use crate::tuplebox::slots::{SlotBox, TupleId}; -use moor_values::util::slice_ref::ByteSource; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; +pub use slotbox::{PageId, SlotBox, SlotBoxError, SlotId, TupleId}; +use thiserror::Error; pub use tuple::Tuple; -pub use tx_tuple::{TupleError, TxTuple}; +pub use tuple_ref::TupleRef; +pub use tx_tuple::TxTuple; +mod slotbox; +mod slotted_page; mod tuple; +mod tuple_ref; mod tx_tuple; -pub struct TupleRef { - sb: Arc, - pub id: TupleId, -} - -impl PartialEq for TupleRef { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} - -impl Eq for TupleRef {} - -impl Hash for TupleRef { - fn hash(&self, state: &mut H) { - let slc = self.sb.get(self.id).expect("Unable to get tuple"); - slc.hash(state) - } -} -impl TupleRef { - pub fn new(sb: Arc, id: TupleId) -> Self { - sb.upcount(id).expect("Unable to add tuple"); - Self { sb, id } - } - - pub fn get(&self) -> Tuple { - Tuple::from_tuple_id(self.sb.clone(), self.id) - } -} - -impl Drop for TupleRef { - fn drop(&mut self) { - self.sb.dncount(self.id).expect("Unable to dncount tuple"); - } -} - -impl Clone for TupleRef { - fn clone(&self) -> Self { - self.sb.upcount(self.id).expect("Unable to upcount tuple"); - Self { - sb: self.sb.clone(), - id: self.id, - } - } -} - -impl ByteSource for TupleRef { - fn as_slice(&self) -> &[u8] { - self.sb.get(self.id).expect("Unable to get tuple").get_ref() - } - - fn len(&self) -> usize { - self.as_slice().len() - } - - fn touch(&self) { - // noop - } +#[derive(Debug, Clone, Eq, PartialEq, Error)] +pub enum TupleError { + #[error("Tuple not found")] + NotFound, + #[error("Tuple already exists")] + Duplicate, } diff --git a/crates/db/src/tuplebox/slots/slotbox.rs b/crates/db/src/tuplebox/tuples/slotbox.rs similarity index 97% rename from crates/db/src/tuplebox/slots/slotbox.rs rename to crates/db/src/tuplebox/tuples/slotbox.rs index a524fd35..cc0823ae 100644 --- a/crates/db/src/tuplebox/slots/slotbox.rs +++ b/crates/db/src/tuplebox/tuples/slotbox.rs @@ -22,6 +22,11 @@ // considering only the reported available "content" area when fitting slots, and there seems // to be a sporadic failure where we end up with a "Page not found" error in the allocator on // free, meaning the page was not found in the used pages list. +// TODO: improve TupleRef so it can hold a direct address to the tuple, and not just an id. +// some swizzling will probably be required. (though at this point we're never paging +// tuples out, so we may not need to swizzle). avoiding the lookup on every reference +// should improve performance massively. +// TODO: rename me, _I_ am the tuplebox. The "slots" are just where my tuples get stored. use std::cmp::max; use std::pin::Pin; @@ -34,8 +39,8 @@ use thiserror::Error; use tracing::error; use crate::tuplebox::pool::{Bid, BufferPool, PagerError}; -pub use crate::tuplebox::slots::slotted_page::SlotId; -use crate::tuplebox::slots::slotted_page::{ +pub use crate::tuplebox::tuples::slotted_page::SlotId; +use crate::tuplebox::tuples::slotted_page::{ slot_index_overhead, slot_page_empty_size, SlottedPage, }; use crate::tuplebox::RelationId; @@ -437,8 +442,8 @@ mod tests { use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; - use crate::tuplebox::slots::slotbox::{SlotBox, SlotBoxError, TupleId}; - use crate::tuplebox::slots::slotted_page::slot_page_empty_size; + use crate::tuplebox::tuples::slotbox::{SlotBox, SlotBoxError, TupleId}; + use crate::tuplebox::tuples::slotted_page::slot_page_empty_size; use crate::tuplebox::RelationId; fn fill_until_full(sb: &mut SlotBox) -> Vec<(TupleId, Vec)> { diff --git a/crates/db/src/tuplebox/slots/slotted_page.rs b/crates/db/src/tuplebox/tuples/slotted_page.rs similarity index 99% rename from crates/db/src/tuplebox/slots/slotted_page.rs rename to crates/db/src/tuplebox/tuples/slotted_page.rs index 498a62b4..2f5f3b76 100644 --- a/crates/db/src/tuplebox/slots/slotted_page.rs +++ b/crates/db/src/tuplebox/tuples/slotted_page.rs @@ -35,7 +35,7 @@ use std::sync::atomic::{AtomicPtr, AtomicU16, AtomicU32}; use atomic_wait::{wait, wake_all, wake_one}; use tracing::error; -use crate::tuplebox::slots::slotbox::SlotBoxError; +use crate::tuplebox::tuples::slotbox::SlotBoxError; pub type SlotId = usize; @@ -697,8 +697,8 @@ impl<'a> Drop for PageReadGuard<'a> { mod tests { use std::sync::atomic::AtomicPtr; - use crate::tuplebox::slots::slotbox::SlotBoxError; - use crate::tuplebox::slots::slotted_page::{ + use crate::tuplebox::tuples::slotbox::SlotBoxError; + use crate::tuplebox::tuples::slotted_page::{ slot_page_empty_size, SlotId, SlotIndexEntry, SlottedPage, }; diff --git a/crates/db/src/tuplebox/tuples/tuple.rs b/crates/db/src/tuplebox/tuples/tuple.rs index 788c58f3..419ffca1 100644 --- a/crates/db/src/tuplebox/tuples/tuple.rs +++ b/crates/db/src/tuplebox/tuples/tuple.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use crate::tuplebox::RelationId; use moor_values::util::slice_ref::SliceRef; -use crate::tuplebox::slots::{SlotBox, TupleId}; use crate::tuplebox::tuples::TupleRef; +use crate::tuplebox::tuples::{SlotBox, TupleId}; define_layout!(tuple_header, LittleEndian, { ts: u64, @@ -87,9 +87,8 @@ impl Tuple { let domain_size = tuple_header::View::new(buffer.as_slice()) .domain_size() .read(); - buffer.slice( - tuple_header::SIZE.unwrap()..tuple_header::SIZE.unwrap() + domain_size as usize, - ) + buffer + .slice(tuple_header::SIZE.unwrap()..tuple_header::SIZE.unwrap() + domain_size as usize) } pub fn codomain(&self) -> SliceRef { diff --git a/crates/db/src/tuplebox/tuples/tuple_ref.rs b/crates/db/src/tuplebox/tuples/tuple_ref.rs new file mode 100644 index 00000000..06f7871a --- /dev/null +++ b/crates/db/src/tuplebox/tuples/tuple_ref.rs @@ -0,0 +1,64 @@ +use crate::tuplebox::tuples::{SlotBox, Tuple, TupleId}; +use moor_values::util::slice_ref::ByteSource; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +pub struct TupleRef { + sb: Arc, + pub id: TupleId, +} + +impl PartialEq for TupleRef { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for TupleRef {} + +impl Hash for TupleRef { + fn hash(&self, state: &mut H) { + let slc = self.sb.get(self.id).expect("Unable to get tuple"); + slc.hash(state) + } +} +impl TupleRef { + pub fn new(sb: Arc, id: TupleId) -> Self { + sb.upcount(id).expect("Unable to add tuple"); + Self { sb, id } + } + + pub fn get(&self) -> Tuple { + Tuple::from_tuple_id(self.sb.clone(), self.id) + } +} + +impl Drop for TupleRef { + fn drop(&mut self) { + self.sb.dncount(self.id).expect("Unable to dncount tuple"); + } +} + +impl Clone for TupleRef { + fn clone(&self) -> Self { + self.sb.upcount(self.id).expect("Unable to upcount tuple"); + Self { + sb: self.sb.clone(), + id: self.id, + } + } +} + +impl ByteSource for TupleRef { + fn as_slice(&self) -> &[u8] { + self.sb.get(self.id).expect("Unable to get tuple").get_ref() + } + + fn len(&self) -> usize { + self.as_slice().len() + } + + fn touch(&self) { + // noop + } +} diff --git a/crates/db/src/tuplebox/tuples/tx_tuple.rs b/crates/db/src/tuplebox/tuples/tx_tuple.rs index 23554092..8a0b9498 100644 --- a/crates/db/src/tuplebox/tuples/tx_tuple.rs +++ b/crates/db/src/tuplebox/tuples/tx_tuple.rs @@ -12,20 +12,10 @@ // this program. If not, see . // -use thiserror::Error; - -use crate::tuplebox::slots::TupleId; +use crate::tuplebox::tuples::TupleId; use crate::tuplebox::tuples::TupleRef; use moor_values::util::slice_ref::SliceRef; -#[derive(Debug, Clone, Eq, PartialEq, Error)] -pub enum TupleError { - #[error("Tuple not found")] - NotFound, - #[error("Tuple already exists")] - Duplicate, -} - /// Possible operations on tuples, in the context of a transaction . #[derive(Clone)] pub enum TxTuple { diff --git a/crates/db/src/tuplebox/tx/transaction.rs b/crates/db/src/tuplebox/tx/transaction.rs index 08c9ef89..4e5cbd98 100644 --- a/crates/db/src/tuplebox/tx/transaction.rs +++ b/crates/db/src/tuplebox/tx/transaction.rs @@ -22,8 +22,8 @@ use tokio::sync::RwLock; use moor_values::util::slice_ref::SliceRef; use crate::tuplebox::base_relation::BaseRelation; -use crate::tuplebox::slots::SlotBox; use crate::tuplebox::tb::TupleBox; +use crate::tuplebox::tuples::SlotBox; use crate::tuplebox::tuples::TupleError; use crate::tuplebox::tx::relvar::RelVar; use crate::tuplebox::tx::working_set::WorkingSet; diff --git a/crates/db/src/tuplebox/tx/working_set.rs b/crates/db/src/tuplebox/tx/working_set.rs index c7d19a8a..795e89d3 100644 --- a/crates/db/src/tuplebox/tx/working_set.rs +++ b/crates/db/src/tuplebox/tx/working_set.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use moor_values::util::slice_ref::SliceRef; -use crate::tuplebox::slots::SlotBox; use crate::tuplebox::tb::{RelationInfo, TupleBox}; -use crate::tuplebox::tuples::{Tuple, TupleError, TxTuple}; +use crate::tuplebox::tuples::{SlotBox, TupleError}; +use crate::tuplebox::tuples::{Tuple, TxTuple}; use crate::tuplebox::RelationId; /// The local tx "working set" of mutations to base relations, and consists of the set of operations @@ -239,7 +239,9 @@ impl WorkingSet { } // Now we have a map of domain -> tuple, so we can just pull out the tuples and return them. - Ok(by_domain.into_values().map(|t| (t.domain(), t.codomain())) + Ok(by_domain + .into_values() + .map(|t| (t.domain(), t.codomain())) .collect()) }