From 54a0e82761015d95bba2514935de094262f328b6 Mon Sep 17 00:00:00 2001 From: Mathieu Dutour Sikiric Date: Sat, 30 Nov 2024 15:10:54 +0100 Subject: [PATCH] Cleanups of the storage definitions. (#2979) * Simplify out the `LocalStackTestContext` that is no longer used. * Remove unused enum in the error types: `MissingDatabase` and `AlreadyExistingDatabase`. * Introduce constructors for the `RocksDbStoreConfig` and similar. * Remove the `pub` attribute when not needed. * Correct the mis-attribution "lru splitting" into "lru caching". * Add a `PathWithStorage` function `new_testing` and remove standalone functions. * Put the constructor definition of `VISIBLE_MAX_VALUE_SIZE` for DynamoDb. * Remove tests that simply reproduce the formula. * The `InvalidTableName` was renamed as `InvalidNamespace`. --- linera-client/src/storage.rs | 39 +----- linera-indexer/lib/src/rocks_db.rs | 20 +-- linera-indexer/lib/src/scylla_db.rs | 16 +-- linera-storage-service/src/common.rs | 4 +- linera-storage-service/src/server.rs | 15 +- linera-views/benches/queue_view.rs | 2 +- linera-views/src/backends/dynamo_db.rs | 166 +++++++++-------------- linera-views/src/backends/lru_caching.rs | 10 +- linera-views/src/backends/rocks_db.rs | 61 +++++---- linera-views/src/backends/scylla_db.rs | 55 +++----- linera-views/src/context.rs | 2 +- 11 files changed, 149 insertions(+), 241 deletions(-) diff --git a/linera-client/src/storage.rs b/linera-client/src/storage.rs index 1a329201a9e..e33d5f39dc1 100644 --- a/linera-client/src/storage.rs +++ b/linera-client/src/storage.rs @@ -12,9 +12,7 @@ use linera_storage_service::{ common::{ServiceStoreConfig, ServiceStoreInternalConfig}, }; #[cfg(feature = "dynamodb")] -use linera_views::dynamo_db::{ - get_config, DynamoDbStore, DynamoDbStoreConfig, DynamoDbStoreInternalConfig, -}; +use linera_views::dynamo_db::{get_config, DynamoDbStore, DynamoDbStoreConfig}; #[cfg(with_storage)] use linera_views::store::LocalAdminKeyValueStore as _; use linera_views::{ @@ -25,15 +23,12 @@ use linera_views::{ use tracing::error; #[cfg(feature = "rocksdb")] use { - linera_views::rocks_db::{ - PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig, - RocksDbStoreInternalConfig, - }, + linera_views::rocks_db::{PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig}, std::path::PathBuf, }; #[cfg(feature = "scylladb")] use { - linera_views::scylla_db::{ScyllaDbStore, ScyllaDbStoreConfig, ScyllaDbStoreInternalConfig}, + linera_views::scylla_db::{ScyllaDbStore, ScyllaDbStoreConfig}, std::num::NonZeroU16, tracing::debug, }; @@ -363,40 +358,18 @@ impl StorageConfigNamespace { StorageConfig::RocksDb { path, spawn_mode } => { let path_buf = path.to_path_buf(); let path_with_guard = PathWithGuard::new(path_buf); - let inner_config = RocksDbStoreInternalConfig { - path_with_guard, - spawn_mode: *spawn_mode, - common_config: common_config.reduced(), - }; - let config = RocksDbStoreConfig { - inner_config, - cache_size: common_config.cache_size, - }; + let config = RocksDbStoreConfig::new(*spawn_mode, path_with_guard, common_config); Ok(StoreConfig::RocksDb(config, namespace)) } #[cfg(feature = "dynamodb")] StorageConfig::DynamoDb { use_localstack } => { let aws_config = get_config(*use_localstack).await?; - let inner_config = DynamoDbStoreInternalConfig { - config: aws_config, - common_config: common_config.reduced(), - }; - let config = DynamoDbStoreConfig { - inner_config, - cache_size: common_config.cache_size, - }; + let config = DynamoDbStoreConfig::new(aws_config, common_config); Ok(StoreConfig::DynamoDb(config, namespace)) } #[cfg(feature = "scylladb")] StorageConfig::ScyllaDb { uri } => { - let inner_config = ScyllaDbStoreInternalConfig { - uri: uri.to_string(), - common_config: common_config.reduced(), - }; - let config = ScyllaDbStoreConfig { - inner_config, - cache_size: common_config.cache_size, - }; + let config = ScyllaDbStoreConfig::new(uri.to_string(), common_config); Ok(StoreConfig::ScyllaDb(config, namespace)) } } diff --git a/linera-indexer/lib/src/rocks_db.rs b/linera-indexer/lib/src/rocks_db.rs index c860eaa7c95..2e960351b28 100644 --- a/linera-indexer/lib/src/rocks_db.rs +++ b/linera-indexer/lib/src/rocks_db.rs @@ -5,11 +5,8 @@ use std::path::PathBuf; use clap::Parser as _; use linera_views::{ - rocks_db::{ - PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig, - RocksDbStoreInternalConfig, - }, - store::{AdminKeyValueStore, CommonStoreInternalConfig}, + rocks_db::{PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig}, + store::{AdminKeyValueStore, CommonStoreConfig}, }; use crate::{ @@ -41,24 +38,17 @@ pub type RocksDbRunner = Runner; impl RocksDbRunner { pub async fn load() -> Result { let config = IndexerConfig::::parse(); - let common_config = CommonStoreInternalConfig { + let common_config = CommonStoreConfig { max_concurrent_queries: config.client.max_concurrent_queries, max_stream_queries: config.client.max_stream_queries, + cache_size: config.client.cache_size, }; let path_buf = config.client.storage.as_path().to_path_buf(); let path_with_guard = PathWithGuard::new(path_buf); // The tests are run in single threaded mode, therefore we need // to use the safe default value of SpawnBlocking. let spawn_mode = RocksDbSpawnMode::SpawnBlocking; - let inner_config = RocksDbStoreInternalConfig { - path_with_guard, - spawn_mode, - common_config, - }; - let store_config = RocksDbStoreConfig { - inner_config, - cache_size: config.client.cache_size, - }; + let store_config = RocksDbStoreConfig::new(spawn_mode, path_with_guard, common_config); let namespace = config.client.namespace.clone(); let root_key = &[]; let store = diff --git a/linera-indexer/lib/src/scylla_db.rs b/linera-indexer/lib/src/scylla_db.rs index dbefd4d642f..1373fa11d39 100644 --- a/linera-indexer/lib/src/scylla_db.rs +++ b/linera-indexer/lib/src/scylla_db.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use linera_views::{ - scylla_db::{ScyllaDbStore, ScyllaDbStoreConfig, ScyllaDbStoreInternalConfig}, - store::{AdminKeyValueStore, CommonStoreInternalConfig}, + scylla_db::{ScyllaDbStore, ScyllaDbStoreConfig}, + store::{AdminKeyValueStore, CommonStoreConfig}, }; use crate::{ @@ -35,20 +35,14 @@ pub type ScyllaDbRunner = Runner; impl ScyllaDbRunner { pub async fn load() -> Result { let config = as clap::Parser>::parse(); - let common_config = CommonStoreInternalConfig { + let common_config = CommonStoreConfig { max_concurrent_queries: config.client.max_concurrent_queries, max_stream_queries: config.client.max_stream_queries, + cache_size: config.client.cache_size, }; let namespace = config.client.table.clone(); let root_key = &[]; - let inner_config = ScyllaDbStoreInternalConfig { - uri: config.client.uri.clone(), - common_config, - }; - let store_config = ScyllaDbStoreConfig { - inner_config, - cache_size: config.client.cache_size, - }; + let store_config = ScyllaDbStoreConfig::new(config.client.uri.clone(), common_config); let store = ScyllaDbStore::connect(&store_config, &namespace, root_key).await?; Self::new(config, store).await } diff --git a/linera-storage-service/src/common.rs b/linera-storage-service/src/common.rs index 889112e46a9..df1e0666515 100644 --- a/linera-storage-service/src/common.rs +++ b/linera-storage-service/src/common.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use linera_base::command::resolve_binary; use linera_views::{ - lru_caching::LruSplittingConfig, + lru_caching::LruCachingConfig, store::{CommonStoreInternalConfig, KeyValueStoreError}, views::MIN_VIEW_TAG, }; @@ -78,7 +78,7 @@ pub struct ServiceStoreInternalConfig { } /// The config type -pub type ServiceStoreConfig = LruSplittingConfig; +pub type ServiceStoreConfig = LruCachingConfig; impl ServiceStoreInternalConfig { pub fn http_address(&self) -> String { diff --git a/linera-storage-service/src/server.rs b/linera-storage-service/src/server.rs index 0fc6c642fb6..b7b2465cd6d 100644 --- a/linera-storage-service/src/server.rs +++ b/linera-storage-service/src/server.rs @@ -14,10 +14,7 @@ use linera_views::{ }; #[cfg(with_rocksdb)] use linera_views::{ - rocks_db::{ - PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig, - RocksDbStoreInternalConfig, - }, + rocks_db::{PathWithGuard, RocksDbSpawnMode, RocksDbStore, RocksDbStoreConfig}, store::AdminKeyValueStore as _, }; use serde::Serialize; @@ -588,15 +585,7 @@ async fn main() { let path_with_guard = PathWithGuard::new(path_buf); // The server is run in multi-threaded mode so we can use the block_in_place. let spawn_mode = RocksDbSpawnMode::get_spawn_mode_from_runtime(); - let inner_config = RocksDbStoreInternalConfig { - path_with_guard, - spawn_mode, - common_config: common_config.reduced(), - }; - let config = RocksDbStoreConfig { - inner_config, - cache_size: common_config.cache_size, - }; + let config = RocksDbStoreConfig::new(spawn_mode, path_with_guard, common_config); let store = RocksDbStore::maybe_create_and_connect(&config, namespace, root_key) .await .expect("store"); diff --git a/linera-views/benches/queue_view.rs b/linera-views/benches/queue_view.rs index 0d3f3ef9c0b..f1ff2f1e935 100644 --- a/linera-views/benches/queue_view.rs +++ b/linera-views/benches/queue_view.rs @@ -14,9 +14,9 @@ use linera_views::rocks_db::RocksDbStore; #[cfg(with_scylladb)] use linera_views::scylla_db::ScyllaDbStore; use linera_views::{ - backends::memory::MemoryStore, bucket_queue_view::BucketQueueView, context::ViewContext, + memory::MemoryStore, queue_view::QueueView, random::{make_deterministic_rng, DeterministicRng}, store::{KeyValueStore, TestKeyValueStore as _}, diff --git a/linera-views/src/backends/dynamo_db.rs b/linera-views/src/backends/dynamo_db.rs index 578c5b695da..8f3d3e7465d 100644 --- a/linera-views/src/backends/dynamo_db.rs +++ b/linera-views/src/backends/dynamo_db.rs @@ -29,11 +29,6 @@ use aws_smithy_types::error::operation::BuildError; use futures::future::{join_all, FutureExt as _}; use linera_base::ensure; use thiserror::Error; -#[cfg(with_testing)] -use { - anyhow::Error, - tokio::sync::{Mutex, MutexGuard}, -}; #[cfg(with_metrics)] use crate::metering::MeteredStore; @@ -41,8 +36,9 @@ use crate::metering::MeteredStore; use crate::store::TestKeyValueStore; use crate::{ batch::SimpleUnorderedBatch, + common::get_uleb128_size, journaling::{DirectWritableKeyValueStore, JournalConsistencyError, JournalingKeyValueStore}, - lru_caching::{LruCachingStore, LruSplittingConfig}, + lru_caching::{LruCachingConfig, LruCachingStore}, store::{ AdminKeyValueStore, CommonStoreInternalConfig, KeyIterable, KeyValueIterable, KeyValueStoreError, ReadableKeyValueStore, WithError, @@ -57,7 +53,7 @@ const LOCALSTACK_ENDPOINT: &str = "LOCALSTACK_ENDPOINT"; pub type Config = aws_sdk_dynamodb::Config; /// Gets the AWS configuration from the environment -pub async fn get_base_config() -> Result { +async fn get_base_config() -> Result { let base_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()) .boxed() .await; @@ -73,7 +69,7 @@ fn get_endpoint_address() -> Option { } /// Gets the localstack config -pub async fn get_localstack_config() -> Result { +async fn get_localstack_config() -> Result { let base_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()) .boxed() .await; @@ -85,9 +81,9 @@ pub async fn get_localstack_config() -> Result Result { +) -> Result { if use_localstack { get_localstack_config().await } else { @@ -95,38 +91,6 @@ pub async fn get_config_internal( } } -/// A type to help tests that need a LocalStack instance. -#[cfg(with_testing)] -pub struct LocalStackTestContext { - config: Config, - _guard: MutexGuard<'static, ()>, -} - -#[cfg(with_testing)] -impl LocalStackTestContext { - /// Creates an instance of [`LocalStackTestContext`], loading the necessary LocalStack - /// configuration. - /// - /// An address to the LocalStack instance must be specified using a `LOCALSTACK_ENDPOINT` - /// environment variable. - /// - /// This also locks the `LOCALSTACK_GUARD` to enforce that only one test has access to the - /// LocalStack instance. - pub async fn new() -> Result { - let config = get_localstack_config().await?; - let _guard = LOCALSTACK_GUARD.lock().await; - - let context = LocalStackTestContext { config, _guard }; - - Ok(context) - } - - /// Creates a new [`aws_sdk_dynamodb::Config`] for tests, using a LocalStack instance. - pub fn dynamo_db_config(&self) -> aws_sdk_dynamodb::Config { - self.config.clone() - } -} - /// The attribute name of the partition key. const PARTITION_ATTRIBUTE: &str = "item_partition"; @@ -155,11 +119,19 @@ const RAW_MAX_VALUE_SIZE: usize = 409600; /// Therefore the actual MAX_VALUE_SIZE might be lower. /// At the maximum the key_size is 1024 bytes (see below) and we pack just one entry. /// So if the key has 1024 bytes this gets us the inequality -/// 1 + 1 + serialized_size(1024)? + serialized_size(x)? <= 400*1024 -/// and so this simplifies to 1 + 1 + (2 + 1024) + (3 + x) <= 400 * 1024 -/// (we write 3 because get_uleb128_size(400*1024) = 3) -/// and so to a maximal value of 408569; -const VISIBLE_MAX_VALUE_SIZE: usize = 408569; +/// `1 + 1 + serialized_size(1024)? + serialized_size(x)? <= 400*1024` +/// and so this simplifies to `1 + 1 + (2 + 1024) + (3 + x) <= 400 * 1024` +/// Note on the following formula: +/// * We write 3 because get_uleb128_size(400*1024) = 3 +/// * We write `1 + 1` because the `SimpleUnorderedBatch` has two entries +/// +/// This gets us a maximal value of 408569; +const VISIBLE_MAX_VALUE_SIZE: usize = RAW_MAX_VALUE_SIZE + - MAX_KEY_SIZE + - get_uleb128_size(RAW_MAX_VALUE_SIZE) + - get_uleb128_size(MAX_KEY_SIZE) + - 1 + - 1; /// Fundamental constant in DynamoDB: The maximum size of a key is 1024 bytes /// See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html @@ -355,9 +327,9 @@ pub struct DynamoDbStoreInternal { #[derive(Debug)] pub struct DynamoDbStoreInternalConfig { /// The AWS configuration - pub config: Config, + config: aws_sdk_dynamodb::Config, /// The common configuration of the key value store - pub common_config: CommonStoreInternalConfig, + common_config: CommonStoreInternalConfig, } impl AdminKeyValueStore for DynamoDbStoreInternal { @@ -541,12 +513,12 @@ impl DynamoDbStoreInternal { /// Namespaces are named table names in DynamoDb [naming /// rules](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.NamingRules), /// so we need to check correctness of the namespace - fn check_namespace(namespace: &str) -> Result<(), InvalidTableName> { + fn check_namespace(namespace: &str) -> Result<(), InvalidNamespace> { if namespace.len() < 3 { - return Err(InvalidTableName::TooShort); + return Err(InvalidNamespace::TooShort); } if namespace.len() > 255 { - return Err(InvalidTableName::TooLong); + return Err(InvalidNamespace::TooLong); } if !namespace.chars().all(|character| { character.is_ascii_alphanumeric() @@ -554,7 +526,7 @@ impl DynamoDbStoreInternal { || character == '-' || character == '_' }) { - return Err(InvalidTableName::InvalidCharacter); + return Err(InvalidNamespace::InvalidCharacter); } Ok(()) } @@ -978,19 +950,19 @@ impl DirectWritableKeyValueStore for DynamoDbStoreInternal { } } -/// Error when validating a table name. +/// Error when validating a namespace #[derive(Debug, Error)] -pub enum InvalidTableName { - /// The table name should be at least 3 characters. - #[error("Table name must have at least 3 characters")] +pub enum InvalidNamespace { + /// The namespace should be at least 3 characters. + #[error("Namespace must have at least 3 characters")] TooShort, - /// The table name should be at most 63 characters. - #[error("Table name must be at most 63 characters")] + /// The namespace should be at most 63 characters. + #[error("Namespace must be at most 63 characters")] TooLong, /// allowed characters are lowercase letters, numbers, periods and hyphens - #[error("Table name must only contain lowercase letters, numbers, periods and hyphens")] + #[error("Namespace must only contain lowercase letters, numbers, periods and hyphens")] InvalidCharacter, } @@ -1049,14 +1021,6 @@ pub enum DynamoDbStoreInternalError { #[error(transparent)] JournalConsistencyError(#[from] JournalConsistencyError), - /// Missing database - #[error("Missing database: {0}")] - MissingDatabase(String), - - /// Already existing database - #[error("Already existing database")] - AlreadyExistingDatabase, - /// The length of the value should be at most 400KB. #[error("The DynamoDB value should be less than 400KB")] ValueLengthTooLarge, @@ -1081,9 +1045,9 @@ pub enum DynamoDbStoreInternalError { #[error(transparent)] BcsError(#[from] bcs::Error), - /// A wrong table name error occurred + /// A wrong namespace error occurred #[error(transparent)] - InvalidTableName(#[from] InvalidTableName), + InvalidNamespace(#[from] InvalidNamespace), /// An error occurred while creating the table. #[error(transparent)] @@ -1150,6 +1114,22 @@ impl KeyValueStoreError for DynamoDbStoreInternalError { const BACKEND: &'static str = "dynamo_db"; } +#[cfg(with_testing)] +impl TestKeyValueStore for JournalingKeyValueStore { + async fn new_test_config() -> Result { + let common_config = CommonStoreInternalConfig { + max_concurrent_queries: Some(TEST_DYNAMO_DB_MAX_CONCURRENT_QUERIES), + max_stream_queries: TEST_DYNAMO_DB_MAX_STREAM_QUERIES, + }; + let use_localstack = true; + let config = get_config_internal(use_localstack).await?; + Ok(DynamoDbStoreInternalConfig { + config, + common_config, + }) + } +} + /// A shared DB client for DynamoDb implementing LruCaching and metrics #[cfg(with_metrics)] pub type DynamoDbStore = MeteredStore< @@ -1169,43 +1149,35 @@ pub type DynamoDbStore = pub type DynamoDbStoreError = ValueSplittingError; /// The config type for DynamoDbStore -pub type DynamoDbStoreConfig = LruSplittingConfig; +pub type DynamoDbStoreConfig = LruCachingConfig; /// Getting a configuration for the system pub async fn get_config(use_localstack: bool) -> Result { Ok(get_config_internal(use_localstack).await?) } -#[cfg(with_testing)] -impl TestKeyValueStore for JournalingKeyValueStore { - async fn new_test_config() -> Result { - let common_config = CommonStoreInternalConfig { - max_concurrent_queries: Some(TEST_DYNAMO_DB_MAX_CONCURRENT_QUERIES), - max_stream_queries: TEST_DYNAMO_DB_MAX_STREAM_QUERIES, - }; - let use_localstack = true; - let config = get_config_internal(use_localstack).await?; - Ok(DynamoDbStoreInternalConfig { +impl DynamoDbStoreConfig { + /// Creates a `DynamoDbStoreConfig` from the input. + pub fn new( + config: Config, + common_config: crate::store::CommonStoreConfig, + ) -> DynamoDbStoreConfig { + let inner_config = DynamoDbStoreInternalConfig { config, - common_config, - }) + common_config: common_config.reduced(), + }; + DynamoDbStoreConfig { + inner_config, + cache_size: common_config.cache_size, + } } } -/// A static lock to prevent multiple tests from using the same LocalStack instance at the same -/// time. -#[cfg(with_testing)] -static LOCALSTACK_GUARD: Mutex<()> = Mutex::const_new(()); - #[cfg(test)] mod tests { use bcs::serialized_size; - use crate::{ - batch::SimpleUnorderedBatch, - common::get_uleb128_size, - dynamo_db::{MAX_KEY_SIZE, RAW_MAX_VALUE_SIZE, VISIBLE_MAX_VALUE_SIZE}, - }; + use crate::common::get_uleb128_size; #[test] fn test_serialization_len() { @@ -1216,12 +1188,4 @@ mod tests { assert_eq!(est_size, serial_size); } } - - #[test] - fn test_raw_visible_sizes() { - let mut vis_computed = RAW_MAX_VALUE_SIZE - MAX_KEY_SIZE; - vis_computed -= serialized_size(&SimpleUnorderedBatch::default()).unwrap(); - vis_computed -= get_uleb128_size(RAW_MAX_VALUE_SIZE) + get_uleb128_size(MAX_KEY_SIZE); - assert_eq!(vis_computed, VISIBLE_MAX_VALUE_SIZE); - } } diff --git a/linera-views/src/backends/lru_caching.rs b/linera-views/src/backends/lru_caching.rs index 3ed68e8de1d..15e9911a37d 100644 --- a/linera-views/src/backends/lru_caching.rs +++ b/linera-views/src/backends/lru_caching.rs @@ -271,7 +271,7 @@ where } /// The configuration type for the `LruCachingStore`. -pub struct LruSplittingConfig { +pub struct LruCachingConfig { /// The inner configuration of the `LruCachingStore`. pub inner_config: C, /// The cache size being used @@ -282,10 +282,10 @@ impl AdminKeyValueStore for LruCachingStore where K: AdminKeyValueStore + Send + Sync, { - type Config = LruSplittingConfig; + type Config = LruCachingConfig; fn get_name() -> String { - format!("lru splitting {}", K::get_name()) + format!("lru caching {}", K::get_name()) } async fn connect( @@ -330,10 +330,10 @@ impl TestKeyValueStore for LruCachingStore where K: TestKeyValueStore + Send + Sync, { - async fn new_test_config() -> Result, K::Error> { + async fn new_test_config() -> Result, K::Error> { let inner_config = K::new_test_config().await?; let cache_size = TEST_CACHE_SIZE; - Ok(LruSplittingConfig { + Ok(LruCachingConfig { inner_config, cache_size, }) diff --git a/linera-views/src/backends/rocks_db.rs b/linera-views/src/backends/rocks_db.rs index 8dee54dca39..bf841f4c38d 100644 --- a/linera-views/src/backends/rocks_db.rs +++ b/linera-views/src/backends/rocks_db.rs @@ -21,7 +21,7 @@ use crate::store::TestKeyValueStore; use crate::{ batch::{Batch, WriteOperation}, common::get_upper_bound, - lru_caching::{LruCachingStore, LruSplittingConfig}, + lru_caching::{LruCachingConfig, LruCachingStore}, store::{ AdminKeyValueStore, CommonStoreInternalConfig, KeyValueStoreError, ReadableKeyValueStore, WithError, WritableKeyValueStore, @@ -42,7 +42,7 @@ const MAX_VALUE_SIZE: usize = 3221225072; const MAX_KEY_SIZE: usize = 8388208; /// The RocksDB client that we use. -pub type DB = rocksdb::DBWithThreadMode; +type DB = rocksdb::DBWithThreadMode; /// The choice of the spawning mode. /// `SpawnBlocking` always works and is the safest. @@ -260,11 +260,11 @@ pub struct RocksDbStoreInternal { #[derive(Clone, Debug)] pub struct RocksDbStoreInternalConfig { /// The path to the storage containing the namespaces - pub path_with_guard: PathWithGuard, + path_with_guard: PathWithGuard, /// The spawn_mode that is chosen - pub spawn_mode: RocksDbSpawnMode, + spawn_mode: RocksDbSpawnMode, /// The common configuration of the key value store - pub common_config: CommonStoreInternalConfig, + common_config: CommonStoreInternalConfig, } impl RocksDbStoreInternal { @@ -508,7 +508,7 @@ impl AdminKeyValueStore for RocksDbStoreInternal { #[cfg(with_testing)] impl TestKeyValueStore for RocksDbStoreInternal { async fn new_test_config() -> Result { - let path_with_guard = create_rocks_db_test_path(); + let path_with_guard = PathWithGuard::new_testing(); let common_config = CommonStoreInternalConfig { max_concurrent_queries: None, max_stream_queries: TEST_ROCKS_DB_MAX_STREAM_QUERIES, @@ -545,18 +545,10 @@ pub enum RocksDbStoreInternalError { #[error("The key must have at most 8M")] KeyTooLong, - /// Missing database - #[error("Missing database: {0}")] - MissingDatabase(String), - - /// Invalid namespace - #[error("Invalid namespace")] + /// Namespace contains forbidden characters + #[error("Namespace contains forbidden characters")] InvalidNamespace, - /// Already existing database - #[error("Already existing database")] - AlreadyExistingDatabase, - /// Filesystem error #[error("Filesystem error: {0}")] FsError(#[from] std::io::Error), @@ -583,15 +575,15 @@ impl PathWithGuard { _dir: None, } } -} -/// Returns the test path for RocksDB without common config. -#[cfg(with_testing)] -fn create_rocks_db_test_path() -> PathWithGuard { - let dir = TempDir::new().unwrap(); - let path_buf = dir.path().to_path_buf(); - let _dir = Some(Arc::new(dir)); - PathWithGuard { path_buf, _dir } + /// Returns the test path for RocksDB without common config. + #[cfg(with_testing)] + pub fn new_testing() -> PathWithGuard { + let dir = TempDir::new().unwrap(); + let path_buf = dir.path().to_path_buf(); + let _dir = Some(Arc::new(dir)); + PathWithGuard { path_buf, _dir } + } } impl KeyValueStoreError for RocksDbStoreInternalError { @@ -612,4 +604,23 @@ pub type RocksDbStore = LruCachingStore; /// The composed config type for the `RocksDbStore` -pub type RocksDbStoreConfig = LruSplittingConfig; +pub type RocksDbStoreConfig = LruCachingConfig; + +impl RocksDbStoreConfig { + /// Creates a new `RocksDbStoreConfig` from the input. + pub fn new( + spawn_mode: RocksDbSpawnMode, + path_with_guard: PathWithGuard, + common_config: crate::store::CommonStoreConfig, + ) -> RocksDbStoreConfig { + let inner_config = RocksDbStoreInternalConfig { + path_with_guard, + spawn_mode, + common_config: common_config.reduced(), + }; + RocksDbStoreConfig { + inner_config, + cache_size: common_config.cache_size, + } + } +} diff --git a/linera-views/src/backends/scylla_db.rs b/linera-views/src/backends/scylla_db.rs index a81e9f2baa4..b0bf46a447d 100644 --- a/linera-views/src/backends/scylla_db.rs +++ b/linera-views/src/backends/scylla_db.rs @@ -33,7 +33,7 @@ use crate::{ batch::UnorderedBatch, common::{get_uleb128_size, get_upper_bound_option}, journaling::{DirectWritableKeyValueStore, JournalConsistencyError, JournalingKeyValueStore}, - lru_caching::{LruCachingStore, LruSplittingConfig}, + lru_caching::{LruCachingConfig, LruCachingStore}, store::{ AdminKeyValueStore, CommonStoreInternalConfig, KeyValueStoreError, ReadableKeyValueStore, WithError, @@ -457,17 +457,9 @@ pub enum ScyllaDbStoreInternalError { #[error(transparent)] ScyllaDbNewSessionError(#[from] scylla::transport::errors::NewSessionError), - /// Table name contains forbidden characters - #[error("Table name contains forbidden characters")] - InvalidTableName, - - /// Missing database - #[error("Missing database: {0}")] - MissingDatabase(String), - - /// Already existing database - #[error("Already existing database")] - AlreadyExistingDatabase, + /// Namespace contains forbidden characters + #[error("Namespace contains forbidden characters")] + InvalidNamespace, /// The journal is not coherent #[error(transparent)] @@ -608,7 +600,7 @@ pub struct ScyllaDbStoreInternalConfig { /// The url to which the requests have to be sent pub uri: String, /// The common configuration of the key value store - pub common_config: CommonStoreInternalConfig, + common_config: CommonStoreInternalConfig, } impl AdminKeyValueStore for ScyllaDbStoreInternal { @@ -827,7 +819,7 @@ impl ScyllaDbStoreInternal { { return Ok(()); } - Err(ScyllaDbStoreInternalError::InvalidTableName) + Err(ScyllaDbStoreInternalError::InvalidNamespace) } } @@ -867,26 +859,21 @@ pub type ScyllaDbStore = LruCachingStore>>; /// The `ScyllaDbStoreConfig` input type -pub type ScyllaDbStoreConfig = LruSplittingConfig; +pub type ScyllaDbStoreConfig = LruCachingConfig; + +impl ScyllaDbStoreConfig { + /// Creates a `ScyllaDbStoreConfig` from the inputs. + pub fn new(uri: String, common_config: crate::store::CommonStoreConfig) -> ScyllaDbStoreConfig { + let inner_config = ScyllaDbStoreInternalConfig { + uri, + common_config: common_config.reduced(), + }; + ScyllaDbStoreConfig { + inner_config, + cache_size: common_config.cache_size, + } + } +} /// The combined error type for the `ScyllaDbStore`. pub type ScyllaDbStoreError = ValueSplittingError; - -#[cfg(test)] -mod tests { - use bcs::serialized_size; - - use crate::{ - batch::UnorderedBatch, - common::get_uleb128_size, - scylla_db::{MAX_KEY_SIZE, RAW_MAX_VALUE_SIZE, VISIBLE_MAX_VALUE_SIZE}, - }; - - #[test] - fn test_raw_visible_sizes() { - let mut vis_computed = RAW_MAX_VALUE_SIZE - MAX_KEY_SIZE; - vis_computed -= serialized_size(&UnorderedBatch::default()).unwrap(); - vis_computed -= get_uleb128_size(RAW_MAX_VALUE_SIZE) + get_uleb128_size(MAX_KEY_SIZE); - assert_eq!(vis_computed, VISIBLE_MAX_VALUE_SIZE); - } -} diff --git a/linera-views/src/context.rs b/linera-views/src/context.rs index 5671b88689b..c880586d345 100644 --- a/linera-views/src/context.rs +++ b/linera-views/src/context.rs @@ -7,9 +7,9 @@ use async_trait::async_trait; use serde::{de::DeserializeOwned, Serialize}; use crate::{ - backends::memory::MemoryStore, batch::{Batch, DeletePrefixExpander}, common::from_bytes_option, + memory::MemoryStore, store::{KeyIterable, KeyValueIterable, KeyValueStoreError, RestrictedKeyValueStore}, views::MIN_VIEW_TAG, };