diff --git a/README.md b/README.md index 91120c3..6ee4db5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Repo link |Crate | Documentation | Description [CQL Model](https://github.com/AndrewSisley/CQLDb/tree/master/cql_model) | [crates.io](https://crates.io/crates/cql_model) | [docs.rs](https://docs.rs/cql_model) | Core CQL database models/interfaces [I16](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_i16) | [crates.io](https://crates.io/crates/cql_i16) | [docs.rs](https://docs.rs/cql_i16) | Signed 16-bit integer storage support [U64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_u64) | [crates.io](https://crates.io/crates/cql_u64) | [docs.rs](https://docs.rs/cql_u64) | Unsigned 64-bit integer storage support -[F64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) | *unpublished* | *unpublished* | 64-bit floating point storage support +[F64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) | [crates.io](https://crates.io/crates/cql_f64) | [docs.rs](https://docs.rs/cql_f64) | 64-bit floating point storage support [NullableF64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_nullable_f64) | [crates.io](https://crates.io/crates/cql_nullable_f64) | [docs.rs](https://docs.rs/cql_nullable_f64) | Nullable 64-bit floating point storage support [TinyText](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_tiny_text) | [crates.io](https://crates.io/crates/cql_tiny_text) | [docs.rs](https://docs.rs/cql_tiny_text) | 255 char utf-8 string storage support diff --git a/cql_storage_types/cql_f64/.test_db/.gitkeep b/cql_storage_types/cql_f64/.test_db/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cql_storage_types/cql_f64/Cargo.toml b/cql_storage_types/cql_f64/Cargo.toml index abdab4d..30772c5 100644 --- a/cql_storage_types/cql_f64/Cargo.toml +++ b/cql_storage_types/cql_f64/Cargo.toml @@ -3,7 +3,7 @@ name = "cql_f64" description = "f64 storage support for CQL Database - a lightweight array-based database" keywords = ["cql", "database", "array", "storage", "nosql"] categories = ["database", "database-implementations", "filesystem", "caching"] -version = "0.1.0" +version = "0.2.0" repository = "https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64" authors = ["Andrew Sisley"] edition = "2018" @@ -13,6 +13,11 @@ license = "MIT OR Apache-2.0" name = "cql_f64" path = "src/f64.rs" +[dev-dependencies] +cql_db = "^0.2.4" +serial_test = "0.3.2" +cql_storage_type_testing_lib = "^0.3.0" + [dependencies] -cql_model = "0.1.0" +cql_model = "^0.2" byteorder = "1" diff --git a/cql_storage_types/cql_f64/benches/constants.rs b/cql_storage_types/cql_f64/benches/constants.rs new file mode 100644 index 0000000..025dada --- /dev/null +++ b/cql_storage_types/cql_f64/benches/constants.rs @@ -0,0 +1 @@ +pub const DATABASE_LOCATION: &str = "./.test_db"; diff --git a/cql_storage_types/cql_f64/benches/read_single.rs b/cql_storage_types/cql_f64/benches/read_single.rs new file mode 100644 index 0000000..d884e8f --- /dev/null +++ b/cql_storage_types/cql_f64/benches/read_single.rs @@ -0,0 +1,53 @@ +#![feature(test)] +mod constants; +extern crate test; + +use constants::DATABASE_LOCATION; +use test::Bencher; +use cql_f64::F64; +use cql_storage_type_testing_lib::benches::read_single; + +#[bench] +fn _1d_f64_single_point_read_location_1(b: &mut Bencher) { + let test_fn = read_single::_1d_read_location_1::(DATABASE_LOCATION, 42.6); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _1d_f64_single_point_read_location_100000(b: &mut Bencher) { + let test_fn = read_single::_1d_read_location_100000::(DATABASE_LOCATION, 42.6); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _4d_f64_single_point_read_location_1_1_1_1(b: &mut Bencher) { + let test_fn = read_single::_4d_read_location_1_1_1_1::(DATABASE_LOCATION, 42.6); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _4d_f64_single_point_read_location_1_1_1_100000(b: &mut Bencher) { + let test_fn = read_single::_4d_read_location_1_1_1_100000::(DATABASE_LOCATION, 5.3); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _4d_f64_single_point_read_location_1_100000_1_1(b: &mut Bencher) { + let test_fn = read_single::_4d_read_location_1_100000_1_1::(DATABASE_LOCATION, 5.3); + + b.iter(|| { + test_fn(); + }); +} diff --git a/cql_storage_types/cql_f64/benches/read_stream.rs b/cql_storage_types/cql_f64/benches/read_stream.rs new file mode 100644 index 0000000..c456275 --- /dev/null +++ b/cql_storage_types/cql_f64/benches/read_stream.rs @@ -0,0 +1,51 @@ +#![feature(test)] +mod constants; +extern crate test; + +use std::io::{ Cursor }; +use constants::DATABASE_LOCATION; +use test::{ Bencher }; +use cql_f64::{ unpack_stream, F64 }; +use cql_storage_type_testing_lib::benches::read_stream; + +#[bench] +fn _1d_f64_stream_read_location_1_to_1(b: &mut Bencher) { + let test_fn = read_stream::_1d_read_empty_location_1_to_1::(DATABASE_LOCATION, &unpack_f64_stream); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _1d_f64_stream_read_location_50000_to_100000(b: &mut Bencher) { + let test_fn = read_stream::_1d_read_empty_location_50000_to_100000::(DATABASE_LOCATION, &unpack_f64_stream); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _4d_f64_stream_read_location_1_1_1_1_to_1_1_1_1(b: &mut Bencher) { + let test_fn = read_stream::_4d_read_empty_location_1_1_1_1_to_1_1_1_1::(DATABASE_LOCATION, &unpack_f64_stream); + + b.iter(|| { + test_fn(); + }); +} + +#[bench] +fn _4d_f64_stream_read_location_1_1_1_50000_to_1_1_1_100000(b: &mut Bencher) { + let test_fn = read_stream::_4d_read_empty_location_1_1_1_50000_to_1_1_1_100000::(DATABASE_LOCATION, &unpack_f64_stream); + + b.iter(|| { + test_fn(); + }); +} + +fn unpack_f64_stream (stream: &mut Cursor>, n_values: usize, result: &mut [f64]) { + unpack_stream(stream, n_values, |idx, value| { + result[idx] = value + }).unwrap() +} diff --git a/cql_storage_types/cql_f64/benches/write_single.rs b/cql_storage_types/cql_f64/benches/write_single.rs new file mode 100644 index 0000000..6a24c73 --- /dev/null +++ b/cql_storage_types/cql_f64/benches/write_single.rs @@ -0,0 +1,53 @@ +#![feature(test)] +mod constants; +extern crate test; + +use constants::DATABASE_LOCATION; +use test::Bencher; +use cql_f64::F64; +use cql_storage_type_testing_lib::benches::write_single; + +#[bench] +fn _1d_f64_single_point_write_location_1(b: &mut Bencher) { + let test_fn = write_single::_1d_write_location_1::(DATABASE_LOCATION); + + b.iter(|| { + test_fn(42.6); + }); +} + +#[bench] +fn _1d_f64_single_point_write_location_100000(b: &mut Bencher) { + let test_fn = write_single::_1d_write_location_100000::(DATABASE_LOCATION); + + b.iter(|| { + test_fn(42.6); + }); +} + +#[bench] +fn _4d_f64_single_point_write_location_1_1_1_1(b: &mut Bencher) { + let test_fn = write_single::_4d_write_location_1_1_1_1::(DATABASE_LOCATION); + + b.iter(|| { + test_fn(5.3); + }); +} + +#[bench] +fn _4d_f64_single_point_write_location_1_1_1_100000(b: &mut Bencher) { + let test_fn = write_single::_4d_write_location_1_1_1_100000::(DATABASE_LOCATION); + + b.iter(|| { + test_fn(5.3); + }); +} + +#[bench] +fn _4d_f64_single_point_write_location_1_100000_1_1(b: &mut Bencher) { + let test_fn = write_single::_4d_write_location_1_100000_1_1::(DATABASE_LOCATION); + + b.iter(|| { + test_fn(5.3); + }); +} diff --git a/cql_storage_types/cql_f64/src/f64.rs b/cql_storage_types/cql_f64/src/f64.rs index 3a9e602..5974fa2 100644 --- a/cql_storage_types/cql_f64/src/f64.rs +++ b/cql_storage_types/cql_f64/src/f64.rs @@ -1,9 +1,88 @@ -#![doc(html_root_url = "https://docs.rs/cql_f64/0.1.0")] +/*! +This crate implements various [CqlType](https://docs.rs/cql_model/0.2/cql_model/trait.CqlType.html) derivatives for storing `f64` values in a CQL database. + +Will allocate 8 bytes per value [linked](https://docs.rs/cql_db/0.2/cql_db/fn.link_dimensions.html). + +# Benchmarks +Benchmarks supplied below are fairly rudimentary (and rounded) and are there to give a rough idea of relative costs. +Full benchmark code can be found in [github](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) and can be run with +`rustup run nightly cargo bench`. + +Operation | Database dimensions | Mean time _unchecked (ns) +--- | --- | --- +Single point read | 1 | 2 610 (+/- 150) +Single point read | 4 | 15 500 (+/- 800) +Single point write | 1 | 2 950 (+/- 250) +Single point write | 4 | 16 500 (+/- 2 000) +Stream read 1 point | 1 | 2 600 (+/- 200) +Stream read 1 point | 4 | 15 400 (+/- 850) +Stream read 50 000 points | 1 | 27 950 000 (+/- 150 000) +Stream read 50 000 points | 4 | 27 930 000 (+/- 200 000) + +# Examples +The following creates a 1D database, writes 2 values to it, and then streams them into an array. +``` +# use std::io::{ Cursor, SeekFrom, Seek }; +# use cql_f64::{ F64, unpack_stream }; +# +# const DATABASE_LOCATION: &str = "./.test_db"; +const N_VALUES_TO_READ: usize = 3; + +# use std::error::Error; +# use std::fs::remove_file; +# fn main() -> Result<(), Box> { +# let _ = remove_file(format!("{}{}", DATABASE_LOCATION, "/db")); +# let _ = remove_file(format!("{}{}", DATABASE_LOCATION, "/ax")); +let base_point = [1]; +let value1 = 1.2; +let value3 = -5.6; + +cql_db::create_db::( + DATABASE_LOCATION, + &[3] +)?; + +cql_db::write_value::( + DATABASE_LOCATION, + &base_point, + value1 +)?; + +cql_db::write_value::( + DATABASE_LOCATION, + &[base_point[0] + 2], + value3 +)?; + +let mut result: [f64; N_VALUES_TO_READ] = [0.0; N_VALUES_TO_READ]; +let mut stream = Cursor::new(Vec::new()); + +cql_db::read_to_stream::( + DATABASE_LOCATION, + &mut stream, + &base_point, + N_VALUES_TO_READ as u64 +)?; + +stream.seek(SeekFrom::Start(0)).unwrap(); +unpack_stream(&mut stream, N_VALUES_TO_READ, |idx, value| { + result[idx] = value +})?; + +assert_eq!(result[0], value1); +assert_eq!(result[1], 0.0); +assert_eq!(result[2], value3); +# Ok(()) +# } +``` +*/ +#![doc(html_root_url = "https://docs.rs/cql_f64/0.2.0")] use std::fs::{File, OpenOptions}; +use std::io; use std::io::{Read, Write, Cursor, SeekFrom, Seek}; use byteorder::{ReadBytesExt, WriteBytesExt, LittleEndian}; -use cql_model::{ CqlType, CqlWritable, CqlReadable }; +use cql_model::{ CqlType, CqlWritable, CqlReadable, CqlStreamReadable }; pub struct F64; @@ -13,27 +92,103 @@ impl CqlType for F64 { } impl CqlWritable for F64 { - fn write_to_db(db_location: &str, value_location: u64, value: f64) { - let mut file = OpenOptions::new().write(true).open(db_location).unwrap(); + fn write_to_db(db_location: &str, value_location: u64, value: f64) -> io::Result<()> { + let mut file = OpenOptions::new().write(true).open(db_location)?; + // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked) file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap(); let mut wtr = vec![]; - wtr.write_f64::(value).unwrap(); - file.write(&wtr).unwrap(); + wtr.write_f64::(value)?; + file.write_all(&wtr) } } impl CqlReadable for F64 { - fn read_from_db(db_location: &str, value_location: u64) -> f64 { - let mut file = File::open(&db_location).unwrap(); + fn read_from_db(db_location: &str, value_location: u64) -> io::Result { + let mut file = File::open(&db_location)?; + // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked) file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap(); let mut buffer = [0; Self::VALUE_SIZE]; - file.read(&mut buffer).unwrap(); + match file.read_exact(&mut buffer) { + Err(e) => { + // ignore io::ErrorKind::UnexpectedEof and continue + if e.kind() != io::ErrorKind::UnexpectedEof { + return Err(e) + } + } + _ => { } + } let mut rdr = Cursor::new(buffer); - rdr.read_f64::().unwrap() + rdr.read_f64::() } } + +impl CqlStreamReadable for F64 { + fn read_to_stream(db_location: &str, stream: &mut dyn Write, value_location: u64, n_values: u64) -> io::Result<()> { + let mut file = File::open(&db_location)?; + + // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked) + file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap(); + + for _i in 0..n_values { + let mut buffer = [0; Self::VALUE_SIZE]; + match file.read_exact(&mut buffer) { + Err(e) => { + // ignore io::ErrorKind::UnexpectedEof and continue (to write '0' bytes to the writer) + if e.kind() != io::ErrorKind::UnexpectedEof { + return Err(e) + } + } + _ => { } + } + stream.write_all(&mut buffer)?; + } + + stream.flush() + } +} + +/// Unpacks `n_values` of f64 from a stream, calling `value_handler` with each value and it's index. +/// +/// # Errors +/// +/// Will return any [I/O errors](https://doc.rust-lang.org/nightly/std/io/enum.ErrorKind.html) encountered during the execution of the function. If an error +/// is returned, it may be that values have already been fed into the `value_handler`. +/// +/// # Panics +/// +/// Function does not actively defend against panics, and may do so if given invalid parameters. If the function panics it may be that values have +/// already been fed into the `value_handler`. +/// +/// # Examples +/// +/// ```ignore +/// cql_db::read_to_stream::( +/// DATABASE_LOCATION, +/// &mut stream, +/// &base_point, +/// N_VALUES_TO_READ as u64 +/// )?; +/// +/// stream.seek(SeekFrom::Start(0)); +/// +/// unpack_stream(&mut stream, N_VALUES_TO_READ, |idx, value| { +/// result[idx] = value +/// })?; +/// ``` +pub fn unpack_stream(stream: &mut Cursor>, n_values: usize, mut value_handler: F) -> io::Result<()> where F: FnMut(usize, f64) { + for index in 0..n_values { + let mut value_buffer = [0; F64::VALUE_SIZE]; + + stream.read_exact(&mut value_buffer)?; + + let mut rdr = Cursor::new(value_buffer); + value_handler(index, rdr.read_f64::()?); + } + + Ok(()) +} diff --git a/cql_storage_types/cql_f64/tests/constants.rs b/cql_storage_types/cql_f64/tests/constants.rs new file mode 100644 index 0000000..025dada --- /dev/null +++ b/cql_storage_types/cql_f64/tests/constants.rs @@ -0,0 +1 @@ +pub const DATABASE_LOCATION: &str = "./.test_db"; diff --git a/cql_storage_types/cql_f64/tests/f64.rs b/cql_storage_types/cql_f64/tests/f64.rs new file mode 100644 index 0000000..6364555 --- /dev/null +++ b/cql_storage_types/cql_f64/tests/f64.rs @@ -0,0 +1,67 @@ +mod constants; + +use serial_test::serial; +use std::io::{ Cursor }; +use constants::DATABASE_LOCATION; +use cql_f64::{ F64, unpack_stream }; +use cql_storage_type_testing_lib::tests; + +#[test] +#[serial] +fn _1d_f64_database_allows_for_single_point_read_writes() { + tests::_1d_database_allows_for_single_point_read_writes::( + DATABASE_LOCATION, + 42.4 + ); +} + +#[test] +#[serial] +fn _4d_f64_database_allows_for_single_point_read_writes() { + tests::_4d_database_allows_for_single_point_read_writes::( + DATABASE_LOCATION, + 5.1 + ); +} + +#[test] +#[serial] +fn _4d_f64_database_allows_for_single_point_read_writes_given_multiple_values_and_overwrites() { + tests::_4d_database_allows_for_single_point_read_writes_given_multiple_values_and_overwrites::( + DATABASE_LOCATION, + 5.1, + -20.0, + 0.56, + 30000.3 + ); +} + +#[test] +#[serial] +fn _1d_f64_database_allows_for_stream_reads() { + tests::_1d_database_allows_for_stream_reads::>, usize, &mut [f64])>( + DATABASE_LOCATION, + 42.423525, + 16.1, + 80.8, + &unpack_f64_stream + ); +} + +#[test] +#[serial] +fn _4d_f64_database_allows_for_stream_reads() { + tests::_4d_database_allows_for_stream_reads::>, usize, &mut [i16])>( + DATABASE_LOCATION, + 42.423525, + 16.1, + 80.8, + &unpack_f64_stream + ); +} + +fn unpack_f64_stream (stream: &mut Cursor>, n_values: usize, result: &mut [f64]) { + unpack_stream(stream, n_values, |idx, value| { + result[idx] = value + }).unwrap() +}