Merge pull request #43 from AndrewSisley/f64

Upgrade f64 to cql_model v.2

README.md

@@ -33,7 +33,7 @@ Repo link |Crate | Documentation | Description
 [CQL Model](https://github.com/AndrewSisley/CQLDb/tree/master/cql_model) | [crates.io](https://crates.io/crates/cql_model) | [docs.rs](https://docs.rs/cql_model) | Core CQL database models/interfaces
 [I16](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_i16) | [crates.io](https://crates.io/crates/cql_i16) | [docs.rs](https://docs.rs/cql_i16) | Signed 16-bit integer storage support
 [U64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_u64) | [crates.io](https://crates.io/crates/cql_u64) | [docs.rs](https://docs.rs/cql_u64) | Unsigned 64-bit integer storage support
-[F64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) | *unpublished* | *unpublished* | 64-bit floating point storage support
+[F64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) | [crates.io](https://crates.io/crates/cql_f64) | [docs.rs](https://docs.rs/cql_f64) | 64-bit floating point storage support
 [NullableF64](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_nullable_f64) | [crates.io](https://crates.io/crates/cql_nullable_f64) | [docs.rs](https://docs.rs/cql_nullable_f64) | Nullable 64-bit floating point storage support
 [TinyText](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_tiny_text) | [crates.io](https://crates.io/crates/cql_tiny_text) | [docs.rs](https://docs.rs/cql_tiny_text) | 255 char utf-8 string storage support
 

cql_storage_types/cql_f64/Cargo.toml

@@ -3,7 +3,7 @@ name = "cql_f64"
 description = "f64 storage support for CQL Database - a lightweight array-based database"
 keywords = ["cql", "database", "array", "storage", "nosql"]
 categories = ["database", "database-implementations", "filesystem", "caching"]
-version = "0.1.0"
+version = "0.2.0"
 repository = "https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64"
 authors = ["Andrew Sisley"]
 edition = "2018"
@@ -13,6 +13,11 @@ license = "MIT OR Apache-2.0"
 name = "cql_f64"
 path = "src/f64.rs"
 
+[dev-dependencies]
+cql_db = "^0.2.4"
+serial_test = "0.3.2"
+cql_storage_type_testing_lib = "^0.3.0"
+
 [dependencies]
-cql_model = "0.1.0"
+cql_model = "^0.2"
 byteorder = "1"

cql_storage_types/cql_f64/benches/constants.rs

@@ -0,0 +1 @@
+pub const DATABASE_LOCATION: &str = "./.test_db";

cql_storage_types/cql_f64/benches/read_single.rs

@@ -0,0 +1,53 @@
+#![feature(test)]
+mod constants;
+extern crate test;
+
+use constants::DATABASE_LOCATION;
+use test::Bencher;
+use cql_f64::F64;
+use cql_storage_type_testing_lib::benches::read_single;
+
+#[bench]
+fn _1d_f64_single_point_read_location_1(b: &mut Bencher) {
+    let test_fn = read_single::_1d_read_location_1::<F64>(DATABASE_LOCATION, 42.6);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _1d_f64_single_point_read_location_100000(b: &mut Bencher) {
+    let test_fn = read_single::_1d_read_location_100000::<F64>(DATABASE_LOCATION, 42.6);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_read_location_1_1_1_1(b: &mut Bencher) {
+    let test_fn = read_single::_4d_read_location_1_1_1_1::<F64>(DATABASE_LOCATION, 42.6);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_read_location_1_1_1_100000(b: &mut Bencher) {
+    let test_fn = read_single::_4d_read_location_1_1_1_100000::<F64>(DATABASE_LOCATION, 5.3);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_read_location_1_100000_1_1(b: &mut Bencher) {
+    let test_fn = read_single::_4d_read_location_1_100000_1_1::<F64>(DATABASE_LOCATION, 5.3);
+
+    b.iter(|| {
+        test_fn();
+    });
+}

cql_storage_types/cql_f64/benches/read_stream.rs

@@ -0,0 +1,51 @@
+#![feature(test)]
+mod constants;
+extern crate test;
+
+use std::io::{ Cursor };
+use constants::DATABASE_LOCATION;
+use test::{ Bencher };
+use cql_f64::{ unpack_stream, F64 };
+use cql_storage_type_testing_lib::benches::read_stream;
+
+#[bench]
+fn _1d_f64_stream_read_location_1_to_1(b: &mut Bencher) {
+    let test_fn = read_stream::_1d_read_empty_location_1_to_1::<F64>(DATABASE_LOCATION, &unpack_f64_stream);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _1d_f64_stream_read_location_50000_to_100000(b: &mut Bencher) {
+    let test_fn = read_stream::_1d_read_empty_location_50000_to_100000::<F64>(DATABASE_LOCATION, &unpack_f64_stream);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _4d_f64_stream_read_location_1_1_1_1_to_1_1_1_1(b: &mut Bencher) {
+    let test_fn = read_stream::_4d_read_empty_location_1_1_1_1_to_1_1_1_1::<F64>(DATABASE_LOCATION, &unpack_f64_stream);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+#[bench]
+fn _4d_f64_stream_read_location_1_1_1_50000_to_1_1_1_100000(b: &mut Bencher) {
+    let test_fn = read_stream::_4d_read_empty_location_1_1_1_50000_to_1_1_1_100000::<F64>(DATABASE_LOCATION, &unpack_f64_stream);
+
+    b.iter(|| {
+        test_fn();
+    });
+}
+
+fn unpack_f64_stream (stream: &mut Cursor<Vec<u8>>, n_values: usize, result: &mut [f64]) {
+    unpack_stream(stream, n_values, |idx, value| {
+        result[idx] = value
+    }).unwrap()
+}

cql_storage_types/cql_f64/benches/write_single.rs

@@ -0,0 +1,53 @@
+#![feature(test)]
+mod constants;
+extern crate test;
+
+use constants::DATABASE_LOCATION;
+use test::Bencher;
+use cql_f64::F64;
+use cql_storage_type_testing_lib::benches::write_single;
+
+#[bench]
+fn _1d_f64_single_point_write_location_1(b: &mut Bencher) {
+    let test_fn = write_single::_1d_write_location_1::<F64>(DATABASE_LOCATION);
+
+    b.iter(|| {
+        test_fn(42.6);
+    });
+}
+
+#[bench]
+fn _1d_f64_single_point_write_location_100000(b: &mut Bencher) {
+    let test_fn = write_single::_1d_write_location_100000::<F64>(DATABASE_LOCATION);
+
+    b.iter(|| {
+        test_fn(42.6);
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_write_location_1_1_1_1(b: &mut Bencher) {
+    let test_fn = write_single::_4d_write_location_1_1_1_1::<F64>(DATABASE_LOCATION);
+
+    b.iter(|| {
+        test_fn(5.3);
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_write_location_1_1_1_100000(b: &mut Bencher) {
+    let test_fn = write_single::_4d_write_location_1_1_1_100000::<F64>(DATABASE_LOCATION);
+
+    b.iter(|| {
+        test_fn(5.3);
+    });
+}
+
+#[bench]
+fn _4d_f64_single_point_write_location_1_100000_1_1(b: &mut Bencher) {
+    let test_fn = write_single::_4d_write_location_1_100000_1_1::<F64>(DATABASE_LOCATION);
+
+    b.iter(|| {
+        test_fn(5.3);
+    });
+}

cql_storage_types/cql_f64/src/f64.rs

@@ -1,9 +1,88 @@
-#![doc(html_root_url = "https://docs.rs/cql_f64/0.1.0")]
+/*!
+This crate implements various [CqlType](https://docs.rs/cql_model/0.2/cql_model/trait.CqlType.html) derivatives for storing `f64` values in a CQL database.
+
+Will allocate 8 bytes per value [linked](https://docs.rs/cql_db/0.2/cql_db/fn.link_dimensions.html).
+
+# Benchmarks
+Benchmarks supplied below are fairly rudimentary (and rounded) and are there to give a rough idea of relative costs.
+Full benchmark code can be found in [github](https://github.com/AndrewSisley/CQLDb/tree/master/cql_storage_types/cql_f64) and can be run with
+`rustup run nightly cargo bench`.
+
+Operation | Database dimensions | Mean time _unchecked (ns)
+--- | --- | ---
+Single point read | 1 | 2 610 (+/- 150)
+Single point read | 4 | 15 500 (+/- 800)
+Single point write | 1 | 2 950 (+/- 250)
+Single point write | 4 | 16 500 (+/- 2 000)
+Stream read 1 point | 1 | 2 600 (+/- 200)
+Stream read 1 point | 4 | 15 400 (+/- 850)
+Stream read 50 000 points | 1 | 27 950 000 (+/- 150 000)
+Stream read 50 000 points | 4 | 27 930 000 (+/- 200 000)
+
+# Examples
+The following creates a 1D database, writes 2 values to it, and then streams them into an array.
+```
+# use std::io::{ Cursor, SeekFrom, Seek };
+# use cql_f64::{ F64, unpack_stream };
+#
+# const DATABASE_LOCATION: &str = "./.test_db";
+const N_VALUES_TO_READ: usize = 3;
+
+# use std::error::Error;
+# use std::fs::remove_file;
+# fn main() -> Result<(), Box<dyn Error>> {
+# let _ = remove_file(format!("{}{}", DATABASE_LOCATION, "/db"));
+# let _ = remove_file(format!("{}{}", DATABASE_LOCATION, "/ax"));
+let base_point = [1];
+let value1 = 1.2;
+let value3 = -5.6;
+
+cql_db::create_db::<F64>(
+    DATABASE_LOCATION,
+    &[3]
+)?;
+
+cql_db::write_value::<F64>(
+    DATABASE_LOCATION,
+    &base_point,
+    value1
+)?;
+
+cql_db::write_value::<F64>(
+    DATABASE_LOCATION,
+    &[base_point[0] + 2],
+    value3
+)?;
+
+let mut result: [f64; N_VALUES_TO_READ] = [0.0; N_VALUES_TO_READ];
+let mut stream = Cursor::new(Vec::new());
+
+cql_db::read_to_stream::<F64>(
+    DATABASE_LOCATION,
+    &mut stream,
+    &base_point,
+    N_VALUES_TO_READ as u64
+)?;
+
+stream.seek(SeekFrom::Start(0)).unwrap();
+unpack_stream(&mut stream, N_VALUES_TO_READ, |idx, value| {
+    result[idx] = value
+})?;
+
+assert_eq!(result[0], value1);
+assert_eq!(result[1], 0.0);
+assert_eq!(result[2], value3);
+# Ok(())
+# }
+```
+*/
+#![doc(html_root_url = "https://docs.rs/cql_f64/0.2.0")]
 use std::fs::{File, OpenOptions};
+use std::io;
 use std::io::{Read, Write, Cursor, SeekFrom, Seek};
 use byteorder::{ReadBytesExt, WriteBytesExt, LittleEndian};
 
-use cql_model::{ CqlType, CqlWritable, CqlReadable };
+use cql_model::{ CqlType, CqlWritable, CqlReadable, CqlStreamReadable };
 
 pub struct F64;
 
@@ -13,27 +92,103 @@ impl CqlType for F64 {
 }
 
 impl CqlWritable for F64 {
-    fn write_to_db(db_location: &str, value_location: u64, value: f64) {
-        let mut file = OpenOptions::new().write(true).open(db_location).unwrap();
+    fn write_to_db(db_location: &str, value_location: u64, value: f64) -> io::Result<()> {
+        let mut file = OpenOptions::new().write(true).open(db_location)?;
 
+        // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked)
         file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap();
 
         let mut wtr = vec![];
-        wtr.write_f64::<LittleEndian>(value).unwrap();
-        file.write(&wtr).unwrap();
+        wtr.write_f64::<LittleEndian>(value)?;
+        file.write_all(&wtr)
     }
 }
 
 impl CqlReadable for F64 {
-    fn read_from_db(db_location: &str, value_location: u64) -> f64 {
-        let mut file = File::open(&db_location).unwrap();
+    fn read_from_db(db_location: &str, value_location: u64) -> io::Result<Self::ValueType> {
+        let mut file = File::open(&db_location)?;
 
+        // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked)
         file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap();
 
         let mut buffer = [0; Self::VALUE_SIZE];
-        file.read(&mut buffer).unwrap();
+        match file.read_exact(&mut buffer) {
+            Err(e) => {
+                // ignore io::ErrorKind::UnexpectedEof and continue
+                if e.kind() != io::ErrorKind::UnexpectedEof {
+                    return Err(e)
+                }
+            }
+            _ => { }
+        }
 
         let mut rdr = Cursor::new(buffer);
-        rdr.read_f64::<LittleEndian>().unwrap()
+        rdr.read_f64::<LittleEndian>()
     }
 }
+
+impl CqlStreamReadable for F64 {
+    fn read_to_stream(db_location: &str, stream: &mut dyn Write, value_location: u64, n_values: u64) -> io::Result<()> {
+        let mut file = File::open(&db_location)?;
+
+        // unwrap should be considered safe by this point, with earlier checks in the cql_db crate (if not deliberately unchecked)
+        file.seek(SeekFrom::Start(value_location * Self::VALUE_SIZE as u64)).unwrap();
+
+        for _i in 0..n_values {
+            let mut buffer = [0; Self::VALUE_SIZE];
+            match file.read_exact(&mut buffer) {
+                Err(e) => {
+                    // ignore io::ErrorKind::UnexpectedEof and continue (to write '0' bytes to the writer)
+                    if e.kind() != io::ErrorKind::UnexpectedEof {
+                        return Err(e)
+                    }
+                }
+                _ => { }
+            }
+            stream.write_all(&mut buffer)?;
+        }
+
+        stream.flush()
+    }
+}
+
+/// Unpacks `n_values` of f64 from a stream, calling `value_handler` with each value and it's index.
+///
+/// # Errors
+///
+/// Will return any [I/O errors](https://doc.rust-lang.org/nightly/std/io/enum.ErrorKind.html) encountered during the execution of the function.  If an error
+/// is returned, it may be that values have already been fed into the `value_handler`.
+///
+/// # Panics
+///
+/// Function does not actively defend against panics, and may do so if given invalid parameters.  If the function panics it may be that values have
+/// already been fed into the `value_handler`.
+///
+/// # Examples
+///
+/// ```ignore
+/// cql_db::read_to_stream::<F64>(
+///     DATABASE_LOCATION,
+///     &mut stream,
+///     &base_point,
+///     N_VALUES_TO_READ as u64
+/// )?;
+///
+/// stream.seek(SeekFrom::Start(0));
+///
+/// unpack_stream(&mut stream, N_VALUES_TO_READ, |idx, value| {
+///     result[idx] = value
+/// })?;
+/// ```
+pub fn unpack_stream<F>(stream: &mut Cursor<Vec<u8>>, n_values: usize, mut value_handler: F) -> io::Result<()> where F: FnMut(usize, f64) {
+    for index in 0..n_values {
+        let mut value_buffer = [0; F64::VALUE_SIZE];
+
+        stream.read_exact(&mut value_buffer)?;
+
+        let mut rdr = Cursor::new(value_buffer);
+        value_handler(index, rdr.read_f64::<LittleEndian>()?);
+    }
+
+    Ok(())
+}

cql_storage_types/cql_f64/tests/constants.rs

@@ -0,0 +1 @@
+pub const DATABASE_LOCATION: &str = "./.test_db";