From 06ca53398398396615ab68e4b96059200149d7fb Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 1 Nov 2024 09:02:32 -0700 Subject: [PATCH] Remove obsolete crutest perf test (#1528) Remove the perf test from crutest. There are other better tests now and this one is just a distraction. Removed the tools/test_perf.sh script and places that referenced it. Fix for: https://github.com/oxidecomputer/crucible/issues/1527 --------- Co-authored-by: Alan Hanson --- .github/buildomat/jobs/build-release.sh | 2 +- Cargo.lock | 1 - crutest/Cargo.toml | 1 - crutest/src/cli.rs | 63 ---- crutest/src/main.rs | 477 ------------------------ crutest/src/protocol.rs | 8 - tools/README.md | 6 - tools/make-nightly.sh | 1 - tools/test_perf.sh | 180 --------- 9 files changed, 1 insertion(+), 738 deletions(-) delete mode 100755 tools/test_perf.sh diff --git a/.github/buildomat/jobs/build-release.sh b/.github/buildomat/jobs/build-release.sh index a8a53f508..7240faf55 100644 --- a/.github/buildomat/jobs/build-release.sh +++ b/.github/buildomat/jobs/build-release.sh @@ -81,7 +81,7 @@ for t in crucible-downstairs crucible-hammer crutest dsc crudd; do done mkdir -p /work/scripts -for s in tools/test_perf.sh tools/crudd-speed-battery.sh tools/dtrace/perf-downstairs-tick.d tools/dtrace/upstairs_info.d tools/test_mem.sh; do +for s in tools/crudd-speed-battery.sh tools/dtrace/perf-downstairs-tick.d tools/dtrace/upstairs_info.d tools/test_mem.sh; do cp "$s" /work/scripts/ done diff --git a/Cargo.lock b/Cargo.lock index 23221aa27..2ac67cdb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1426,7 +1426,6 @@ dependencies = [ "crucible-common", "crucible-protocol", "crucible-workspace-hack", - "csv", "dsc-client", "futures", "futures-core", diff --git a/crutest/Cargo.toml b/crutest/Cargo.toml index 50588cb38..a0aac6a7b 100644 --- a/crutest/Cargo.toml +++ b/crutest/Cargo.toml @@ -15,7 +15,6 @@ crucible-client-types.workspace = true crucible-common.workspace = true crucible-protocol.workspace = true crucible.workspace = true -csv.workspace = true dsc-client.workspace = true human_bytes.workspace = true futures-core.workspace = true diff --git a/crutest/src/cli.rs b/crutest/src/cli.rs index 6c0e09f16..39bbe960d 100644 --- a/crutest/src/cli.rs +++ b/crutest/src/cli.rs @@ -145,24 +145,6 @@ enum CliCommand { Info, /// Report if the Upstairs is ready for IO IsActive, - /// Run the client perf test - Perf { - /// Number of IOs to execute for each test phase - #[clap(long, short, default_value = "5000", action)] - count: usize, - /// Size in blocks of each IO - #[clap(long, default_value = "1", action)] - io_size: usize, - /// Number of outstanding IOs at the same time - #[clap(long, default_value = "1", action)] - io_depth: usize, - /// Number of read test loops to do. - #[clap(long, default_value = "2", action)] - read_loops: usize, - /// Number of write test loops to do. - #[clap(long, default_value = "2", action)] - write_loops: usize, - }, /// Quit the CLI Quit, /// Read from a given block offset @@ -522,22 +504,6 @@ async fn cmd_to_msg( CliCommand::Info => { fw.send(CliMessage::InfoPlease).await?; } - CliCommand::Perf { - count, - io_size, - io_depth, - read_loops, - write_loops, - } => { - fw.send(CliMessage::Perf( - count, - io_size, - io_depth, - read_loops, - write_loops, - )) - .await?; - } CliCommand::Quit => { println!("The quit command has nothing to send"); return Ok(()); @@ -936,35 +902,6 @@ async fn process_cli_command( Err(e) => fw.send(CliMessage::Error(e)).await, } } - CliMessage::Perf(count, io_size, io_depth, read_loops, write_loops) => { - if let Some(ri) = ri_option { - perf_header(); - match perf_workload( - volume, - ri, - None, - count, - io_size, - io_depth, - read_loops, - write_loops, - ) - .await - { - Ok(_) => fw.send(CliMessage::DoneOk).await, - Err(e) => { - let msg = format!("{}", e); - let e = CrucibleError::GenericError(msg); - fw.send(CliMessage::Error(e)).await - } - } - } else { - fw.send(CliMessage::Error(CrucibleError::GenericError( - "Info not initialized".to_string(), - ))) - .await - } - } CliMessage::RandRead => { if let Some(ri) = ri_option { let mut rng = rand_chacha::ChaCha8Rng::from_entropy(); diff --git a/crutest/src/main.rs b/crutest/src/main.rs index 6a62f0062..04ad8151d 100644 --- a/crutest/src/main.rs +++ b/crutest/src/main.rs @@ -2,7 +2,6 @@ use anyhow::{anyhow, bail, Result}; use bytes::Bytes; use clap::Parser; -use csv::WriterBuilder; use futures::stream::FuturesOrdered; use futures::StreamExt; use human_bytes::human_bytes; @@ -15,7 +14,6 @@ use signal_hook::consts::signal::*; use signal_hook_tokio::Signals; use slog::{info, o, warn, Logger}; use std::fmt; -use std::fs::File; use std::io::Write; use std::net::{IpAddr, SocketAddr}; use std::num::NonZeroU64; @@ -88,24 +86,6 @@ enum Workload { Generic, Nothing, One, - /// Run the perf test, random writes, then random reads - Perf { - /// Size in blocks of each IO - #[clap(long, default_value_t = 1, action)] - io_size: usize, - /// Number of outstanding IOs at the same time. - #[clap(long, default_value_t = 1, action)] - io_depth: usize, - /// Output file for IO times - #[clap(long, global = true, name = "PERF", action)] - perf_out: Option, - /// Number of read test loops to do. - #[clap(long, default_value_t = 2, action)] - read_loops: usize, - /// Number of write test loops to do. - #[clap(long, default_value_t = 2, action)] - write_loops: usize, - }, /// Measure performance with a random read workload RandRead { #[clap(flatten)] @@ -1190,60 +1170,6 @@ async fn main() -> Result<()> { println!("One test"); one_workload(&volume, &mut region_info).await?; } - Workload::Perf { - io_size, - io_depth, - perf_out, - read_loops, - write_loops, - } => { - // Pathetic. This tiny wait is just so all my output from the - // test will be after all the upstairs messages have finised. - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; - println!("Perf test"); - let count = opt.count.unwrap_or(5000); - - // NOTICE: The graphing code REQUIRES the data to be in a - // specific format, where the first 7 columns are as described - // below, and the 8 through whatever are the data samples from - // the performance test. - let mut opt_wtr = None; - let mut wtr; - if let Some(perf_out) = perf_out { - wtr = WriterBuilder::new() - .flexible(true) - .from_path(perf_out) - .unwrap(); - wtr.serialize(( - "type", - "total_time_ns", - "io_depth", - "io_size", - "count", - "es", - "ec", - "times", - ))?; - opt_wtr = Some(wtr); - } - - // The header for all perf tests - perf_header(); - perf_workload( - &volume, - ®ion_info, - opt_wtr, - count, - io_depth, - io_size, - write_loops, - read_loops, - ) - .await?; - if opt.quit { - return Ok(()); - } - } Workload::RandRead { cfg } => { rand_read_write_workload( &volume, @@ -2931,326 +2857,6 @@ async fn dirty_workload( Ok(()) } -/* - * Print the perf header. - */ -pub fn perf_header() { - println!( - "{:>8} {:7} {:5} {:4} {:>7} {:>7} {:>7} {:>7} {:>8} {:>5} {:>5}", - "TEST", - "SECONDS", - "COUNT", - "DPTH", - "IOPS", - "MEAN", - "P95", - "P99", - "MAX", - "ES", - "EC", - ); -} - -/* - * Take the Vec of Durations for IOs and write it out in CSV format using - * the provided CSV Writer. - */ -#[allow(clippy::too_many_arguments)] -pub fn perf_csv( - wtr: &mut csv::Writer, - msg: &str, - count: usize, - io_depth: usize, - io_size: usize, - duration: Duration, - iotimes: &[Duration], - es: u64, - ec: u64, -) { - // Convert all Durations to u64 nanoseconds. - let times = iotimes - .iter() - .map(|x| (x.as_secs() * 100000000) + x.subsec_nanos() as u64) - .collect::>(); - - let time_in_nsec = - duration.as_secs() * 100000000 + duration.subsec_nanos() as u64; - - wtr.serialize(Record { - label: msg.to_string(), - total_time: time_in_nsec, - io_depth, - io_size, - count, - es, - ec, - time: times, - }) - .unwrap(); - wtr.flush().unwrap(); -} - -// Percentile -// Given a SORTED vec of f32's (I'm trusting you to provide that), -// and a value between 1 and 99 (the desired percentile), -// determine which index (or which indexes to average) contain our desired -// percentile. -// Once we have the value at our index (or the average of two values at the -// desired indices), return that to the caller. -// -// Remember, the array index is one less (zero-based index) -fn percentile(times: &[f32], perc: u8) -> Result { - if times.is_empty() { - bail!("Array for percentile too short"); - } - if perc == 0 || perc >= 100 { - bail!("Requested percentile not: 0 < {} < 100", perc); - } - - let position = times.len() as f32 * (perc as f32 / 100.0); - - if position == position.trunc() { - // Our position is a whole number. - // We use the rounded up position as our second index because the - // array index is zero based. - let index_two = position.ceil() as usize; - let index_one = index_two - 1; - - Ok((times[index_one] + times[index_two]) / 2.0) - } else { - // Our position is not an integer, so round up to get the correct - // position for our percentile. However, since we need to subtract - // one to get the zero-based index, we can just round down here. - // This is the same as rounding up, then subtracting one. - let index = position.trunc() as usize; - Ok(times[index]) - } -} - -/* - * Display the summary results from a perf run. - */ -fn perf_summary( - msg: &str, - count: usize, - io_depth: usize, - times: &[Duration], - total_time: Duration, - es: u64, - ec: u64, -) { - // Convert all the Durations into floats. - let mut times = times - .iter() - .map(|x| x.as_secs() as f32 + (x.subsec_nanos() as f32 / 1e9)) - .collect::>(); - times.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - - // Determine the number of seconds as a float elapsed to perform - // all the IOs. Then, divide the total operations performed to get - // the average IOPs - let time_f = - total_time.as_secs() as f32 + (total_time.subsec_nanos() as f32 / 1e9); - println!( - "{:>8} {:>7.2} {:5} {:4} {:>7.2} {:.5} {:.5} {:.5} {:8.5} {:>5} {:>5}", - msg, - time_f, - count, - io_depth, - count as f32 / time_f, - statistical::mean(×), - percentile(×, 95).unwrap(), - percentile(×, 99).unwrap(), - times.last().unwrap(), - es, - ec, - ); -} - -#[derive(Debug, Serialize, Deserialize)] -struct Record { - label: String, - total_time: u64, - io_depth: usize, - io_size: usize, - count: usize, - es: u64, - ec: u64, - time: Vec, -} -/** - * A simple IO test in two stages: 100% random writes, then 100% random - * reads. The caller can select: - * io_depth: The number of outstanding IOs issued at a time - * blocks_per_io: The size of each io (in multiple of block size). - * count: The number of loops to perform for each test (all IOs - * in io_depth are considered as a single loop). - * write_loop: The number of times to do the 100% write loop. - * read_loop: The number of times to do the 100% read loop. - * - * A summary is printed at the end of each stage. - */ -#[allow(clippy::too_many_arguments)] -async fn perf_workload( - volume: &Volume, - ri: &RegionInfo, - mut wtr: Option>, - count: usize, - io_depth: usize, - blocks_per_io: usize, - write_loop: usize, - read_loop: usize, -) -> Result<()> { - // Before we start, make sure the work queues are empty. - loop { - let wc = volume.query_work_queue().await?; - if wc.up_count + wc.ds_count == 0 { - break; - } - tokio::time::sleep(Duration::from_secs(2)).await; - } - - let mut rng = rand::thread_rng(); - let io_size = blocks_per_io * ri.volume_info.block_size as usize; - - let write_buffers: Vec = - (0..io_depth) - .map(|_| { - let mut out = BytesMut::with_capacity(io_size); - out.extend((0..io_size).map(|_| -> u8 { - rng.sample(rand::distributions::Standard) - })); - out - }) - .collect(); - - let mut read_buffers: Vec = (0..io_depth) - .map(|_| Buffer::new(blocks_per_io, ri.volume_info.block_size as usize)) - .collect(); - - let mut es = 0; - for sv in ri.volume_info.volumes.iter() { - es += sv.blocks_per_extent; - } - let ec = ri.volume_info.total_blocks() as u64 / es; - - // To make a random block offset, we take the total block count and subtract - // the IO size in blocks (so that we don't overspill the region) - let offset_mod = (ri.volume_info.total_blocks() - blocks_per_io) as u64; - for _ in 0..write_loop { - let mut wtime = Vec::with_capacity(count); - let big_start = Instant::now(); - for _ in 0..count { - let burst_start = Instant::now(); - let mut write_futures = FuturesOrdered::new(); - - for write_buffer in write_buffers.iter().take(io_depth) { - let offset: u64 = rng.gen::() % offset_mod; - let future = volume.write_to_byte_offset( - offset * ri.volume_info.block_size, - write_buffer.clone(), - ); - write_futures.push_back(future); - } - - while let Some(result) = write_futures.next().await { - result?; - } - wtime.push(burst_start.elapsed()); - } - let big_end = big_start.elapsed(); - - volume.flush(None).await?; - perf_summary("rwrites", count, io_depth, &wtime, big_end, es, ec); - if let Some(wtr) = wtr.as_mut() { - perf_csv( - wtr, - "rwrite", - count, - io_depth, - blocks_per_io, - big_end, - &wtime, - es, - ec, - ); - } - - // Before we loop or end, make sure the work queues are empty. - loop { - let wc = volume.query_work_queue().await?; - if wc.up_count + wc.ds_count == 0 { - break; - } - tokio::time::sleep(Duration::from_secs(2)).await; - } - } - - let mut rtime = Vec::with_capacity(count); - for _ in 0..read_loop { - let big_start = Instant::now(); - for _ in 0..count { - let burst_start = Instant::now(); - let mut read_futures = FuturesOrdered::new(); - - for mut read_buffer in read_buffers.drain(0..io_depth) { - let offset: u64 = rng.gen::() % offset_mod; - let future = { - let volume = volume.clone(); - let bs = ri.volume_info.block_size; - tokio::spawn(async move { - volume - .read_from_byte_offset( - offset * bs, - &mut read_buffer, - ) - .await?; - Ok(read_buffer) - }) - }; - read_futures.push_back(future); - } - - while let Some(result) = read_futures.next().await { - let result: Result = result?; - let read_buffer = result?; - read_buffers.push(read_buffer); - } - - rtime.push(burst_start.elapsed()); - } - let big_end = big_start.elapsed(); - - perf_summary("rreads", count, io_depth, &rtime, big_end, es, ec); - - if let Some(wtr) = wtr.as_mut() { - perf_csv( - wtr, - "rread", - count, - io_depth, - blocks_per_io, - big_end, - &rtime, - es, - ec, - ); - } - - volume.flush(None).await?; - - // Before we finish, make sure the work queues are empty. - loop { - let wc = volume.query_work_queue().await?; - if wc.up_count + wc.ds_count == 0 { - break; - } - tokio::time::sleep(tokio::time::Duration::from_secs(4)).await; - } - } - Ok(()) -} - /// Prints a pleasant summary of the given region fn print_region_description(ri: &RegionInfo, encrypted: bool) { println!("region info:"); @@ -5086,87 +4692,4 @@ mod test { ValidateStatus::Good ); } - - #[test] - fn test_95_small() { - // Test of one element - let fv = vec![10.0]; - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 10.0); - } - - #[test] - fn test_perc_bad_perc() { - // Should fail on a bad percentile value - let fv = vec![10.0]; - let res = percentile(&fv, 0); - assert!(res.is_err()); - } - - #[test] - fn test_perc_bad_big_perc() { - // Should fail on a bad percentile value - let fv = vec![10.0]; - let res = percentile(&fv, 100); - assert!(res.is_err()); - } - - #[test] - fn test_95_2() { - // Determine the 95th percentile value with 2 elements - // We must round up. - let fv = vec![10.0, 20.0]; - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 20.0); - } - - #[test] - fn test_95_10() { - // Determine the 95th percentile value with 10 elements - // We must round up. - let fv = vec![1.1, 2.2, 3.3, 4.4, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; - - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 10.0); - } - #[test] - fn test_95_20() { - // Determine the 95th percentile value with 20 elements - // There is a whole number position for this array, so we must - // return the average of two elements. - let fv = vec![ - 1.1, 2.2, 3.3, 4.4, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, - 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, - ]; - - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 19.5); - } - #[test] - fn test_95_21() { - // Determine the 95th percentile value with 21 elements - let fv = vec![ - 1.1, 2.2, 3.3, 4.4, 5.0, 6.0, 7.0, 8.0, 9.0, 8.0, 10.0, 11.0, 12.0, - 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, - ]; - - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 19.0); - } - #[test] - fn test_perc_mixed() { - // Determine the 95th, 90th, and 20th percentile values - let fv = vec![ - 43.0, 54.0, 56.0, 61.0, 62.0, 66.0, 68.0, 69.0, 69.0, 70.0, 71.0, - 72.0, 77.0, 78.0, 79.0, 85.0, 87.0, 88.0, 89.0, 93.0, 95.0, 96.0, - 98.0, 99.0, 99.4, - ]; - - let pp = percentile(&fv, 95).unwrap(); - assert_eq!(pp, 99.0); - let pp = percentile(&fv, 90).unwrap(); - assert_eq!(pp, 98.0); - let pp = percentile(&fv, 20).unwrap(); - assert_eq!(pp, 64.0); - } } diff --git a/crutest/src/protocol.rs b/crutest/src/protocol.rs index febe14e78..6e6b2b1d9 100644 --- a/crutest/src/protocol.rs +++ b/crutest/src/protocol.rs @@ -42,7 +42,6 @@ pub enum CliMessage { InfoPlease, IsActive, MyUuid(Uuid), - Perf(usize, usize, usize, usize, usize), Read(usize, usize), RandRead, ReadResponse(usize, Result), @@ -316,13 +315,6 @@ mod tests { Ok(()) } - #[test] - fn rt_perf() -> Result<()> { - let input = CliMessage::Perf(2, 3, 4, 2, 2); - assert_eq!(input, round_trip(&input)?); - Ok(()) - } - // ZZZ tests for readresponse, Error #[test] fn correctly_detect_truncated_message() -> Result<()> { diff --git a/tools/README.md b/tools/README.md index 2fd5bf4a1..f874a1aac 100644 --- a/tools/README.md +++ b/tools/README.md @@ -47,12 +47,6 @@ This runs a selection of tests from this directory and reports their results. It is intended to be a test for Crucible that runs nightly and does deeper/longer tests than what we do as part of every push. -## test_perf.sh -A test that creates three downstairs regions of ~100G each and then runs -the crutest perf test using those regions. -A variety of extent size and extent counts are used (always the same total -region size of ~100G). - ## test_repair.sh A test to break, then repair a downstairs region that is out of sync with the other regions, in a loop diff --git a/tools/make-nightly.sh b/tools/make-nightly.sh index a060e710d..038d78a5d 100755 --- a/tools/make-nightly.sh +++ b/tools/make-nightly.sh @@ -27,7 +27,6 @@ tar cavf out/crucible-nightly.tar.gz \ tools/test_live_repair.sh \ tools/test_fail_live_repair.sh \ tools/test_mem.sh \ - tools/test_perf.sh \ tools/test_replay.sh \ tools/test_repair.sh \ tools/test_replace_special.sh \ diff --git a/tools/test_perf.sh b/tools/test_perf.sh deleted file mode 100755 index c5a14bb8e..000000000 --- a/tools/test_perf.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash - -# Perf test shell script. - -set -o errexit -set -o pipefail - -# Control-C to cleanup. -trap ctrl_c INT -function ctrl_c() { - echo "Stopping at your request" - if [[ -n "$dsc" ]]; then - "$dsc" cmd shutdown > /dev/null 2>&1 || true - fi - exit 1 -} - -usage () { - echo "Usage: $0 [f] [-b #] [-c #] [-g ] [-r #] [-w #]" >&2 - echo " -b block_size Block size for the region (default 4096)" >&2 - echo " -c count Count of IOs in the perf test (default 30000)" >&2 - echo " -f Write to all blocks once before testing" >&2 - echo " -g REGION_DIR Directory where regions will be created" >&2 - echo " (default /var/tmp/dsc)" >&2 - echo " -r read_loops Number of read loops to do (default 2)" >&2 - echo " -w write_loops Number of write loops to do (default 2)" >&2 -} - -block_size=4096 -count=30000 -prefill=0 -read_loops=2 -write_loops=2 -REGION_ROOT=${REGION_ROOT:-/var/tmp/test_perf} -region_dir="/var/tmp/dsc" - -while getopts 'b:c:g:hfr:w:' opt; do - case "$opt" in - b) block_size=$OPTARG - echo "Using block size $block_size" - ;; - c) count=$OPTARG - echo "Using count of $count" - ;; - f) prefill=1 - echo "Region will be filled once before testing" - ;; - g) region_dir=$OPTARG - echo "Using region dir of $region_dir" - ;; - h) usage - exit 0 - ;; - r) read_loops=$OPTARG - echo "Using read_loops: $read_loops" - ;; - w) write_loops=$OPTARG - echo "Using write_loops: $write_loops" - ;; - *) echo "Invalid option" - usage - exit 1 - ;; - esac -done - -# Create a region with the given extent_size ($1) and extent_count ($2) -# Once created, run the client perf test on that region, recording the -# output to a csv file. -function perf_round() { - if [[ $# -ne 2 ]]; then - echo "Missing EC and ES for perf_round()" >&2 - exit 1 - fi - es=$1 - ec=$2 - - echo Create region with ES:"$es" EC:"$ec" BS:"$block_size" - - "$dsc" create --ds-bin "$downstairs" --cleanup \ - --extent-size "$es" --extent-count "$ec" \ - --region-dir "$region_dir" \ - --block-size "$block_size" - - "$dsc" start --ds-bin "$downstairs" \ - --region-dir "$region_dir" & - dsc_pid=$! - sleep 5 - if ! pgrep -P $dsc_pid; then - echo "Failed to start dsc" - exit 1 - fi - # Sometimes, this command fails. When it does, gather a bit more - # information to help us debug it. And, throw in a sleep/retry. - if ! "$dsc" cmd disable-restart-all; then - echo "Failed to disable auto-restart on dsc" - echo "dsc_pid: $dsc_pid" - ps -ef | grep $dsc_pid - echo "ps grep downstairs" - ps -ef | grep downstairs - echo "ps grep dsc" - ps -ef | grep dsc - sleep 20 - if ! "$dsc" cmd disable-restart-all; then - echo "Failed twice to disable auto-restart on dsc" - exit 1 - fi - fi - - # Args for crutest. Using the default IP:port for dsc - gen=1 - args="-t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 -c $count -q" - - echo "A new loop begins at $(date)" | tee -a "$outfile" - - if [[ "$prefill" -eq 1 ]]; then - # Fill the region - echo "$ct" fill $args --skip-verify -g "$gen" | tee -a "$outfile" - "$ct" fill $args --skip-verify -g "$gen" | tee -a "$outfile" - echo "IOPs for es=$es ec=$ec" >> "$outfile" - (( gen += 1 )) - echo "" | tee -a "$outfile" - fi - - # Test after the fill - echo "$ct" perf -g "$gen" $args --write-loops "$write_loops" --read-loops "$read_loops" --perf-out /tmp/perf-ES-"$es"-EC-"$ec".csv | tee -a "$outfile" - "$ct" perf -g "$gen" $args \ - --write-loops "$write_loops" --read-loops "$read_loops" \ - --perf-out /tmp/perf-ES-"$es"-EC-"$ec".csv 2>&1 | tee -a "$outfile" - echo "" >> "$outfile" - - echo Perf test completed, stop all downstairs - set +o errexit - "$dsc" cmd shutdown - wait $dsc_pid - unset dsc_pid - set -o errexit -} - -ROOT=$(cd "$(dirname "$0")/.." && pwd) -BINDIR=${BINDIR:-$ROOT/target/release} - -echo "$ROOT" -cd "$ROOT" || (echo failed to cd "$ROOT"; exit 1) - -ct="$BINDIR/crutest" -dsc="$BINDIR/dsc" -downstairs="$BINDIR/crucible-downstairs" -outfile="/tmp/perfout.txt" - -for bin in $dsc $ct $downstairs; do - if [[ ! -f "$bin" ]]; then - echo "Can't find crucible binary at $bin" >&2 - exit 1 - fi -done - -if pgrep -fl -U "$(id -u)" "$downstairs"; then - echo "Downstairs already running" >&2 - echo Run: pkill -f -U "$(id -u)" crucible-downstairs >&2 - exit 1 -fi - -echo "Perf test begins at $(date)" > "$outfile" - -# ES EC -perf_round 16384 640 -perf_round 16384 1280 - -# Print out a nice summary of all the perf results. This depends -# on the header and client perf output matching specific strings. -# A hack, yes, sorry. We are in bash hell here. -echo "" -grep TEST $outfile | head -1 -grep rwrites $outfile || true -echo "" -grep TEST $outfile | head -1 -grep rreads $outfile || true - -echo "Perf test finished on $(date)" | tee -a "$outfile"