From adac441859cf279602691761120d264cb8eaf660 Mon Sep 17 00:00:00 2001 From: Hao Hou Date: Thu, 31 Oct 2019 16:27:23 -0600 Subject: [PATCH] Plotters Integration This change integrates the pure Rust plotting crate, Plotters, to Criterion. This will allows the user doesn't relies on Gnuplot to generate the benchmark data visualization. After the change, Criterion won't have any change unless: 1. Gnuplot is not installed: Preivously we completely disable HTML report, now we using Plotters to generate the report visualization instead. 2. If `--prefer-plotters` is passed to benchmark, Criterion will prefer Plotters based plotting backend. Additional Notes: 1. This change refactors the plotting code to use dynamic dispatch 2. The Plotters based plotting backend is implemented 3. Commandline options is added --- .gitignore | 2 + Cargo.toml | 6 + src/html/mod.rs | 180 ++++------ src/lib.rs | 71 +++- .../{ => gnuplot_backend}/distributions.rs | 0 src/plot/gnuplot_backend/mod.rs | 223 +++++++++++++ src/plot/{ => gnuplot_backend}/pdf.rs | 0 src/plot/{ => gnuplot_backend}/regression.rs | 0 src/plot/{ => gnuplot_backend}/summary.rs | 0 src/plot/{ => gnuplot_backend}/t_test.rs | 0 src/plot/mod.rs | 131 ++++---- src/plot/plotters_backend/distributions.rs | 294 +++++++++++++++++ src/plot/plotters_backend/mod.rs | 189 +++++++++++ src/plot/plotters_backend/pdf.rs | 307 ++++++++++++++++++ src/plot/plotters_backend/regression.rs | 235 ++++++++++++++ src/plot/plotters_backend/summary.rs | 254 +++++++++++++++ src/plot/plotters_backend/t_test.rs | 59 ++++ 17 files changed, 1762 insertions(+), 189 deletions(-) rename src/plot/{ => gnuplot_backend}/distributions.rs (100%) create mode 100644 src/plot/gnuplot_backend/mod.rs rename src/plot/{ => gnuplot_backend}/pdf.rs (100%) rename src/plot/{ => gnuplot_backend}/regression.rs (100%) rename src/plot/{ => gnuplot_backend}/summary.rs (100%) rename src/plot/{ => gnuplot_backend}/t_test.rs (100%) create mode 100644 src/plot/plotters_backend/distributions.rs create mode 100644 src/plot/plotters_backend/mod.rs create mode 100644 src/plot/plotters_backend/pdf.rs create mode 100644 src/plot/plotters_backend/regression.rs create mode 100644 src/plot/plotters_backend/summary.rs create mode 100644 src/plot/plotters_backend/t_test.rs diff --git a/.gitignore b/.gitignore index fccbc2448..a8a47d521 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .criterion Cargo.lock target + +**/.*.sw* diff --git a/Cargo.toml b/Cargo.toml index c0d7c2d1c..657393b9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,12 @@ rand_core = { version = "0.5", default-features = false } rand = "0.7" rayon = "1.1" +[dependencies.plotters] +# This should be changed to Plotters' next release +git = "https://github.com/38/plotters.git" +default-features = false +features = ["svg"] + [dev-dependencies] tempdir = "0.3.7" approx = "0.3" diff --git a/src/html/mod.rs b/src/html/mod.rs index 80c4a2bce..539a2d3e4 100644 --- a/src/html/mod.rs +++ b/src/html/mod.rs @@ -1,41 +1,22 @@ use crate::report::{make_filename_safe, BenchmarkId, MeasurementData, Report, ReportContext}; use crate::stats::bivariate::regression::Slope; -use crate::stats::bivariate::Data; use crate::estimate::Statistic; use crate::format; use crate::fs; use crate::measurement::ValueFormatter; -use crate::plot; +use crate::plot::{PlotContext, PlotData, Plotter}; use crate::Estimate; use criterion_plot::Size; use serde::Serialize; +use std::cell::RefCell; use std::cmp::Ordering; use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; -use std::process::Child; use tinytemplate::TinyTemplate; const THUMBNAIL_SIZE: Option = Some(Size(450, 300)); -fn wait_on_gnuplot(children: Vec) { - let start = ::std::time::Instant::now(); - let child_count = children.len(); - for child in children { - match child.wait_with_output() { - Ok(ref out) if out.status.success() => {} - Ok(out) => error!("Error in Gnuplot: {}", String::from_utf8_lossy(&out.stderr)), - Err(e) => error!("Got IO error while waiting for Gnuplot to complete: {}", e), - } - } - let elapsed = &start.elapsed(); - info!( - "Waiting for {} gnuplot processes took {}", - child_count, - format::time(crate::DurationExt::to_nanos(elapsed) as f64) - ); -} - fn debug_context(path: &str, context: &S) { if crate::debug_enabled() { let mut context_path = PathBuf::from(path); @@ -278,9 +259,10 @@ struct IndexContext<'a> { pub struct Html { templates: TinyTemplate<'static>, + plotter: RefCell>, } impl Html { - pub fn new() -> Html { + pub(crate) fn new(plotter: Box) -> Html { let mut templates = TinyTemplate::new(); templates .add_template("report_link", include_str!("report_link.html.tt")) @@ -295,7 +277,8 @@ impl Html { .add_template("summary_report", include_str!("summary_report.html.tt")) .expect("Unable to parse summary_report template"); - Html { templates } + let plotter = RefCell::new(plotter); + Html { templates, plotter } } } impl Report for Html { @@ -420,7 +403,6 @@ impl Report for Html { }) .collect::>(); - let mut all_plots = vec![]; let group_id = all_ids[0].group_id.clone(); let data = self.load_summary_data(&context.output_directory, &all_ids); @@ -466,13 +448,13 @@ impl Report for Html { let subgroup_id = BenchmarkId::new(group_id.clone(), Some(function_id.clone()), None, None); - all_plots.extend(self.generate_summary( + self.generate_summary( &subgroup_id, &*samples_with_function, context, formatter, false, - )); + ); } } @@ -487,13 +469,13 @@ impl Report for Html { let subgroup_id = BenchmarkId::new(group_id.clone(), None, Some(value_str.clone()), None); - all_plots.extend(self.generate_summary( + self.generate_summary( &subgroup_id, &*samples_with_value, context, formatter, false, - )); + ); } } @@ -522,14 +504,14 @@ impl Report for Html { // function name, then value. This one has to be a stable sort. all_data.sort_by_key(|(id, _)| id.function_id.as_ref()); - all_plots.extend(self.generate_summary( + self.generate_summary( &BenchmarkId::new(group_id, None, None, None), &*(all_data), context, formatter, true, - )); - wait_on_gnuplot(all_plots) + ); + self.plotter.borrow_mut().wait(); } fn final_summary(&self, report_context: &ReportContext) { @@ -645,21 +627,31 @@ impl Html { formatter: &dyn ValueFormatter, measurements: &MeasurementData<'_>, ) { - let mut gnuplots = vec![ - // Probability density plots - plot::pdf(id, context, formatter, measurements, None), - plot::pdf_small(id, context, formatter, measurements, THUMBNAIL_SIZE), - // Linear regression plots - plot::regression(id, context, formatter, measurements, None), - plot::regression_small(id, context, formatter, measurements, THUMBNAIL_SIZE), - ]; - gnuplots.extend(plot::abs_distributions( + let plot_ctx = PlotContext { id, context, - formatter, + size: None, + is_thumbnail: false, + }; + + let plot_data = PlotData { measurements, - None, - )); + formatter, + comparison: None, + }; + + let plot_ctx_small = plot_ctx.thumbnail(true).size(THUMBNAIL_SIZE); + + self.plotter.borrow_mut().pdf(plot_ctx, plot_data); + self.plotter.borrow_mut().pdf(plot_ctx_small, plot_data); + self.plotter.borrow_mut().regression(plot_ctx, plot_data); + self.plotter + .borrow_mut() + .regression(plot_ctx_small, plot_data); + + self.plotter + .borrow_mut() + .abs_distributions(plot_ctx, plot_data); if let Some(ref comp) = measurements.comparison { try_else_return!(fs::mkdirp(&format!( @@ -674,47 +666,21 @@ impl Html { id.as_directory_name() ))); - let base_data = Data::new(&comp.base_iter_counts, &comp.base_sample_times); - gnuplots.append(&mut vec![ - plot::regression_comparison( - id, - context, - formatter, - measurements, - comp, - &base_data, - None, - ), - plot::regression_comparison_small( - id, - context, - formatter, - measurements, - comp, - &base_data, - THUMBNAIL_SIZE, - ), - plot::pdf_comparison(id, context, formatter, measurements, comp, None), - plot::pdf_comparison_small( - id, - context, - formatter, - measurements, - comp, - THUMBNAIL_SIZE, - ), - plot::t_test(id, context, measurements, comp, None), - ]); - gnuplots.extend(plot::rel_distributions( - id, - context, - measurements, - comp, - None, - )); + let comp_data = plot_data.comparison(&comp); + + self.plotter.borrow_mut().pdf(plot_ctx, comp_data); + self.plotter.borrow_mut().pdf(plot_ctx_small, comp_data); + self.plotter.borrow_mut().regression(plot_ctx, comp_data); + self.plotter + .borrow_mut() + .regression(plot_ctx_small, comp_data); + self.plotter.borrow_mut().t_test(plot_ctx, comp_data); + self.plotter + .borrow_mut() + .rel_distributions(plot_ctx, comp_data); } - wait_on_gnuplot(gnuplots); + self.plotter.borrow_mut().wait(); } fn load_summary_data<'a>( @@ -749,8 +715,13 @@ impl Html { report_context: &ReportContext, formatter: &dyn ValueFormatter, full_summary: bool, - ) -> Vec { - let mut gnuplots = vec![]; + ) { + let plot_ctx = PlotContext { + id, + context: report_context, + size: None, + is_thumbnail: false, + }; try_else_return!( fs::mkdirp(&format!( @@ -758,21 +729,10 @@ impl Html { report_context.output_directory, id.as_directory_name() )), - || gnuplots + || {} ); - let violin_path = format!( - "{}/{}/report/violin.svg", - report_context.output_directory, - id.as_directory_name() - ); - gnuplots.push(plot::violin( - formatter, - id.as_title(), - data, - &violin_path, - report_context.plot_config.summary_scale, - )); + self.plotter.borrow_mut().violin(plot_ctx, formatter, data); let value_types: Vec<_> = data.iter().map(|&&(ref id, _)| id.value_type()).collect(); let mut line_path = None; @@ -781,22 +741,10 @@ impl Html { if let Some(value_type) = value_types[0] { let values: Vec<_> = data.iter().map(|&&(ref id, _)| id.as_number()).collect(); if values.iter().any(|x| x != &values[0]) { - let path = format!( - "{}/{}/report/lines.svg", - report_context.output_directory, - id.as_directory_name() - ); - - gnuplots.push(plot::line_comparison( - formatter, - id.as_title(), - data, - &path, - value_type, - report_context.plot_config.summary_scale, - )); - - line_path = Some(path); + self.plotter + .borrow_mut() + .line_comparison(plot_ctx, formatter, data, value_type); + line_path = Some(plot_ctx.line_comparison_path()); } } } @@ -813,7 +761,7 @@ impl Html { thumbnail_width: THUMBNAIL_SIZE.unwrap().0, thumbnail_height: THUMBNAIL_SIZE.unwrap().1, - violin_plot: Some(violin_path), + violin_plot: Some(plot_ctx.violin_path()), line_chart: line_path, benchmarks, @@ -831,9 +779,7 @@ impl Html { .templates .render("summary_report", &context) .expect("Failed to render summary report template"); - try_else_return!(fs::save_string(&text, report_path,), || gnuplots); - - gnuplots + try_else_return!(fs::save_string(&text, report_path,), || {}); } } diff --git a/src/lib.rs b/src/lib.rs index 7b54f6d62..07bd2b897 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,6 +87,7 @@ use crate::csv_report::FileCsvReport; use crate::estimate::{Distributions, Estimates, Statistic}; use crate::html::Html; use crate::measurement::{Measurement, WallTime}; +use crate::plot::{Gnuplot, Plotter, PlottersBackend}; use crate::plotting::Plotting; use crate::profiler::{ExternalProfiler, Profiler}; use crate::report::{CliReport, Report, ReportContext, Reports}; @@ -629,6 +630,7 @@ pub enum Baseline { /// benchmark. pub struct Criterion { config: BenchmarkConfig, + prefer_plotters: bool, plotting: Plotting, filter: Option, report: Box, @@ -680,6 +682,10 @@ impl Default for Criterion { significance_level: 0.05, warm_up_time: Duration::new(3, 0), }, + // In the initial version, we just prefer the Gnuplot backend by default, but we can + // adjust adjust later. And we may remove this if we decide completely relies on + // Plotters for plotting. + prefer_plotters: false, plotting, filter: None, report: Box::new(Reports::new(reports)), @@ -703,6 +709,7 @@ impl Criterion { pub fn with_measurement(self, m: M2) -> Criterion { Criterion { config: self.config, + prefer_plotters: self.prefer_plotters, plotting: self.plotting, filter: self.filter, report: self.report, @@ -728,6 +735,19 @@ impl Criterion { } } + /// Changes the preferred plotting backend to Plotters based one. + /// + /// By default Criterion will use gnuplot if possible and use plotters as the fallback plotting + /// backend. + /// If plotters backend is preferred, Criterion will use plotters backend regardless if gnuplot + /// is installed. + pub fn prefer_plotters(self) -> Criterion { + Criterion { + prefer_plotters: true, + ..self + } + } + /// Changes the default size of the sample for benchmarks run with this runner. /// /// A bigger sample should yield more accurate results if paired with a sufficiently large @@ -841,26 +861,32 @@ impl Criterion { self } - /// Enables plotting - pub fn with_plots(mut self) -> Criterion { + fn create_plotter(&self) -> Box { use criterion_plot::VersionError; - self.plotting = match criterion_plot::version() { - Ok(_) => { - let mut reports: Vec> = vec![]; - reports.push(Box::new(CliReport::new(false, false, false))); - reports.push(Box::new(FileCsvReport)); - reports.push(Box::new(Html::new())); - self.report = Box::new(Reports::new(reports)); - Plotting::Enabled - } - Err(e) => { + match (self.prefer_plotters, criterion_plot::version()) { + (false, Ok(_)) => Box::new(Gnuplot::default()), + (false, Err(e)) => { match e { - VersionError::Exec(_) => println!("Gnuplot not found, disabling plotting"), - e => println!("Gnuplot not found or not usable, disabling plotting\n{}", e), + VersionError::Exec(_) => println!("Gnuplot not found, using plotters backend"), + e => println!( + "Gnuplot not found or not usable, using plotters backend\n{}", + e + ), } - Plotting::NotAvailable + Box::new(PlottersBackend::default()) } - }; + _ => Box::new(PlottersBackend::default()), + } + } + + /// Enables plotting + pub fn with_plots(mut self) -> Criterion { + self.plotting = Plotting::Enabled; + let mut reports: Vec> = vec![]; + reports.push(Box::new(CliReport::new(false, false, false))); + reports.push(Box::new(FileCsvReport)); + reports.push(Box::new(Html::new(self.create_plotter()))); + self.report = Box::new(Reports::new(reports)); self } @@ -1013,6 +1039,9 @@ impl Criterion { .arg(Arg::with_name("bench") .hidden(true) .long("bench")) + .arg(Arg::with_name("prefer-plotters") + .long("prefer-plotters") + .help("Set the preferred plotting backend to Plotters, otherwise Criterion will only use plotters backend when gnuplot isn't available.")) .arg(Arg::with_name("version") .hidden(true) .short("V") @@ -1048,6 +1077,10 @@ To test that the benchmarks work, run `cargo test --benches` _ => enable_text_coloring = stdout_isatty, } + if matches.is_present("prefer-plotters") { + self.prefer_plotters = true; + } + if matches.is_present("noplot") || matches.is_present("test") { self = self.without_plots(); } else { @@ -1176,7 +1209,7 @@ To test that the benchmarks work, run `cargo test --benches` } if self.profile_time.is_none() { - reports.push(Box::new(Html::new())); + reports.push(Box::new(Html::new(self.create_plotter()))); } self.report = Box::new(Reports::new(reports)); @@ -1423,6 +1456,10 @@ mod plotting { Unset, Disabled, Enabled, + // Not sure if we should make Plotters backend behind a feature swith, but if this is the + // case, we definitely want to keep this. Otherwise, we should have at least plotters + // backend as the default plotter when Gnuplot is not available. + #[allow(dead_code)] NotAvailable, } diff --git a/src/plot/distributions.rs b/src/plot/gnuplot_backend/distributions.rs similarity index 100% rename from src/plot/distributions.rs rename to src/plot/gnuplot_backend/distributions.rs diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs new file mode 100644 index 000000000..5ecf7e714 --- /dev/null +++ b/src/plot/gnuplot_backend/mod.rs @@ -0,0 +1,223 @@ +use std::iter; +use std::path::PathBuf; +use std::process::Child; + +use crate::stats::univariate::Sample; +use criterion_plot::prelude::*; + +mod distributions; +mod pdf; +mod regression; +mod summary; +mod t_test; +pub(crate) use self::distributions::*; +pub(crate) use self::pdf::*; +pub(crate) use self::regression::*; +pub(crate) use self::summary::*; +pub(crate) use self::t_test::*; + +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ValueType}; +use crate::stats::bivariate::Data; + +use super::{PlotContext, PlotData, Plotter}; +use crate::format; + +fn escape_underscores(string: &str) -> String { + string.replace("_", "\\_") +} + +static DEFAULT_FONT: &str = "Helvetica"; +static KDE_POINTS: usize = 500; +static SIZE: Size = Size(1280, 720); + +const LINEWIDTH: LineWidth = LineWidth(2.); +const POINT_SIZE: PointSize = PointSize(0.75); + +const DARK_BLUE: Color = Color::Rgb(31, 120, 180); +const DARK_ORANGE: Color = Color::Rgb(255, 127, 0); +const DARK_RED: Color = Color::Rgb(227, 26, 28); + +fn debug_script(path: &PathBuf, figure: &Figure) { + if crate::debug_enabled() { + let mut script_path = path.clone(); + script_path.set_extension("gnuplot"); + println!("Writing gnuplot script to {:?}", script_path); + let result = figure.save(script_path.as_path()); + if let Err(e) = result { + error!("Failed to write debug output: {}", e); + } + } +} + +/// Private +trait Append { + /// Private + fn append_(self, item: T) -> Self; +} + +// NB I wish this was in the standard library +impl Append for Vec { + fn append_(mut self, item: T) -> Vec { + self.push(item); + self + } +} + +#[derive(Default)] +pub(crate) struct Gnuplot { + process_list: Vec, +} + +impl Plotter for Gnuplot { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.push(if ctx.is_thumbnail { + if let Some(cmp) = data.comparison { + pdf_comparison_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + pdf_small(ctx.id, ctx.context, data.formatter, data.measurements, size) + } + } else if let Some(cmp) = data.comparison { + pdf_comparison( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + size, + ) + } else { + pdf(ctx.id, ctx.context, data.formatter, data.measurements, size) + }); + } + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.push(if ctx.is_thumbnail { + if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression_comparison_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + &base_data, + size, + ) + } else { + regression_small(ctx.id, ctx.context, data.formatter, data.measurements, size) + } + } else if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression_comparison( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + cmp, + &base_data, + size, + ) + } else { + regression(ctx.id, ctx.context, data.formatter, data.measurements, size) + }); + } + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + self.process_list.extend(abs_distributions( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + size, + )); + } + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + if let Some(cmp) = data.comparison { + self.process_list.extend(rel_distributions( + ctx.id, + ctx.context, + data.measurements, + cmp, + size, + )); + } else { + error!("Comparison data is not provided for a relative distribution figure"); + } + } + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let size = ctx.size.map(|(w, h)| Size(w, h)); + if let Some(cmp) = data.comparison { + self.process_list + .push(t_test(ctx.id, ctx.context, data.measurements, cmp, size)); + } else { + error!("Comparison data is not provided for t_test plot"); + } + } + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + value_type: ValueType, + ) { + let path = ctx.line_comparison_path(); + self.process_list.push(line_comparison( + formatter, + ctx.id.as_title(), + all_curves, + &path, + value_type, + ctx.context.plot_config.summary_scale, + )); + } + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + ) { + let violin_path = ctx.violin_path(); + + self.process_list.push(violin( + formatter, + ctx.id.as_title(), + all_curves, + &violin_path, + ctx.context.plot_config.summary_scale, + )); + } + + fn wait(&mut self) { + let start = std::time::Instant::now(); + let child_count = self.process_list.len(); + for child in self.process_list.drain(..) { + match child.wait_with_output() { + Ok(ref out) if out.status.success() => {} + Ok(out) => error!("Error in Gnuplot: {}", String::from_utf8_lossy(&out.stderr)), + Err(e) => error!("Got IO error while waiting for Gnuplot to complete: {}", e), + } + } + let elapsed = &start.elapsed(); + info!( + "Waiting for {} gnuplot processes took {}", + child_count, + format::time(crate::DurationExt::to_nanos(elapsed) as f64) + ); + } +} diff --git a/src/plot/pdf.rs b/src/plot/gnuplot_backend/pdf.rs similarity index 100% rename from src/plot/pdf.rs rename to src/plot/gnuplot_backend/pdf.rs diff --git a/src/plot/regression.rs b/src/plot/gnuplot_backend/regression.rs similarity index 100% rename from src/plot/regression.rs rename to src/plot/gnuplot_backend/regression.rs diff --git a/src/plot/summary.rs b/src/plot/gnuplot_backend/summary.rs similarity index 100% rename from src/plot/summary.rs rename to src/plot/gnuplot_backend/summary.rs diff --git a/src/plot/t_test.rs b/src/plot/gnuplot_backend/t_test.rs similarity index 100% rename from src/plot/t_test.rs rename to src/plot/gnuplot_backend/t_test.rs diff --git a/src/plot/mod.rs b/src/plot/mod.rs index 0261013a3..59fa6a768 100644 --- a/src/plot/mod.rs +++ b/src/plot/mod.rs @@ -1,68 +1,89 @@ -use std::iter; -use std::path::PathBuf; - -use crate::stats::univariate::Sample; -use criterion_plot::prelude::*; - -mod distributions; -mod pdf; -mod regression; -mod summary; -mod t_test; -pub(crate) use self::distributions::*; -pub(crate) use self::pdf::*; -pub(crate) use self::regression::*; -pub(crate) use self::summary::*; -pub(crate) use self::t_test::*; - -fn escape_underscores(string: &str) -> String { - string.replace("_", "\\_") +mod gnuplot_backend; +mod plotters_backend; + +pub(crate) use gnuplot_backend::Gnuplot; +pub(crate) use plotters_backend::PlottersBackend; + +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext, ValueType}; + +#[derive(Clone, Copy)] +pub(crate) struct PlotContext<'a> { + pub(crate) id: &'a BenchmarkId, + pub(crate) context: &'a ReportContext, + pub(crate) size: Option<(usize, usize)>, + pub(crate) is_thumbnail: bool, } -static DEFAULT_FONT: &str = "Helvetica"; -static KDE_POINTS: usize = 500; -static SIZE: Size = Size(1280, 720); - -const LINEWIDTH: LineWidth = LineWidth(2.); -const POINT_SIZE: PointSize = PointSize(0.75); - -const DARK_BLUE: Color = Color::Rgb(31, 120, 180); -const DARK_ORANGE: Color = Color::Rgb(255, 127, 0); -const DARK_RED: Color = Color::Rgb(227, 26, 28); - -fn debug_script(path: &PathBuf, figure: &Figure) { - if crate::debug_enabled() { - let mut script_path = path.clone(); - script_path.set_extension("gnuplot"); - println!("Writing gnuplot script to {:?}", script_path); - let result = figure.save(script_path.as_path()); - if let Err(e) = result { - error!("Failed to write debug output: {}", e); +impl<'a> PlotContext<'a> { + pub fn size(mut self, s: Option) -> PlotContext<'a> { + if let Some(s) = s { + self.size = Some((s.0, s.1)); } + self } -} -/*fn get_max(values: &[f64]) -> f64 { - assert!(!values.is_empty()); - let mut elems = values.iter(); + pub fn thumbnail(mut self, value: bool) -> PlotContext<'a> { + self.is_thumbnail = value; + self + } - match elems.next() { - Some(&head) => elems.fold(head, |a, &b| a.max(b)), - // NB `unreachable!` because `Sample` is guaranteed to have at least one data point - None => unreachable!(), + pub fn line_comparison_path(&self) -> String { + format!( + "{}/{}/report/lines.svg", + self.context.output_directory, + self.id.as_directory_name() + ) } -}*/ -/// Private -trait Append { - /// Private - fn append_(self, item: T) -> Self; + pub fn violin_path(&self) -> String { + format!( + "{}/{}/report/violin.svg", + self.context.output_directory, + self.id.as_directory_name() + ) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct PlotData<'a> { + pub(crate) formatter: &'a dyn ValueFormatter, + pub(crate) measurements: &'a MeasurementData<'a>, + pub(crate) comparison: Option<&'a ComparisonData>, } -// NB I wish this was in the standard library -impl Append for Vec { - fn append_(mut self, item: T) -> Vec { - self.push(item); +impl<'a> PlotData<'a> { + pub fn comparison(mut self, comp: &'a ComparisonData) -> PlotData<'a> { + self.comparison = Some(comp); self } } + +pub(crate) trait Plotter { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + value_type: ValueType, + ); + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + ); + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); + + fn wait(&mut self); +} diff --git a/src/plot/plotters_backend/distributions.rs b/src/plot/plotters_backend/distributions.rs new file mode 100644 index 000000000..840a8b11c --- /dev/null +++ b/src/plot/plotters_backend/distributions.rs @@ -0,0 +1,294 @@ +use super::*; +use crate::estimate::Statistic; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, MeasurementData, ReportContext}; +use crate::stats::Distribution; +use crate::Estimate; + +fn abs_distribution( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + statistic: Statistic, + distribution: &Distribution, + estimate: &Estimate, + size: Option<(u32, u32)>, +) { + let ci = estimate.confidence_interval; + let typical = ci.upper_bound; + let mut ci_values = [ci.lower_bound, ci.upper_bound, estimate.point_estimate]; + let unit = formatter.scale_values(typical, &mut ci_values); + let (lb, ub, p) = (ci_values[0], ci_values[1], ci_values[2]); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let mut scaled_xs: Vec = distribution.iter().cloned().collect(); + let _ = formatter.scale_values(typical, &mut scaled_xs); + let scaled_xs_sample = Sample::new(&scaled_xs); + let (kde_xs, ys) = kde::sweep(scaled_xs_sample, KDE_POINTS, Some((start, end))); + + let n_p = kde_xs.iter().enumerate().find(|&(_, &x)| x >= p).unwrap().0; + let y_p = ys[n_p - 1] + + (ys[n_p] - ys[n_p - 1]) / (kde_xs[n_p] - kde_xs[n_p - 1]) * (p - kde_xs[n_p - 1]); + + let start = kde_xs + .iter() + .enumerate() + .find(|&(_, &x)| x >= lb) + .unwrap() + .0; + let end = kde_xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let kde_xs_sample = Sample::new(&kde_xs); + + let path = context.report_path(id, &format!("{}.svg", statistic)); + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let x_range = plotters::data::fitting_range(kde_xs_sample.iter()); + let mut y_range = plotters::data::fitting_range(ys.iter()); + + y_range.end *= 1.1; + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption( + format!("{}:{}", id.as_title(), statistic), + (DEFAULT_FONT, 20), + ) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc(format!("Average time ({})", unit)) + .y_desc("Density (a.u.)") + .x_label_formatter(&|&v| pretty_print_float(v, true)) + .y_label_formatter(&|&v| pretty_print_float(v, true)) + .draw() + .unwrap(); + + chart + .draw_series(LineSeries::new( + kde_xs.iter().zip(ys.iter()).map(|(&x, &y)| (x, y)), + &DARK_BLUE, + )) + .unwrap() + .label("Bootstrap distribution") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(AreaSeries::new( + kde_xs + .iter() + .zip(ys.iter()) + .skip(start) + .take(len) + .map(|(&x, &y)| (x, y)), + 0.0, + DARK_BLUE.mix(0.25).filled().stroke_width(3), + )) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(p, 0.0), (p, y_p)], + DARK_BLUE.filled().stroke_width(3), + ))) + .unwrap() + .label("Point estimate") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperRight) + .draw() + .unwrap(); +} + +pub(crate) fn abs_distributions( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + measurements + .distributions + .iter() + .for_each(|(&statistic, distribution)| { + abs_distribution( + id, + context, + formatter, + statistic, + distribution, + &measurements.absolute_estimates[&statistic], + size, + ) + }) +} + +fn rel_distribution( + id: &BenchmarkId, + context: &ReportContext, + statistic: Statistic, + distribution: &Distribution, + estimate: &Estimate, + noise_threshold: f64, + size: Option<(u32, u32)>, +) { + let ci = estimate.confidence_interval; + let (lb, ub) = (ci.lower_bound, ci.upper_bound); + + let start = lb - (ub - lb) / 9.; + let end = ub + (ub - lb) / 9.; + let (xs, ys) = kde::sweep(distribution, KDE_POINTS, Some((start, end))); + let xs_ = Sample::new(&xs); + + let p = estimate.point_estimate; + let n_p = xs.iter().enumerate().find(|&(_, &x)| x >= p).unwrap().0; + let y_p = ys[n_p - 1] + (ys[n_p] - ys[n_p - 1]) / (xs[n_p] - xs[n_p - 1]) * (p - xs[n_p - 1]); + + let start = xs.iter().enumerate().find(|&(_, &x)| x >= lb).unwrap().0; + let end = xs + .iter() + .enumerate() + .rev() + .find(|&(_, &x)| x <= ub) + .unwrap() + .0; + let len = end - start; + + let x_min = xs_.min(); + let x_max = xs_.max(); + + let (fc_start, fc_end) = if noise_threshold < x_min || -noise_threshold > x_max { + let middle = (x_min + x_max) / 2.; + + (middle, middle) + } else { + ( + if -noise_threshold < x_min { + x_min + } else { + -noise_threshold + }, + if noise_threshold > x_max { + x_max + } else { + noise_threshold + }, + ) + }; + let y_range = plotters::data::fitting_range(ys.iter()); + let path = context.report_path(id, &format!("change/{}.svg", statistic)); + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption( + format!("{}:{}", id.as_title(), statistic), + (DEFAULT_FONT, 20), + ) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_min..x_max, y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc("Relative change (%)") + .y_desc("Density (a.u.)") + .x_label_formatter(&|&v| pretty_print_float(v, true)) + .y_label_formatter(&|&v| pretty_print_float(v, true)) + .draw() + .unwrap(); + + chart + .draw_series(LineSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + &DARK_BLUE, + )) + .unwrap() + .label("Bootstrap distribution") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(AreaSeries::new( + xs.iter() + .zip(ys.iter()) + .skip(start) + .take(len) + .map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.25).filled().stroke_width(3), + )) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(p, 0.0), (p, y_p)], + DARK_BLUE.filled().stroke_width(3), + ))) + .unwrap() + .label("Point estimate") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(std::iter::once(Rectangle::new( + [(fc_start, y_range.start), (fc_end, y_range.end)], + DARK_RED.mix(0.1).filled(), + ))) + .unwrap() + .label("Noise threshold") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.25).filled()) + }); + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperRight) + .draw() + .unwrap(); +} + +pub(crate) fn rel_distributions( + id: &BenchmarkId, + context: &ReportContext, + _measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + comparison + .relative_distributions + .iter() + .for_each(|(&statistic, distribution)| { + rel_distribution( + id, + context, + statistic, + distribution, + &comparison.relative_estimates[&statistic], + comparison.noise_threshold, + size, + ) + }); +} diff --git a/src/plot/plotters_backend/mod.rs b/src/plot/plotters_backend/mod.rs new file mode 100644 index 000000000..c3ea361cd --- /dev/null +++ b/src/plot/plotters_backend/mod.rs @@ -0,0 +1,189 @@ +use super::{PlotContext, PlotData, Plotter}; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ValueType}; +use plotters::data::float::pretty_print_float; +use plotters::prelude::*; + +use crate::kde; +use crate::stats::bivariate::Data; +use crate::stats::univariate::Sample; + +static DEFAULT_FONT: FontFamily = FontFamily::SansSerif; +static KDE_POINTS: usize = 500; +static SIZE: (u32, u32) = (960, 540); +static POINT_SIZE: u32 = 3; + +const DARK_BLUE: RGBColor = RGBColor(31, 120, 180); +const DARK_ORANGE: RGBColor = RGBColor(255, 127, 0); +const DARK_RED: RGBColor = RGBColor(227, 26, 28); + +mod distributions; +mod pdf; +mod regression; +mod summary; +mod t_test; + +fn convert_size(size: Option<(usize, usize)>) -> Option<(u32, u32)> { + if let Some((w, h)) = size { + return Some((w as u32, h as u32)); + } + None +} +#[derive(Default)] +pub struct PlottersBackend; + +#[allow(unused_variables)] +impl Plotter for PlottersBackend { + fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + if let Some(cmp) = data.comparison { + let (path, title) = if ctx.is_thumbnail { + ( + ctx.context.report_path(ctx.id, "relative_pdf_small.svg"), + None, + ) + } else { + ( + ctx.context.report_path(ctx.id, "both/pdf.svg"), + Some(ctx.id.as_title()), + ) + }; + pdf::pdf_comparison_figure( + path.as_ref(), + title, + data.formatter, + data.measurements, + cmp, + convert_size(ctx.size), + ); + return; + } + if ctx.is_thumbnail { + pdf::pdf_small( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } else { + pdf::pdf( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + } + + fn regression(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let (title, path) = match (data.comparison.is_some(), ctx.is_thumbnail) { + (true, true) => ( + None, + ctx.context + .report_path(ctx.id, "relative_regression_small.svg"), + ), + (true, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "both/regression.svg"), + ), + (false, true) => ( + None, + ctx.context.report_path(ctx.id, "regression_small.svg"), + ), + (false, false) => ( + Some(ctx.id.as_title()), + ctx.context.report_path(ctx.id, "regression.svg"), + ), + }; + + if let Some(cmp) = data.comparison { + let base_data = Data::new(&cmp.base_iter_counts, &cmp.base_sample_times); + regression::regression_comparison_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + cmp, + &base_data, + convert_size(ctx.size), + ); + } else { + regression::regression_figure( + title, + path.as_path(), + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + } + + fn abs_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + distributions::abs_distributions( + ctx.id, + ctx.context, + data.formatter, + data.measurements, + convert_size(ctx.size), + ); + } + + fn rel_distributions(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + distributions::rel_distributions( + ctx.id, + ctx.context, + data.measurements, + data.comparison.unwrap(), + convert_size(ctx.size), + ); + } + + fn line_comparison( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + value_type: ValueType, + ) { + let path = ctx.line_comparison_path(); + summary::line_comparison( + formatter, + ctx.id.as_title(), + all_curves, + &path, + value_type, + ctx.context.plot_config.summary_scale, + ); + } + + fn violin( + &mut self, + ctx: PlotContext<'_>, + formatter: &dyn ValueFormatter, + all_curves: &[&(&BenchmarkId, Vec)], + ) { + let violin_path = ctx.violin_path(); + + summary::violin( + formatter, + ctx.id.as_title(), + all_curves, + &violin_path, + ctx.context.plot_config.summary_scale, + ); + } + + fn t_test(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>) { + let title = ctx.id.as_title(); + let path = ctx.context.report_path(ctx.id, "change/t-test.svg"); + t_test::t_test( + path.as_path(), + title, + data.comparison.unwrap(), + convert_size(ctx.size), + ); + } + + fn wait(&mut self) {} +} diff --git a/src/plot/plotters_backend/pdf.rs b/src/plot/plotters_backend/pdf.rs new file mode 100644 index 000000000..b44fcc0e2 --- /dev/null +++ b/src/plot/plotters_backend/pdf.rs @@ -0,0 +1,307 @@ +use super::*; +use crate::measurement::ValueFormatter; +use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext}; +use plotters::data; +use plotters::style::RGBAColor; +use std::path::Path; + +pub(crate) fn pdf_comparison_figure( + path: &Path, + title: Option<&str>, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + let base_avg_times = Sample::new(&comparison.base_avg_times); + let typical = base_avg_times.max().max(measurements.avg_times.max()); + let mut scaled_base_avg_times: Vec = comparison.base_avg_times.clone(); + let unit = formatter.scale_values(typical, &mut scaled_base_avg_times); + let scaled_base_avg_times = Sample::new(&scaled_base_avg_times); + + let mut scaled_new_avg_times: Vec = (&measurements.avg_times as &Sample) + .iter() + .cloned() + .collect(); + let _ = formatter.scale_values(typical, &mut scaled_new_avg_times); + let scaled_new_avg_times = Sample::new(&scaled_new_avg_times); + + let base_mean = scaled_base_avg_times.mean(); + let new_mean = scaled_new_avg_times.mean(); + + let (base_xs, base_ys, base_y_mean) = + kde::sweep_and_estimate(scaled_base_avg_times, KDE_POINTS, None, base_mean); + let (xs, ys, y_mean) = + kde::sweep_and_estimate(scaled_new_avg_times, KDE_POINTS, None, new_mean); + + let x_range = data::fitting_range(base_xs.iter().chain(xs.iter())); + let y_range = data::fitting_range(base_ys.iter().chain(ys.iter())); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range.clone(), y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density (a.u.)") + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .x_labels(5) + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + base_xs.iter().zip(base_ys.iter()).map(|(x, y)| (*x, *y)), + y_range.start, + DARK_RED.mix(0.5).filled(), + )) + .unwrap() + .label("Base PDF") + .legend(|(x, y)| Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_RED.mix(0.5).filled())); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + y_range.start, + DARK_BLUE.mix(0.5).filled(), + )) + .unwrap() + .label("New PDF") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(base_mean, 0.0), (base_mean, base_y_mean)], + DARK_RED.filled().stroke_width(2), + ))) + .unwrap() + .label("Base Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_RED)); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(new_mean, 0.0), (new_mean, y_mean)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap() + .label("New Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + if title.is_some() { + chart.configure_series_labels().draw().unwrap(); + } +} + +pub(crate) fn pdf_small( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let avg_times = &*measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec = (avg_times as &Sample).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + let mean = scaled_avg_times.mean(); + + let (xs, ys, mean_y) = kde::sweep_and_estimate(scaled_avg_times, KDE_POINTS, None, mean); + let xs_ = Sample::new(&xs); + let ys_ = Sample::new(&ys); + + let y_limit = ys_.max() * 1.1; + + let path = context.report_path(id, "pdf_small.svg"); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(xs_.min()..xs_.max(), 0.0..y_limit) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density (a.u.)") + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .x_labels(5) + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.25).filled(), + )) + .unwrap(); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(mean, 0.0), (mean, mean_y)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap(); +} + +pub(crate) fn pdf( + id: &BenchmarkId, + context: &ReportContext, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let avg_times = &measurements.avg_times; + let typical = avg_times.max(); + let mut scaled_avg_times: Vec = (avg_times as &Sample).iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_avg_times); + let scaled_avg_times = Sample::new(&scaled_avg_times); + + let mean = scaled_avg_times.mean(); + + let iter_counts = measurements.iter_counts(); + let &max_iters = iter_counts + .iter() + .max_by_key(|&&iters| iters as u64) + .unwrap(); + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let y_scale = 10f64.powi(-exponent); + + let y_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let (xs, ys) = kde::sweep(&scaled_avg_times, KDE_POINTS, None); + let (lost, lomt, himt, hist) = avg_times.fences(); + let mut fences = [lost, lomt, himt, hist]; + let _ = formatter.scale_values(typical, &mut fences); + let [lost, lomt, himt, hist] = fences; + + let path = context.report_path(id, "pdf.svg"); + + let xs_ = Sample::new(&xs); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(&path, (size.0 as u32, size.1 as u32)).into_drawing_area(); + + let range = data::fitting_range(ys.iter()); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption(id.as_title(), (DEFAULT_FONT, 20)) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Right, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(xs_.min()..xs_.max(), 0.0..max_iters) + .unwrap() + .set_secondary_coord(xs_.min()..xs_.max(), 0.0..range.end); + + chart + .configure_mesh() + .disable_mesh() + .y_desc(y_label) + .x_desc(format!("Average Time ({})", unit)) + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y * y_scale, true)) + .draw() + .unwrap(); + + chart + .configure_secondary_axes() + .y_desc("Density (a.u.)") + .x_label_formatter(&|&x| pretty_print_float(x, true)) + .y_label_formatter(&|&y| pretty_print_float(y, true)) + .draw() + .unwrap(); + + chart + .draw_secondary_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + DARK_BLUE.mix(0.5).filled(), + )) + .unwrap() + .label("PDF") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.5).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(mean, 0.0), (mean, max_iters)], + &DARK_BLUE, + ))) + .unwrap() + .label("Mean") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart + .draw_series(vec![ + PathElement::new(vec![(lomt, 0.0), (lomt, max_iters)], &DARK_ORANGE), + PathElement::new(vec![(himt, 0.0), (himt, max_iters)], &DARK_ORANGE), + PathElement::new(vec![(lost, 0.0), (lost, max_iters)], &DARK_RED), + PathElement::new(vec![(hist, 0.0), (hist, max_iters)], &DARK_RED), + ]) + .unwrap(); + use crate::stats::univariate::outliers::tukey::Label; + + let mut draw_data_point_series = + |filter: &dyn Fn(&Label) -> bool, color: RGBAColor, name: &str| { + chart + .draw_series( + avg_times + .iter() + .zip(scaled_avg_times.iter()) + .zip(iter_counts.iter()) + .filter_map(|(((_, label), t), i)| { + if filter(&label) { + Some(Circle::new((*t, *i), POINT_SIZE, color.filled())) + } else { + None + } + }), + ) + .unwrap() + .label(name) + .legend(move |(x, y)| Circle::new((x + 10, y), POINT_SIZE, color.filled())); + }; + + draw_data_point_series( + &|l| !l.is_outlier(), + DARK_BLUE.to_rgba(), + "\"Clean\" sample", + ); + draw_data_point_series( + &|l| l.is_mild(), + RGBColor(255, 127, 0).to_rgba(), + "Mild outliers", + ); + draw_data_point_series(&|l| l.is_severe(), DARK_RED.to_rgba(), "Severe outliers"); + chart.configure_series_labels().draw().unwrap(); +} diff --git a/src/plot/plotters_backend/regression.rs b/src/plot/plotters_backend/regression.rs new file mode 100644 index 000000000..d56ff3a03 --- /dev/null +++ b/src/plot/plotters_backend/regression.rs @@ -0,0 +1,235 @@ +use super::*; + +use std::path::Path; + +use crate::estimate::Statistic; +use crate::stats::bivariate::regression::Slope; +use crate::stats::bivariate::Data; +use crate::{ConfidenceInterval, Estimate}; + +pub(crate) fn regression_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + size: Option<(u32, u32)>, +) { + let slope_estimate = &measurements.absolute_estimates[&Statistic::Slope]; + let slope_dist = &measurements.distributions[&Statistic::Slope]; + let (lb, ub) = + slope_dist.confidence_interval(slope_estimate.confidence_interval.confidence_level); + + let data = &measurements.data; + let (max_iters, typical) = (data.x().max(), data.y().max()); + let mut scaled_y: Vec = data.y().iter().cloned().collect(); + let unit = formatter.scale_values(typical, &mut scaled_y); + let scaled_y = Sample::new(&scaled_y); + + let point_estimate = Slope::fit(&measurements.data).0; + let mut scaled_points = [point_estimate * max_iters, lb * max_iters, ub * max_iters]; + let _ = formatter.scale_values(typical, &mut scaled_points); + let [point, lb, ub] = scaled_points; + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + + let x_scale = 10f64.powi(-exponent); + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let x_range = plotters::data::fitting_range(data.x().iter()); + let y_range = plotters::data::fitting_range(scaled_y.iter()); + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .x_desc(x_label) + .y_desc(format!("Total sample time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series( + data.x() + .iter() + .zip(scaled_y.iter()) + .map(|(x, y)| Circle::new((*x, *y), POINT_SIZE, DARK_BLUE.filled())), + ) + .unwrap() + .label("Sample") + .legend(|(x, y)| Circle::new((x + 10, y), POINT_SIZE, DARK_BLUE.filled())); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(0.0, 0.0), (max_iters, point)], + &DARK_BLUE, + ))) + .unwrap() + .label("Linear regression") + .legend(|(x, y)| { + PathElement::new( + vec![(x, y), (x + 20, y)], + DARK_BLUE.filled().stroke_width(2), + ) + }); + + chart + .draw_series(std::iter::once(Polygon::new( + vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)], + DARK_BLUE.mix(0.25).filled(), + ))) + .unwrap() + .label("Confidence interval") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} + +pub(crate) fn regression_comparison_figure( + title: Option<&str>, + path: &Path, + formatter: &dyn ValueFormatter, + measurements: &MeasurementData<'_>, + comparison: &ComparisonData, + base_data: &Data<'_, f64, f64>, + size: Option<(u32, u32)>, +) { + let data = &measurements.data; + let max_iters = base_data.x().max().max(data.x().max()); + let typical = base_data.y().max().max(data.y().max()); + + let exponent = (max_iters.log10() / 3.).floor() as i32 * 3; + let x_scale = 10f64.powi(-exponent); + + let x_label = if exponent == 0 { + "Iterations".to_owned() + } else { + format!("Iterations (x 10^{})", exponent) + }; + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: base_lb, + upper_bound: base_ub, + .. + }, + point_estimate: base_point, + .. + } = comparison.base_estimates[&Statistic::Slope]; + + let Estimate { + confidence_interval: + ConfidenceInterval { + lower_bound: lb, + upper_bound: ub, + .. + }, + point_estimate: point, + .. + } = comparison.base_estimates[&Statistic::Slope]; + + let mut points = [ + base_lb * max_iters, + base_point * max_iters, + base_ub * max_iters, + lb * max_iters, + point * max_iters, + ub * max_iters, + ]; + let unit = formatter.scale_values(typical, &mut points); + let [base_lb, base_point, base_ub, lb, point, ub] = points; + + let y_max = point.max(base_point); + + let size = size.unwrap_or(SIZE); + let root_area = SVGBackend::new(path, size).into_drawing_area(); + + let mut cb = ChartBuilder::on(&root_area); + if let Some(title) = title { + cb.caption(title, (DEFAULT_FONT, 20)); + } + + let mut chart = cb + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(0.0..max_iters, 0.0..y_max) + .unwrap(); + + chart + .configure_mesh() + .x_desc(x_label) + .y_desc(format!("Total sample time ({})", unit)) + .x_label_formatter(&|x| pretty_print_float(x * x_scale, true)) + .line_style_2(&TRANSPARENT) + .draw() + .unwrap(); + + chart + .draw_series(vec![ + PathElement::new(vec![(0.0, 0.0), (max_iters, base_point)], &DARK_RED).into_dyn(), + Polygon::new( + vec![(0.0, 0.0), (max_iters, base_lb), (max_iters, base_ub)], + DARK_RED.mix(0.25).filled(), + ) + .into_dyn(), + ]) + .unwrap() + .label("Base Sample") + .legend(|(x, y)| { + PathElement::new(vec![(x, y), (x + 20, y)], DARK_RED.filled().stroke_width(2)) + }); + + chart + .draw_series(vec![ + PathElement::new(vec![(0.0, 0.0), (max_iters, point)], &DARK_BLUE).into_dyn(), + Polygon::new( + vec![(0.0, 0.0), (max_iters, lb), (max_iters, ub)], + DARK_BLUE.mix(0.25).filled(), + ) + .into_dyn(), + ]) + .unwrap() + .label("New Sample") + .legend(|(x, y)| { + PathElement::new( + vec![(x, y), (x + 20, y)], + DARK_BLUE.filled().stroke_width(2), + ) + }); + + if title.is_some() { + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); + } +} diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs new file mode 100644 index 000000000..728076e93 --- /dev/null +++ b/src/plot/plotters_backend/summary.rs @@ -0,0 +1,254 @@ +use super::*; +use crate::AxisScale; +use itertools::Itertools; +use plotters::coord::{AsRangedCoord, Shift}; +use std::cmp::Ordering; + +const NUM_COLORS: usize = 8; +static COMPARISON_COLORS: [RGBColor; NUM_COLORS] = [ + RGBColor(178, 34, 34), + RGBColor(46, 139, 87), + RGBColor(0, 139, 139), + RGBColor(255, 215, 0), + RGBColor(0, 0, 139), + RGBColor(220, 20, 60), + RGBColor(139, 0, 139), + RGBColor(0, 255, 127), +]; + +pub fn line_comparison( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec)], + path: &str, + value_type: ValueType, + axis_scale: AxisScale, +) { + let (unit, series_data) = line_comparision_series_data(formatter, all_curves); + + let x_range = + plotters::data::fitting_range(series_data.iter().map(|(_, xs, _)| xs.iter()).flatten()); + let y_range = + plotters::data::fitting_range(series_data.iter().map(|(_, _, ys)| ys.iter()).flatten()); + let root_area = SVGBackend::new(&path, SIZE) + .into_drawing_area() + .titled(&format!("{}: Comparision", title), (DEFAULT_FONT, 20)) + .unwrap(); + + match axis_scale { + AxisScale::Linear => { + draw_line_comarision_figure(root_area, unit, x_range, y_range, value_type, series_data) + } + AxisScale::Logarithmic => draw_line_comarision_figure( + root_area, + unit, + LogRange(x_range), + LogRange(y_range), + value_type, + series_data, + ), + } +} + +fn draw_line_comarision_figure, YR: AsRangedCoord>( + root_area: DrawingArea, + y_unit: &str, + x_range: XR, + y_range: YR, + value_type: ValueType, + data: Vec<(Option<&String>, Vec, Vec)>, +) { + let input_suffix = match value_type { + ValueType::Bytes => " Size (Bytes)", + ValueType::Elements => " Size (Elements)", + ValueType::Value => "", + }; + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .x_desc(format!("Input{}", input_suffix)) + .y_desc(format!("Average time ({})", y_unit)) + .draw() + .unwrap(); + + for (id, (name, xs, ys)) in (0..).zip(data.into_iter()) { + let series = chart + .draw_series( + LineSeries::new( + xs.into_iter().zip(ys.into_iter()), + COMPARISON_COLORS[id % NUM_COLORS].filled(), + ) + .point_size(POINT_SIZE), + ) + .unwrap(); + if let Some(name) = name { + series.label(name).legend(move |(x, y)| { + Rectangle::new( + [(x, y - 5), (x + 20, y + 5)], + COMPARISON_COLORS[id % NUM_COLORS].filled(), + ) + }); + } + } + + chart + .configure_series_labels() + .position(SeriesLabelPosition::UpperLeft) + .draw() + .unwrap(); +} + +#[allow(clippy::type_complexity)] +fn line_comparision_series_data<'a>( + formatter: &dyn ValueFormatter, + all_curves: &[&(&'a BenchmarkId, Vec)], +) -> (&'static str, Vec<(Option<&'a String>, Vec, Vec)>) { + let max = all_curves + .iter() + .map(|&&(_, ref data)| Sample::new(data).mean()) + .fold(::std::f64::NAN, f64::max); + + let mut dummy = [1.0]; + let unit = formatter.scale_values(max, &mut dummy); + + let mut series_data = vec![]; + + // This assumes the curves are sorted. It also assumes that the benchmark IDs all have numeric + // values or throughputs and that value is sensible (ie. not a mix of bytes and elements + // or whatnot) + for (key, group) in &all_curves.iter().group_by(|&&&(ref id, _)| &id.function_id) { + let mut tuples: Vec<_> = group + .map(|&&(ref id, ref sample)| { + // Unwrap is fine here because it will only fail if the assumptions above are not true + // ie. programmer error. + let x = id.as_number().unwrap(); + let y = Sample::new(sample).mean(); + + (x, y) + }) + .collect(); + tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less))); + let function_name = key.as_ref(); + let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); + formatter.scale_values(max, &mut ys); + series_data.push((function_name, xs, ys)); + } + (unit, series_data) +} + +pub fn violin( + formatter: &dyn ValueFormatter, + title: &str, + all_curves: &[&(&BenchmarkId, Vec)], + path: &str, + axis_scale: AxisScale, +) { + let all_curves_vec = all_curves.iter().rev().cloned().collect::>(); + let all_curves: &[&(&BenchmarkId, Vec)] = &*all_curves_vec; + + let mut kdes = all_curves + .iter() + .map(|&&(ref id, ref sample)| { + let (x, mut y) = kde::sweep(Sample::new(sample), KDE_POINTS, None); + let y_max = Sample::new(&y).max(); + for y in y.iter_mut() { + *y /= y_max; + } + + (id.as_title(), x, y) + }) + .collect::>(); + + let mut xs = kdes + .iter() + .flat_map(|&(_, ref x, _)| x.iter()) + .filter(|&&x| x > 0.); + let (mut min, mut max) = { + let &first = xs.next().unwrap(); + (first, first) + }; + for &e in xs { + if e < min { + min = e; + } else if e > max { + max = e; + } + } + let mut dummy = [1.0]; + let unit = formatter.scale_values(max, &mut dummy); + kdes.iter_mut().for_each(|&mut (_, ref mut xs, _)| { + formatter.scale_values(max, xs); + }); + + let x_range = plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten()); + let y_range = -0.5..all_curves.len() as f64 - 0.5; + + let size = (960, 150 + (18 * all_curves.len() as u32)); + + let root_area = SVGBackend::new(&path, size) + .into_drawing_area() + .titled(&format!("{}: Violin plot", title), (DEFAULT_FONT, 20)) + .unwrap(); + + match axis_scale { + AxisScale::Linear => draw_violin_figure(root_area, unit, x_range, y_range, kdes), + AxisScale::Logarithmic => { + draw_violin_figure(root_area, unit, LogRange(x_range), y_range, kdes) + } + } +} + +#[allow(clippy::type_complexity)] +fn draw_violin_figure, YR: AsRangedCoord>( + root_area: DrawingArea, + unit: &'static str, + x_range: XR, + y_range: YR, + data: Vec<(&str, Box<[f64]>, Box<[f64]>)>, +) { + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .set_label_area_size(LabelAreaPosition::Left, (10).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_width().min(40)) + .build_ranged(x_range, y_range) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Input") + .x_desc(format!("Average time ({})", unit)) + .y_label_style((DEFAULT_FONT, 10)) + .y_label_formatter(&|v: &f64| data[v.round() as usize].0.to_string()) + .y_labels(data.len()) + .draw() + .unwrap(); + + for (i, (_, x, y)) in data.into_iter().enumerate() { + let base = i as f64; + + chart + .draw_series(AreaSeries::new( + x.iter().zip(y.iter()).map(|(x, y)| (*x, base + *y / 2.0)), + base, + &DARK_BLUE.mix(0.25), + )) + .unwrap(); + + chart + .draw_series(AreaSeries::new( + x.iter().zip(y.iter()).map(|(x, y)| (*x, base - *y / 2.0)), + base, + &DARK_BLUE.mix(0.25), + )) + .unwrap(); + } +} diff --git a/src/plot/plotters_backend/t_test.rs b/src/plot/plotters_backend/t_test.rs new file mode 100644 index 000000000..741cc9ebf --- /dev/null +++ b/src/plot/plotters_backend/t_test.rs @@ -0,0 +1,59 @@ +use super::*; +use crate::report::ComparisonData; +use std::path::Path; + +pub(crate) fn t_test( + path: &Path, + title: &str, + comparison: &ComparisonData, + size: Option<(u32, u32)>, +) { + let t = comparison.t_value; + let (xs, ys) = kde::sweep(&comparison.t_distribution, KDE_POINTS, None); + + let x_range = plotters::data::fitting_range(xs.iter()); + let mut y_range = plotters::data::fitting_range(ys.iter()); + y_range.start = 0.0; + y_range.end *= 1.1; + + let root_area = SVGBackend::new(&path, size.unwrap_or(SIZE)).into_drawing_area(); + + let mut chart = ChartBuilder::on(&root_area) + .margin((5).percent()) + .caption(format!("{}: Welch t test", title), (DEFAULT_FONT, 20)) + .set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60)) + .set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40)) + .build_ranged(x_range, y_range.clone()) + .unwrap(); + + chart + .configure_mesh() + .disable_mesh() + .y_desc("Density") + .x_desc("t score") + .draw() + .unwrap(); + + chart + .draw_series(AreaSeries::new( + xs.iter().zip(ys.iter()).map(|(x, y)| (*x, *y)), + 0.0, + &DARK_BLUE.mix(0.25), + )) + .unwrap() + .label("t distribution") + .legend(|(x, y)| { + Rectangle::new([(x, y - 5), (x + 20, y + 5)], DARK_BLUE.mix(0.25).filled()) + }); + + chart + .draw_series(std::iter::once(PathElement::new( + vec![(t, 0.0), (t, y_range.end)], + DARK_BLUE.filled().stroke_width(2), + ))) + .unwrap() + .label("t statistic") + .legend(|(x, y)| PathElement::new(vec![(x, y), (x + 20, y)], &DARK_BLUE)); + + chart.configure_series_labels().draw().unwrap(); +}