From e0b6e21a7bc1b93abd6fe9a1d3bbb05c9b10ccd3 Mon Sep 17 00:00:00 2001 From: Brook Heisler Date: Sat, 13 Jan 2018 19:24:09 -0600 Subject: [PATCH] Add editorconfig file and standardize on unix-style line endings. --- .editorconfig | 8 + CHANGELOG.md | 98 ++++---- CONTRIBUTING.md | 92 +++---- benches/bench_main.rs | 40 +-- benches/compare_functions.rs | 72 +++--- benches/external_process.rs | 44 ++-- book/src/analysis.md | 110 ++++----- book/src/criterion_rs.md | 48 ++-- book/src/faq.md | 128 +++++----- book/src/getting_started.md | 334 +++++++++++++------------- src/macros.rs | 250 +++++++++---------- stats/benches/bivariate_bootstrap.rs | 116 ++++----- stats/benches/bivariate_regression.rs | 112 ++++----- stats/benches/common_bench.rs | 40 +-- stats/benches/univariate_bootstrap.rs | 84 +++---- stats/benches/univariate_kde.rs | 104 ++++---- stats/benches/univariate_sample.rs | 220 ++++++++--------- stats/src/lib.rs | 224 ++++++++--------- 18 files changed, 1066 insertions(+), 1058 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..8f7f66599 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 +indent_style = space diff --git a/CHANGELOG.md b/CHANGELOG.md index da92ce6d0..3d5febee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,50 +1,50 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) -and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -## [0.1.2] -### Changed -- Criterion.rs is now stable-compatible! -- Criterion.rs now includes its own stable-compatible `black_box` function. - Some benchmarks may now be affected by dead-code-elimination where they - previously weren't and may have to be updated. -- Criterion.rs now uses `serde` to save results. Existing results files will - be automatically removed when benchmarks are run. -- Redesigned the command-line output to highlight the important information - and reduce noise. - -### Added -- Running benchmarks with the variable "CRITERION_DEBUG" in the environment will - cause Criterion.rs to generate extra debug output and save the gnuplot scripts - alongside the generated plots. - -### Fixed -- Don't panic on IO errors or gnuplot failures -- Fix generation of invalid gnuplot scripts when benchmarking over inputs and inputs include values <= 0. -- Bug where benchmarks would run one sample fewer than was configured. - -### Removed -- Generated plots will no longer use log-scale. - -## [0.1.1] -### Added -- A changelog file. -- Added a chapter to the book on how Criterion.rs collects and analyzes data. -- Added macro rules to generate a test harness for use with `cargo bench`. - Benchmarks defined without these macros should continue to work. -- New contribution guidelines -- Criterion.rs can selectively run benchmarks. See the Command-line page for -more details - -## 0.1.0 - 2017-12-02 -### Added -- Initial release on Crates.io. - - -[Unreleased]: https://github.com/japaric/criterion.rs/compare/0.1.2...HEAD -[0.1.1]: https://github.com/japaric/criterion.rs/compare/0.1.0...0.1.1 +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.2] +### Changed +- Criterion.rs is now stable-compatible! +- Criterion.rs now includes its own stable-compatible `black_box` function. + Some benchmarks may now be affected by dead-code-elimination where they + previously weren't and may have to be updated. +- Criterion.rs now uses `serde` to save results. Existing results files will + be automatically removed when benchmarks are run. +- Redesigned the command-line output to highlight the important information + and reduce noise. + +### Added +- Running benchmarks with the variable "CRITERION_DEBUG" in the environment will + cause Criterion.rs to generate extra debug output and save the gnuplot scripts + alongside the generated plots. + +### Fixed +- Don't panic on IO errors or gnuplot failures +- Fix generation of invalid gnuplot scripts when benchmarking over inputs and inputs include values <= 0. +- Bug where benchmarks would run one sample fewer than was configured. + +### Removed +- Generated plots will no longer use log-scale. + +## [0.1.1] +### Added +- A changelog file. +- Added a chapter to the book on how Criterion.rs collects and analyzes data. +- Added macro rules to generate a test harness for use with `cargo bench`. + Benchmarks defined without these macros should continue to work. +- New contribution guidelines +- Criterion.rs can selectively run benchmarks. See the Command-line page for +more details + +## 0.1.0 - 2017-12-02 +### Added +- Initial release on Crates.io. + + +[Unreleased]: https://github.com/japaric/criterion.rs/compare/0.1.2...HEAD +[0.1.1]: https://github.com/japaric/criterion.rs/compare/0.1.0...0.1.1 [0.1.2]: https://github.com/japaric/criterion.rs/compare/0.1.1...0.1.2 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 255d4735b..b7119609c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,47 +1,47 @@ -# Contributing to Criterion.rs - -## Ideas, Experiences and Questions - -The easiest way to contribute to Criterion.rs is to use it and report your experiences, ask questions and contribute ideas. We'd love to hear your thoughts on how to make Criterion.rs better, or your comments on why you are or are not currently using it. - -Issues, ideas, requests and questions should be posted on the issue tracker at: - -https://github.com/japaric/criterion.rs/issues - -## Code - -Pull requests are welcome, though please raise an issue for discussion first if none exists. We're happy to assist new contributors. - -If you're not sure what to work on, try checking the [good first issue label](https://github.com/japaric/criterion.rs/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - -To make changes to the code, fork the repo and clone it: - -`git clone git@github.com:your-username/criterion.rs.git` - -You'll probably want to install [gnuplot](http://www.gnuplot.info/) as well. See the gnuplot website for installation instructions. - -Then make your changes to the code. When you're done, run the tests: - -``` -cargo test --all -cargo bench -``` - -It's a good idea to run clippy and fix any warnings as well: - -``` -cargo install clippy -cargo clippy --all -``` - -Don't forget to update the CHANGELOG.md file and any appropriate documentation. Once you're finished, push to your fork and submit a pull request. We try to respond to new issues and pull requests quickly, so if there hasn't been any response for more than a few days feel free to ping @bheisler. - -Some things that will increase the chance that your pull request is accepted: - -* Write tests -* Clearly document public methods -* Write a good commit message - -## Code of Conduct - +# Contributing to Criterion.rs + +## Ideas, Experiences and Questions + +The easiest way to contribute to Criterion.rs is to use it and report your experiences, ask questions and contribute ideas. We'd love to hear your thoughts on how to make Criterion.rs better, or your comments on why you are or are not currently using it. + +Issues, ideas, requests and questions should be posted on the issue tracker at: + +https://github.com/japaric/criterion.rs/issues + +## Code + +Pull requests are welcome, though please raise an issue for discussion first if none exists. We're happy to assist new contributors. + +If you're not sure what to work on, try checking the [good first issue label](https://github.com/japaric/criterion.rs/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + +To make changes to the code, fork the repo and clone it: + +`git clone git@github.com:your-username/criterion.rs.git` + +You'll probably want to install [gnuplot](http://www.gnuplot.info/) as well. See the gnuplot website for installation instructions. + +Then make your changes to the code. When you're done, run the tests: + +``` +cargo test --all +cargo bench +``` + +It's a good idea to run clippy and fix any warnings as well: + +``` +cargo install clippy +cargo clippy --all +``` + +Don't forget to update the CHANGELOG.md file and any appropriate documentation. Once you're finished, push to your fork and submit a pull request. We try to respond to new issues and pull requests quickly, so if there hasn't been any response for more than a few days feel free to ping @bheisler. + +Some things that will increase the chance that your pull request is accepted: + +* Write tests +* Clearly document public methods +* Write a good commit message + +## Code of Conduct + We follow the [Rust Code of Conduct](http://www.rust-lang.org/conduct.html). \ No newline at end of file diff --git a/benches/bench_main.rs b/benches/bench_main.rs index dbca24b32..42702cf78 100644 --- a/benches/bench_main.rs +++ b/benches/bench_main.rs @@ -1,21 +1,21 @@ -#[macro_use] -extern crate criterion; -extern crate walkdir; - -mod no_plots; -mod compare_functions; -mod external_process; -mod iter_with_large_drop; -mod iter_with_large_setup; -mod iter_with_setup; -mod with_inputs; - -criterion_main!{ - no_plots::benches, - compare_functions::fibonaccis, - external_process::benches, - iter_with_large_drop::benches, - iter_with_large_setup::benches, - iter_with_setup::benches, - with_inputs::benches +#[macro_use] +extern crate criterion; +extern crate walkdir; + +mod no_plots; +mod compare_functions; +mod external_process; +mod iter_with_large_drop; +mod iter_with_large_setup; +mod iter_with_setup; +mod with_inputs; + +criterion_main!{ + no_plots::benches, + compare_functions::fibonaccis, + external_process::benches, + iter_with_large_drop::benches, + iter_with_large_setup::benches, + iter_with_setup::benches, + with_inputs::benches } \ No newline at end of file diff --git a/benches/compare_functions.rs b/benches/compare_functions.rs index 5d38b0142..c8559fdac 100644 --- a/benches/compare_functions.rs +++ b/benches/compare_functions.rs @@ -1,37 +1,37 @@ -use criterion::Criterion; -use criterion::Fun; - -fn fibonacci_slow(n: u64) -> u64 { - match n { - 0 | 1 => 1, - n => fibonacci_slow(n-1) + fibonacci_slow(n-2), - } -} - -fn fibonacci_fast(n: u64) -> u64 { - let mut a = 0u64; - let mut b = 1u64; - let mut c : u64; - - if n == 0 { - return 0 - } - - for _ in 0..(n+1) { - c = a + b; - a = b; - b = c; - } - b -} - -fn compare_fibonaccis(c: &mut Criterion) { - let fib_slow = Fun::new("Recursive", |b, i| b.iter(|| fibonacci_slow(*i))); - let fib_fast = Fun::new("Iterative", |b, i| b.iter(|| fibonacci_fast(*i))); - - let functions = vec!(fib_slow, fib_fast); - - c.bench_functions("Fibonacci", functions, &20); -} - +use criterion::Criterion; +use criterion::Fun; + +fn fibonacci_slow(n: u64) -> u64 { + match n { + 0 | 1 => 1, + n => fibonacci_slow(n-1) + fibonacci_slow(n-2), + } +} + +fn fibonacci_fast(n: u64) -> u64 { + let mut a = 0u64; + let mut b = 1u64; + let mut c : u64; + + if n == 0 { + return 0 + } + + for _ in 0..(n+1) { + c = a + b; + a = b; + b = c; + } + b +} + +fn compare_fibonaccis(c: &mut Criterion) { + let fib_slow = Fun::new("Recursive", |b, i| b.iter(|| fibonacci_slow(*i))); + let fib_fast = Fun::new("Iterative", |b, i| b.iter(|| fibonacci_fast(*i))); + + let functions = vec!(fib_slow, fib_fast); + + c.bench_functions("Fibonacci", functions, &20); +} + criterion_group!(fibonaccis, compare_fibonaccis); \ No newline at end of file diff --git a/benches/external_process.rs b/benches/external_process.rs index f41c158e0..26d82b0fb 100644 --- a/benches/external_process.rs +++ b/benches/external_process.rs @@ -1,23 +1,23 @@ -use criterion::Criterion; -use std::process::Command; -use std::process::Stdio; - -fn create_command() -> Command { - let mut command = Command::new("python3"); - command.arg("benches/external_process.py"); - command -} - -fn python_fibonacci(c: &mut Criterion) { - let has_python3 = Command::new("python3") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .output().is_ok(); - - if has_python3 { - c.bench_program("fibonacci-python", create_command()); - } -} - +use criterion::Criterion; +use std::process::Command; +use std::process::Stdio; + +fn create_command() -> Command { + let mut command = Command::new("python3"); + command.arg("benches/external_process.py"); + command +} + +fn python_fibonacci(c: &mut Criterion) { + let has_python3 = Command::new("python3") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .output().is_ok(); + + if has_python3 { + c.bench_program("fibonacci-python", create_command()); + } +} + criterion_group!(benches, python_fibonacci); \ No newline at end of file diff --git a/book/src/analysis.md b/book/src/analysis.md index 507be00c5..335671cf1 100644 --- a/book/src/analysis.md +++ b/book/src/analysis.md @@ -1,56 +1,56 @@ -# Analysis Process # - -This page details the data collection and analysis process used by Criterion.rs. This is a bit more advanced than the user guide; it is assumed the reader is somewhat familiar with statistical concepts. In particular, the reader should know what bootstrap sampling means. - -So, without further ado, let's start with a general overview. Each benchmark in Criterion.rs goes through four phases: - -* Warmup - The routine is executed repeatedly to fill the CPU and OS caches and (if applicable) give the JIT time to compile the code -* Measurement - The routine is executed repeatedly and the execution times are recorded -* Analysis - The recorded samples are analyzed and distilled into meaningful statistics, which are then reported to the user -* Comparison - The performance of the current run is compared to the stored data from the last run to determine whether it has changed, and if so by how much - -## Warmup ## - -The first step in the process is warmup. In this phase, the routine is executed repeatedly to give the OS, CPU and JIT time to adapt to the new workload. This helps prevent things like cold caches and JIT compilation time from throwing off the measurements later. The warmup period is controlled by the `warm_up_time` value in the Criterion struct. - -The warmup period is quite simple. The routine is executed once, then twice, four times and so on until the total accumulated execution time is greater than the configured warm up time. The number of iterations that were completed during this period is recorded, along with the elapsed time. - -## Measurement ## - -The measurement phase is when Criterion.rs collects the performance data that will be analyzed and used in later stages. This phase is mainly controlled by the `measurement_time` value in the Criterion struct. - -The measurements are done in a number of samples (see the `sample_size` parameter). Each sample consists of one or more (typically many) iterations of the routine. The elapsed time between the beginning and the end of the iterations, divided by the number of iterations, gives an estimate of the time taken by each iteration. - -As measurement progresses, the sample iteration counts are increased. Suppose that the first sample contains 10 iterations. The second sample will contain 20, the third will contain 30 and so on. More formally, the iteration counts are calculated like so: - -`iterations = [d, 2d, 3d, ... Nd]` - -Where `N` is the total number of samples and `d` is a factor, calculated from the rough estimate of iteration time measured during the warmup period, which is used to scale the number of iterations to meet the configured measurement time. Note that `d` cannot be less than 1, and therefore the actual measurment time may exceed the configured measurement time if the iteration time is large or the configured measurement time is small. - -Note that Criterion.rs does not measure each individual iteration, only the complete sample. The resulting samples are stored for use in later stages. The sample data is also written to the local disk so that it can be used in the comparison phase of future benchmark runs. - -## Analysis ## - -During this phase Criterion.rs calculates useful statistics from the samples collected during the measurement phase. - -### Outlier Classification ### - -The first step in analysis is outlier classification. Each sample is classified using a modified version of Tukey's Method, which will be summarized here. First, the interquartile range (IQR) is calculated from the difference between the 25th and 75th percentile. In Tukey's Method, values less than (25th percentile - 1.5 * IQR) or greater than (75th percentile + 1.5 * IQR) are considered outliers. Criterion.rs creates additional fences at (25pct - 3 * IQR) and (75pct + 3 * IQR); values outside that range are considered severe outliers. - -Outlier classification is important because the analysis method used to estimate the average iteration time is sensitive to outliers. Thus, when Criterion.rs detects outliers, a warning is printed to inform the user that the benchmark may be less reliable. Additionally, a plot is generated showing which data points are considered outliers, where the fences are, etc. - -Note, however, that outlier samples are _not_ dropped from the data, and are used in the following analysis steps along with all other samples. - -### Linear Regression ### - -The samples collected from a good benchmark should form a rough line when plotted on a chart showing the number of iterations and the time for each sample. The slope of that line gives an estimate of the time per iteration. A single estimate is difficult to interpret, however, since it contains no context. A confidence interval is generally more helpful. In order to generate a confidence interval, a large number of bootstrap samples are generated from the measured samples. A line is fitted to each of the bootstrap samples, and the result is a statistical distribution of slopes that gives a reliable confidence interval around the single estimate calculated from the measured samples. - -This resampling process is repeated to generate the mean, standard deviation, median and median absolute deviation of the measured iteration times as well. All of this information is printed to the user and charts are generated. Finally, if there are saved statistics from a previous run, the two benchmark runs are compared. - -## Comparison ## - -In the comparison phase, the statistics calculated from the current benchmark run are compared against those saved by the previous run to determine if the performance has changed in the meantime, and if so, by how much. - -Once again, Criterion.rs generates many bootstrap samples, based on the measured samples from the two runs. The new and old bootstrap samples are compared and their T score is calculated using a T-test. The fraction of the bootstrapped T scores which are more extreme than the T score calculated by comparing the two measured samples gives the probability that the observed difference between the two sets of samples is merely by chance. Thus, if that probability is very low or zero, Criterion.rs can be confident that there is truly a difference in execution time between the two samples. In that case, the mean and median differences are bootstrapped and printed for the user, and the entire process begins again with the next benchmark. - +# Analysis Process # + +This page details the data collection and analysis process used by Criterion.rs. This is a bit more advanced than the user guide; it is assumed the reader is somewhat familiar with statistical concepts. In particular, the reader should know what bootstrap sampling means. + +So, without further ado, let's start with a general overview. Each benchmark in Criterion.rs goes through four phases: + +* Warmup - The routine is executed repeatedly to fill the CPU and OS caches and (if applicable) give the JIT time to compile the code +* Measurement - The routine is executed repeatedly and the execution times are recorded +* Analysis - The recorded samples are analyzed and distilled into meaningful statistics, which are then reported to the user +* Comparison - The performance of the current run is compared to the stored data from the last run to determine whether it has changed, and if so by how much + +## Warmup ## + +The first step in the process is warmup. In this phase, the routine is executed repeatedly to give the OS, CPU and JIT time to adapt to the new workload. This helps prevent things like cold caches and JIT compilation time from throwing off the measurements later. The warmup period is controlled by the `warm_up_time` value in the Criterion struct. + +The warmup period is quite simple. The routine is executed once, then twice, four times and so on until the total accumulated execution time is greater than the configured warm up time. The number of iterations that were completed during this period is recorded, along with the elapsed time. + +## Measurement ## + +The measurement phase is when Criterion.rs collects the performance data that will be analyzed and used in later stages. This phase is mainly controlled by the `measurement_time` value in the Criterion struct. + +The measurements are done in a number of samples (see the `sample_size` parameter). Each sample consists of one or more (typically many) iterations of the routine. The elapsed time between the beginning and the end of the iterations, divided by the number of iterations, gives an estimate of the time taken by each iteration. + +As measurement progresses, the sample iteration counts are increased. Suppose that the first sample contains 10 iterations. The second sample will contain 20, the third will contain 30 and so on. More formally, the iteration counts are calculated like so: + +`iterations = [d, 2d, 3d, ... Nd]` + +Where `N` is the total number of samples and `d` is a factor, calculated from the rough estimate of iteration time measured during the warmup period, which is used to scale the number of iterations to meet the configured measurement time. Note that `d` cannot be less than 1, and therefore the actual measurment time may exceed the configured measurement time if the iteration time is large or the configured measurement time is small. + +Note that Criterion.rs does not measure each individual iteration, only the complete sample. The resulting samples are stored for use in later stages. The sample data is also written to the local disk so that it can be used in the comparison phase of future benchmark runs. + +## Analysis ## + +During this phase Criterion.rs calculates useful statistics from the samples collected during the measurement phase. + +### Outlier Classification ### + +The first step in analysis is outlier classification. Each sample is classified using a modified version of Tukey's Method, which will be summarized here. First, the interquartile range (IQR) is calculated from the difference between the 25th and 75th percentile. In Tukey's Method, values less than (25th percentile - 1.5 * IQR) or greater than (75th percentile + 1.5 * IQR) are considered outliers. Criterion.rs creates additional fences at (25pct - 3 * IQR) and (75pct + 3 * IQR); values outside that range are considered severe outliers. + +Outlier classification is important because the analysis method used to estimate the average iteration time is sensitive to outliers. Thus, when Criterion.rs detects outliers, a warning is printed to inform the user that the benchmark may be less reliable. Additionally, a plot is generated showing which data points are considered outliers, where the fences are, etc. + +Note, however, that outlier samples are _not_ dropped from the data, and are used in the following analysis steps along with all other samples. + +### Linear Regression ### + +The samples collected from a good benchmark should form a rough line when plotted on a chart showing the number of iterations and the time for each sample. The slope of that line gives an estimate of the time per iteration. A single estimate is difficult to interpret, however, since it contains no context. A confidence interval is generally more helpful. In order to generate a confidence interval, a large number of bootstrap samples are generated from the measured samples. A line is fitted to each of the bootstrap samples, and the result is a statistical distribution of slopes that gives a reliable confidence interval around the single estimate calculated from the measured samples. + +This resampling process is repeated to generate the mean, standard deviation, median and median absolute deviation of the measured iteration times as well. All of this information is printed to the user and charts are generated. Finally, if there are saved statistics from a previous run, the two benchmark runs are compared. + +## Comparison ## + +In the comparison phase, the statistics calculated from the current benchmark run are compared against those saved by the previous run to determine if the performance has changed in the meantime, and if so, by how much. + +Once again, Criterion.rs generates many bootstrap samples, based on the measured samples from the two runs. The new and old bootstrap samples are compared and their T score is calculated using a T-test. The fraction of the bootstrapped T scores which are more extreme than the T score calculated by comparing the two measured samples gives the probability that the observed difference between the two sets of samples is merely by chance. Thus, if that probability is very low or zero, Criterion.rs can be confident that there is truly a difference in execution time between the two samples. In that case, the mean and median differences are bootstrapped and printed for the user, and the entire process begins again with the next benchmark. + This process can be extremely sensitive to changes, especially when combined with a small, highly deterministic benchmark routine. In these circumstances even very small changes (eg. differences in the load from background processes) can change the measurements enough that the comparison process detects an optimization or regression. Since these sorts of unpredictable fluctuations are rarely of interest while benchmarking, there is also a configurable noise threshold. Optimizations or regressions within (for example) +-1% are considered noise and ignored. It is best to benchmark on a quiet computer where possible to minimize this noise, but it is not always possible to eliminate it entirely. \ No newline at end of file diff --git a/book/src/criterion_rs.md b/book/src/criterion_rs.md index 3cfeb25c1..c03312b5c 100644 --- a/book/src/criterion_rs.md +++ b/book/src/criterion_rs.md @@ -1,25 +1,25 @@ -# Criterion.rs # - -Criterion.rs is a statistics-driven micro-benchmarking tool. It is a Rust port of [Haskell's Criterion](https://hackage.haskell.org/package/criterion) library. - -Criterion.rs benchmarks collect and store statistical information from run to run and can automatically detect performance regressions as well as measuring optimizations. - -Criterion.rs is free and open source. You can find the source on [GitHub](https://github.com/japaric/criterion.rs). Issues and feature requests can be posted on [the issue tracker](https://github.com/japaric/criterion.rs/issues). - -## API Docs ## - -In addition to this book, you may also wish to read [the API documentation](http://japaric.github.io/criterion.rs/criterion/). - -## License ## - -Criterion.rs is dual-licensed under the [Apache 2.0](https://github.com/japaric/criterion.rs/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/japaric/criterion.rs/blob/master/LICENSE-MIT) licenses. - -## Debug Output ## - -To enable debug output in Criterion.rs, define the environment variable `CRITERION_DEBUG`. For example (in bash): - -```bash -CRITERION_DEBUG=1 cargo bench -``` - +# Criterion.rs # + +Criterion.rs is a statistics-driven micro-benchmarking tool. It is a Rust port of [Haskell's Criterion](https://hackage.haskell.org/package/criterion) library. + +Criterion.rs benchmarks collect and store statistical information from run to run and can automatically detect performance regressions as well as measuring optimizations. + +Criterion.rs is free and open source. You can find the source on [GitHub](https://github.com/japaric/criterion.rs). Issues and feature requests can be posted on [the issue tracker](https://github.com/japaric/criterion.rs/issues). + +## API Docs ## + +In addition to this book, you may also wish to read [the API documentation](http://japaric.github.io/criterion.rs/criterion/). + +## License ## + +Criterion.rs is dual-licensed under the [Apache 2.0](https://github.com/japaric/criterion.rs/blob/master/LICENSE-APACHE) and the [MIT](https://github.com/japaric/criterion.rs/blob/master/LICENSE-MIT) licenses. + +## Debug Output ## + +To enable debug output in Criterion.rs, define the environment variable `CRITERION_DEBUG`. For example (in bash): + +```bash +CRITERION_DEBUG=1 cargo bench +``` + This will enable extra debug output. Criterion.rs will also save the gnuplot scripts alongside the generated plot files. When raising issues with Criterion.rs (especially when reporting issues with the plot generation) please run your benchmarks with this option enabled and provide the additional output and relevant gnuplot scripts. \ No newline at end of file diff --git a/book/src/faq.md b/book/src/faq.md index a60138f53..b3f4e1a9a 100644 --- a/book/src/faq.md +++ b/book/src/faq.md @@ -1,65 +1,65 @@ -## Frequently Asked Questions - -### How Should I Run Criterion.rs Benchmarks In A CI Pipeline? - -Criterion.rs benchmarks can be run as part of a CI pipeline just as they -normally would on the command line - simply run `cargo bench`. - -To compare the master branch to a pull request, you could run the benchmarks on -the master branch to set a baseline, then run them again with the pull request -branch. An example script for Travis-CI might be: - -```bash -#!/usr/bin/env bash - -if [ "${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" != "master" ] && [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then - REMOTE_URL="$(git config --get remote.origin.url)"; - cd ${TRAVIS_BUILD_DIR}/.. && \ - git clone ${REMOTE_URL} "${TRAVIS_REPO_SLUG}-bench" && \ - cd "${TRAVIS_REPO_SLUG}-bench" && \ - # Bench master - git checkout master && \ - cargo bench && \ - # Bench pull request - git checkout ${TRAVIS_COMMIT} && \ - cargo bench; -fi -``` - -(Thanks to [BeachApe](https://beachape.com/blog/2016/11/02/rust-performance-testing-on-travis-ci/) for the script on which this is based.) - -Note that cloud CI providers like Travis-CI and Appveyor introduce a great deal -of noise into the benchmarking process. For example, unpredictable load on the -physical hosts of their build VM's. Benchmarks measured on such services tend -to be unreliable, so you should be skeptical of the results. In particular, -benchmarks that detect performance regressions should not cause the build to -fail, and apparent performance regressions should be verified manually before -rejecting a pull request. - -### `cargo bench -- --verbose` Panics - -This occurs because the `libtest` benchmark harness implicitly added to your -crate is executing before the Criterion.rs benchmarks, and it panics when -presented with a command-line argument it doesn't expect. There are two ways to -work around this at present: - -You could run only your Criterion benchmark, like so: - -`cargo bench --bench my_benchmark -- --verbose` - -Note that `my_benchmark` here corresponds to the name of your benchmark in your -`Cargo.toml` file. - -Another option is to disable benchmarks for your lib or app crate. For example, -for library crates, you could add this to your `Cargo.toml` file: - -```toml -[lib] -bench = false -``` - -Of course, this only works if you define all of your benchmarks in the -`benches` directory. - -See [Rust Issue #47241](https://github.com/rust-lang/rust/issues/47241) for +## Frequently Asked Questions + +### How Should I Run Criterion.rs Benchmarks In A CI Pipeline? + +Criterion.rs benchmarks can be run as part of a CI pipeline just as they +normally would on the command line - simply run `cargo bench`. + +To compare the master branch to a pull request, you could run the benchmarks on +the master branch to set a baseline, then run them again with the pull request +branch. An example script for Travis-CI might be: + +```bash +#!/usr/bin/env bash + +if [ "${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" != "master" ] && [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then + REMOTE_URL="$(git config --get remote.origin.url)"; + cd ${TRAVIS_BUILD_DIR}/.. && \ + git clone ${REMOTE_URL} "${TRAVIS_REPO_SLUG}-bench" && \ + cd "${TRAVIS_REPO_SLUG}-bench" && \ + # Bench master + git checkout master && \ + cargo bench && \ + # Bench pull request + git checkout ${TRAVIS_COMMIT} && \ + cargo bench; +fi +``` + +(Thanks to [BeachApe](https://beachape.com/blog/2016/11/02/rust-performance-testing-on-travis-ci/) for the script on which this is based.) + +Note that cloud CI providers like Travis-CI and Appveyor introduce a great deal +of noise into the benchmarking process. For example, unpredictable load on the +physical hosts of their build VM's. Benchmarks measured on such services tend +to be unreliable, so you should be skeptical of the results. In particular, +benchmarks that detect performance regressions should not cause the build to +fail, and apparent performance regressions should be verified manually before +rejecting a pull request. + +### `cargo bench -- --verbose` Panics + +This occurs because the `libtest` benchmark harness implicitly added to your +crate is executing before the Criterion.rs benchmarks, and it panics when +presented with a command-line argument it doesn't expect. There are two ways to +work around this at present: + +You could run only your Criterion benchmark, like so: + +`cargo bench --bench my_benchmark -- --verbose` + +Note that `my_benchmark` here corresponds to the name of your benchmark in your +`Cargo.toml` file. + +Another option is to disable benchmarks for your lib or app crate. For example, +for library crates, you could add this to your `Cargo.toml` file: + +```toml +[lib] +bench = false +``` + +Of course, this only works if you define all of your benchmarks in the +`benches` directory. + +See [Rust Issue #47241](https://github.com/rust-lang/rust/issues/47241) for more details. \ No newline at end of file diff --git a/book/src/getting_started.md b/book/src/getting_started.md index 00ee934c9..45c1ddce1 100644 --- a/book/src/getting_started.md +++ b/book/src/getting_started.md @@ -1,168 +1,168 @@ -# Getting Started # - -### Step 1 - Add Dependency to cargo.toml ### - -To enable Criterion.rs benchmarks, add the following to your `cargo.toml` file: - -```toml -[dev-dependencies] -criterion = "0.1.2" - -[[bench]] -name = "my_benchmark" -harness = false -``` - -This adds a development dependency on Criterion.rs, and declares a benchmark called `my_benchmark` without the standard benchmarking harness. It's important to disable the standard benchmark harness, because we'll later add our own and we don't want them to conflict. - -### Step 2 - Add Benchmark ### - -As an example, we'll benchmark an implementation of the Fibonacci function. Create a benchmark file at `$PROJECT/benches/my_benchmark.rs` with the following contents (see the Details section below for an explanation of this code): - -```rust -#[macro_use] -extern crate criterion; - -use criterion::Criterion; - -fn fibonacci(n: u64) -> u64 { - match n { - 0 => 1, - 1 => 1, - n => fibonacci(n-1) + fibonacci(n-2), - } -} - -fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("fib 20", |b| b.iter(|| fibonacci(20))); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); -``` - -### Step 3 - Run Benchmark ### - -To run this benchmark, use the following command: - -`cargo bench` - -You should see output similar to this: - -``` - Running target/release/deps/example-423eedc43b2b3a93 -Benchmarking fib 20 -Benchmarking fib 20: Warming up for 3.0000 s -Benchmarking fib 20: Collecting 100 samples in estimated 5.0658 s (188100 iterations) -Benchmarking fib 20: Analyzing -fib 20 time: [26.029 us 26.251 us 26.505 us] -Found 11 outliers among 99 measurements (11.11%) - 6 (6.06%) high mild - 5 (5.05%) high severe -slope [26.029 us 26.505 us] R^2 [0.8745662 0.8728027] -mean [26.106 us 26.561 us] std. dev. [808.98 ns 1.4722 us] -median [25.733 us 25.988 us] med. abs. dev. [234.09 ns 544.07 ns] -``` - -### Details ### - -Let's go back and walk through that benchmark code in more detail. - -```rust -#[macro use] -extern crate criterion; - -use criterion::Criterion; -``` - -First, we declare the criterion crate and import the [Criterion type](http://japaric.github.io/criterion.rs/criterion/struct.Criterion.html). Criterion is the main type for the Criterion.rs library. It provides methods to configure and define groups of benchmarks. - -```rust -fn fibonacci(n: u64) -> u64 { - match n { - 0 => 1, - 1 => 1, - n => fibonacci(n-1) + fibonacci(n-2), - } -} -``` - -Second, we define the function to benchmark. In normal usage, this would be imported from elsewhere in your crate, but for simplicity we'll just define it right here. - -```rust -fn criterion_benchmark(c: &mut Criterion) { -``` - -Here we create a function to contain our benchmark code. The name of the benchmark function doesn't matter, but it should be clear and understandable. - -```rust - c.bench_function("fib 20", |b| b.iter(|| fibonacci(20))); -} -``` - -This is where the real work happens. The `bench_function` method defines a benchmark with a name and a closure. The name should be unique among all of the benchmarks for your project. The closure must accept one argument, a [Bencher](http://japaric.github.io/criterion.rs/criterion/struct.Bencher.html). The bencher performs the benchmark - in this case, it simply calls our `fibonacci` function in a loop. There are a number of other benchmark functions, including the option to benchmark with arguments, to benchmark external programs and to compare the performance of two functions. See the API documentation for details on all of the different benchmarking options. - -```rust -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); -``` - -Here we invoke the `criterion_group!` [(link)](http://japaric.github.io/criterion.rs/criterion/macro.criterion_group.html) macro to generate a benchmark group called benches, containing the `criterion_benchmark` function defined earlier. Finally, we invoke the `criterion_main!` [(link)](http://japaric.github.io/criterion.rs/criterion/macro.criterion_main.html) macro to generate a main function which executes the `benches` group. See the API documentation for more information on these macros. - -### Step 4 - Optimize ### - -This fibonacci function is quite inefficient. We can do better: - -```rust -fn fibonacci(n: u64) -> u64 { - let mut a = 0u64; - let mut b = 1u64; - let mut c = 0u64; - - if n == 0 { - return 0 - } - - for _ in 0..(n+1) { - c = a + b; - a = b; - b = c; - } - return b; -} -``` - -Running the benchmark now produces output like this: - -``` - Running target/release/deps/example-423eedc43b2b3a93 -Benchmarking fib 20 -Benchmarking fib 20: Warming up for 3.0000 s -Benchmarking fib 20: Collecting 100 samples in estimated 5.0000 s (13548862800 iterations) -Benchmarking fib 20: Analyzing -fib 20 time: [353.59 ps 356.19 ps 359.07 ps] - change: [-99.999% -99.999% -99.999%] (p = 0.00 < 0.05) - Performance has improved. -Found 6 outliers among 99 measurements (6.06%) - 4 (4.04%) high mild - 2 (2.02%) high severe -slope [353.59 ps 359.07 ps] R^2 [0.8734356 0.8722124] -mean [356.57 ps 362.74 ps] std. dev. [10.672 ps 20.419 ps] -median [351.57 ps 355.85 ps] med. abs. dev. [4.6479 ps 10.059 ps] -``` - -As you can see, Criterion is statistically confident that our optimization has made an improvement. If we introduce a performance regression, Criterion will instead print a message indicating this. - -### Known Limitations ### - -There are currently a number of limitations to the use of Criterion.rs relative to the standard benchmark harness. - -First, it is necessary for Criterion.rs to provide its own `main` function using the `criterion_main` macro. This means that it's not currently possible to include benchmarks in the `src/` directory. - -Second, Criterion.rs provides a stable-compatible replacement for the `black_box` function provided by the standard test crate. This replacement is not as reliable as the official one, and it may allow dead-code-elimination to affect the benchmarks in some circumstances. If you're using a Nightly build of Rust, you can add the `real_blackbox` feature to your dependency on Criterion.rs to use the standard `black_box` function instead. - -Example: - -```toml -criterion = { version = '...', features=['real_blackbox'] } +# Getting Started # + +### Step 1 - Add Dependency to cargo.toml ### + +To enable Criterion.rs benchmarks, add the following to your `cargo.toml` file: + +```toml +[dev-dependencies] +criterion = "0.1.2" + +[[bench]] +name = "my_benchmark" +harness = false +``` + +This adds a development dependency on Criterion.rs, and declares a benchmark called `my_benchmark` without the standard benchmarking harness. It's important to disable the standard benchmark harness, because we'll later add our own and we don't want them to conflict. + +### Step 2 - Add Benchmark ### + +As an example, we'll benchmark an implementation of the Fibonacci function. Create a benchmark file at `$PROJECT/benches/my_benchmark.rs` with the following contents (see the Details section below for an explanation of this code): + +```rust +#[macro_use] +extern crate criterion; + +use criterion::Criterion; + +fn fibonacci(n: u64) -> u64 { + match n { + 0 => 1, + 1 => 1, + n => fibonacci(n-1) + fibonacci(n-2), + } +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("fib 20", |b| b.iter(|| fibonacci(20))); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); +``` + +### Step 3 - Run Benchmark ### + +To run this benchmark, use the following command: + +`cargo bench` + +You should see output similar to this: + +``` + Running target/release/deps/example-423eedc43b2b3a93 +Benchmarking fib 20 +Benchmarking fib 20: Warming up for 3.0000 s +Benchmarking fib 20: Collecting 100 samples in estimated 5.0658 s (188100 iterations) +Benchmarking fib 20: Analyzing +fib 20 time: [26.029 us 26.251 us 26.505 us] +Found 11 outliers among 99 measurements (11.11%) + 6 (6.06%) high mild + 5 (5.05%) high severe +slope [26.029 us 26.505 us] R^2 [0.8745662 0.8728027] +mean [26.106 us 26.561 us] std. dev. [808.98 ns 1.4722 us] +median [25.733 us 25.988 us] med. abs. dev. [234.09 ns 544.07 ns] +``` + +### Details ### + +Let's go back and walk through that benchmark code in more detail. + +```rust +#[macro use] +extern crate criterion; + +use criterion::Criterion; +``` + +First, we declare the criterion crate and import the [Criterion type](http://japaric.github.io/criterion.rs/criterion/struct.Criterion.html). Criterion is the main type for the Criterion.rs library. It provides methods to configure and define groups of benchmarks. + +```rust +fn fibonacci(n: u64) -> u64 { + match n { + 0 => 1, + 1 => 1, + n => fibonacci(n-1) + fibonacci(n-2), + } +} +``` + +Second, we define the function to benchmark. In normal usage, this would be imported from elsewhere in your crate, but for simplicity we'll just define it right here. + +```rust +fn criterion_benchmark(c: &mut Criterion) { +``` + +Here we create a function to contain our benchmark code. The name of the benchmark function doesn't matter, but it should be clear and understandable. + +```rust + c.bench_function("fib 20", |b| b.iter(|| fibonacci(20))); +} +``` + +This is where the real work happens. The `bench_function` method defines a benchmark with a name and a closure. The name should be unique among all of the benchmarks for your project. The closure must accept one argument, a [Bencher](http://japaric.github.io/criterion.rs/criterion/struct.Bencher.html). The bencher performs the benchmark - in this case, it simply calls our `fibonacci` function in a loop. There are a number of other benchmark functions, including the option to benchmark with arguments, to benchmark external programs and to compare the performance of two functions. See the API documentation for details on all of the different benchmarking options. + +```rust +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); +``` + +Here we invoke the `criterion_group!` [(link)](http://japaric.github.io/criterion.rs/criterion/macro.criterion_group.html) macro to generate a benchmark group called benches, containing the `criterion_benchmark` function defined earlier. Finally, we invoke the `criterion_main!` [(link)](http://japaric.github.io/criterion.rs/criterion/macro.criterion_main.html) macro to generate a main function which executes the `benches` group. See the API documentation for more information on these macros. + +### Step 4 - Optimize ### + +This fibonacci function is quite inefficient. We can do better: + +```rust +fn fibonacci(n: u64) -> u64 { + let mut a = 0u64; + let mut b = 1u64; + let mut c = 0u64; + + if n == 0 { + return 0 + } + + for _ in 0..(n+1) { + c = a + b; + a = b; + b = c; + } + return b; +} +``` + +Running the benchmark now produces output like this: + +``` + Running target/release/deps/example-423eedc43b2b3a93 +Benchmarking fib 20 +Benchmarking fib 20: Warming up for 3.0000 s +Benchmarking fib 20: Collecting 100 samples in estimated 5.0000 s (13548862800 iterations) +Benchmarking fib 20: Analyzing +fib 20 time: [353.59 ps 356.19 ps 359.07 ps] + change: [-99.999% -99.999% -99.999%] (p = 0.00 < 0.05) + Performance has improved. +Found 6 outliers among 99 measurements (6.06%) + 4 (4.04%) high mild + 2 (2.02%) high severe +slope [353.59 ps 359.07 ps] R^2 [0.8734356 0.8722124] +mean [356.57 ps 362.74 ps] std. dev. [10.672 ps 20.419 ps] +median [351.57 ps 355.85 ps] med. abs. dev. [4.6479 ps 10.059 ps] +``` + +As you can see, Criterion is statistically confident that our optimization has made an improvement. If we introduce a performance regression, Criterion will instead print a message indicating this. + +### Known Limitations ### + +There are currently a number of limitations to the use of Criterion.rs relative to the standard benchmark harness. + +First, it is necessary for Criterion.rs to provide its own `main` function using the `criterion_main` macro. This means that it's not currently possible to include benchmarks in the `src/` directory. + +Second, Criterion.rs provides a stable-compatible replacement for the `black_box` function provided by the standard test crate. This replacement is not as reliable as the official one, and it may allow dead-code-elimination to affect the benchmarks in some circumstances. If you're using a Nightly build of Rust, you can add the `real_blackbox` feature to your dependency on Criterion.rs to use the standard `black_box` function instead. + +Example: + +```toml +criterion = { version = '...', features=['real_blackbox'] } ``` \ No newline at end of file diff --git a/src/macros.rs b/src/macros.rs index d587a2b46..4629d2caa 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,126 +1,126 @@ -//! Contains macros which together define a benchmark harness that can be used -//! in place of the standard benchmark harness. This allows the user to run -//! Criterion.rs benchmarks with `cargo bench`. - -/// Macro used to define a benchmark group for the benchmark harness; see the -/// criterion! macro for more details. -/// -/// This is used to define a benchmark group; a collection of related benchmarks -/// which share a common configuration. Accepts two forms which can be seen -/// below. -/// -/// # Examples: -/// -/// Complete form: -/// -/// ``` -/// # #[macro_use] -/// # extern crate criterion; -/// # use criterion::Criterion; -/// # fn bench_method1(c: &mut Criterion) { -/// # } -/// # -/// # fn bench_method2(c: &mut Criterion) { -/// # } -/// # -/// criterion_group!{ -/// name = benches; -/// config = Criterion::default(); -/// targets = bench_method1, bench_method2 -/// } -/// # -/// # fn main() {} -/// ``` -/// -/// In this form, all of the options are clearly spelled out. This expands to -/// a function named benches, which uses the given config expression to create -/// an instance of the Criterion struct. This is then passed by mutable -/// reference to the targets. -/// -/// Compact Form: -/// -/// ``` -/// # #[macro_use] -/// # extern crate criterion; -/// # use criterion::Criterion; -/// # fn bench_method1(c: &mut Criterion) { -/// # } -/// # -/// # fn bench_method2(c: &mut Criterion) { -/// # } -/// # -/// criterion_group!(benches, bench_method1, bench_method2); -/// # -/// # fn main() {} -/// ``` -/// In this form, the first parameter is the name of the group and subsequent -/// parameters are the target methods. The Criterion struct will be created using -/// the `Criterion::default()` function. If you wish to customize the -/// configuration, use the complete form and provide your own configuration -/// function. -#[macro_export] -macro_rules! criterion_group { - (name = $name:ident; config = $config:expr; targets = $( $target:path ),+ ) => { - pub fn $name() { - $( - let mut criterion: Criterion = $config; - criterion.configure_from_args(); - $target(&mut criterion); - )+ - } - }; - ($name:ident, $( $target:path ),+ ) => { - criterion_group!{ - name = $name; - config = Criterion::default(); - targets = $( $target ),+ - } - } -} - -/// Macro which expands to a benchmark harness. -/// -/// Currently, using Criterion.rs requires disabling the benchmark harness -/// generated automatically by rustc. This can be done like so: -/// -/// ```toml -/// [[bench]] -/// name = "my_bench" -/// harness = false -/// ``` -/// -/// In this case, `my_bench` must be a rust file inside the 'benches' directory, -/// like so: -/// -/// `benches/my_bench.rs` -/// -/// Since we've disabled the default benchmark harness, we need to add our own: -/// -/// ```ignore -/// #[macro_use] -/// extern crate criterion; -/// use criterion::Criterion; -/// fn bench_method1(c: &mut Criterion) { -/// } -/// -/// fn bench_method2(c: &mut Criterion) { -/// } -/// -/// criterion_group!(benches, bench_method1, bench_method2); -/// criterion_main!(benches); -/// ``` -/// -/// The `criterion_main` macro expands to a `main` function which runs all of the -/// benchmarks in the given groups. -/// -#[macro_export] -macro_rules! criterion_main { - ( $( $group:path ),+ ) => { - fn main() { - criterion::init_logging(); - $( - $group(); - )+ - } - } +//! Contains macros which together define a benchmark harness that can be used +//! in place of the standard benchmark harness. This allows the user to run +//! Criterion.rs benchmarks with `cargo bench`. + +/// Macro used to define a benchmark group for the benchmark harness; see the +/// criterion! macro for more details. +/// +/// This is used to define a benchmark group; a collection of related benchmarks +/// which share a common configuration. Accepts two forms which can be seen +/// below. +/// +/// # Examples: +/// +/// Complete form: +/// +/// ``` +/// # #[macro_use] +/// # extern crate criterion; +/// # use criterion::Criterion; +/// # fn bench_method1(c: &mut Criterion) { +/// # } +/// # +/// # fn bench_method2(c: &mut Criterion) { +/// # } +/// # +/// criterion_group!{ +/// name = benches; +/// config = Criterion::default(); +/// targets = bench_method1, bench_method2 +/// } +/// # +/// # fn main() {} +/// ``` +/// +/// In this form, all of the options are clearly spelled out. This expands to +/// a function named benches, which uses the given config expression to create +/// an instance of the Criterion struct. This is then passed by mutable +/// reference to the targets. +/// +/// Compact Form: +/// +/// ``` +/// # #[macro_use] +/// # extern crate criterion; +/// # use criterion::Criterion; +/// # fn bench_method1(c: &mut Criterion) { +/// # } +/// # +/// # fn bench_method2(c: &mut Criterion) { +/// # } +/// # +/// criterion_group!(benches, bench_method1, bench_method2); +/// # +/// # fn main() {} +/// ``` +/// In this form, the first parameter is the name of the group and subsequent +/// parameters are the target methods. The Criterion struct will be created using +/// the `Criterion::default()` function. If you wish to customize the +/// configuration, use the complete form and provide your own configuration +/// function. +#[macro_export] +macro_rules! criterion_group { + (name = $name:ident; config = $config:expr; targets = $( $target:path ),+ ) => { + pub fn $name() { + $( + let mut criterion: Criterion = $config; + criterion.configure_from_args(); + $target(&mut criterion); + )+ + } + }; + ($name:ident, $( $target:path ),+ ) => { + criterion_group!{ + name = $name; + config = Criterion::default(); + targets = $( $target ),+ + } + } +} + +/// Macro which expands to a benchmark harness. +/// +/// Currently, using Criterion.rs requires disabling the benchmark harness +/// generated automatically by rustc. This can be done like so: +/// +/// ```toml +/// [[bench]] +/// name = "my_bench" +/// harness = false +/// ``` +/// +/// In this case, `my_bench` must be a rust file inside the 'benches' directory, +/// like so: +/// +/// `benches/my_bench.rs` +/// +/// Since we've disabled the default benchmark harness, we need to add our own: +/// +/// ```ignore +/// #[macro_use] +/// extern crate criterion; +/// use criterion::Criterion; +/// fn bench_method1(c: &mut Criterion) { +/// } +/// +/// fn bench_method2(c: &mut Criterion) { +/// } +/// +/// criterion_group!(benches, bench_method1, bench_method2); +/// criterion_main!(benches); +/// ``` +/// +/// The `criterion_main` macro expands to a `main` function which runs all of the +/// benchmarks in the given groups. +/// +#[macro_export] +macro_rules! criterion_main { + ( $( $group:path ),+ ) => { + fn main() { + criterion::init_logging(); + $( + $group(); + )+ + } + } } \ No newline at end of file diff --git a/stats/benches/bivariate_bootstrap.rs b/stats/benches/bivariate_bootstrap.rs index dea1c39de..26c2ca468 100644 --- a/stats/benches/bivariate_bootstrap.rs +++ b/stats/benches/bivariate_bootstrap.rs @@ -1,59 +1,59 @@ -#[macro_use] -extern crate criterion; -extern crate criterion_stats as stats; -extern crate rand; - -use criterion::Criterion; - -mod common_bench; - -macro_rules! bench { - ($ty:ident) => { - pub mod $ty { - use stats::bivariate::Data; - use stats::bivariate::regression::{Slope, StraightLine}; - use criterion::Criterion; - - const NRESAMPLES: usize = 100_000; - const SAMPLE_SIZE: usize = 100; - - pub fn straight_line(c: &mut Criterion) { - let x = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); - let y = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); - let data = Data::new(&x, &y); - - c.bench_function( - &format!("bivariate_bootstrap_straight_line_{}", stringify!($ty)), - |b| b.iter(|| { - data.bootstrap(NRESAMPLES, |d| (StraightLine::fit(d),)) - }) - ); - } - - pub fn slope(c: &mut Criterion) { - let x = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); - let y = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); - let data = Data::new(&x, &y); - - c.bench_function( - &format!("bivariate_bootstrap_slope_{}", stringify!($ty)), - |b| b.iter(|| { - data.bootstrap(NRESAMPLES, |d| (Slope::fit(d),)) - }) - ); - } - } - } -} - -mod bench { - bench!(f32); - bench!(f64); -} - -criterion_group!( - name = benches; - config = common_bench::reduced_samples(); - targets = bench::f32::slope, bench::f32::straight_line, - bench::f64::slope, bench::f64::straight_line); +#[macro_use] +extern crate criterion; +extern crate criterion_stats as stats; +extern crate rand; + +use criterion::Criterion; + +mod common_bench; + +macro_rules! bench { + ($ty:ident) => { + pub mod $ty { + use stats::bivariate::Data; + use stats::bivariate::regression::{Slope, StraightLine}; + use criterion::Criterion; + + const NRESAMPLES: usize = 100_000; + const SAMPLE_SIZE: usize = 100; + + pub fn straight_line(c: &mut Criterion) { + let x = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); + let y = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); + let data = Data::new(&x, &y); + + c.bench_function( + &format!("bivariate_bootstrap_straight_line_{}", stringify!($ty)), + |b| b.iter(|| { + data.bootstrap(NRESAMPLES, |d| (StraightLine::fit(d),)) + }) + ); + } + + pub fn slope(c: &mut Criterion) { + let x = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); + let y = ::common_bench::vec_sized::(SAMPLE_SIZE).unwrap(); + let data = Data::new(&x, &y); + + c.bench_function( + &format!("bivariate_bootstrap_slope_{}", stringify!($ty)), + |b| b.iter(|| { + data.bootstrap(NRESAMPLES, |d| (Slope::fit(d),)) + }) + ); + } + } + } +} + +mod bench { + bench!(f32); + bench!(f64); +} + +criterion_group!( + name = benches; + config = common_bench::reduced_samples(); + targets = bench::f32::slope, bench::f32::straight_line, + bench::f64::slope, bench::f64::straight_line); criterion_main!(benches); \ No newline at end of file diff --git a/stats/benches/bivariate_regression.rs b/stats/benches/bivariate_regression.rs index 0abb701de..ba86de078 100644 --- a/stats/benches/bivariate_regression.rs +++ b/stats/benches/bivariate_regression.rs @@ -1,57 +1,57 @@ -#[macro_use] -extern crate criterion; -extern crate criterion_stats as stats; -extern crate rand; - -mod common_bench; - -use criterion::Criterion; - -macro_rules! bench { - ($ty:ident) => { - pub mod $ty { - use stats::bivariate::regression::{Slope, StraightLine}; - use stats::bivariate::Data; - use criterion::Criterion; - - pub fn slope(c: &mut Criterion) { - let x = ::common_bench::vec::<$ty>(); - let y = ::common_bench::vec(); - let data = Data::new(&x, &y); - - c.bench_function( - &format!("bivariate_regression_slope_{}", stringify!($ty)), - |b| b.iter(|| { - Slope::fit(data) - }) - ); - - } - - pub fn straight_line(c: &mut Criterion) { - let x = ::common_bench::vec::<$ty>(); - let y = ::common_bench::vec(); - let data = Data::new(&x, &y); - - c.bench_function( - &format!("bivariate_regression_straight_line_{}", stringify!($ty)), - |b| b.iter(|| { - StraightLine::fit(data) - }) - ); - } - } - } -} - -mod bench { - bench!(f32); - bench!(f64); -} - -criterion_group!( - name = benches; - config = common_bench::reduced_samples(); - targets = bench::f32::slope, bench::f32::straight_line, - bench::f64::slope, bench::f64::straight_line); +#[macro_use] +extern crate criterion; +extern crate criterion_stats as stats; +extern crate rand; + +mod common_bench; + +use criterion::Criterion; + +macro_rules! bench { + ($ty:ident) => { + pub mod $ty { + use stats::bivariate::regression::{Slope, StraightLine}; + use stats::bivariate::Data; + use criterion::Criterion; + + pub fn slope(c: &mut Criterion) { + let x = ::common_bench::vec::<$ty>(); + let y = ::common_bench::vec(); + let data = Data::new(&x, &y); + + c.bench_function( + &format!("bivariate_regression_slope_{}", stringify!($ty)), + |b| b.iter(|| { + Slope::fit(data) + }) + ); + + } + + pub fn straight_line(c: &mut Criterion) { + let x = ::common_bench::vec::<$ty>(); + let y = ::common_bench::vec(); + let data = Data::new(&x, &y); + + c.bench_function( + &format!("bivariate_regression_straight_line_{}", stringify!($ty)), + |b| b.iter(|| { + StraightLine::fit(data) + }) + ); + } + } + } +} + +mod bench { + bench!(f32); + bench!(f64); +} + +criterion_group!( + name = benches; + config = common_bench::reduced_samples(); + targets = bench::f32::slope, bench::f32::straight_line, + bench::f64::slope, bench::f64::straight_line); criterion_main!(benches); \ No newline at end of file diff --git a/stats/benches/common_bench.rs b/stats/benches/common_bench.rs index cd708cfa1..c796726e4 100644 --- a/stats/benches/common_bench.rs +++ b/stats/benches/common_bench.rs @@ -1,21 +1,21 @@ -#![allow(dead_code)] -use rand::{Rand, thread_rng, XorShiftRng, Rng}; -use criterion::Criterion; - -pub fn vec() -> Vec where T: Rand { - const SIZE: usize = 1_000_000; - - vec_sized(SIZE).unwrap() -} - -pub fn vec_sized(size: usize) -> Option> where T: Rand { - let mut rng: XorShiftRng = thread_rng().gen(); - - Some((0..size).map(|_| rng.gen()).collect()) -} - -pub fn reduced_samples() -> Criterion { - let mut c = Criterion::default(); - c.sample_size(20); - c +#![allow(dead_code)] +use rand::{Rand, thread_rng, XorShiftRng, Rng}; +use criterion::Criterion; + +pub fn vec() -> Vec where T: Rand { + const SIZE: usize = 1_000_000; + + vec_sized(SIZE).unwrap() +} + +pub fn vec_sized(size: usize) -> Option> where T: Rand { + let mut rng: XorShiftRng = thread_rng().gen(); + + Some((0..size).map(|_| rng.gen()).collect()) +} + +pub fn reduced_samples() -> Criterion { + let mut c = Criterion::default(); + c.sample_size(20); + c } \ No newline at end of file diff --git a/stats/benches/univariate_bootstrap.rs b/stats/benches/univariate_bootstrap.rs index d13151071..23318bff8 100644 --- a/stats/benches/univariate_bootstrap.rs +++ b/stats/benches/univariate_bootstrap.rs @@ -1,43 +1,43 @@ -#[macro_use] -extern crate criterion; -extern crate criterion_stats as stats; -extern crate rand; - -mod common_bench; - -use criterion::Criterion; - -macro_rules! bench { - ($ty:ident) => { - pub mod $ty { - use stats::univariate::Sample; - use criterion::Criterion; - - const NRESAMPLES: usize = 100_000; - const SAMPLE_SIZE: usize = 100; - - pub fn mean(c: &mut Criterion) { - let v = ::common_bench::vec_sized::<$ty>(SAMPLE_SIZE).unwrap(); - let sample = Sample::new(&v); - - c.bench_function( - &format!("univariate_bootstrap_mean_{}", stringify!($ty)), - |b| b.iter(|| { - sample.bootstrap(NRESAMPLES, |s| (s.mean(),)) - }) - ); - } - } - } -} - -mod bench { - bench!(f32); - bench!(f64); -} - -criterion_group!( - name = benches; - config = common_bench::reduced_samples(); - targets = bench::f32::mean, bench::f64::mean); +#[macro_use] +extern crate criterion; +extern crate criterion_stats as stats; +extern crate rand; + +mod common_bench; + +use criterion::Criterion; + +macro_rules! bench { + ($ty:ident) => { + pub mod $ty { + use stats::univariate::Sample; + use criterion::Criterion; + + const NRESAMPLES: usize = 100_000; + const SAMPLE_SIZE: usize = 100; + + pub fn mean(c: &mut Criterion) { + let v = ::common_bench::vec_sized::<$ty>(SAMPLE_SIZE).unwrap(); + let sample = Sample::new(&v); + + c.bench_function( + &format!("univariate_bootstrap_mean_{}", stringify!($ty)), + |b| b.iter(|| { + sample.bootstrap(NRESAMPLES, |s| (s.mean(),)) + }) + ); + } + } + } +} + +mod bench { + bench!(f32); + bench!(f64); +} + +criterion_group!( + name = benches; + config = common_bench::reduced_samples(); + targets = bench::f32::mean, bench::f64::mean); criterion_main!(benches); \ No newline at end of file diff --git a/stats/benches/univariate_kde.rs b/stats/benches/univariate_kde.rs index d2763a459..448eebe14 100644 --- a/stats/benches/univariate_kde.rs +++ b/stats/benches/univariate_kde.rs @@ -1,53 +1,53 @@ -#[macro_use] -extern crate criterion; -extern crate criterion_stats as stats; -extern crate itertools; -extern crate rand; - -mod common_bench; - -macro_rules! bench { - ($ty:ident) => { - pub mod $ty { - use stats::univariate::Sample; - use stats::univariate::kde::kernel::Gaussian; - use stats::univariate::kde::{Bandwidth, Kde}; - use criterion::Criterion; - - const KDE_POINTS: usize = 100; - const SAMPLE_SIZE: usize = 100_000; - - fn call(c: &mut Criterion) { - let data = ::common_bench::vec::<$ty>(); - let kde = Kde::new(Sample::new(&data), Gaussian, Bandwidth::Silverman); - let x = Sample::new(&data).mean(); - - c.bench_function(&format!("univariate_kde_call_{}", stringify!($ty)), |b| b.iter(|| { - kde.estimate(x) - })); - } - - fn map(c: &mut Criterion) { - let data = ::common_bench::vec_sized(SAMPLE_SIZE).unwrap(); - let kde = Kde::new(Sample::new(&data), Gaussian, Bandwidth::Silverman); - let xs: Vec<_> = ::itertools::linspace::<$ty>(0., 1., KDE_POINTS).collect(); - - c.bench_function(&format!("univariate_kde_map_{}", stringify!($ty)), |b| b.iter(|| { - kde.map(&xs) - })); - } - - criterion_group!( - name = benches; - config = ::common_bench::reduced_samples(); - targets = call, map); - } - } -} - -mod bench { - bench!(f32); - bench!(f64); -} - +#[macro_use] +extern crate criterion; +extern crate criterion_stats as stats; +extern crate itertools; +extern crate rand; + +mod common_bench; + +macro_rules! bench { + ($ty:ident) => { + pub mod $ty { + use stats::univariate::Sample; + use stats::univariate::kde::kernel::Gaussian; + use stats::univariate::kde::{Bandwidth, Kde}; + use criterion::Criterion; + + const KDE_POINTS: usize = 100; + const SAMPLE_SIZE: usize = 100_000; + + fn call(c: &mut Criterion) { + let data = ::common_bench::vec::<$ty>(); + let kde = Kde::new(Sample::new(&data), Gaussian, Bandwidth::Silverman); + let x = Sample::new(&data).mean(); + + c.bench_function(&format!("univariate_kde_call_{}", stringify!($ty)), |b| b.iter(|| { + kde.estimate(x) + })); + } + + fn map(c: &mut Criterion) { + let data = ::common_bench::vec_sized(SAMPLE_SIZE).unwrap(); + let kde = Kde::new(Sample::new(&data), Gaussian, Bandwidth::Silverman); + let xs: Vec<_> = ::itertools::linspace::<$ty>(0., 1., KDE_POINTS).collect(); + + c.bench_function(&format!("univariate_kde_map_{}", stringify!($ty)), |b| b.iter(|| { + kde.map(&xs) + })); + } + + criterion_group!( + name = benches; + config = ::common_bench::reduced_samples(); + targets = call, map); + } + } +} + +mod bench { + bench!(f32); + bench!(f64); +} + criterion_main!(bench::f32::benches, bench::f64::benches); \ No newline at end of file diff --git a/stats/benches/univariate_sample.rs b/stats/benches/univariate_sample.rs index 0b6cf4e55..7050ba4be 100644 --- a/stats/benches/univariate_sample.rs +++ b/stats/benches/univariate_sample.rs @@ -1,111 +1,111 @@ -#[macro_use] -extern crate criterion; -extern crate criterion_stats as stats; -extern crate rand; -extern crate cast; - -mod common_bench; - -macro_rules! stat { - ($ty:ident <- $($stat:ident),+) => { - $( - fn $stat(c: &mut Criterion) { - let v = ::common_bench::vec::<$ty>(); - let s = ::stats::univariate::Sample::new(&v); - - c.bench_function( - &format!("stat_{}_{}", stringify!($ty), stringify!($stat)), - |b| b.iter(|| s.$stat())); - } - )+ - } -} - -macro_rules! stat_none { - ($ty:ident <- $($stat:ident),+) => { - $( - fn $stat(c: &mut Criterion) { - let v = ::common_bench::vec::<$ty>(); - let s = ::stats::univariate::Sample::new(&v); - - c.bench_function( - &format!("stat_none_{}_{}", stringify!($ty), stringify!($stat)), - |b| b.iter(|| s.$stat(None))); - } - )+ - } -} - -macro_rules! fast_stat { - ($ty:ident <- $(($stat:ident, $aux_stat:ident)),+) => { - $( - fn $stat(c: &mut Criterion) { - let v = ::common_bench::vec::<$ty>(); - let s = ::stats::univariate::Sample::new(&v); - let aux = Some(s.$aux_stat()); - - c.bench_function( - &format!("fast_stat_{}_{}", stringify!($ty), stringify!($stat)), - |b| b.iter(|| s.$stat(aux))); - } - )+ - } -} - -macro_rules! bench { - ($ty:ident) => { - pub mod $ty { - pub trait SampleExt { - fn base_percentiles(&self) -> ::stats::univariate::Percentiles<$ty> where - usize: ::cast::From<$ty, Output=Result>; - - fn iqr(&self) -> $ty where - usize: ::cast::From<$ty, Output=Result>, - { - self.base_percentiles().iqr() - } - - fn median(&self) -> $ty where - usize: ::cast::From<$ty, Output=Result>, - { - self.base_percentiles().median() - } - } - impl SampleExt for ::stats::univariate::Sample<$ty> { - fn base_percentiles(&self) -> ::stats::univariate::Percentiles<$ty> where - usize: ::cast::From<$ty, Output=Result> { - self.percentiles() - } - } - - use criterion::Criterion; - - stat!($ty <- iqr, max, mean, median, median_abs_dev_pct, min, std_dev_pct, sum); - stat_none!($ty <- median_abs_dev, std_dev, var); - - criterion_group!( - name = benches; - config = ::common_bench::reduced_samples(); - targets = iqr, max, mean, median, median_abs_dev_pct, min, - std_dev_pct, sum, median_abs_dev, std_dev, var); - - pub mod fast { - use criterion::Criterion; - use super::SampleExt; - - fast_stat!($ty <- (median_abs_dev, median), (std_dev, mean), (var, mean)); - criterion_group!( - name = benches; - config = ::common_bench::reduced_samples(); - targets = median_abs_dev, std_dev, var); - } - } - } -} - -bench!(f64); - -criterion_main!( - f64::benches, - f64::fast::benches +#[macro_use] +extern crate criterion; +extern crate criterion_stats as stats; +extern crate rand; +extern crate cast; + +mod common_bench; + +macro_rules! stat { + ($ty:ident <- $($stat:ident),+) => { + $( + fn $stat(c: &mut Criterion) { + let v = ::common_bench::vec::<$ty>(); + let s = ::stats::univariate::Sample::new(&v); + + c.bench_function( + &format!("stat_{}_{}", stringify!($ty), stringify!($stat)), + |b| b.iter(|| s.$stat())); + } + )+ + } +} + +macro_rules! stat_none { + ($ty:ident <- $($stat:ident),+) => { + $( + fn $stat(c: &mut Criterion) { + let v = ::common_bench::vec::<$ty>(); + let s = ::stats::univariate::Sample::new(&v); + + c.bench_function( + &format!("stat_none_{}_{}", stringify!($ty), stringify!($stat)), + |b| b.iter(|| s.$stat(None))); + } + )+ + } +} + +macro_rules! fast_stat { + ($ty:ident <- $(($stat:ident, $aux_stat:ident)),+) => { + $( + fn $stat(c: &mut Criterion) { + let v = ::common_bench::vec::<$ty>(); + let s = ::stats::univariate::Sample::new(&v); + let aux = Some(s.$aux_stat()); + + c.bench_function( + &format!("fast_stat_{}_{}", stringify!($ty), stringify!($stat)), + |b| b.iter(|| s.$stat(aux))); + } + )+ + } +} + +macro_rules! bench { + ($ty:ident) => { + pub mod $ty { + pub trait SampleExt { + fn base_percentiles(&self) -> ::stats::univariate::Percentiles<$ty> where + usize: ::cast::From<$ty, Output=Result>; + + fn iqr(&self) -> $ty where + usize: ::cast::From<$ty, Output=Result>, + { + self.base_percentiles().iqr() + } + + fn median(&self) -> $ty where + usize: ::cast::From<$ty, Output=Result>, + { + self.base_percentiles().median() + } + } + impl SampleExt for ::stats::univariate::Sample<$ty> { + fn base_percentiles(&self) -> ::stats::univariate::Percentiles<$ty> where + usize: ::cast::From<$ty, Output=Result> { + self.percentiles() + } + } + + use criterion::Criterion; + + stat!($ty <- iqr, max, mean, median, median_abs_dev_pct, min, std_dev_pct, sum); + stat_none!($ty <- median_abs_dev, std_dev, var); + + criterion_group!( + name = benches; + config = ::common_bench::reduced_samples(); + targets = iqr, max, mean, median, median_abs_dev_pct, min, + std_dev_pct, sum, median_abs_dev, std_dev, var); + + pub mod fast { + use criterion::Criterion; + use super::SampleExt; + + fast_stat!($ty <- (median_abs_dev, median), (std_dev, mean), (var, mean)); + criterion_group!( + name = benches; + config = ::common_bench::reduced_samples(); + targets = median_abs_dev, std_dev, var); + } + } + } +} + +bench!(f64); + +criterion_main!( + f64::benches, + f64::fast::benches ); \ No newline at end of file diff --git a/stats/src/lib.rs b/stats/src/lib.rs index b0c9b2957..085b5f085 100644 --- a/stats/src/lib.rs +++ b/stats/src/lib.rs @@ -1,112 +1,112 @@ -//! [Criterion]'s statistics library. -//! -//! [Criterion]: https://github.com/japaric/criterion.rs -//! -//! **WARNING** This library is criterion's implementation detail and there no plans to stabilize -//! it. In other words, the API may break at any time without notice. - -#![deny(missing_docs)] -#![deny(warnings)] -#![cfg_attr(feature = "cargo-clippy", allow(used_underscore_binding))] - -extern crate cast; -extern crate num_traits; -extern crate num_cpus; -extern crate rand; -extern crate thread_scoped; - -#[cfg(test)] #[macro_use] extern crate approx; -#[cfg(test)] #[macro_use] extern crate quickcheck; -#[cfg(test)] extern crate itertools; - -#[cfg(test)] mod test; - -pub mod bivariate; -pub mod tuple; -pub mod univariate; - -mod float; - -use std::mem; -use std::ops::Deref; - -use float::Float; -use univariate::Sample; - -/// The bootstrap distribution of some parameter -pub struct Distribution(Box<[A]>); - -impl Distribution where A: Float { - /// Computes the confidence interval of the population parameter using percentiles - /// - /// # Panics - /// - /// Panics if the `confidence_level` is not in the `(0, 1)` range. - pub fn confidence_interval(&self, confidence_level: A) -> (A, A) - where usize: cast::From>, - { - let _0 = A::cast(0); - let _1 = A::cast(1); - let _50 = A::cast(50); - - assert!(confidence_level > _0 && confidence_level < _1); - - let percentiles = self.percentiles(); - - // FIXME(privacy) this should use the `at_unchecked()` method - ( - percentiles.at(_50 * (_1 - confidence_level)), - percentiles.at(_50 * (_1 + confidence_level)), - ) - } - - /// Computes the "likelihood" of seeing the value `t` or "more extreme" values in the - /// distribution. - pub fn p_value(&self, t: A, tails: &Tails) -> A { - use std::cmp; - - let n = self.0.len(); - let hits = self.0.iter().filter(|&&x| x < t).count(); - - let tails = A::cast(match *tails { - Tails::One => 1, - Tails::Two => 2, - }); - - A::cast(cmp::min(hits, n - hits)) / A::cast(n) * tails - } -} - -impl Deref for Distribution { - type Target = Sample; - - fn deref(&self) -> &Sample { - let slice: &[_] = &self.0; - - unsafe { - mem::transmute(slice) - } - } -} - -/// Number of tails for significance testing -pub enum Tails { - /// One tailed test - One, - /// Two tailed test - Two, -} - -fn dot(xs: &[A], ys: &[A]) -> A - where A: Float -{ - xs.iter().zip(ys).fold(A::cast(0), |acc, (&x, &y)| acc + x * y) -} - -fn sum(xs: &[A]) -> A - where A: Float -{ - use std::ops::Add; - - xs.iter().cloned().fold(A::cast(0), Add::add) -} +//! [Criterion]'s statistics library. +//! +//! [Criterion]: https://github.com/japaric/criterion.rs +//! +//! **WARNING** This library is criterion's implementation detail and there no plans to stabilize +//! it. In other words, the API may break at any time without notice. + +#![deny(missing_docs)] +#![deny(warnings)] +#![cfg_attr(feature = "cargo-clippy", allow(used_underscore_binding))] + +extern crate cast; +extern crate num_traits; +extern crate num_cpus; +extern crate rand; +extern crate thread_scoped; + +#[cfg(test)] #[macro_use] extern crate approx; +#[cfg(test)] #[macro_use] extern crate quickcheck; +#[cfg(test)] extern crate itertools; + +#[cfg(test)] mod test; + +pub mod bivariate; +pub mod tuple; +pub mod univariate; + +mod float; + +use std::mem; +use std::ops::Deref; + +use float::Float; +use univariate::Sample; + +/// The bootstrap distribution of some parameter +pub struct Distribution(Box<[A]>); + +impl Distribution where A: Float { + /// Computes the confidence interval of the population parameter using percentiles + /// + /// # Panics + /// + /// Panics if the `confidence_level` is not in the `(0, 1)` range. + pub fn confidence_interval(&self, confidence_level: A) -> (A, A) + where usize: cast::From>, + { + let _0 = A::cast(0); + let _1 = A::cast(1); + let _50 = A::cast(50); + + assert!(confidence_level > _0 && confidence_level < _1); + + let percentiles = self.percentiles(); + + // FIXME(privacy) this should use the `at_unchecked()` method + ( + percentiles.at(_50 * (_1 - confidence_level)), + percentiles.at(_50 * (_1 + confidence_level)), + ) + } + + /// Computes the "likelihood" of seeing the value `t` or "more extreme" values in the + /// distribution. + pub fn p_value(&self, t: A, tails: &Tails) -> A { + use std::cmp; + + let n = self.0.len(); + let hits = self.0.iter().filter(|&&x| x < t).count(); + + let tails = A::cast(match *tails { + Tails::One => 1, + Tails::Two => 2, + }); + + A::cast(cmp::min(hits, n - hits)) / A::cast(n) * tails + } +} + +impl Deref for Distribution { + type Target = Sample; + + fn deref(&self) -> &Sample { + let slice: &[_] = &self.0; + + unsafe { + mem::transmute(slice) + } + } +} + +/// Number of tails for significance testing +pub enum Tails { + /// One tailed test + One, + /// Two tailed test + Two, +} + +fn dot(xs: &[A], ys: &[A]) -> A + where A: Float +{ + xs.iter().zip(ys).fold(A::cast(0), |acc, (&x, &y)| acc + x * y) +} + +fn sum(xs: &[A]) -> A + where A: Float +{ + use std::ops::Add; + + xs.iter().cloned().fold(A::cast(0), Add::add) +}