Skip to content

Commit

Permalink
Added benchmark results
Browse files Browse the repository at this point in the history
  • Loading branch information
@TobiasJacob
TobiasJacob committed May 29, 2024
1 parent 4b9a683 commit 0a71a44
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 45 deletions.
59 changes: 59 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,65 @@ $$

and can be highlighted to the user.

### Benchmarks

There are some benchmarks included. The CirclesWithLines and StairsWithLines are ridiculously hard problems. This one compares all 4 slovers. BGFS and GradientBasedSolver are the fastest. However, the GradientBasedSolver is usually to inaccurate, while the BGFS solver is the only one that solves all the problems.


```console
cargo test --release test_benchmark1 -- --ignored --nocapture

running 1 test
Benchmark: CirclesWithLines
n: 3, primitives: 5, constraints: 7, solver: GradientBasedSolver , solved: false, error: 0.00, duration: 3ms
n: 3, primitives: 5, constraints: 7, solver: GaussNewtonSolver , solved: false, error: 0.00, duration: 32ms
n: 3, primitives: 5, constraints: 7, solver: LevenbergMarquardtSolver, solved: false, error: 0.00, duration: 3ms
n: 3, primitives: 5, constraints: 7, solver: BFGSSolver , solved: true, error: 0.00, duration: 0ms
n: 5, primitives: 9, constraints: 13, solver: GradientBasedSolver , solved: false, error: 0.00, duration: 7ms
n: 5, primitives: 9, constraints: 13, solver: GaussNewtonSolver , solved: false, error: 0.00, duration: 74ms
n: 5, primitives: 9, constraints: 13, solver: LevenbergMarquardtSolver, solved: false, error: 0.00, duration: 7ms
n: 5, primitives: 9, constraints: 13, solver: BFGSSolver , solved: true, error: 0.00, duration: 1ms
n: 10, primitives: 19, constraints: 28, solver: GradientBasedSolver , solved: false, error: 0.00, duration: 13ms
n: 10, primitives: 19, constraints: 28, solver: GaussNewtonSolver , solved: false, error: 1.00, duration: 299ms
n: 10, primitives: 19, constraints: 28, solver: LevenbergMarquardtSolver, solved: false, error: 1.00, duration: 29ms
n: 10, primitives: 19, constraints: 28, solver: BFGSSolver , solved: true, error: 0.00, duration: 4ms
Benchmark: StairsWithLines
n: 3, primitives: 5, constraints: 5, solver: GradientBasedSolver , solved: false, error: 0.00, duration: 3ms
n: 3, primitives: 5, constraints: 5, solver: GaussNewtonSolver , solved: false, error: 0.00, duration: 25ms
n: 3, primitives: 5, constraints: 5, solver: LevenbergMarquardtSolver, solved: false, error: 0.00, duration: 3ms
n: 3, primitives: 5, constraints: 5, solver: BFGSSolver , solved: true, error: 0.00, duration: 0ms
n: 5, primitives: 9, constraints: 9, solver: GradientBasedSolver , solved: false, error: 0.07, duration: 6ms
n: 5, primitives: 9, constraints: 9, solver: GaussNewtonSolver , solved: false, error: 0.00, duration: 61ms
n: 5, primitives: 9, constraints: 9, solver: LevenbergMarquardtSolver, solved: false, error: 0.00, duration: 7ms
n: 5, primitives: 9, constraints: 9, solver: BFGSSolver , solved: true, error: 0.00, duration: 0ms
n: 10, primitives: 19, constraints: 19, solver: GradientBasedSolver , solved: false, error: 0.64, duration: 13ms
n: 10, primitives: 19, constraints: 19, solver: GaussNewtonSolver , solved: false, error: 0.00, duration: 259ms
n: 10, primitives: 19, constraints: 19, solver: LevenbergMarquardtSolver, solved: false, error: 0.00, duration: 29ms
n: 10, primitives: 19, constraints: 19, solver: BFGSSolver , solved: true, error: 0.00, duration: 2ms
```

So we run some extended benchmarks on them.

```console
cargo test --release test_benchmark2 -- --ignored --nocapture

running 1 test
Benchmark: CirclesWithLines
n: 30, primitives: 59, constraints: 88, solver: BFGSSolver , solved: true, error: 0.00, duration: 21ms
n: 50, primitives: 99, constraints: 148, solver: BFGSSolver , solved: true, error: 0.00, duration: 41ms
n: 100, primitives: 199, constraints: 298, solver: BFGSSolver , solved: true, error: 0.00, duration: 94ms
n: 300, primitives: 599, constraints: 898, solver: BFGSSolver , solved: true, error: 0.00, duration: 625ms
Benchmark: StairsWithLines
n: 30, primitives: 59, constraints: 59, solver: BFGSSolver , solved: true, error: 0.00, duration: 18ms
n: 50, primitives: 99, constraints: 99, solver: BFGSSolver , solved: true, error: 0.00, duration: 49ms
n: 100, primitives: 199, constraints: 199, solver: BFGSSolver , solved: true, error: 0.00, duration: 214ms
n: 300, primitives: 599, constraints: 599, solver: BFGSSolver , solved: true, error: 0.00, duration: 3845ms
```

See how it is eben able to solve ridiculously large problems with 200 primitives in less than 1 second. It can also go higher than this.

Keep in mind that the problems are intentionally bad conditioned. A real world problem would be much easier to solve.

## Usage

Check out the examples folder at [src/examples](src/examples) for more examples.
Expand Down
40 changes: 12 additions & 28 deletions src/examples/benchmarks/circle_with_lines_benchmark.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,7 @@ use crate::{

use super::{Benchmark, BenchmarkFactory};

// This creates a stairs with lines problem that has a lot of constraints
// It tries to build a sketch like this with n steps (e.g. 6 steps):
// _
// _|
// _|
// _|
// _|
// |

// This creates a circle problem from lines
pub fn circle(n: usize) -> Vec<Vector2<f64>> {
let mut points = Vec::new();
for i in 0..n {
Expand Down Expand Up @@ -51,25 +43,17 @@ impl BenchmarkFactory for CirclesWithLinesBenchmarkFactory {
point_references.push(point);
}

sketch
.borrow_mut()
.add_constraint(ConstraintCell::FixPoint(Rc::new(RefCell::new(
FixPoint::new(
point_references[0].clone(),
Vector2::new(reference_points[0].x, reference_points[0].y),
),
))))
.unwrap();

sketch
.borrow_mut()
.add_constraint(ConstraintCell::FixPoint(Rc::new(RefCell::new(
FixPoint::new(
point_references[1].clone(),
Vector2::new(reference_points[1].x, reference_points[1].y),
),
))))
.unwrap();
for i in 0..n {
sketch
.borrow_mut()
.add_constraint(ConstraintCell::FixPoint(Rc::new(RefCell::new(
FixPoint::new(
point_references[i].clone(),
Vector2::new(reference_points[i].x, reference_points[i].y),
),
))))
.unwrap();
}

for i in 0..n - 1 {
let line = Rc::new(RefCell::new(Line::new(
Expand Down
65 changes: 59 additions & 6 deletions src/examples/benchmarks/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ pub trait Benchmark {
#[cfg(test)]
mod tests {
use crate::solvers::{
bfgs_solver::BFGSSolver, gradient_based_solver::GradientBasedSolver, Solver,
bfgs_solver::BFGSSolver, gauss_newton_solver::GaussNewtonSolver,
gradient_based_solver::GradientBasedSolver, levenberg_marquardt::LevenbergMarquardtSolver,
Solver,
};

use super::{
Expand All @@ -27,8 +29,8 @@ mod tests {

#[ignore]
#[test]
// Run the benchmark manually with `cargo test --release test_benchmark -- --ignored --nocapture`
pub fn test_benchmark() {
// Run the benchmark manually with `cargo test --release test_benchmark1 -- --ignored --nocapture`
pub fn test_benchmark1() {
let benchmarks: Vec<(&str, Box<dyn BenchmarkFactory>)> = vec![
(
"CirclesWithLines",
Expand All @@ -37,9 +39,61 @@ mod tests {
("StairsWithLines", Box::new(StairsWithLinesBenchmarkFactory)),
];
let solvers: Vec<(&str, Box<dyn Solver>)> = vec![
// ("GradientBasedSolver", Box::new(GradientBasedSolver::new())),
("BFGSSolver ", Box::new(BFGSSolver::new())),
(
"GradientBasedSolver ",
Box::new(GradientBasedSolver::new()),
),
(
"GaussNewtonSolver ",
Box::new(GaussNewtonSolver::new()),
),
(
"LevenbergMarquardtSolver",
Box::new(LevenbergMarquardtSolver::new()),
),
("BFGSSolver ", Box::new(BFGSSolver::new())),
];

for (benchmark_name, benchmark) in benchmarks.iter() {
println!("Benchmark: {}", benchmark_name);
for n in &[3, 5, 10] {
for (solver_name, solver) in &solvers {
// Measure the time it takes to solve the benchmark
let benchmark = benchmark.new_benchmark(*n);
let sketch = benchmark.get_sketch();
let start = std::time::Instant::now();
solver.solve(sketch.clone()).unwrap();
let duration = start.elapsed();
let solved = benchmark.check(1e-6);
let error = sketch.borrow_mut().get_loss();
println!(
"n: {:4}, \tprimitives: {:4}, \tconstraints:{:4}, \tsolver: {},\tsolved: {},\terror: {:.2},\tduration: {}ms",
n,
sketch.borrow().get_num_primitives(),
sketch.borrow().get_num_constraints(),
solver_name,
solved,
error,
duration.as_millis()
);
}
}
}
}

#[ignore]
#[test]
// Run the benchmark manually with `cargo test --release test_benchmark -- --ignored --nocapture`
pub fn test_benchmark2() {
let benchmarks: Vec<(&str, Box<dyn BenchmarkFactory>)> = vec![
(
"CirclesWithLines",
Box::new(CirclesWithLinesBenchmarkFactory),
),
("StairsWithLines", Box::new(StairsWithLinesBenchmarkFactory)),
];
let solvers: Vec<(&str, Box<dyn Solver>)> =
vec![("BFGSSolver ", Box::new(BFGSSolver::new()))];

for (benchmark_name, benchmark) in benchmarks.iter() {
println!("Benchmark: {}", benchmark_name);
Expand All @@ -63,7 +117,6 @@ mod tests {
error,
duration.as_millis()
);
return;
}
}
}
Expand Down
29 changes: 18 additions & 11 deletions src/solvers/bfgs_solver.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,29 @@ impl Solver for BFGSSolver {
// println!("Gradient: {:?}", gradient);

loss = sketch.borrow_mut().get_loss();
println!("Loss: {:?}", loss);
println!("Alpha: {:?}", alpha);
// println!("Loss: {:?}", loss);
// println!("Alpha: {:?}", alpha);

let p = -(&h) * &gradient;
assert!(
p.iter().all(|x| x.is_finite()),
"p contains non-finite values"
);

alpha = alpha * 2.0;
loop {
let new_data = &data + 20.0 * alpha * &p;
sketch.borrow_mut().set_data(new_data);
let new_loss = sketch.borrow_mut().get_loss();
if new_loss <= loss {
break;
}
alpha = alpha * 0.5;
if alpha < 1e-10 {
return Ok(());
}
}

let mut best_alpha = 0.0;
for i in 0..self.alpha_search_steps {
let new_data = &data + alpha * i as f64 * &p;
Expand All @@ -90,14 +105,6 @@ impl Solver for BFGSSolver {
}
}

// This means we are already at the minimum
if best_alpha == 0.0 {
alpha = alpha * 0.5;
sketch.borrow_mut().set_data(data.clone());
continue;
}
alpha = alpha * 1.5;

let s = best_alpha * &p;

let new_data = &data + &s;
Expand All @@ -109,7 +116,7 @@ impl Solver for BFGSSolver {

let mut s_dot_y = s.dot(&y);
if s_dot_y.abs() < 1e-16 {
println!("s_dot_y is too small");
// println!("s_dot_y is too small");
s_dot_y += 1e-6;
}
let factor = s_dot_y + (y.transpose() * &h * &y)[(0, 0)];
Expand Down

0 comments on commit 0a71a44

Please sign in to comment.