From e4d939aa34a3d8ebca379789b8ce1d3b88a42e43 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Mon, 19 Apr 2021 20:23:46 +1200 Subject: [PATCH 1/7] update readme ablaom -> JuliaAI --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6a766e6..156bfb0 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ | Linux | Coverage | | :-----------: | :------: | -| [![Build status](https://github.com/ablaom/EarlyStopping.jl/workflows/CI/badge.svg)](https://github.com/ablaom/EarlyStopping.jl/actions)| [![codecov.io](http://codecov.io/github/ablaom/EarlyStopping.jl/coverage.svg?branch=master)](http://codecov.io/github/ablaom/EarlyStopping.jl?branch=master) | +| [![Build status](https://github.com/JuliaAI/EarlyStopping.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/EarlyStopping.jl/actions)| [![codecov.io](http://codecov.io/github/JuliaAI/EarlyStopping.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaAI/EarlyStopping.jl?branch=master) | A small package for applying early stopping criteria to loss-generating iterative algorithms, with a view to training and optimizing machine learning models. -The basis of [IterationControl.jl](https://github.com/ablaom/IterationControl.jl), +The basis of [IterationControl.jl](https://github.com/JuliaAI/IterationControl.jl), a package externally controlling iterative algorithms. Includes the stopping criteria surveyed in [Prechelt, Lutz From 311b37b4f7f889b133f04ea3484d61688a656dd2 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Sat, 16 Oct 2021 18:30:17 -0400 Subject: [PATCH 2/7] Add Warmup Stopping Criteria This commits adds a Warmup critieria to wait for n loss updates before applying a criteria --- README.md | 5 +++-- src/EarlyStopping.jl | 1 + src/criteria.jl | 44 ++++++++++++++++++++++++++++++++++++++++++++ test/criteria.jl | 7 +++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 156bfb0..7c2654d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A small package for applying early stopping criteria to loss-generating iterative algorithms, with a view to training and optimizing machine learning models. -The basis of [IterationControl.jl](https://github.com/JuliaAI/IterationControl.jl), +The basis of [IterationControl.jl](https://github.com/JuliaAI/IterationControl.jl), a package externally controlling iterative algorithms. Includes the stopping criteria surveyed in [Prechelt, Lutz @@ -87,11 +87,12 @@ criterion | description | `TimeLimit(t=0.5)` | Stop after `t` hours | `NumberLimit(n=100)` | Stop after `n` loss updates (excl. "training losses") | `NumberSinceBest(n=6)`| Stop after `n` loss updates (excl. "training losses") | -`Threshold(value=0.0)`| Stop when `loss < value` | +`Threshold(value=0.0)`| Stop when `loss < value` | `GL(alpha=2.0)` | Stop after "Generalization Loss" exceeds `alpha` | ``GL_α`` `PQ(alpha=0.75, k=5)` | Stop after "Progress-modified GL" exceeds `alpha` | ``PQ_α`` `Patience(n=5)` | Stop after `n` consecutive loss increases | ``UP_s`` `Disjunction(c...)` | Stop when any of the criteria `c` apply | +`Warmup(c, n)` | Wait for `n` loss updates before checking criteria `c`| ## Criteria tracking both training and out-of-sample losses diff --git a/src/EarlyStopping.jl b/src/EarlyStopping.jl index 2c41007..a7598e3 100644 --- a/src/EarlyStopping.jl +++ b/src/EarlyStopping.jl @@ -17,6 +17,7 @@ export StoppingCriterion, NumberLimit, Threshold, Disjunction, + Warmup, criteria, stopping_time, EarlyStopper, diff --git a/src/criteria.jl b/src/criteria.jl index 16e0c33..e8fa486 100644 --- a/src/criteria.jl +++ b/src/criteria.jl @@ -423,10 +423,54 @@ done(criterion::Threshold, state) = state < criterion.value needs_loss(::Type{<:Threshold}) = true +""" + Warmup(c::StoppingCriterion, n) +Wait for `n` updates before checking stopping criterion `c` +""" +struct Warmup{C} <: StoppingCriterion where {C <: StoppingCriterion} + criterion::C + n::Int + function Warmup(criterion::C, n::N) where {C <: StoppingCriterion, N <: Integer} + n > 0 || throw(ArgumentError("`n` must be positive. ")) + new{C}(criterion, Int(n)) + end +end +# Initialize inner state for type-stability, and record first observation +update(criterion::Warmup, loss) = (1, update(criterion.criterion, loss)) + +# Catch uninitialized state +update(c::Warmup, loss, ::Nothing) = update(c, loss) +update_training(c::Warmup, loss, ::Nothing) = update(c, loss) + +# Handle update vs update_training +update(c::Warmup, loss, state) = _update(update, c, loss, state) +update_training(c::Warmup, loss, state) = _update(update_training, c, loss, state) + +# Dispatch update and update_training here +function _update(f::Function, criterion::Warmup, loss, state) + n, inner = state + n += 1 + if n <= criterion.n + # Skip inner criterion + return n, inner + elseif n == criterion.n+1 + # First step of inner criterion + return n, f(criterion.criterion, loss) + else + # Step inner criterion + return n, f(criterion.criterion, loss, inner) + end +end +# Default warmup for testing +Warmup() = Warmup(InvalidValue(), 1) +function done(criterion::Warmup, state) + # Only check if inner criterion is done after n updates + return state[1] <= criterion.n ? false : done(criterion.criterion, state[2]) +end ## NOT A NUMBER (deprecated) diff --git a/test/criteria.jl b/test/criteria.jl index 5b88735..dfc4abf 100644 --- a/test/criteria.jl +++ b/test/criteria.jl @@ -209,6 +209,13 @@ end end end +@testset "Warmup" begin + @test_throws ArgumentError Warmup(Patience(), 0) + for n in 1:(length(losses)-1) + @test stopping_time(Warmup(NumberLimit(1), n), losses) == n+1 + end +end + # # DEPRECATED From 2f87bb6c315b95e044f976bfd70e261a2511a0fc Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Sat, 16 Oct 2021 18:49:58 -0400 Subject: [PATCH 3/7] Add dispatch to inner criteria's message --- src/criteria.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/criteria.jl b/src/criteria.jl index e8fa486..ca6c195 100644 --- a/src/criteria.jl +++ b/src/criteria.jl @@ -472,6 +472,8 @@ function done(criterion::Warmup, state) return state[1] <= criterion.n ? false : done(criterion.criterion, state[2]) end +message(c::Warmup, state) = message(c.criterion, state[2]) + ## NOT A NUMBER (deprecated) From 20ee19e742cb74872a282a70b5d0ea392ae4a39f Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Mon, 18 Oct 2021 09:12:01 -0400 Subject: [PATCH 4/7] fix: add needs_loss and needs_training_losses to Warmup --- src/criteria.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/criteria.jl b/src/criteria.jl index ca6c195..4ef44e2 100644 --- a/src/criteria.jl +++ b/src/criteria.jl @@ -447,6 +447,8 @@ update_training(c::Warmup, loss, ::Nothing) = update(c, loss) # Handle update vs update_training update(c::Warmup, loss, state) = _update(update, c, loss, state) update_training(c::Warmup, loss, state) = _update(update_training, c, loss, state) +needs_loss(::Type{<:Warmup{C}}) where C = needs_loss(C) +needs_training_losses(::Type{<:Warmup{C}}) where C = needs_training_losses(C) # Dispatch update and update_training here function _update(f::Function, criterion::Warmup, loss, state) From fa3780b65d7cebb878e143f6e7c98976901986b2 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Mon, 18 Oct 2021 10:27:11 -0400 Subject: [PATCH 5/7] Add more tests and fix init for update_training Should dispatch to update_training, not update when initalizing from update_training --- src/criteria.jl | 5 +++-- test/criteria.jl | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/criteria.jl b/src/criteria.jl index 4ef44e2..3521fe7 100644 --- a/src/criteria.jl +++ b/src/criteria.jl @@ -438,11 +438,12 @@ struct Warmup{C} <: StoppingCriterion where {C <: StoppingCriterion} end # Initialize inner state for type-stability, and record first observation +update(c::Warmup, loss, ::Nothing) = update(c, loss) update(criterion::Warmup, loss) = (1, update(criterion.criterion, loss)) # Catch uninitialized state -update(c::Warmup, loss, ::Nothing) = update(c, loss) -update_training(c::Warmup, loss, ::Nothing) = update(c, loss) +update_training(c::Warmup, loss, ::Nothing) = update_training(c, loss) +update_training(c::Warmup, loss) = (1, update_training(c.criterion, loss)) # Handle update vs update_training update(c::Warmup, loss, state) = _update(update, c, loss, state) diff --git a/test/criteria.jl b/test/criteria.jl index dfc4abf..de96b16 100644 --- a/test/criteria.jl +++ b/test/criteria.jl @@ -214,6 +214,30 @@ end for n in 1:(length(losses)-1) @test stopping_time(Warmup(NumberLimit(1), n), losses) == n+1 end + + # Test message + @testset "message" begin + stopper = Warmup(Patience(2), 2) + stopper_ref = Warmup(Patience(2), 2) + state, state_ref = nothing, nothing + for loss = losses + state = EarlyStopping.update(stopper, loss, state) + state_ref = EarlyStopping.update(stopper_ref, loss, state_ref) + @test message(stopper, state) == message(stopper_ref, state_ref) + end + end + + @testset "training" begin + stopper = Warmup(PQ(), 3) + is_training = @show map(x -> x%3 > 0, 1:length(losses)) + + # Feed 2 training losses + 1 non-training to criteria with/without + stop_time = stopping_time(stopper, losses, is_training) + ref_stop_time = stopping_time(stopper.criterion, losses[3:end], is_training) + + # PQ only counts training loss updates + @test round(stop_time/3, RoundUp) == ref_stop_time + end end From c3af46308e75c56c747ed5f614ba6760400d5011 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Mon, 18 Oct 2021 16:06:01 -0400 Subject: [PATCH 6/7] fix: provide n as a keyword argument --- README.md | 2 +- src/criteria.jl | 7 ++++--- test/criteria.jl | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7c2654d..da016a4 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ criterion | description | `PQ(alpha=0.75, k=5)` | Stop after "Progress-modified GL" exceeds `alpha` | ``PQ_α`` `Patience(n=5)` | Stop after `n` consecutive loss increases | ``UP_s`` `Disjunction(c...)` | Stop when any of the criteria `c` apply | -`Warmup(c, n)` | Wait for `n` loss updates before checking criteria `c`| +`Warmup(c; n=1)` | Wait for `n` loss updates before checking criteria `c`| ## Criteria tracking both training and out-of-sample losses diff --git a/src/criteria.jl b/src/criteria.jl index 3521fe7..753b13b 100644 --- a/src/criteria.jl +++ b/src/criteria.jl @@ -437,6 +437,10 @@ struct Warmup{C} <: StoppingCriterion where {C <: StoppingCriterion} end end +# Constructors for Warmup +Warmup() = Warmup(InvalidValue()) # Default for testing +Warmup(c; n = 1) = Warmup(c, n) # Provide kwargs interface + # Initialize inner state for type-stability, and record first observation update(c::Warmup, loss, ::Nothing) = update(c, loss) update(criterion::Warmup, loss) = (1, update(criterion.criterion, loss)) @@ -467,9 +471,6 @@ function _update(f::Function, criterion::Warmup, loss, state) end end -# Default warmup for testing -Warmup() = Warmup(InvalidValue(), 1) - function done(criterion::Warmup, state) # Only check if inner criterion is done after n updates return state[1] <= criterion.n ? false : done(criterion.criterion, state[2]) diff --git a/test/criteria.jl b/test/criteria.jl index de96b16..466aebf 100644 --- a/test/criteria.jl +++ b/test/criteria.jl @@ -217,7 +217,7 @@ end # Test message @testset "message" begin - stopper = Warmup(Patience(2), 2) + stopper = Warmup(Patience(2); n = 2) stopper_ref = Warmup(Patience(2), 2) state, state_ref = nothing, nothing for loss = losses From 3acfbbb72369348b11540b488cf2b30f967cc383 Mon Sep 17 00:00:00 2001 From: "Anthony Blaom, PhD" Date: Tue, 19 Oct 2021 11:31:31 +1300 Subject: [PATCH 7/7] bump 0.1.9 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b819504..8ae14ba 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "EarlyStopping" uuid = "792122b4-ca99-40de-a6bc-6742525f08b6" authors = ["Anthony D. Blaom "] -version = "0.1.8" +version = "0.1.9" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"