Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add error messages and unit tests for setups where min_budget >= max_budget #46

Merged
merged 7 commits into from
Aug 1, 2023
32 changes: 20 additions & 12 deletions src/dehb/optimizers/dehb.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None,
crossover_prob=None, strategy=None, min_budget=None,
max_budget=None, eta=None, min_clip=None, max_clip=None,
boundary_fix_type='random', max_age=np.inf, **kwargs):
# Miscellaneous
self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './'
os.makedirs(self.output_path, exist_ok=True)
self.logger = logger
log_suffix = time.strftime("%x %X %Z")
log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_')
self.logger.add(
"{}/dehb_{}.log".format(self.output_path, log_suffix),
**_logger_props
)
self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix)

Bronzila marked this conversation as resolved.
Show resolved Hide resolved
# Benchmark related variables
self.cs = cs
self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False
Expand Down Expand Up @@ -59,7 +71,14 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None,
# Hyperband related variables
self.min_budget = min_budget
self.max_budget = max_budget
assert self.max_budget > self.min_budget, "only (Max Budget > Min Budget) supported!"
if self.max_budget <= self.min_budget:
self.logger.error("Only (Max Budget > Min Budget) is supported for DEHB.")
if self.max_budget == self.min_budget:
self.logger.error(
"If you have a fixed fidelity, " \
"you can instead run DE. For more information checkout: " \
"https://automl.github.io/DEHB/references/de")
raise AssertionError()
Bronzila marked this conversation as resolved.
Show resolved Hide resolved
self.eta = eta
self.min_clip = min_clip
self.max_clip = max_clip
Expand All @@ -75,17 +94,6 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None,
-np.linspace(start=self.max_SH_iter - 1,
stop=0, num=self.max_SH_iter))

# Miscellaneous
self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './'
os.makedirs(self.output_path, exist_ok=True)
self.logger = logger
log_suffix = time.strftime("%x %X %Z")
log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_')
self.logger.add(
"{}/dehb_{}.log".format(self.output_path, log_suffix),
**_logger_props
)
self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix)
# Updating DE parameter list
self.de_params.update({"output_path": self.output_path})

Expand Down
23 changes: 15 additions & 8 deletions tests/test_dehb.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import pytest
import time
import typing

import ConfigSpace
import numpy as np
import time
import pytest
from src.dehb.optimizers.dehb import DEHB


def create_toy_searchspace():
"""Creates a toy searchspace with a single hyperparameter.

Expand Down Expand Up @@ -66,8 +68,7 @@ class TestBudgetExhaustion():
evaluations and number of brackets to run.
"""
def test_runtime_exhaustion(self):
"""Test for runtime budget exhaustion.
"""
"""Test for runtime budget exhaustion."""
cs = create_toy_searchspace()
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
objective_function=objective_function)
Expand All @@ -77,8 +78,7 @@ def test_runtime_exhaustion(self):
assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted"

def test_fevals_exhaustion(self):
"""Test for function evaluations budget exhaustion.
"""
"""Test for function evaluations budget exhaustion."""
cs = create_toy_searchspace()
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
objective_function=objective_function)
Expand All @@ -88,8 +88,7 @@ def test_fevals_exhaustion(self):
assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted"

def test_brackets_exhaustion(self):
"""Test for bracket budget exhaustion.
"""
"""Test for bracket budget exhaustion."""
cs = create_toy_searchspace()
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
objective_function=objective_function)
Expand All @@ -98,3 +97,11 @@ def test_brackets_exhaustion(self):

assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted"

class TestInitialization:
"""Class that bundles all tests regarding the initialization of DEHB."""
def test_higher_min_budget(self):
"""Test that verifies, that DEHB breaks if min_budget > max_budget."""
cs = create_toy_searchspace()
with pytest.raises(AssertionError):
create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3,
objective_function=objective_function)
Bronzila marked this conversation as resolved.
Show resolved Hide resolved
Loading