From 9608e926bf85827ed69b91ec40fd70c15ff40a4d Mon Sep 17 00:00:00 2001 From: Bronzila Date: Tue, 25 Jul 2023 17:08:40 +0200 Subject: [PATCH 1/8] Add error messages and unit tests for setups where min_budget >= max_budget --- src/dehb/optimizers/dehb.py | 37 +++++++++++++++++++++++++------------ tests/test_dehb.py | 23 +++++++++++++++-------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index aa496cb..c025e7e 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -27,6 +27,18 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, crossover_prob=None, strategy=None, min_budget=None, max_budget=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs): + # Miscellaneous + self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' + os.makedirs(self.output_path, exist_ok=True) + self.logger = logger + log_suffix = time.strftime("%x %X %Z") + log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_') + self.logger.add( + "{}/dehb_{}.log".format(self.output_path, log_suffix), + **_logger_props + ) + self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix) + # Benchmark related variables self.cs = cs self.configspace = True if isinstance(self.cs, ConfigSpace.ConfigurationSpace) else False @@ -59,7 +71,19 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, # Hyperband related variables self.min_budget = min_budget self.max_budget = max_budget - assert self.max_budget > self.min_budget, "only (Max Budget > Min Budget) supported!" + if self.max_budget <= self.min_budget: + self.logger.error("Only (Max Budget > Min Budget) is supported for DEHB.") + if self.max_budget == self.min_budget: + self.logger.error( + "If you have a fixed fidelity, " \ + "you can instead run DE as follows: " \ + "AsyncDE(confispace, f=target_function, dimensions=" \ + f"{self.dimensions}, pop_size={self.dimensions * 2}," \ + f"max_age={self.max_age}, mutation_factor=" \ + f"{self.mutation_factor}, crossover_prob={self.crossover_prob},"\ + f"strategy={self.strategy}, budget={self.max_budget}," \ + f"boundary_fix_type={self.fix_type})") + sys.exit() self.eta = eta self.min_clip = min_clip self.max_clip = max_clip @@ -75,17 +99,6 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, -np.linspace(start=self.max_SH_iter - 1, stop=0, num=self.max_SH_iter)) - # Miscellaneous - self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' - os.makedirs(self.output_path, exist_ok=True) - self.logger = logger - log_suffix = time.strftime("%x %X %Z") - log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_') - self.logger.add( - "{}/dehb_{}.log".format(self.output_path, log_suffix), - **_logger_props - ) - self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix) # Updating DE parameter list self.de_params.update({"output_path": self.output_path}) diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 47f45af..7984e89 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -1,10 +1,12 @@ -import pytest +import time import typing + import ConfigSpace import numpy as np -import time +import pytest from src.dehb.optimizers.dehb import DEHB + def create_toy_searchspace(): """Creates a toy searchspace with a single hyperparameter. @@ -66,8 +68,7 @@ class TestBudgetExhaustion(): evaluations and number of brackets to run. """ def test_runtime_exhaustion(self): - """Test for runtime budget exhaustion. - """ + """Test for runtime budget exhaustion.""" cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) @@ -77,8 +78,7 @@ def test_runtime_exhaustion(self): assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted" def test_fevals_exhaustion(self): - """Test for function evaluations budget exhaustion. - """ + """Test for function evaluations budget exhaustion.""" cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) @@ -88,8 +88,7 @@ def test_fevals_exhaustion(self): assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted" def test_brackets_exhaustion(self): - """Test for bracket budget exhaustion. - """ + """Test for bracket budget exhaustion.""" cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) @@ -98,3 +97,11 @@ def test_brackets_exhaustion(self): assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted" +class TestInitialization: + """Class that bundles all tests regarding the initialization of DEHB.""" + def test_higher_min_budget(self): + """Test that verifies, that DEHB breaks if min_budget > max_budget.""" + cs = create_toy_searchspace() + with pytest.raises(SystemExit): + create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3, + objective_function=objective_function) From 3010e51e51c12c36f439d0a7c72e1d5144bfd325 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Tue, 25 Jul 2023 17:22:13 +0200 Subject: [PATCH 2/8] Fix spelling error --- src/dehb/optimizers/dehb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index c025e7e..0df013b 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -77,7 +77,7 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, self.logger.error( "If you have a fixed fidelity, " \ "you can instead run DE as follows: " \ - "AsyncDE(confispace, f=target_function, dimensions=" \ + "AsyncDE(cs=configspace, f=target_function, dimensions=" \ f"{self.dimensions}, pop_size={self.dimensions * 2}," \ f"max_age={self.max_age}, mutation_factor=" \ f"{self.mutation_factor}, crossover_prob={self.crossover_prob},"\ From bafb5364afa3e07920c3d2b1e6be96769746f48c Mon Sep 17 00:00:00 2001 From: Bronzila Date: Tue, 25 Jul 2023 17:25:11 +0200 Subject: [PATCH 3/8] Add whitespaces to error messages to beautify formatting --- src/dehb/optimizers/dehb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index 0df013b..6416f58 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -78,10 +78,10 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, "If you have a fixed fidelity, " \ "you can instead run DE as follows: " \ "AsyncDE(cs=configspace, f=target_function, dimensions=" \ - f"{self.dimensions}, pop_size={self.dimensions * 2}," \ + f"{self.dimensions}, pop_size={self.dimensions * 2}, " \ f"max_age={self.max_age}, mutation_factor=" \ - f"{self.mutation_factor}, crossover_prob={self.crossover_prob},"\ - f"strategy={self.strategy}, budget={self.max_budget}," \ + f"{self.mutation_factor}, crossover_prob={self.crossover_prob}, "\ + f"strategy={self.strategy}, budget={self.max_budget}, " \ f"boundary_fix_type={self.fix_type})") sys.exit() self.eta = eta From 30c345167e79369b4ce71b3edb31d403eea356e2 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Thu, 27 Jul 2023 11:16:18 +0200 Subject: [PATCH 4/8] Adjust sys.exit() to AssertionError for clearer user-communication --- src/dehb/optimizers/dehb.py | 2 +- tests/test_dehb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index 6416f58..e3e6a6b 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -83,7 +83,7 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, f"{self.mutation_factor}, crossover_prob={self.crossover_prob}, "\ f"strategy={self.strategy}, budget={self.max_budget}, " \ f"boundary_fix_type={self.fix_type})") - sys.exit() + raise AssertionError() self.eta = eta self.min_clip = min_clip self.max_clip = max_clip diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 7984e89..10f0965 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -102,6 +102,6 @@ class TestInitialization: def test_higher_min_budget(self): """Test that verifies, that DEHB breaks if min_budget > max_budget.""" cs = create_toy_searchspace() - with pytest.raises(SystemExit): + with pytest.raises(AssertionError): create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3, objective_function=objective_function) From c51a6941820f1f2dcaee74a7088f395350ebc6c6 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Thu, 27 Jul 2023 11:33:10 +0200 Subject: [PATCH 5/8] Adjust error message to point user to DE documentation --- src/dehb/optimizers/dehb.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index e3e6a6b..dbd4a6b 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -76,13 +76,8 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, if self.max_budget == self.min_budget: self.logger.error( "If you have a fixed fidelity, " \ - "you can instead run DE as follows: " \ - "AsyncDE(cs=configspace, f=target_function, dimensions=" \ - f"{self.dimensions}, pop_size={self.dimensions * 2}, " \ - f"max_age={self.max_age}, mutation_factor=" \ - f"{self.mutation_factor}, crossover_prob={self.crossover_prob}, "\ - f"strategy={self.strategy}, budget={self.max_budget}, " \ - f"boundary_fix_type={self.fix_type})") + "you can instead run DE. For more information checkout: " \ + "https://automl.github.io/DEHB/references/de") raise AssertionError() self.eta = eta self.min_clip = min_clip From 5ad146bd57ce1d3e200dac05ea85817fed5c70a8 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 29 Jul 2023 18:53:07 +0200 Subject: [PATCH 6/8] Move logger setup to extra function and implement new unit test for min = max budget --- src/dehb/optimizers/dehb.py | 24 ++++++++++++++---------- tests/test_dehb.py | 9 ++++++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/dehb/optimizers/dehb.py b/src/dehb/optimizers/dehb.py index dbd4a6b..5ec8985 100644 --- a/src/dehb/optimizers/dehb.py +++ b/src/dehb/optimizers/dehb.py @@ -28,16 +28,7 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, max_budget=None, eta=None, min_clip=None, max_clip=None, boundary_fix_type='random', max_age=np.inf, **kwargs): # Miscellaneous - self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' - os.makedirs(self.output_path, exist_ok=True) - self.logger = logger - log_suffix = time.strftime("%x %X %Z") - log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_') - self.logger.add( - "{}/dehb_{}.log".format(self.output_path, log_suffix), - **_logger_props - ) - self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix) + self._setup_logger(kwargs) # Benchmark related variables self.cs = cs @@ -104,6 +95,19 @@ def __init__(self, cs=None, f=None, dimensions=None, mutation_factor=None, self.inc_config = None self.history = [] + def _setup_logger(self, kwargs): + """Sets up the logger.""" + self.output_path = kwargs['output_path'] if 'output_path' in kwargs else './' + os.makedirs(self.output_path, exist_ok=True) + self.logger = logger + log_suffix = time.strftime("%x %X %Z") + log_suffix = log_suffix.replace("/", '-').replace(":", '-').replace(" ", '_') + self.logger.add( + "{}/dehb_{}.log".format(self.output_path, log_suffix), + **_logger_props + ) + self.log_filename = "{}/dehb_{}.log".format(self.output_path, log_suffix) + def reset(self): self.inc_score = np.inf self.inc_config = None diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 10f0965..4cce7bf 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -40,7 +40,7 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget dim = len(configspace.get_hyperparameters()) return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_budget=min_budget, - max_budget=max_budget, eta=eta, n_workers=1) + max_budget=max_budget, eta=eta, n_workers=2) def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): @@ -105,3 +105,10 @@ def test_higher_min_budget(self): with pytest.raises(AssertionError): create_toy_optimizer(configspace=cs, min_budget=28, max_budget=27, eta=3, objective_function=objective_function) + + def test_equal_min_max_budget(self): + """Test that verifies, that DEHB breaks if min_budget == max_budget.""" + cs = create_toy_searchspace() + with pytest.raises(AssertionError): + create_toy_optimizer(configspace=cs, min_budget=27, max_budget=27, eta=3, + objective_function=objective_function) From f8130b33db28871bff817700b28b8a038b016ef0 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sun, 30 Jul 2023 14:54:37 +0200 Subject: [PATCH 7/8] Adjust dev requirements in current setup to updated pipelines --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 120faf9..1d015b5 100644 --- a/setup.py +++ b/setup.py @@ -48,12 +48,10 @@ def read_file(filepath: str) -> str: "pytest-xdist", "pytest-timeout", # Docs - "automl_sphinx_theme", + "mkdocs-material", + "mkdocstrings", # Others - "isort", - "black", - "pydocstyle", - "flake8", + "ruff", "pre-commit", ] } @@ -79,6 +77,7 @@ def read_file(filepath: str) -> str: "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Natural Language :: English", "Intended Audience :: Developers", "Intended Audience :: Education", From 0c6b26a385838ba1853504d4ac08b3ec4a2d97e2 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Mon, 31 Jul 2023 12:37:14 +0200 Subject: [PATCH 8/8] Use single worker for test toy optimizer --- tests/test_dehb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 4cce7bf..8a96e03 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -40,7 +40,7 @@ def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget dim = len(configspace.get_hyperparameters()) return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_budget=min_budget, - max_budget=max_budget, eta=eta, n_workers=2) + max_budget=max_budget, eta=eta, n_workers=1) def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs):