From d5c271a0f1428cd4fcab251b4db6fd6e8a149e76 Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Mon, 3 Jul 2023 18:14:33 +0200 Subject: [PATCH 01/14] Add first basic unit tests --- tests/test_dehb.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_myfile.py | 20 ---------------- 2 files changed, 54 insertions(+), 20 deletions(-) create mode 100644 tests/test_dehb.py delete mode 100644 tests/test_myfile.py diff --git a/tests/test_dehb.py b/tests/test_dehb.py new file mode 100644 index 0000000..3b9044e --- /dev/null +++ b/tests/test_dehb.py @@ -0,0 +1,54 @@ +import pytest +import ConfigSpace +import numpy as np +import time +from dehb.optimizers.dehb import DEHB + +def create_toy_searchspace(): + cs = ConfigSpace.ConfigurationSpace() + cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) + return cs + +def create_toy_optimizer(configspace, min_budget, max_budget, eta, objective_function): + dim = len(configspace.get_hyperparameters()) + return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_budget=min_budget, + max_budget=max_budget, eta=eta, n_workers=1) + + +def objective_function(x, budget, **kwargs): + y = np.random.uniform() + cost = 5 + result = { + "fitness": y, + "cost": cost + } + return result + +class TestBudgetExhaustion(): + def test_runtime_exhaustion(self): + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.start = time.time() - 10 + + assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted" + + def test_fevals_exhaustion(self): + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.traj.append("Just needed for the test") + + assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted" + + def test_brackets_exhaustion(self): + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.iteration_counter = 5 + + assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted" + diff --git a/tests/test_myfile.py b/tests/test_myfile.py deleted file mode 100644 index 83f44ef..0000000 --- a/tests/test_myfile.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest - -from dehb.myfile import MyClass - - -def test_oreos(): - """ - Should add `a` and the param `x` - """ - myclass = MyClass(a=3, b={}) - assert myclass.oreos(2) == 5 - - -@pytest.mark.parametrize("value", [0, -1, -10]) -def test_construction_with_negative_a_raises_error(value): - """ - Should raise a ValueError with a negative `a` - """ - with pytest.raises(ValueError): - MyClass(a=value, b={}) From c79e096adb17e917da20239bd6aaa7f48fed344b Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 15:27:53 +0200 Subject: [PATCH 02/14] Add Coveralls to project for coverage analysis and badge --- .github/workflows/pytest.yml | 3 +++ pyproject.toml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f4d86e7..5e2f76f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -73,6 +73,9 @@ jobs: run: | pytest ${{ env.pytest-args }} ${{ env.test-dir }} + - name: Coveralls GitHub Action + uses: coverallsapp/github-action@v2 + - name: Check for files left behind by test run: | before="${{ steps.status-before.outputs.BEFORE }}" diff --git a/pyproject.toml b/pyproject.toml index 11d7e9f..9f119d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,10 @@ [tool.pytest.ini_options] testpaths = ["tests"] # path to the test directory minversion = "3.8" -addopts = "--cov=dehb" # Should be package name +addopts = "--cov=dehb --cov-report=lcov" # Should be package name +pythonpath = [ + "." +] [tool.coverage.run] branch = true From 4f1ec8abd0c75a5ce46cd79342419dbf5e2a5421 Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 15:41:58 +0200 Subject: [PATCH 03/14] Remove coverage file after sending it to Coveralls --- .github/workflows/pytest.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5e2f76f..674106c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -76,6 +76,11 @@ jobs: - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 + - name: Remove Coverage file + uses: JesseTG/rm@v1.0.3 + with: + path: /coverage.lcov + - name: Check for files left behind by test run: | before="${{ steps.status-before.outputs.BEFORE }}" From 49d2936dcd84022dba1cc3ac360f0b398b55d82d Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 15:55:07 +0200 Subject: [PATCH 04/14] Add coverage badge pointing to master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fd44a5..30606cf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # DEHB: Evolutionary Hyperband for Scalable, Robust and Efficient Hyperparameter Optimization - +[![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) ### Installation ```bash # from pypi From 85ea2d5088ae04e73778842798fb05802dead31d Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 16:08:58 +0200 Subject: [PATCH 05/14] Remove creation of redundant code coverage report --- .github/workflows/pytest.yml | 5 ----- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 674106c..5e2f76f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -76,11 +76,6 @@ jobs: - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 - - name: Remove Coverage file - uses: JesseTG/rm@v1.0.3 - with: - path: /coverage.lcov - - name: Check for files left behind by test run: | before="${{ steps.status-before.outputs.BEFORE }}" diff --git a/pyproject.toml b/pyproject.toml index 9f119d1..726cc29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ [tool.pytest.ini_options] testpaths = ["tests"] # path to the test directory minversion = "3.8" -addopts = "--cov=dehb --cov-report=lcov" # Should be package name +addopts = "--cov=dehb" # Should be package name pythonpath = [ "." ] From 00c9f921a54e63ac39b37d182500801d8ca199ad Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 16:13:43 +0200 Subject: [PATCH 06/14] Add coverage file with proper remove command again, since the other file cannot be read --- .github/workflows/pytest.yml | 5 +++++ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5e2f76f..75761a6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -76,6 +76,11 @@ jobs: - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 + - name: Remove Coverage file + uses: JesseTG/rm@v1.0.3 + with: + path: coverage.lcov + - name: Check for files left behind by test run: | before="${{ steps.status-before.outputs.BEFORE }}" diff --git a/pyproject.toml b/pyproject.toml index 726cc29..9f119d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ [tool.pytest.ini_options] testpaths = ["tests"] # path to the test directory minversion = "3.8" -addopts = "--cov=dehb" # Should be package name +addopts = "--cov=dehb --cov-report=lcov" # Should be package name pythonpath = [ "." ] From a5a21edf262434d541832a79ffa6d552c2171cec Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 16:20:45 +0200 Subject: [PATCH 07/14] Remove .coverage file ahead of Coveralls action to fix windows memory error --- .github/workflows/pytest.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 75761a6..d1ed7fa 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,6 +72,11 @@ jobs: - name: Tests run: | pytest ${{ env.pytest-args }} ${{ env.test-dir }} + + - name: Remove unused coverage file to avoid memory error for windows + uses: JesseTG/rm@v1.0.3 + with: + path: .coverage - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 From 1a65ba0c4b5d6e99100256029eb0675c84364f4b Mon Sep 17 00:00:00 2001 From: Janis Fix Date: Tue, 4 Jul 2023 16:33:55 +0200 Subject: [PATCH 08/14] Add test workflow status badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 30606cf..3f27386 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # DEHB: Evolutionary Hyperband for Scalable, Robust and Efficient Hyperparameter Optimization +[![Documentation](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml) [![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) ### Installation ```bash From 6adc8580c42c4c6d52c28a384b3d37274d9fe234 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Thu, 6 Jul 2023 11:34:22 +0200 Subject: [PATCH 09/14] Adjust workflow to send coverage data only once to Coveralls --- .github/workflows/pytest.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d1ed7fa..3cb985b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,14 +72,10 @@ jobs: - name: Tests run: | pytest ${{ env.pytest-args }} ${{ env.test-dir }} - - - name: Remove unused coverage file to avoid memory error for windows - uses: JesseTG/rm@v1.0.3 - with: - path: .coverage - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 + if: ${{ matrix.os == "ubuntu-latest"}}$ - name: Remove Coverage file uses: JesseTG/rm@v1.0.3 From c7a8855f4b95be73b8a46246c5c7d7249a2eeb00 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Thu, 6 Jul 2023 11:38:56 +0200 Subject: [PATCH 10/14] Adjust if clause to use single quotes --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3cb985b..7d90207 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -75,7 +75,7 @@ jobs: - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 - if: ${{ matrix.os == "ubuntu-latest"}}$ + if: matrix.os == 'ubuntu-latest' - name: Remove Coverage file uses: JesseTG/rm@v1.0.3 From c53046f93f199cee3decf0df659ad011ecf14cef Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 8 Jul 2023 16:03:42 +0200 Subject: [PATCH 11/14] Add license badge to README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f27386..d83935d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # DEHB: Evolutionary Hyperband for Scalable, Robust and Efficient Hyperparameter Optimization -[![Documentation](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Tests](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml) [![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) ### Installation ```bash From cbeac3b00ed02da90771a2ee064829f935c3daca Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 8 Jul 2023 16:09:18 +0200 Subject: [PATCH 12/14] Adjust tests to only run for 4 different python versions and 3 different os Remove conda and dist tests from template, since they are not crucial for this repository. --- .github/workflows/pytest.yml | 81 ++---------------------------------- 1 file changed, 3 insertions(+), 78 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7d90207..f0f7778 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: @@ -75,7 +75,7 @@ jobs: - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2 - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' - name: Remove Coverage file uses: JesseTG/rm@v1.0.3 @@ -91,79 +91,4 @@ jobs: echo "git status from after: $after" echo "Not all generated files have been deleted!" exit 1 - fi - -# Testing with conda - conda-tests: - name: conda-${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} # Default to using bash on all and load (-l) .bashrc which miniconda uses - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Conda install - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - - - name: Install ${{ env.package-name }} - run: | - python -V - python -m pip install --upgrade pip - python -m pip install wheel - python -m pip install -e ".${{ env.extra-requires }}" - - - name: Tests - run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} - - # Testing a dist install - dist-test: - name: dist-${{ matrix.python-version }}-${{ matrix.os }} - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Create sdist - id: sdist - run: | - python -m pip install --upgrade pip - python setup.py sdist - echo "${{env.package-name}}" - echo "sdist_name=$(ls -t dist/${{ env.package-name }}-*.tar.gz | head -n 1)" >> $GITHUB_ENV - - - name: Install ${{ env.package-name }} - run: | - python -m pip install ${{ env.sdist_name }}${{ env.extra-requires }} - - - name: Tests - run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} + fi \ No newline at end of file From d8f57fd1d1472337c3275ad9fad218495b68ea85 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Sat, 8 Jul 2023 16:32:58 +0200 Subject: [PATCH 13/14] Add badges for PyPi and python versions --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d83935d..d42f7f1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Tests](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml) [![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) +[![PyPI](https://img.shields.io/pypi/v/dehb)](https://pypi.org/project/dehb/) +[![Static Badge](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20-blue)](https://pypi.org/project/dehb/) ### Installation ```bash # from pypi From 233162d35bc44f76aa124738b28bade8be994a11 Mon Sep 17 00:00:00 2001 From: Bronzila Date: Tue, 11 Jul 2023 11:04:27 +0200 Subject: [PATCH 14/14] Add docstrings and typing to unittests --- tests/test_dehb.py | 56 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/test_dehb.py b/tests/test_dehb.py index 3b9044e..1170830 100644 --- a/tests/test_dehb.py +++ b/tests/test_dehb.py @@ -1,21 +1,56 @@ import pytest +import typing import ConfigSpace import numpy as np import time from dehb.optimizers.dehb import DEHB def create_toy_searchspace(): + """Creates a toy searchspace with a single hyperparameter. + + Can be used in order to instantiate a DEHB instance for simple unittests not + requiring a proper configuration space for optimization. + + + Returns: + ConfigurationSpace: Toy searchspace + """ cs = ConfigSpace.ConfigurationSpace() - cs.add_hyperparameter(ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) + cs.add_hyperparameter( + ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) return cs -def create_toy_optimizer(configspace, min_budget, max_budget, eta, objective_function): +def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float, + max_budget: float, eta: int, + objective_function: typing.Callable): + """Creates a DEHB instance. + + Args: + configspace (ConfigurationSpace): Searchspace to use + min_budget (float): Minimum budget for DEHB + max_budget (float): Maximum budget for DEHB + eta (int): Eta parameter of DEHB + objective_function (Callable): Function to optimize + + Returns: + _type_: _description_ + """ dim = len(configspace.get_hyperparameters()) - return DEHB(f=objective_function, cs=configspace, dimensions=dim, min_budget=min_budget, - max_budget=max_budget, eta=eta, n_workers=1) + return DEHB(f=objective_function, cs=configspace, dimensions=dim, + min_budget=min_budget, + max_budget=max_budget, eta=eta, n_workers=1) + +def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): + """Toy objective function. -def objective_function(x, budget, **kwargs): + Args: + x (ConfigSpace.Configuration): Configuration to evaluate + budget (float): Budget to evaluate x on + + Returns: + dict: Result dictionary + """ y = np.random.uniform() cost = 5 result = { @@ -25,7 +60,14 @@ def objective_function(x, budget, **kwargs): return result class TestBudgetExhaustion(): + """Class that bundles all Budget exhaustion tests. + + These tests include budget exhaustion tests for runtime, number of function + evaluations and number of brackets to run. + """ def test_runtime_exhaustion(self): + """Test for runtime budget exhaustion. + """ cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) @@ -35,6 +77,8 @@ def test_runtime_exhaustion(self): assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted" def test_fevals_exhaustion(self): + """Test for function evaluations budget exhaustion. + """ cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function) @@ -44,6 +88,8 @@ def test_fevals_exhaustion(self): assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted" def test_brackets_exhaustion(self): + """Test for bracket budget exhaustion. + """ cs = create_toy_searchspace() dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, objective_function=objective_function)