diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f4d86e7..f0f7778 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: @@ -73,6 +73,15 @@ jobs: run: | pytest ${{ env.pytest-args }} ${{ env.test-dir }} + - name: Coveralls GitHub Action + uses: coverallsapp/github-action@v2 + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' + + - name: Remove Coverage file + uses: JesseTG/rm@v1.0.3 + with: + path: coverage.lcov + - name: Check for files left behind by test run: | before="${{ steps.status-before.outputs.BEFORE }}" @@ -82,79 +91,4 @@ jobs: echo "git status from after: $after" echo "Not all generated files have been deleted!" exit 1 - fi - -# Testing with conda - conda-tests: - name: conda-${{ matrix.python-version }}-${{ matrix.os }} - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} # Default to using bash on all and load (-l) .bashrc which miniconda uses - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Conda install - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - - - name: Install ${{ env.package-name }} - run: | - python -V - python -m pip install --upgrade pip - python -m pip install wheel - python -m pip install -e ".${{ env.extra-requires }}" - - - name: Tests - run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} - - # Testing a dist install - dist-test: - name: dist-${{ matrix.python-version }}-${{ matrix.os }} - - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Create sdist - id: sdist - run: | - python -m pip install --upgrade pip - python setup.py sdist - echo "${{env.package-name}}" - echo "sdist_name=$(ls -t dist/${{ env.package-name }}-*.tar.gz | head -n 1)" >> $GITHUB_ENV - - - name: Install ${{ env.package-name }} - run: | - python -m pip install ${{ env.sdist_name }}${{ env.extra-requires }} - - - name: Tests - run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} + fi \ No newline at end of file diff --git a/README.md b/README.md index 6fd44a5..d42f7f1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # DEHB: Evolutionary Hyperband for Scalable, Robust and Efficient Hyperparameter Optimization - +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Tests](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml) +[![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB) +[![PyPI](https://img.shields.io/pypi/v/dehb)](https://pypi.org/project/dehb/) +[![Static Badge](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20-blue)](https://pypi.org/project/dehb/) ### Installation ```bash # from pypi diff --git a/pyproject.toml b/pyproject.toml index 11d7e9f..9f119d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,10 @@ [tool.pytest.ini_options] testpaths = ["tests"] # path to the test directory minversion = "3.8" -addopts = "--cov=dehb" # Should be package name +addopts = "--cov=dehb --cov-report=lcov" # Should be package name +pythonpath = [ + "." +] [tool.coverage.run] branch = true diff --git a/tests/test_dehb.py b/tests/test_dehb.py new file mode 100644 index 0000000..1170830 --- /dev/null +++ b/tests/test_dehb.py @@ -0,0 +1,100 @@ +import pytest +import typing +import ConfigSpace +import numpy as np +import time +from dehb.optimizers.dehb import DEHB + +def create_toy_searchspace(): + """Creates a toy searchspace with a single hyperparameter. + + Can be used in order to instantiate a DEHB instance for simple unittests not + requiring a proper configuration space for optimization. + + + Returns: + ConfigurationSpace: Toy searchspace + """ + cs = ConfigSpace.ConfigurationSpace() + cs.add_hyperparameter( + ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) + return cs + +def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float, + max_budget: float, eta: int, + objective_function: typing.Callable): + """Creates a DEHB instance. + + Args: + configspace (ConfigurationSpace): Searchspace to use + min_budget (float): Minimum budget for DEHB + max_budget (float): Maximum budget for DEHB + eta (int): Eta parameter of DEHB + objective_function (Callable): Function to optimize + + Returns: + _type_: _description_ + """ + dim = len(configspace.get_hyperparameters()) + return DEHB(f=objective_function, cs=configspace, dimensions=dim, + min_budget=min_budget, + max_budget=max_budget, eta=eta, n_workers=1) + + +def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): + """Toy objective function. + + Args: + x (ConfigSpace.Configuration): Configuration to evaluate + budget (float): Budget to evaluate x on + + Returns: + dict: Result dictionary + """ + y = np.random.uniform() + cost = 5 + result = { + "fitness": y, + "cost": cost + } + return result + +class TestBudgetExhaustion(): + """Class that bundles all Budget exhaustion tests. + + These tests include budget exhaustion tests for runtime, number of function + evaluations and number of brackets to run. + """ + def test_runtime_exhaustion(self): + """Test for runtime budget exhaustion. + """ + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.start = time.time() - 10 + + assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted" + + def test_fevals_exhaustion(self): + """Test for function evaluations budget exhaustion. + """ + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.traj.append("Just needed for the test") + + assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted" + + def test_brackets_exhaustion(self): + """Test for bracket budget exhaustion. + """ + cs = create_toy_searchspace() + dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, + objective_function=objective_function) + + dehb.iteration_counter = 5 + + assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted" + diff --git a/tests/test_myfile.py b/tests/test_myfile.py deleted file mode 100644 index 83f44ef..0000000 --- a/tests/test_myfile.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest - -from dehb.myfile import MyClass - - -def test_oreos(): - """ - Should add `a` and the param `x` - """ - myclass = MyClass(a=3, b={}) - assert myclass.oreos(2) == 5 - - -@pytest.mark.parametrize("value", [0, -1, -10]) -def test_construction_with_negative_a_raises_error(value): - """ - Should raise a ValueError with a negative `a` - """ - with pytest.raises(ValueError): - MyClass(a=value, b={})