-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #37 from automl/unittests
Repair unittest and test coverage pipeline
- Loading branch information
Showing
5 changed files
with
120 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -45,7 +45,7 @@ jobs: | |
strategy: | ||
fail-fast: false | ||
matrix: | ||
python-version: ["3.10"] | ||
python-version: ["3.8", "3.9", "3.10", "3.11"] | ||
os: ["ubuntu-latest", "macos-latest", "windows-latest"] | ||
|
||
steps: | ||
|
@@ -73,6 +73,15 @@ jobs: | |
run: | | ||
pytest ${{ env.pytest-args }} ${{ env.test-dir }} | ||
- name: Coveralls GitHub Action | ||
uses: coverallsapp/github-action@v2 | ||
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' | ||
|
||
- name: Remove Coverage file | ||
uses: JesseTG/[email protected] | ||
with: | ||
path: coverage.lcov | ||
|
||
- name: Check for files left behind by test | ||
run: | | ||
before="${{ steps.status-before.outputs.BEFORE }}" | ||
|
@@ -82,79 +91,4 @@ jobs: | |
echo "git status from after: $after" | ||
echo "Not all generated files have been deleted!" | ||
exit 1 | ||
fi | ||
# Testing with conda | ||
conda-tests: | ||
name: conda-${{ matrix.python-version }}-${{ matrix.os }} | ||
runs-on: ${{ matrix.os }} | ||
defaults: | ||
run: | ||
shell: bash -l {0} # Default to using bash on all and load (-l) .bashrc which miniconda uses | ||
|
||
strategy: | ||
fail-fast: false | ||
matrix: | ||
python-version: ["3.8", "3.9", "3.10"] | ||
os: ["ubuntu-latest", "macos-latest", "windows-latest"] | ||
|
||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v2 | ||
|
||
- name: Conda install | ||
uses: conda-incubator/setup-miniconda@v2 | ||
with: | ||
auto-update-conda: true | ||
python-version: ${{ matrix.python-version }} | ||
|
||
- name: Install ${{ env.package-name }} | ||
run: | | ||
python -V | ||
python -m pip install --upgrade pip | ||
python -m pip install wheel | ||
python -m pip install -e ".${{ env.extra-requires }}" | ||
- name: Tests | ||
run: | | ||
pytest ${{ env.pytest-args }} ${{ env.test-dir }} | ||
# Testing a dist install | ||
dist-test: | ||
name: dist-${{ matrix.python-version }}-${{ matrix.os }} | ||
|
||
runs-on: ${{ matrix.os }} | ||
defaults: | ||
run: | ||
shell: bash | ||
|
||
strategy: | ||
fail-fast: false | ||
matrix: | ||
python-version: ["3.8", "3.9", "3.10"] | ||
os: ["ubuntu-latest", "macos-latest", "windows-latest"] | ||
|
||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v2 | ||
|
||
- name: Setup Python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
|
||
- name: Create sdist | ||
id: sdist | ||
run: | | ||
python -m pip install --upgrade pip | ||
python setup.py sdist | ||
echo "${{env.package-name}}" | ||
echo "sdist_name=$(ls -t dist/${{ env.package-name }}-*.tar.gz | head -n 1)" >> $GITHUB_ENV | ||
- name: Install ${{ env.package-name }} | ||
run: | | ||
python -m pip install ${{ env.sdist_name }}${{ env.extra-requires }} | ||
- name: Tests | ||
run: | | ||
pytest ${{ env.pytest-args }} ${{ env.test-dir }} | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
import pytest | ||
import typing | ||
import ConfigSpace | ||
import numpy as np | ||
import time | ||
from dehb.optimizers.dehb import DEHB | ||
|
||
def create_toy_searchspace(): | ||
"""Creates a toy searchspace with a single hyperparameter. | ||
Can be used in order to instantiate a DEHB instance for simple unittests not | ||
requiring a proper configuration space for optimization. | ||
Returns: | ||
ConfigurationSpace: Toy searchspace | ||
""" | ||
cs = ConfigSpace.ConfigurationSpace() | ||
cs.add_hyperparameter( | ||
ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False)) | ||
return cs | ||
|
||
def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float, | ||
max_budget: float, eta: int, | ||
objective_function: typing.Callable): | ||
"""Creates a DEHB instance. | ||
Args: | ||
configspace (ConfigurationSpace): Searchspace to use | ||
min_budget (float): Minimum budget for DEHB | ||
max_budget (float): Maximum budget for DEHB | ||
eta (int): Eta parameter of DEHB | ||
objective_function (Callable): Function to optimize | ||
Returns: | ||
_type_: _description_ | ||
""" | ||
dim = len(configspace.get_hyperparameters()) | ||
return DEHB(f=objective_function, cs=configspace, dimensions=dim, | ||
min_budget=min_budget, | ||
max_budget=max_budget, eta=eta, n_workers=1) | ||
|
||
|
||
def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs): | ||
"""Toy objective function. | ||
Args: | ||
x (ConfigSpace.Configuration): Configuration to evaluate | ||
budget (float): Budget to evaluate x on | ||
Returns: | ||
dict: Result dictionary | ||
""" | ||
y = np.random.uniform() | ||
cost = 5 | ||
result = { | ||
"fitness": y, | ||
"cost": cost | ||
} | ||
return result | ||
|
||
class TestBudgetExhaustion(): | ||
"""Class that bundles all Budget exhaustion tests. | ||
These tests include budget exhaustion tests for runtime, number of function | ||
evaluations and number of brackets to run. | ||
""" | ||
def test_runtime_exhaustion(self): | ||
"""Test for runtime budget exhaustion. | ||
""" | ||
cs = create_toy_searchspace() | ||
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, | ||
objective_function=objective_function) | ||
|
||
dehb.start = time.time() - 10 | ||
|
||
assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted" | ||
|
||
def test_fevals_exhaustion(self): | ||
"""Test for function evaluations budget exhaustion. | ||
""" | ||
cs = create_toy_searchspace() | ||
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, | ||
objective_function=objective_function) | ||
|
||
dehb.traj.append("Just needed for the test") | ||
|
||
assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted" | ||
|
||
def test_brackets_exhaustion(self): | ||
"""Test for bracket budget exhaustion. | ||
""" | ||
cs = create_toy_searchspace() | ||
dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3, | ||
objective_function=objective_function) | ||
|
||
dehb.iteration_counter = 5 | ||
|
||
assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted" | ||
|
This file was deleted.
Oops, something went wrong.