Merge pull request #37 from automl/unittests

Repair unittest and test coverage pipeline

.github/workflows/pytest.yml

@@ -45,7 +45,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     steps:
@@ -73,6 +73,15 @@ jobs:
       run: |
         pytest ${{ env.pytest-args }} ${{ env.test-dir }}
 
+    - name: Coveralls GitHub Action
+      uses: coverallsapp/github-action@v2
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
+
+    - name: Remove Coverage file
+      uses: JesseTG/[email protected]
+      with:
+        path: coverage.lcov
+
     - name: Check for files left behind by test
       run: |
         before="${{ steps.status-before.outputs.BEFORE }}"
@@ -82,79 +91,4 @@ jobs:
             echo "git status from after: $after"
             echo "Not all generated files have been deleted!"
             exit 1
-        fi
-
-# Testing with conda
-  conda-tests:
-    name: conda-${{ matrix.python-version }}-${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: bash -l {0}  # Default to using bash on all and load (-l) .bashrc which miniconda uses
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Conda install
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        auto-update-conda: true
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install ${{ env.package-name }}
-      run: |
-        python -V
-        python -m pip install --upgrade pip
-        python -m pip install wheel
-        python -m pip install -e ".${{ env.extra-requires }}"
-
-    - name: Tests
-      run: |
-          pytest ${{ env.pytest-args }} ${{ env.test-dir }}
-
-  # Testing a dist install
-  dist-test:
-    name: dist-${{ matrix.python-version }}-${{ matrix.os }}
-
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: bash
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Create sdist
-      id: sdist
-      run: |
-        python -m pip install --upgrade pip
-        python setup.py sdist
-        echo "${{env.package-name}}"
-        echo "sdist_name=$(ls -t dist/${{ env.package-name }}-*.tar.gz | head -n 1)" >> $GITHUB_ENV
-
-    - name: Install ${{ env.package-name }}
-      run: |
-        python -m pip install ${{ env.sdist_name }}${{ env.extra-requires }}
-
-    - name: Tests
-      run: |
-        pytest ${{ env.pytest-args }} ${{ env.test-dir }}
+        fi

README.md

@@ -1,5 +1,9 @@
 # DEHB: Evolutionary Hyperband for Scalable, Robust and Efficient Hyperparameter Optimization
-
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Tests](https://github.com/automl/DEHB/actions/workflows/pytest.yml/badge.svg)](https://github.com/automl/DEHB/actions/workflows/pytest.yml)
+[![Coverage Status](https://coveralls.io/repos/github/automl/DEHB/badge.svg)](https://coveralls.io/github/automl/DEHB)
+[![PyPI](https://img.shields.io/pypi/v/dehb)](https://pypi.org/project/dehb/)
+[![Static Badge](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20-blue)](https://pypi.org/project/dehb/)
 ### Installation
 ```bash
 # from pypi

pyproject.toml

@@ -4,7 +4,10 @@
 [tool.pytest.ini_options]
 testpaths = ["tests"]             # path to the test directory
 minversion = "3.8"
-addopts = "--cov=dehb" # Should be package name
+addopts = "--cov=dehb --cov-report=lcov" # Should be package name
+pythonpath = [
+  "."
+]
 
 [tool.coverage.run]
 branch = true

tests/test_dehb.py

@@ -0,0 +1,100 @@
+import pytest
+import typing
+import ConfigSpace
+import numpy as np
+import time
+from dehb.optimizers.dehb import DEHB
+
+def create_toy_searchspace():
+    """Creates a toy searchspace with a single hyperparameter.
+
+    Can be used in order to instantiate a DEHB instance for simple unittests not
+    requiring a proper configuration space for optimization.
+
+
+    Returns:
+        ConfigurationSpace: Toy searchspace
+    """
+    cs = ConfigSpace.ConfigurationSpace()
+    cs.add_hyperparameter(
+        ConfigSpace.UniformFloatHyperparameter("x0", lower=3, upper=10, log=False))
+    return cs
+
+def create_toy_optimizer(configspace: ConfigSpace.ConfigurationSpace, min_budget: float,
+                         max_budget: float, eta: int,
+                         objective_function: typing.Callable):
+    """Creates a DEHB instance.
+
+    Args:
+        configspace (ConfigurationSpace): Searchspace to use
+        min_budget (float): Minimum budget for DEHB
+        max_budget (float): Maximum budget for DEHB
+        eta (int): Eta parameter of DEHB
+        objective_function (Callable): Function to optimize
+
+    Returns:
+        _type_: _description_
+    """
+    dim = len(configspace.get_hyperparameters())
+    return DEHB(f=objective_function, cs=configspace, dimensions=dim,
+                min_budget=min_budget,
+                max_budget=max_budget, eta=eta, n_workers=1)
+
+
+def objective_function(x: ConfigSpace.Configuration, budget: float, **kwargs):
+    """Toy objective function.
+
+    Args:
+        x (ConfigSpace.Configuration): Configuration to evaluate
+        budget (float): Budget to evaluate x on
+
+    Returns:
+        dict: Result dictionary
+    """
+    y = np.random.uniform()
+    cost = 5
+    result = {
+        "fitness": y,
+        "cost": cost
+    }
+    return result
+
+class TestBudgetExhaustion():
+    """Class that bundles all Budget exhaustion tests.
+
+    These tests include budget exhaustion tests for runtime, number of function
+    evaluations and number of brackets to run.
+    """
+    def test_runtime_exhaustion(self):
+        """Test for runtime budget exhaustion.
+        """
+        cs = create_toy_searchspace()
+        dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
+                                        objective_function=objective_function)
+
+        dehb.start = time.time() - 10
+
+        assert dehb._is_run_budget_exhausted(total_cost=1), "Run budget should be exhausted"
+
+    def test_fevals_exhaustion(self):
+        """Test for function evaluations budget exhaustion.
+        """
+        cs = create_toy_searchspace()
+        dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
+                                    objective_function=objective_function)
+
+        dehb.traj.append("Just needed for the test")
+
+        assert dehb._is_run_budget_exhausted(fevals=1), "Run budget should be exhausted"
+
+    def test_brackets_exhaustion(self):
+        """Test for bracket budget exhaustion.
+        """
+        cs = create_toy_searchspace()
+        dehb = create_toy_optimizer(configspace=cs, min_budget=3, max_budget=27, eta=3,
+                                        objective_function=objective_function)
+
+        dehb.iteration_counter = 5
+
+        assert dehb._is_run_budget_exhausted(brackets=1), "Run budget should be exhausted"
+