Version 1.3.3

* Hotfix: Since multi-objective implementation depends on normalized costs, it now is ensured that the
cached costs are updated everytime a new entry is added.
* Removed mac-specific files.
* Added entry point for cli.
* Added `ConfigSpace` to third known parties s.t. sorting should be the same across different
operating systems.
* Fixed bugs in makefile in which tools were specified incorrectly.
* Executed isort/black on examples and tests.
* Updated README.
* Fixed a problem, which incremented time twice before taking log (#833).
* New wrapper for multi-objective models (base_uncorrelated_mo_model). Makes it easier for
developing new multi-objective models.
* Raise error if acquisition function is incompatible with the epm models.
* Restricting pynisher.

Co-authored-by: Difan Deng <[email protected]>
Co-authored-by: Deyao Chen <[email protected]>
Co-authored-by: BastianZim

.github/stale.yml

@@ -17,6 +17,6 @@ markComment: >
   This issue has been automatically marked as stale because it has not had
   recent activity. It will be closed if no further activity occurs. Thank you
   for your contributions.
-  
+
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false

.gitignore

@@ -133,4 +133,7 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-*smac3-output_*
+*smac3-output_*
+
+# macOS files
+.DS_Store

Makefile

@@ -49,7 +49,7 @@ check-black:
 
 check-isort:
 	$(ISORT) ${SOURCE_DIR} --check || :
-	$(BLACK) ${EXAMPLES_DIR} --check || :
+	$(ISORT) ${EXAMPLES_DIR} --check || :
 	$(ISORT) ${TESTS_DIR} --check || :
 
 check-pydocstyle:
@@ -60,7 +60,7 @@ check-mypy:
 
 check-flake8:
 	$(FLAKE8) ${SOURCE_DIR} || :
-	$(BLACK) ${EXAMPLES_DIR} --check || :
+	$(FLAKE8) ${EXAMPLES_DIR} --check || :
 	$(FLAKE8) ${TESTS_DIR} || :
 
 check: check-black check-isort check-mypy check-flake8 check-pydocstyle
@@ -75,7 +75,7 @@ format-black:
 
 format-isort:
 	$(ISORT) ${SOURCE_DIR}
-	$(BLACK) ${EXAMPLES_DIR}
+	$(ISORT) ${EXAMPLES_DIR}
 	$(ISORT) ${TESTS_DIR}
 
 format: format-black format-isort

README.md

@@ -12,7 +12,7 @@ arbitrary algorithms, including hyperparameter optimization of Machine Learning
 Bayesian Optimization in combination with an aggressive racing mechanism to
 efficiently decide which of two configurations performs better.
 
-SMAC3 is written in Python3 and continuously tested with Python 3.7, 3.8 and 3.9. Its Random
+SMAC3 is written in Python3 and continuously tested with Python 3.7, 3.8, 3.9, and 3.10. Its Random
 Forest is written in C++. In further texts, SMAC is representatively mentioned for SMAC3.
 
 [Documention](https://automl.github.io/SMAC3)

changelog.md

@@ -1,3 +1,20 @@
+# 1.3.3
+* Hotfix: Since multi-objective implementation depends on normalized costs, it now is ensured that the
+cached costs are updated everytime a new entry is added.
+* Removed mac-specific files.
+* Added entry point for cli.
+* Added `ConfigSpace` to third known parties s.t. sorting should be the same across different
+operating systems.
+* Fixed bugs in makefile in which tools were specified incorrectly.
+* Executed isort/black on examples and tests.
+* Updated README.
+* Fixed a problem, which incremented time twice before taking log (#833).
+* New wrapper for multi-objective models (base_uncorrelated_mo_model). Makes it easier for
+developing new multi-objective models.
+* Raise error if acquisition function is incompatible with the epm models.
+* Restricting pynisher.
+
+
 # 1.3.2
 * Added stale bot support.
 * If package version 0.0.0 via `get_distribution` is found, the version of the module is used

examples/commandline/spear_qcp/target_algorithm/scripts/genericWrapper.py

@@ -705,7 +705,9 @@ def __init__(self):
         self.required = []
         self.args = Arguments()
 
-    def add_argument(self, parameter_name, dest, default=None, help="", type=str, required=False):  # pylint: disable=built-in
+    def add_argument(
+        self, parameter_name, dest, default=None, help="", type=str, required=False
+    ):  # pylint: disable=built-in
         """
         adds arguments to parse from command line
         Args:

examples/python/plot_gb_non_deterministic.py

@@ -16,14 +16,14 @@
 logging.basicConfig(level=logging.INFO)
 
 import numpy as np
-from sklearn.datasets import make_hastie_10_2
-from sklearn.ensemble import GradientBoostingClassifier
-from sklearn.model_selection import KFold, cross_val_score
-
 from ConfigSpace.hyperparameters import (
     UniformFloatHyperparameter,
     UniformIntegerHyperparameter,
 )
+from sklearn.datasets import make_hastie_10_2
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import KFold, cross_val_score
+
 from smac.configspace import ConfigurationSpace
 from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.scenario.scenario import Scenario

examples/python/plot_mlp_mf.py

@@ -22,18 +22,18 @@
 
 import warnings
 
-import numpy as np
-from sklearn.datasets import load_digits
-from sklearn.exceptions import ConvergenceWarning
-from sklearn.model_selection import StratifiedKFold, cross_val_score
-from sklearn.neural_network import MLPClassifier
-
 import ConfigSpace as CS
+import numpy as np
 from ConfigSpace.hyperparameters import (
     CategoricalHyperparameter,
     UniformFloatHyperparameter,
     UniformIntegerHyperparameter,
 )
+from sklearn.datasets import load_digits
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.model_selection import StratifiedKFold, cross_val_score
+from sklearn.neural_network import MLPClassifier
+
 from smac.configspace import ConfigurationSpace
 from smac.facade.smac_mf_facade import SMAC4MF
 from smac.scenario.scenario import Scenario

examples/python/plot_scalarized_multi_objective.py

@@ -17,15 +17,15 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-from sklearn import datasets, svm
-from sklearn.model_selection import cross_val_score
-
 from ConfigSpace.conditions import InCondition
 from ConfigSpace.hyperparameters import (
     CategoricalHyperparameter,
     UniformFloatHyperparameter,
     UniformIntegerHyperparameter,
 )
+from sklearn import datasets, svm
+from sklearn.model_selection import cross_val_score
+
 from smac.configspace import ConfigurationSpace
 from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.scenario.scenario import Scenario

examples/python/plot_sgd_instances.py

@@ -21,15 +21,14 @@
 import warnings
 
 import numpy as np
-from sklearn import datasets
-from sklearn.exceptions import ConvergenceWarning
-from sklearn.linear_model import SGDClassifier
-from sklearn.model_selection import StratifiedKFold, cross_val_score
-
 from ConfigSpace.hyperparameters import (
     CategoricalHyperparameter,
     UniformFloatHyperparameter,
 )
+from sklearn import datasets
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.linear_model import SGDClassifier
+from sklearn.model_selection import StratifiedKFold, cross_val_score
 
 # Import ConfigSpace and different types of parameters
 from smac.configspace import ConfigurationSpace

examples/python/plot_simple_multi_objective.py

@@ -8,9 +8,9 @@
 __license__ = "3-clause BSD"
 
 import numpy as np
+from ConfigSpace.hyperparameters import UniformFloatHyperparameter
 from matplotlib import pyplot as plt
 
-from ConfigSpace.hyperparameters import UniformFloatHyperparameter
 from smac.configspace import ConfigurationSpace
 from smac.facade.smac_bb_facade import SMAC4BB
 from smac.scenario.scenario import Scenario

examples/python/plot_svm_cv.py

@@ -16,15 +16,15 @@
 logging.basicConfig(level=logging.INFO)
 
 import numpy as np
-from sklearn import datasets, svm
-from sklearn.model_selection import cross_val_score
-
 from ConfigSpace.conditions import InCondition
 from ConfigSpace.hyperparameters import (
     CategoricalHyperparameter,
     UniformFloatHyperparameter,
     UniformIntegerHyperparameter,
 )
+from sklearn import datasets, svm
+from sklearn.model_selection import cross_val_score
+
 from smac.configspace import ConfigurationSpace
 from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.scenario.scenario import Scenario

examples/python/plot_svm_eips.py

@@ -0,0 +1,122 @@
+"""
+SVM with EIPS as acquisition functions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+An example to optimize a simple SVM on the IRIS-benchmark with EIPS (EI per seconds)
+acquisition function. Since EIPS requires two types of objections: EI values and the predicted
+time used for the configurations. We need to fit the data
+with a multi-objective model
+"""
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+import numpy as np
+from sklearn import datasets, svm
+from sklearn.model_selection import cross_val_score
+
+from ConfigSpace.hyperparameters import UniformFloatHyperparameter, CategoricalHyperparameter
+
+from smac.configspace import ConfigurationSpace
+from smac.facade.smac_ac_facade import SMAC4AC
+
+# Import SMAC-utilities
+from smac.scenario.scenario import Scenario
+
+# EIPS related
+from smac.optimizer.acquisition import EIPS
+from smac.runhistory.runhistory2epm import RunHistory2EPM4EIPS
+from smac.epm.uncorrelated_mo_rf_with_instances import UncorrelatedMultiObjectiveRandomForestWithInstances
+
+__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
+__license__ = "3-clause BSD"
+
+iris = datasets.load_iris()
+
+
+# Target Algorithm
+def svm_from_cfg(cfg):
+    """Creates a SVM based on a configuration and evaluates it on the
+    iris-dataset using cross-validation. Note here random seed is fixed
+
+    Parameters:
+    -----------
+    cfg: Configuration (ConfigSpace.ConfigurationSpace.Configuration)
+        Configuration containing the parameters.
+        Configurations are indexable!
+
+    Returns:
+    --------
+    A crossvalidated mean score for the svm on the loaded data-set.
+    """
+    # For deactivated parameters, the configuration stores None-values.
+    # This is not accepted by the SVM, so we remove them.
+    cfg = {k: cfg[k] for k in cfg if cfg[k]}
+    # And for gamma, we set it to a fixed value or to "auto" (if used)
+    if "gamma" in cfg:
+        cfg["gamma"] = cfg["gamma_value"] if cfg["gamma"] == "value" else "auto"
+        cfg.pop("gamma_value", None)  # Remove "gamma_value"
+
+    clf = svm.SVC(**cfg, random_state=42)
+
+    scores = cross_val_score(clf, iris.data, iris.target, cv=5)
+    return 1 - np.mean(scores)  # Minimize!
+
+
+if __name__ == "__main__":
+    # Build Configuration Space which defines all parameters and their ranges
+    cs = ConfigurationSpace()
+
+    # We define a few possible types of SVM-kernels and add them as "kernel" to our cs
+    kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default_value="poly")
+    cs.add_hyperparameter(kernel)
+
+    # There are some hyperparameters shared by all kernels
+    C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0, log=True)
+    shrinking = CategoricalHyperparameter("shrinking", [True, False], default_value=True)
+    cs.add_hyperparameters([C, shrinking])
+
+    # Scenario object
+    scenario = Scenario(
+        {
+            "run_obj": "quality",  # we optimize quality (alternatively runtime)
+            "runcount-limit": 50,  # max. number of function evaluations
+            "cs": cs,  # configuration space
+            "deterministic": True,
+        }
+    )
+
+    # Example call of the function
+    # It returns: Status, Cost, Runtime, Additional Infos
+    def_value = svm_from_cfg(cs.get_default_configuration())
+    print("Default Value: %.2f" % def_value)
+
+    # Optimize, using a SMAC-object
+    print("Optimizing! Depending on your machine, this might take a few minutes.")
+
+    # Besides the kwargs used for initializing UncorrelatedMultiObjectiveRandomForestWithInstances,
+    # we also need kwargs for initializing the model insides UncorrelatedMultiObjectiveModel
+    model_kwargs = {"target_names": ["loss", "time"], "model_kwargs": {"seed": 1}}
+    smac = SMAC4AC(
+        scenario=scenario,
+        model=UncorrelatedMultiObjectiveRandomForestWithInstances,
+        rng=np.random.RandomState(42),
+        model_kwargs=model_kwargs,
+        tae_runner=svm_from_cfg,
+        acquisition_function=EIPS,
+        runhistory2epm=RunHistory2EPM4EIPS
+    )
+
+    incumbent = smac.optimize()
+
+    inc_value = svm_from_cfg(incumbent)
+    print("Optimized Value: %.2f" % (inc_value))
+
+    # We can also validate our results (though this makes a lot more sense with instances)
+    smac.validate(
+        config_mode="inc",  # We can choose which configurations to evaluate
+        # instance_mode='train+test',  # Defines what instances to validate
+        repetitions=100,  # Ignored, unless you set "deterministic" to "false" in line 95
+        n_jobs=1,
+    )  # How many cores to use in parallel for optimization

examples/python/plot_synthetic_function.py

@@ -16,7 +16,6 @@
 logging.basicConfig(level=logging.INFO)
 
 import numpy as np
-
 from ConfigSpace.hyperparameters import UniformFloatHyperparameter
 
 # Import ConfigSpace and different types of parameters

examples/python/user_prior_mlp.py → examples/python/plot_user_prior_mlp.py

@@ -19,26 +19,25 @@
 logging.basicConfig(level=logging.INFO)
 
 import warnings
-import numpy as np
 
 import ConfigSpace as CS
+import numpy as np
 from ConfigSpace.hyperparameters import (
-    CategoricalHyperparameter,
-    UniformIntegerHyperparameter,
     BetaIntegerHyperparameter,
+    CategoricalHyperparameter,
     NormalFloatHyperparameter,
+    UniformIntegerHyperparameter,
 )
-
 from sklearn.datasets import load_digits
 from sklearn.exceptions import ConvergenceWarning
-from sklearn.model_selection import cross_val_score, StratifiedKFold
+from sklearn.model_selection import StratifiedKFold, cross_val_score
 from sklearn.neural_network import MLPClassifier
 
 from smac.configspace import ConfigurationSpace
-from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.facade.smac_bb_facade import SMAC4BB
-from smac.scenario.scenario import Scenario
+from smac.facade.smac_hpo_facade import SMAC4HPO
 from smac.initial_design.random_configuration_design import RandomConfigurations
+from smac.scenario.scenario import Scenario
 
 __copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
 __license__ = "3-clause BSD"
@@ -135,7 +134,8 @@ def mlp_from_cfg(cfg, seed):
         }
     )
 
-    # The rate at which SMAC forgets the prior. The higher the value, the more the prior is considered.
+    # The rate at which SMAC forgets the prior.
+    # The higher the value, the more the prior is considered.
     # Defaults to # n_iterations / 10
     user_prior_kwargs = {"decay_beta": 1.5}
 
@@ -144,9 +144,11 @@ def mlp_from_cfg(cfg, seed):
         scenario=scenario,
         rng=np.random.RandomState(42),
         tae_runner=mlp_from_cfg,
-        user_priors=True,  # This flag is required to conduct the optimisation using priors over the optimum
+        # This flag is required to conduct the optimisation using priors over the optimum
+        user_priors=True,
         user_prior_kwargs=user_prior_kwargs,
-        initial_design=RandomConfigurations,  # Using random configurations will cause the initialization to be samples drawn from the prior
+        # Using random configurations will cause the initialization to be samples drawn from the prior
+        initial_design=RandomConfigurations,
     )
 
     # Example call of the function with default values

pyproject.toml

@@ -34,6 +34,7 @@ src_paths = ["smac", "tests"]
 known_types = ["typing", "abc"] # We put these in their own section "types"
 known_test = ["tests"]
 known_first_party = ["smac"]
+known_third_party = ["ConfigSpace"]
 sections = [
     "FUTURE",
     "TYPES",