diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b405ff36..dd60a230 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,31 +18,19 @@ jobs: matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-latest, macos-latest, windows-latest] - exclude: - - os: macos-latest # Segmentation fault on github actions that we can not reproduce in the wild - python-version: '3.8' + defaults: + run: + shell: bash runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v4 - + - run: pipx install poetry - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'poetry' + cache-dependency-path: '**/pyproject.toml' + - run: poetry install + - run: poetry run pytest -m "" # Run all markers - # Temporary bugfix see https://github.com/pre-commit/pre-commit/issues/2178 - - name: Pin virtualenv version - run: pip install virtualenv==20.10.0 - - - name: Install Poetry - uses: abatilo/actions-poetry@v2.3.0 - with: - poetry-version: 1.3.2 - - - name: Run poetry install - run: poetry install - - - name: Run pytest - timeout-minutes: 15 - run: poetry run pytest -m "all_examples or runtime or neps_api or summary_csv" diff --git a/neps/search_spaces/architecture/cfg.py b/neps/search_spaces/architecture/cfg.py index 1a6bd758..f7815f6d 100644 --- a/neps/search_spaces/architecture/cfg.py +++ b/neps/search_spaces/architecture/cfg.py @@ -1,14 +1,17 @@ +from __future__ import annotations import itertools import math import sys from collections import defaultdict, deque from functools import partial from queue import LifoQueue -from typing import Deque, Tuple +from typing import Deque, Tuple, Hashable import numpy as np -from nltk import CFG +from nltk import CFG, Production from nltk.grammar import Nonterminal +from scipy.integrate._ivp.radau import P +from torch import Value class Grammar(CFG): @@ -192,10 +195,12 @@ def sampler( for i in range(0, n) ] - def _sampler(self, symbol=None, user_priors: bool = False): + def _sampler(self, symbol=None, user_priors: bool = False, *, _cache: dict[Hashable, str] | None = None): # simple sampler where each production is sampled uniformly from all possible productions # Tree choses if return tree or list of terminals # recursive implementation + if _cache is None: + _cache = {} # init the sequence tree = "(" + str(symbol) @@ -208,12 +213,19 @@ def _sampler(self, symbol=None, user_priors: bool = False): production = choice(productions, probs=self._prior[str(symbol)]) else: production = choice(productions) + for sym in production.rhs(): if isinstance(sym, str): - # if terminal then add string to sequence + ## if terminal then add string to sequence tree = tree + " " + sym else: - tree = tree + " " + self._sampler(sym, user_priors=user_priors) + ")" + cached = _cache.get(sym) + if cached is None: + cached = self._sampler(sym, user_priors=user_priors, _cache=_cache) + _cache[sym] = cached + + tree = tree + " " + cached + ")" + return tree def sampler_maxMin_func(self, symbol: str = None, largest: bool = True): @@ -284,88 +296,83 @@ def _convergent_sampler( return tree, depth, num_prod def compute_prior(self, string_tree: str, log: bool = True) -> float: - def skip_char(char: str) -> bool: - if char in [" ", "\t", "\n"]: - return True - # special case: "(" is (part of) a terminal - if ( - i != 0 - and char == "(" - and string_tree[i - 1] == " " - and string_tree[i + 1] == " " - ): - return False - if char == "(": - return True - return False - - def find_longest_match( - i: int, string_tree: str, symbols: list, max_match: int - ) -> int: - # search for longest matching symbol and add it - # assumes that the longest match is the true match - j = min(i + max_match, len(string_tree) - 1) - while j > i and j < len(string_tree): - if string_tree[i:j] in symbols: - break - j -= 1 - if j == i: - raise Exception(f"Terminal or nonterminal at position {i} does not exist") - return j - prior_prob = 1.0 if not log else 0.0 symbols = self.nonterminals + self.terminals - max_match = max(map(len, symbols)) - find_longest_match_func = partial( - find_longest_match, - string_tree=string_tree, - symbols=symbols, - max_match=max_match, - ) + q_production_rules: list[tuple[list, int]] = [] + non_terminal_productions: dict[str, list[Production]] = { + sym: self.productions(lhs=Nonterminal(sym)) + for sym in self.nonterminals + } - q_production_rules: LifoQueue = LifoQueue() + _symbols_by_size = sorted(symbols, key=len, reverse=True) + _longest = len(_symbols_by_size[0]) i = 0 - while i < len(string_tree): + _tree_len = len(string_tree) + while i < _tree_len: char = string_tree[i] - if skip_char(char): - pass - elif char == ")" and not string_tree[i - 1] == " ": + if char in " \t\n": + i += 1 + continue + + if char == "(": + if i == 0: + i += 1 + continue + + # special case: "(" is (part of) a terminal + if string_tree[i - 1: i + 2] != " ( ": + i += 1 + continue + + if char == ")" and not string_tree[i - 1] == " ": # closing symbol of production - production = q_production_rules.get(block=False)[0][0] - idx = self.productions(production.lhs()).index(production) + production = q_production_rules.pop()[0][0] + lhs_production = production.lhs() + + idx = self.productions(lhs=lhs_production).index(production) if log: - prior_prob += np.log(self.prior[str(production.lhs())][idx] + 1e-1000) + prior_prob += np.log(self.prior[(lhs_production)][idx] + 1e-15) else: - prior_prob *= self.prior[str(production.lhs())][idx] + prior_prob *= self.prior[str(lhs_production)][idx] + i+=1 + continue + + _s = string_tree[i : i + _longest] + for sym in _symbols_by_size: + if _s.startswith(sym): + break else: - j = find_longest_match_func(i) - sym = string_tree[i:j] - i = j - 1 + raise RuntimeError(f"Terminal or nonterminal at position {i} does not exist") + + i += len(sym) - 1 - if sym in self.terminals: - q_production_rules.queue[-1][0] = [ + if sym in self.terminals: + _productions, _count = q_production_rules[-1] + new_productions = [ + production + for production in _productions + if production.rhs()[_count] == sym + ] + q_production_rules[-1] = (new_productions, _count + 1) + elif sym in self.nonterminals: + if len(q_production_rules) > 0: + _productions, _count = q_production_rules[-1] + new_productions = [ production - for production in q_production_rules.queue[-1][0] - if production.rhs()[q_production_rules.queue[-1][1]] == sym + for production in _productions + if str(production.rhs()[_count]) + == sym ] - q_production_rules.queue[-1][1] += 1 - elif sym in self.nonterminals: - if not q_production_rules.empty(): - q_production_rules.queue[-1][0] = [ - production - for production in q_production_rules.queue[-1][0] - if str(production.rhs()[q_production_rules.queue[-1][1]]) - == sym - ] - q_production_rules.queue[-1][1] += 1 - q_production_rules.put([self.productions(lhs=Nonterminal(sym)), 0]) - else: - raise Exception(f"Unknown symbol {sym}") + q_production_rules[-1] = (new_productions, _count + 1) + + q_production_rules.append((non_terminal_productions[sym], 0)) + else: + raise Exception(f"Unknown symbol {sym}") i += 1 - if not q_production_rules.empty(): + if len(q_production_rules) > 0: raise Exception(f"Error in prior computation for {string_tree}") return prior_prob diff --git a/neps/search_spaces/architecture/graph_grammar.py b/neps/search_spaces/architecture/graph_grammar.py index 553dff4b..e21e94d8 100644 --- a/neps/search_spaces/architecture/graph_grammar.py +++ b/neps/search_spaces/architecture/graph_grammar.py @@ -6,6 +6,7 @@ from functools import partial from typing import Any, ClassVar, Mapping from typing_extensions import override, Self +from neps.utils.types import NotSet, _NotSet import networkx as nx import numpy as np @@ -41,6 +42,7 @@ class GraphParameter(ParameterWithPrior[nx.DiGraph, str], MutatableParameter): DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] = {"not_in_use": 1.0} default_confidence_choice = "not_in_use" has_prior: bool + input_kwargs: dict[str, Any] @property @abstractmethod @@ -71,10 +73,6 @@ def __eq__(self, other: Any) -> bool: @abstractmethod def compute_prior(self, normalized_value: float) -> float: ... - @override - def serialize_value(self) -> str: - return self.id # type: ignore - @override def set_value(self, value: str | None) -> None: # NOTE(eddiebergman): Not entirely sure how this should be done @@ -156,9 +154,29 @@ def normalized_to_value(self, normalized_value: float) -> nx.DiGraph: @override def clone(self) -> Self: - # NOTE(eddiebergman): We don't have any safe way better than a deepcopy - # I think - return deepcopy(self) + new_self = self.__class__(**self.input_kwargs) + + # HACK(eddiebergman): It seems the subclasses all have these and + # so we just copy over those attributes, deepcloning anything that is mutable + if self._value is not None: + _attrs_that_subclasses_use_to_reoresent_a_value = ( + ("_value", True), + ("string_tree", False), + ("string_tree_list", False), + ("nxTree", False), + ("_function_id", False), + ) + for _attr, is_mutable in _attrs_that_subclasses_use_to_reoresent_a_value: + retrieved_attr = getattr(self, _attr, NotSet) + if retrieved_attr is NotSet: + continue + + if is_mutable: + setattr(new_self, _attr, deepcopy(retrieved_attr)) + else: + setattr(new_self, _attr, retrieved_attr) + + return new_self class GraphGrammar(GraphParameter, CoreGraphGrammar): hp_name = "graph_grammar" @@ -207,7 +225,7 @@ def __init__( @override def sample(self, *, user_priors: bool = False) -> Self: - copy_self = deepcopy(self) + copy_self = self.clone() copy_self.reset() copy_self.string_tree = copy_self.grammars[0].sampler(1, user_priors=user_priors)[0] _ = copy_self.value # required for checking if graph is valid! @@ -386,7 +404,7 @@ def create_graph_from_string(self, child: str): raise NotImplementedError -class GraphGrammarRepetitive(CoreGraphGrammar, GraphParameter): +class GraphGrammarRepetitive(GraphParameter, CoreGraphGrammar): hp_name = "graph_grammar_repetitive" def __init__( @@ -487,7 +505,7 @@ def crossover( @override def sample(self, *, user_priors: bool = False) -> Self: - copy_self = deepcopy(self) + copy_self = self.clone() copy_self.reset() copy_self.string_tree_list = [grammar.sampler(1)[0] for grammar in copy_self.grammars] copy_self.string_tree = copy_self.assemble_trees( @@ -614,7 +632,7 @@ def recursive_worker( ) -class GraphGrammarMultipleRepetitive(CoreGraphGrammar, GraphParameter): +class GraphGrammarMultipleRepetitive(GraphParameter, CoreGraphGrammar): hp_name = "graph_grammar_multiple_repetitive" def __init__( @@ -734,7 +752,7 @@ def _identify_macro_grammar(grammar, terminal_to_sublanguage_map): @override def sample(self, *, user_priors: bool = False) -> Self: - copy_self = deepcopy(self) + copy_self = self.clone() copy_self.reset() copy_self.string_tree_list = [ grammar.sampler(1, user_priors=user_priors)[0] diff --git a/neps/search_spaces/hyperparameters/numerical.py b/neps/search_spaces/hyperparameters/numerical.py index 9d23ee16..9aaaf6d1 100644 --- a/neps/search_spaces/hyperparameters/numerical.py +++ b/neps/search_spaces/hyperparameters/numerical.py @@ -22,6 +22,7 @@ from __future__ import annotations +from functools import lru_cache from typing import TYPE_CHECKING, Any, ClassVar, Literal, Mapping, TypeVar from typing_extensions import Self, override @@ -38,6 +39,22 @@ T = TypeVar("T", int, float) +# OPTIM(eddiebergman): When calculating priors over and over, +# creating this scipy.rvs is surprisingly slow. Since we do not +# mutate them, we just cache them. This is done across instances so +# we also can access this cache with new copies of the hyperparameters. +@lru_cache(maxsize=128, typed=False) +def _get_truncnorm_prior_and_std( + low: int | float, + high: int | float, + default: int | float, + confidence_score: float, +) -> tuple[TruncNorm, float]: + std = (high - low) * confidence_score + a, b = (low - default) / std, (high - default) / std + return scipy.stats.truncnorm(a, b), float(std) + + class NumericalParameter(ParameterWithPrior[T, T], MutatableParameter): """A numerical hyperparameter is bounded by a lower and upper value. @@ -222,10 +239,12 @@ def _get_truncnorm_prior_and_std(self) -> tuple[TruncNorm, float]: default = self.default assert default is not None - - std = (high - low) * self.default_confidence_score - a, b = (low - default) / std, (high - default) / std - return scipy.stats.truncnorm(a, b), float(std) + return _get_truncnorm_prior_and_std( + low=low, + high=high, + default=default, + confidence_score=self.default_confidence_score, + ) def to_integer(self) -> IntegerParameter: """Convert the numerical hyperparameter to an integer hyperparameter.""" diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index a70cc494..724ac4e3 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -96,7 +96,7 @@ def set_seed(seed=123): def MNIST( batch_size: int = 256, n_train_size: float = 0.9, - data_reduction_factor: float = 0.3, + data_reduction_factor: float = 0.5, ) -> Tuple[DataLoader, DataLoader, DataLoader]: # Download MNIST training and test datasets if not already downloaded. train_dataset = torchvision.datasets.MNIST( @@ -261,7 +261,7 @@ def run_pipeline(lr, optim, weight_decay): # Load the MNIST dataset for training, validation, and testing. train_loader, validation_loader, test_loader = MNIST( - batch_size=96, n_train_size=0.9, data_reduction_factor=0.3 + batch_size=96, n_train_size=0.6, data_reduction_factor=0.75 ) scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.75) @@ -336,7 +336,7 @@ def run_pipeline(lr, optim, weight_decay): if __name__ == "__main__": """ When running this code without any arguments, it will by default - run bayesian optimization with 10 evaluations of 9 epochs each: + run bayesian optimization with 3 evaluations total. ```bash python neps_tblogger_tutorial.py diff --git a/pyproject.toml b/pyproject.toml index 82ae55c7..421c7eda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -262,7 +262,7 @@ max-args = 10 # Changed from default of 5 [tool.pytest.ini_options] addopts = "--basetemp ./tests_tmpdir -m 'neps_api or core_examples'" markers = [ - "all_examples", + "ci_examples", "core_examples", "regression_all", "runtime", diff --git a/tests/regression_objectives.py b/tests/regression_objectives.py index 499d40c1..6655fcbb 100644 --- a/tests/regression_objectives.py +++ b/tests/regression_objectives.py @@ -4,13 +4,11 @@ from pathlib import Path from typing import Any, Callable -import jahs_bench import numpy as np from typing_extensions import Literal import neps from neps.search_spaces.search_space import SearchSpace, pipeline_space_from_configspace -from tests.joint_config_space import joint_config_space class RegressionObjectiveBase: @@ -68,6 +66,7 @@ def evaluation_func(self): """ If the optimizer is cost aware, return the evaluation function with cost """ + import jahs_bench self.benchmark = jahs_bench.Benchmark( task=self.task, kind="surrogate", download=True, save_dir=self.save_dir @@ -116,6 +115,8 @@ def __init__( required for the benchmark to run will be read from. """ super().__init__(optimizer=optimizer, task=task) + from tests.joint_config_space import joint_config_space + self.save_dir = Path(save_dir) self.benchmark = None diff --git a/tests/test_examples.py b/tests/test_examples.py index bf6f2483..6b5e2299 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -25,12 +25,12 @@ def no_logs_gte_error(caplog): core_examples = [ # Run locally and on github actions - "basic_usage/hyperparameters", + "basic_usage/hyperparameters", # NOTE: This needs to be first for the test below... "basic_usage/analyse", "experimental/expert_priors_for_architecture_and_hyperparameters", "efficiency/multi_fidelity", ] -all_examples = core_examples + [ # Run on github actions +ci_examples = [ # Run on github actions "basic_usage/architecture_and_hyperparameters", "experimental/hierarchical_architecture", "efficiency/expert_priors_for_hyperparameters", @@ -43,7 +43,7 @@ def no_logs_gte_error(caplog): examples_folder = Path(__file__, "..", "..", "neps_examples").resolve() core_examples_scripts = [examples_folder / f"{example}.py" for example in core_examples] -all_examples_scripts = [examples_folder / f"{example}.py" for example in all_examples] +ci_examples_scripts = [examples_folder / f"{example}.py" for example in ci_examples] @pytest.mark.core_examples @@ -56,7 +56,7 @@ def test_core_examples(example): runpy.run_path(example, run_name="__main__") -@pytest.mark.all_examples -@pytest.mark.parametrize("example", all_examples_scripts, ids=all_examples) -def test_all_examples(example): +@pytest.mark.ci_examples +@pytest.mark.parametrize("example", ci_examples_scripts, ids=ci_examples) +def test_ci_examples(example): test_core_examples(example) diff --git a/tests/test_regression.py b/tests/test_regression.py index 41359695..6223d2fd 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -5,6 +5,10 @@ from tests.settings import OPTIMIZERS +pytest.skip( + "Pretty slow and will be reworked", + allow_module_level=True +) @pytest.fixture(autouse=True) def use_tmpdir(tmp_path, request):