add tests for checking optimizer loading via run_args

tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml

@@ -4,7 +4,7 @@ run_pipeline:
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
 root_directory: "tests_tmpdir/test_run_args_by_neps_run/results2"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml

@@ -0,0 +1,30 @@
+# args of optimizer from searcher kwargs (neps.run) and from run_args (yaml)
+
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: hyperband
+  name: my_hyperband
+  eta: 8
+  initial_design_type: max_budget
+  use_priors: false
+
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml

@@ -0,0 +1,41 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "priorband"
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml

@@ -0,0 +1,23 @@
+run_pipeline:
+  path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py"
+  name: run_pipeline
+pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
+root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo"
+
+max_evaluations_total: 5
+max_cost_total:
+
+monitoring:
+  overwrite_working_directory: true
+  post_run_summary: false
+  development_stage_id: None
+  task_id: None
+
+parallelization_setup:
+  max_evaluations_per_run: None
+  continue_until_max_evaluation_completed:
+
+searcher:
+  strategy: "bayesian_optimization"
+
+pre_load_hooks: None

tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml

@@ -4,7 +4,7 @@ run_pipeline:
 pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml"
 root_directory: "tests_tmpdir/test_run_args_by_neps_run/results1"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml

@@ -7,7 +7,7 @@ pipeline_space:
   name: "pipeline_space"
 root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results"
 
-max_evaluations_total: 1
+max_evaluations_total: 5
 max_cost_total:
 
 monitoring:

tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py

@@ -22,9 +22,20 @@ def run_pipeline(learning_rate, epochs, optimizer, batch_size):
 )
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Run NEPS optimization with run_args.yml."
-    )
-    parser.add_argument("run_args", type=str, help="Path to the YAML configuration file.")
-    args = parser.parse_args()
-    neps.run(run_args=args.run_args)
+    if __name__ == "__main__":
+        parser = argparse.ArgumentParser(
+            description="Run NEPS optimization with run_args.yml.")
+        parser.add_argument("run_args", type=str,
+                            help="Path to the YAML configuration file.")
+        parser.add_argument("--kwargs_flag", action="store_true",
+                            help="Additional keyword arguments")
+        args = parser.parse_args()
+
+        hyperband_args_optimizer = {"random_interleave_prob": 0.9,
+                                    "sample_default_first": False,
+                                    "sample_default_at_target": False}
+
+        if args.kwargs_flag:
+            neps.run(run_args=args.run_args, **hyperband_args_optimizer)
+        else:
+            neps.run(run_args=args.run_args)

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml

@@ -0,0 +1,11 @@
+searcher_name: my_hyperband
+searcher_alg: hyperband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 8
+  initial_design_type: max_budget
+  use_priors: false
+  random_interleave_prob: 0.9
+  sample_default_first: false
+  sample_default_at_target: false

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml

@@ -0,0 +1,23 @@
+searcher_name: custom_priorband
+searcher_alg: priorband
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  eta: 3
+  initial_design_type: max_budget
+  prior_confidence: medium
+  random_interleave_prob: 0.0
+  sample_default_first: true
+  sample_default_at_target: false
+  prior_weight_type: geometric
+  inc_sample_type: mutation
+  inc_mutation_rate: 0.5
+  inc_mutation_std: 0.25
+  inc_style: dynamic
+  model_based: true
+  modelling_type: joint
+  initial_design_size: 5
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation

tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml

@@ -0,0 +1,13 @@
+searcher_name: custom_bayesian_optimization
+searcher_alg: bayesian_optimization
+searcher_selection: user-run_args-yaml
+neps_decision_tree: false
+searcher_args:
+  initial_design_size: 10
+  surrogate_model: gp
+  acquisition: EI
+  log_prior_weighted: false
+  acquisition_sampler: mutation
+  random_interleave_prob: 0.0
+  disable_priors: true
+  sample_default_first: false

tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml

@@ -0,0 +1,11 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: true
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+optimizer:
+  choices: ["a", "b", "c"]
+batch_size: 64

tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml

@@ -0,0 +1,15 @@
+epochs:
+  lower: 1
+  upper: 3
+  is_fidelity: True
+learning_rate:
+  lower: 1e-6
+  upper: 1e-1
+  log: False
+  default: 1e-3
+  default_confidence: "low"
+optimizer:
+  choices: ["a", "b", "c"]
+  default: "b"
+  default_confidence: "high"
+batch_size: 64

tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py

@@ -2,22 +2,65 @@
 import subprocess
 import os
 import sys
+import yaml
 BASE_PATH = "tests/test_yaml_run_args/test_run_args_by_neps_run/"
 
 
 @pytest.mark.neps_api
-@pytest.mark.parametrize("yaml_file", [
-    "config.yaml",
-    "loading_pipeline_space.yaml",
-    "loading_optimizer.yaml"
+@pytest.mark.parametrize("config", [
+    {"file_name": "config.yaml"},
+    {"file_name": "loading_pipeline_space.yaml"},
+    {"file_name": "loading_optimizer.yaml"},
+    {"file_name": "config_select_bo.yaml", "check_optimizer": True, "optimizer_path":
+        "select_bo_run_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo"
+                    "/.optimizer_info.yaml"},
+    {"file_name": "config_priorband_with_args.yaml", "check_optimizer": True,
+     "optimizer_path": "priorband_args_run_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband"
+                    "/.optimizer_info.yaml"},
+    {"file_name": "config_hyperband_mixed_args.yaml", "check_optimizer": True,
+     "optimizer_path": "hyperband_searcher_kwargs_yaml_args.yaml",
+     "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband"
+                    "/.optimizer_info.yaml", "args": True}
 ])
-def test_run_with_yaml(yaml_file: str) -> None:
+def test_run_with_yaml(config: dict) -> None:
     """Test "neps.run" with various run_args.yaml settings to simulate loading options
     for variables."""
-    assert os.path.exists(BASE_PATH + yaml_file), f"{yaml_file} does not exist."
+    file_name = config["file_name"]
+    check_optimizer = config.pop("check_optimizer", False)
+    assert os.path.exists(os.path.join(BASE_PATH, file_name)), (f"{file_name} "
+                                                                f"does not exist.")
+
+    cmd = [sys.executable, os.path.join(BASE_PATH, 'neps_run.py'),
+           os.path.join(BASE_PATH, file_name)]
+    if "args" in config:
+        cmd.append('--kwargs_flag')
 
     try:
-        subprocess.check_call([sys.executable, BASE_PATH + 'neps_run.py', BASE_PATH +
-                               yaml_file])
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
-        pytest.fail(f"NePS run failed for configuration: {yaml_file}")
+        pytest.fail(f"NePS run failed for configuration: {file_name}")
+
+    if check_optimizer:
+        optimizer_path = config.pop("optimizer_path")
+        result_path = config.pop("result_path")
+        compare_generated_yaml(result_path, optimizer_path)
+
+
+def compare_generated_yaml(result_path, optimizer_path):
+
+    assert os.path.exists(result_path), \
+        "Generated YAML file does not exist."
+
+    assert os.path.exists(BASE_PATH + "optimizer_yamls/" + optimizer_path), \
+        "Solution YAML file does not exist."
+
+    with open(result_path, 'r') as gen_file:
+        generated_content = yaml.safe_load(gen_file)
+
+    with open(BASE_PATH + "optimizer_yamls/" + optimizer_path, 'r') as ref_file:
+        reference_content = yaml.safe_load(ref_file)
+
+    assert generated_content == reference_content, \
+        "The generated YAML does not match the reference YAML"