refactor: Integrate pipeline space into run args yaml + new design fo…

…r defining constant parameter (#96)

docs/doc_yamls/architecture_search_space.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+from torch import nn
+import neps
+from neps.search_spaces.architecture import primitives as ops
+from neps.search_spaces.architecture import topologies as topos
+from neps.search_spaces.architecture.primitives import AbstractPrimitive
+
+
+class DownSampleBlock(AbstractPrimitive):
+    def __init__(self, in_channels: int, out_channels: int):
+        super().__init__(locals())
+        self.conv_a = ReLUConvBN(
+            in_channels, out_channels, kernel_size=3, stride=2, padding=1
+        )
+        self.conv_b = ReLUConvBN(
+            out_channels, out_channels, kernel_size=3, stride=1, padding=1
+        )
+        self.downsample = nn.Sequential(
+            nn.AvgPool2d(kernel_size=2, stride=2, padding=0),
+            nn.Conv2d(
+                in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False
+            ),
+        )
+
+    def forward(self, inputs):
+        basicblock = self.conv_a(inputs)
+        basicblock = self.conv_b(basicblock)
+        residual = self.downsample(inputs)
+        return residual + basicblock
+
+
+class ReLUConvBN(AbstractPrimitive):
+    def __init__(self, in_channels, out_channels, kernel_size, stride, padding):
+        super().__init__(locals())
+
+        self.kernel_size = kernel_size
+        self.op = nn.Sequential(
+            nn.ReLU(inplace=False),
+            nn.Conv2d(
+                in_channels,
+                out_channels,
+                kernel_size,
+                stride=stride,
+                padding=padding,
+                dilation=1,
+                bias=False,
+            ),
+            nn.BatchNorm2d(out_channels, affine=True, track_running_stats=True),
+        )
+
+    def forward(self, x):
+        return self.op(x)
+
+
+class AvgPool(AbstractPrimitive):
+    def __init__(self, **kwargs):
+        super().__init__(kwargs)
+        self.op = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
+
+    def forward(self, x):
+        return self.op(x)
+
+
+primitives = {
+    "Sequential15": topos.get_sequential_n_edge(15),
+    "DenseCell": topos.get_dense_n_node_dag(4),
+    "down": {"op": DownSampleBlock},
+    "avg_pool": {"op": AvgPool},
+    "id": {"op": ops.Identity},
+    "conv3x3": {"op": ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1},
+    "conv1x1": {"op": ReLUConvBN, "kernel_size": 1, "stride": 1, "padding": 0},
+}
+
+
+structure = {
+    "S": ["Sequential15(C, C, C, C, C, down, C, C, C, C, C, down, C, C, C, C, C)"],
+    "C": ["DenseCell(OPS, OPS, OPS, OPS, OPS, OPS)"],
+    "OPS": ["id", "conv3x3", "conv1x1", "avg_pool"],
+}
+
+
+def set_recursive_attribute(op_name, predecessor_values):
+    in_channels = 16 if predecessor_values is None else predecessor_values["out_channels"]
+    out_channels = in_channels * 2 if op_name == "DownSampleBlock" else in_channels
+    return dict(in_channels=in_channels, out_channels=out_channels)
+
+
+pipeline_space = dict(
+    architecture=neps.ArchitectureParameter(
+        set_recursive_attribute=set_recursive_attribute,
+        structure=structure,
+        primitives=primitives,
+    ),
+    optimizer=neps.CategoricalParameter(choices=["sgd", "adam"]),
+    learning_rate=neps.FloatParameter(lower=10e-7, upper=10e-3, log=True),
+)
+

docs/doc_yamls/customizing_neps_optimizer.yaml

@@ -0,0 +1,24 @@
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  optimizer:
+    choices: [adam, sgd, adamw]
+  epochs: 50
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget
+searcher:
+  algorithm: bayesian_optimization    # name linked with neps keywords, more information click here..?
+  # Specific arguments depending on the searcher
+  initial_design_size: 7
+  surrogate_model: gp
+  acquisition: EI
+  acquisition_sampler: random
+  random_interleave_prob: 0.1
+

docs/doc_yamls/defining_hooks.yaml

@@ -0,0 +1,24 @@
+# Basic NEPS Configuration Example
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  epochs:
+    lower: 5
+    upper: 20
+    is_fidelity: True
+  optimizer:
+    choices: [adam, sgd, adamw]
+  batch_size: 64
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget
+
+pre_load_hooks:
+    hook1: path/to/your/hooks.py # (function_name: Path to the function's file)
+    hook2: path/to/your/hooks.py # Different function name from the same file source

docs/doc_yamls/full_configuration_template.yaml

@@ -0,0 +1,42 @@
+# Full Configuration Template for NePS
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  epochs:
+    lower: 5
+    upper: 20
+    is_fidelity: True
+  optimizer:
+    choices: [adam, sgd, adamw]
+  batch_size: 64
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20             # Budget
+max_cost_total:
+
+# Debug and Monitoring
+overwrite_working_directory: True
+post_run_summary: True
+development_stage_id:
+task_id:
+
+# Parallelization Setup
+max_evaluations_per_run:
+continue_until_max_evaluation_completed: False
+
+# Error Handling
+loss_value_on_error:
+cost_value_on_error:
+ignore_errors:
+
+# Customization Options
+searcher: bayesian_optimization       # Internal key to select a NePS optimizer.
+
+# Hooks
+pre_load_hooks:

docs/doc_yamls/loading_own_optimizer.yaml

@@ -0,0 +1,22 @@
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  optimizer:
+    choices: [adam, sgd, adamw]
+  epochs: 50
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget
+searcher:
+  path: path/to/your/searcher.py  # Path to the class
+  name: CustomOptimizer           # class name within the file
+  # Specific arguments depending on your searcher
+  initial_design_size: 7
+  surrogate_model: gp
+  acquisition: EI

docs/doc_yamls/loading_pipeline_space_dict.yaml

@@ -0,0 +1,11 @@
+# Loading pipeline space from a python dict
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline                   # Function name within the file
+
+pipeline_space:
+  path: path/to/your/search_space.py  # Path to the dict file
+  name: pipeline_space                  # Name of the dict instance
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget

docs/doc_yamls/outsourcing_optimizer.yaml

@@ -0,0 +1,18 @@
+# Optimizer settings from YAML configuration
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  optimizer:
+    choices: [adam, sgd, adamw]
+  epochs: 50
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget
+
+searcher: path/to/your/searcher_setup.yaml

docs/doc_yamls/outsourcing_pipeline_space.yaml

@@ -0,0 +1,10 @@
+# Pipeline space settings from YAML
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline              # Function name within the file
+
+pipeline_space: path/to/your/pipeline_space.yaml
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20               # Budget
+

docs/doc_yamls/pipeline_space.yaml

@@ -0,0 +1,22 @@
+# pipeline_space including priors and fidelity
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+    default: 1e-2
+    default_confidence: "medium"
+  epochs:
+    lower: 5
+    upper: 20
+    is_fidelity: True
+  dropout_rate:
+    lower: 0.1
+    upper: 0.5
+    default: 0.2
+    default_confidence: "high"
+  optimizer:
+    choices: [adam, sgd, adamw]
+    default: adam
+    # default confidence low
+  batch_size: 64

docs/doc_yamls/run_pipeline.py

@@ -0,0 +1,7 @@
+
+
+def example_pipeline(learning_rate, optimizer, epochs):
+    model = initialize_model()
+    training_loss = train_model(model, optimizer, learning_rate, epochs)
+    evaluation_loss = evaluate_model(model)
+    return {"loss": evaluation_loss, "training_loss": training_loss}

docs/doc_yamls/run_pipeline_architecture.py

@@ -0,0 +1,24 @@
+from torch import nn
+
+
+def example_pipeline(architecture, optimizer, learning_rate):
+    in_channels = 3
+    base_channels = 16
+    n_classes = 10
+    out_channels_factor = 4
+
+    # E.g., in shape = (N, 3, 32, 32) => out shape = (N, 10)
+    model = architecture.to_pytorch()
+    model = nn.Sequential(
+        nn.Conv2d(in_channels, base_channels, 3, padding=1, bias=False),
+        nn.BatchNorm2d(base_channels),
+        model,
+        nn.BatchNorm2d(base_channels * out_channels_factor),
+        nn.ReLU(inplace=True),
+        nn.AdaptiveAvgPool2d(1),
+        nn.Flatten(),
+        nn.Linear(base_channels * out_channels_factor, n_classes),
+    )
+    training_loss = train_model(model, optimizer, learning_rate)
+    evaluation_loss = evaluate_model(model)
+    return {"loss": evaluation_loss, "training_loss": training_loss}

docs/doc_yamls/run_pipeline_big_search_space.py

@@ -0,0 +1,6 @@
+
+def example_pipeline(learning_rate, optimizer, epochs, batch_size, dropout_rate):
+    model = initialize_model(dropout_rate)
+    training_loss = train_model(model, optimizer, learning_rate, epochs, batch_size)
+    evaluation_loss = evaluate_model(model)
+    return {"loss": evaluation_loss, "training_loss": training_loss}

docs/doc_yamls/run_pipeline_extended.py

@@ -0,0 +1,6 @@
+
+def example_pipeline(learning_rate, optimizer, epochs, batch_size):
+    model = initialize_model()
+    training_loss = train_model(model, optimizer, learning_rate, epochs, batch_size)
+    evaluation_loss = evaluate_model(model)
+    return {"loss": evaluation_loss, "training_loss": training_loss}

docs/doc_yamls/set_up_optimizer.yaml

@@ -0,0 +1,11 @@
+algorithm: bayesian_optimization
+# Specific arguments depending on the searcher
+initial_design_size: 7
+surrogate_model: gp
+acquisition: EI
+log_prior_weighted: false
+acquisition_sampler: random
+random_interleave_prob: 0.1
+disable_priors: false
+prior_confidence: high
+sample_default_first: false

docs/doc_yamls/simple_example.yaml

@@ -0,0 +1,16 @@
+# Basic NePS Configuration Example
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  epochs:
+    lower: 5
+    upper: 20
+    is_fidelity: True
+  optimizer:
+    choices: [adam, sgd, adamw]
+  batch_size: 64
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20             # Budget

docs/doc_yamls/simple_example_including_run_pipeline.yaml

@@ -0,0 +1,20 @@
+# Simple NePS configuration including run_pipeline
+run_pipeline:
+  path: path/to/your/run_pipeline.py  # Path to the function file
+  name: example_pipeline                   # Function name within the file
+
+pipeline_space:
+  learning_rate:
+    lower: 1e-5
+    upper: 1e-1
+    log: True  # Log scale for learning rate
+  epochs:
+    lower: 5
+    upper: 20
+    is_fidelity: True
+  optimizer:
+    choices: [adam, sgd, adamw]
+  batch_size: 64
+
+root_directory: path/to/results       # Directory for result storage
+max_evaluations_total: 20             # Budget