code/docs improved (#7)

* wip * update 5.12.24 * code/docs improvements * code/docs improvements(fixed)
automl · Dec 18, 2024 · 06dd6e7 · 06dd6e7
1 parent 4439186
commit 06dd6e7
Show file tree

Hide file tree

Showing 23 changed files with 326 additions and 265 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # Quick-Tune-Tool
 
 [![image](https://img.shields.io/pypi/l/quicktunetool.svg)](https://pypi.python.org/pypi/quicktunetool)
-[![image](https://img.shields.io/pypi/pyversions/quikctunetool.svg)](https://pypi.python.org/pypi/quicktunetool)
+[![image](https://img.shields.io/pypi/pyversions/quicktunetool.svg)](https://pypi.python.org/pypi/quicktunetool)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md)
 
@@ -31,7 +31,7 @@ from qtt.finetune.image.classification import fn
 task_info, metafeat = extract_task_info_metafeat("path/to/dataset")
 
 # Initialize the optimizer
-optimizer = get_pretrained_optimizer("mtlbm/micro")
+optimizer = get_pretrained_optimizer("mtlbm/full")
 optimizer.setup(128, metafeat)
 
 # Create QuickTuner instance and run

diff --git a/docs/reference/optimizers/index.md b/docs/reference/optimizers/index.md
@@ -20,5 +20,5 @@ This class is intended to be extended for specific optimization strategies, with
 
 ### Available Optimizers
 
-- [**`RandomOptimizer`**][qtt.optimizers.random]
+- [**`RandomOptimizer`**][qtt.optimizers.rndm]
 - [**`QuickOptimizer`**][qtt.optimizers.quick]
diff --git a/docs/reference/optimizers/random.md b/docs/reference/optimizers/random.md
@@ -1 +1 @@
-::: qtt.optimizers.random
+::: qtt.optimizers.rndm
diff --git a/examples/define_search_space.py b/examples/define_search_space.py
@@ -3,7 +3,7 @@
 This examples shows how to define a search space. We use
 [ConfigSpace](https://github.com/automl/ConfigSpace).
 
-This search space is defined for a computer vision classification task and includes
+This search space is defined for a iamge classification task and includes
 various hyperparameters that can be optimized.
 
 First import the necessary modules:
@@ -17,53 +17,53 @@
     OrdinalHyperparameter,
 )
 
-cs = ConfigurationSpace("cv-classification")
+cs = ConfigurationSpace("image-classification")
 
 """
 ## Finetuning Parameters
 The finetuning parameters in this configuration space are designed to control how a
 pre-trained model is fine-tuned on a new dataset. Here's a breakdown of each finetuning
 parameter:
 
-1. **`pct_to_freeze`** (Percentage of Model to Freeze):
+1. **`pct-to-freeze`** (Percentage of Model to Freeze):
 This parameter controls the fraction of the model's layers that will be frozen during
 training. Freezing a layer means that its weights will not be updated. Where `0.0`
 means no layers are frozen, and `1.0` means all layers are frozen, except for the
 final classification layer.
 
-2. **`layer_decay`** (Layer-wise Learning Rate Decay):
+2. **`layer-decay`** (Layer-wise Learning Rate Decay):
 Layer-wise decay is a technique where deeper layers of the model use lower learning
 rates than layers closer to the output.
 
-3. **`linear_probing`**:
+3. **`linear-probing`**:
 When linear probing is enabled, it means the training is focused on updating only the
 final classification layer (linear layer), while keeping the rest of the model frozen.
 
-4. **`stoch_norm`** ([Stochastic Normalization](https://proceedings.neurips.cc//paper_files/paper/2020/hash/bc573864331a9e42e4511de6f678aa83-Abstract.html)):
+4. **`stoch-norm`** ([Stochastic Normalization](https://proceedings.neurips.cc//paper_files/paper/2020/hash/bc573864331a9e42e4511de6f678aa83-Abstract.html)):
 Enabling stochastic normalization during training.
 
-5. **`sp_reg`** ([Starting Point Regularization](https://arxiv.org/abs/1802.01483)):
+5. **`sp-reg`** ([Starting Point Regularization](https://arxiv.org/abs/1802.01483)):
 This parameter controls the amount of regularization applied to the weights of the model
 towards the pretrained model.
 
-6. **`delta_reg`** ([DELTA Regularization](https://arxiv.org/abs/1901.09229)):
+6. **`delta-reg`** ([DELTA Regularization](https://arxiv.org/abs/1901.09229)):
 DELTA regularization aims to preserve the outer layer outputs of the target network.
 
-7. **`bss_reg`** ([Batch Spectral Shrinkage Regularization](https://proceedings.neurips.cc/paper_files/paper/2019/hash/c6bff625bdb0393992c9d4db0c6bbe45-Abstract.html)):
+7. **`bss-reg`** ([Batch Spectral Shrinkage Regularization](https://proceedings.neurips.cc/paper_files/paper/2019/hash/c6bff625bdb0393992c9d4db0c6bbe45-Abstract.html)):
 Batch Spectral Shrinkage (BSS) regularization penalizes the spectral norm of the model's weight matrices.
    
-8. **`cotuning_reg`** ([Co-tuning Regularization](https://proceedings.neurips.cc/paper/2020/hash/c8067ad1937f728f51288b3eb986afaa-Abstract.html)):
+8. **`cotuning-reg`** ([Co-tuning Regularization](https://proceedings.neurips.cc/paper/2020/hash/c8067ad1937f728f51288b3eb986afaa-Abstract.html)):
 This parameter controls the strength of co-tuning, a method that aligns the
 representation of new data with the pre-trained model's representations
 """
-freeze = OrdinalHyperparameter("pct_to_freeze", [0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
-ld = OrdinalHyperparameter("layer_decay", [0.0, 0.65, 0.75])
-lp = OrdinalHyperparameter("linear_probing", [False, True])
-sn = OrdinalHyperparameter("stoch_norm", [False, True])
-sr = OrdinalHyperparameter("sp_reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
-d_reg = OrdinalHyperparameter("delta_reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
-bss = OrdinalHyperparameter("bss_reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
-cot = OrdinalHyperparameter("cotuning_reg", [0.0, 0.5, 1.0, 2.0, 4.0])
+freeze = OrdinalHyperparameter("pct-to-freeze", [0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
+ld = OrdinalHyperparameter("layer-decay", [0.0, 0.65, 0.75])
+lp = OrdinalHyperparameter("linear-probing", [False, True])
+sn = OrdinalHyperparameter("stoch-norm", [False, True])
+sr = OrdinalHyperparameter("sp-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
+d_reg = OrdinalHyperparameter("delta-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
+bss = OrdinalHyperparameter("bss-reg", [0.0, 0.0001, 0.001, 0.01, 0.1])
+cot = OrdinalHyperparameter("cotuning-reg", [0.0, 0.5, 1.0, 2.0, 4.0])
 
 """
 ## Regularization Parameters
@@ -82,35 +82,35 @@
 
 """
 mix = OrdinalHyperparameter("mixup", [0.0, 0.2, 0.4, 1.0, 2.0, 4.0, 8.0])
-mix_p = OrdinalHyperparameter("mixup_prob", [0.0, 0.25, 0.5, 0.75, 1.0])
+mix_p = OrdinalHyperparameter("mixup-prob", [0.0, 0.25, 0.5, 0.75, 1.0])
 cut = OrdinalHyperparameter("cutmix", [0.0, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0])
 drop = OrdinalHyperparameter("drop", [0.0, 0.1, 0.2, 0.3, 0.4])
 smooth = OrdinalHyperparameter("smoothing", [0.0, 0.05, 0.1])
-clip = OrdinalHyperparameter("clip_grad", [0, 1, 10])
+clip = OrdinalHyperparameter("clip-grad", [0, 1, 10])
 
 """
 ## Optimization Parameters
 """
 amp = OrdinalHyperparameter("amp", [False, True])
 opt = Categorical("opt", ["sgd", "momentum", "adam", "adamw", "adamp"])
-betas = Categorical("opt_betas", ["(0.9, 0.999)", "(0.0, 0.99)", "(0.9, 0.99)", "(0.0, 0.999)"])
+betas = Categorical("opt-betas", ["0.9 0.999", "0.0 0.99", "0.9 0.99", "0.0 0.999"])
 lr = OrdinalHyperparameter("lr", [1e-05, 5e-05, 0.0001, 0.0005, 0.001, 0.005, 0.01])
 w_ep = OrdinalHyperparameter("warmup_epochs", [0, 5, 10])
-w_lr = OrdinalHyperparameter("warmup_lr", [0.0, 1e-05, 1e-06])
-wd = OrdinalHyperparameter("weight_decay", [0, 1e-05, 0.0001, 0.001, 0.01, 0.1])
-bs = OrdinalHyperparameter("batch_size", [2, 4, 8, 16, 32, 64, 128, 256, 512])
+w_lr = OrdinalHyperparameter("warmup-lr", [0.0, 1e-05, 1e-06])
+wd = OrdinalHyperparameter("weight-decay", [0, 1e-05, 0.0001, 0.001, 0.01, 0.1])
+bs = OrdinalHyperparameter("batch-size", [2, 4, 8, 16, 32, 64, 128, 256, 512])
 mom = OrdinalHyperparameter("momentum", [0.0, 0.8, 0.9, 0.95, 0.99])
 sched = Categorical("sched", ["cosine", "step", "multistep", "plateau"])
-pe = OrdinalHyperparameter("patience_epochs", [2, 5, 10])
-dr = OrdinalHyperparameter("decay_rate", [0.1, 0.5])
-de = OrdinalHyperparameter("decay_epochs", [10, 20])
+pe = OrdinalHyperparameter("patience-epochs", [2, 5, 10])
+dr = OrdinalHyperparameter("decay-rate", [0.1, 0.5])
+de = OrdinalHyperparameter("decay-epochs", [10, 20])
 da = Categorical(
-    "data_augmentation",
-    ["auto_augment", "random_augment", "trivial_augment", "none"],
+    "data-augmentation",
+    ["auto-augment", "random-augment", "trivial-augment", "none"],
 )
-aa = Categorical("auto_augment", ["v0", "original"])
-ra_nops = OrdinalHyperparameter("ra_num_ops", [2, 3])
-ra_mag = OrdinalHyperparameter("ra_magnitude", [9, 17])
+aa = Categorical("auto-augment", ["v0", "original"])
+ra_nops = OrdinalHyperparameter("ra-num-ops", [2, 3])
+ra_mag = OrdinalHyperparameter("ra-magnitude", [9, 17])
 cond_1 = EqualsCondition(pe, sched, "plateau")
 cond_2 = OrConjunction(
     EqualsCondition(dr, sched, "step"),
@@ -126,9 +126,9 @@
     EqualsCondition(betas, opt, "adamw"),
     EqualsCondition(betas, opt, "adamp"),
 )
-cond_6 = EqualsCondition(ra_nops, da, "random_augment")
-cond_7 = EqualsCondition(ra_mag, da, "random_augment")
-cond_8 = EqualsCondition(aa, da, "auto_augment")
+cond_6 = EqualsCondition(ra_nops, da, "random-augment")
+cond_7 = EqualsCondition(ra_mag, da, "random-augment")
+cond_8 = EqualsCondition(aa, da, "auto-augment")
 cs.add(
     mix,
     mix_p,
@@ -222,3 +222,5 @@
     ],
 )
 cs.add(model)
+
+cs.to_yaml("space.yaml")
diff --git a/examples/quicktuning.py b/examples/quicktuning.py
@@ -1,6 +1,5 @@
-"""A quick example of using QuickCVCLSTuner to tune vision classifiers on a dataset."""
+"""A quick example of using a special QuickTuner to tune image classifiers on a new dataset."""
 
-from qtt import QuickTuner_ImageCLS
+from qtt import QuickImageCLSTuner
 
-tuner = QuickTuner_ImageCLS("path/to/dataset")
-tuner.run(fevals=100, time_budget=3600)
+tuner = QuickImageCLSTuner("path/to/dataset")
diff --git a/examples/step_by_step.py b/examples/step_by_step.py
@@ -1,42 +1,20 @@
 from qtt import QuickOptimizer, QuickTuner
-from qtt.predictors import PerfPredictor, CostPredictor
-from qtt.finetune.cv.classification import finetune_script, extract_task_info_metafeat
+from qtt.finetune.image.classification import extract_image_dataset_metafeat, fn
 import pandas as pd
 from ConfigSpace import ConfigurationSpace
 
-config = pd.read_csv("config.csv", index_col=0)  # pipeline configurations
-meta = pd.read_csv("meta.csv", index_col=0)  # if meta-features are available
-curve = pd.read_csv("curve.csv", index_col=0)  # learning curves
-cost = pd.read_csv("cost.csv", index_col=0)  # runtime costs
+pipeline = pd.read_csv("pipeline.csv", index_col=0)
+curve = pd.read_csv("curve.csv", index_col=0)
+cost = pd.read_csv("cost.csv", index_col=0)
+meta = pd.read_csv("meta.csv", index_col=0)
+cs = ConfigurationSpace.from_yaml("space.yaml")
 
-X = pd.concat([config, meta], axis=1)
-curve = curve.values  # predictors expect curves as numpy arrays
-cost = cost.values  # predictors expect costs as numpy arrays
+config = pd.merge(pipeline, meta, on="dataset")
+config.drop(("dataset"), axis=1, inplace=True)
+opt = QuickOptimizer(cs, 50, cost_aware=True)
 
-perf_predictor = PerfPredictor().fit(X, curve)
-cost_predictor = CostPredictor().fit(X, cost)
+ti, mf = extract_image_dataset_metafeat("path/to/dataset")
+opt.setup(128, mf)
 
-# Define/Load the search space
-cs = ConfigurationSpace()  # ConfigurationSpace.from_json("cs.json")
-
-# Define the optimizer
-optimizer = QuickOptimizer(
-    cs=cs,
-    max_fidelity=50,
-    perf_predictor=perf_predictor,
-    cost_predictor=cost_predictor,
-)
-
-task_info, metafeat = extract_task_info_metafeat("path/to/dataset")
-
-
-optimizer.setup(
-    512,
-    metafeat=metafeat,
-)
-# Define the tuner
-tuner = QuickTuner(
-    optimizer=optimizer,
-    f=finetune_script,
-)
-tuner.run(trial_info=task_info, fevals=100, time_budget=3600)
+qt = QuickTuner(opt, fn)
+qt.run(100, trial_info=ti)
diff --git a/pyproject.toml b/pyproject.toml
@@ -39,9 +39,5 @@ dependencies = [
 "Bug Reports" = "https://github.com/automl/quicktunetool/issues"
 "Source" = "https://github.com/automl/quicktunetool"
 
-[project.scripts]
-train = "qtt.finetune.cv.classification.train:main_cli"
-validate = "qtt.finetune.cv.classification.validate:main_cli"
-
 [tool.setuptools.packages.find]
 where = ["src"]
diff --git a/src/qtt/finetune/image/classification/fn.py b/src/qtt/finetune/image/classification/fn.py
@@ -35,73 +35,84 @@
 trial_args = ["train-split", "val-split", "num-classes"]
 
 
-def fn(trial: dict, trial_info: dict):
+def fn(trial: dict, trial_info: dict) -> dict:
+    """
+    Fine-tune a pretrained model on a image dataset.
+    Using the [fimm library](<https://github.com/rapanti/fimm>).
+
+    Args:
+        trial (dict): A dictionary containing trial-specific configurations. Mandatory keys include:
+            - "config-id" (str): Unique identifier for the trial configuration.
+            - "config" (dict): Hyperparameter settings for the trial, as a dictionary.
+            - "fidelity" (int): Specifies the fidelity level for the trial (e.g., epoch count or number of samples).
+        trial_info (dict): A dictionary with additional trial metadata. Mandatory keys include:
+            - "data-dir" (str): Path to the directory containing the image dataset.
+            - "output-dir" (str): Path to the directory where training results and logs are saved.
+            - "train-split" (str): Path to the training split folder.
+            - "val-split" (str): Path to the validation split folder.
+            - "num-classes" (int): Number of classes in the dataset.
+
+    Returns:
+        dict: Updated trial dictionary with:
+            - "status" (bool): Indicates whether the training process was successful.
+            - "score" (float): Final evaluation score (top-1 accuracy as a decimal).
+            - "cost" (float): Time taken for the training process in seconds.
+    """
+
     if not find_spec("fimm"):
         raise ImportError(
-            "You need to install fimm to run this script. Run `pip install fimm` in your terminal."
+            "You need to install fimm to run this script. Run `pip install fimm` in your console."
         )
 
-    config = trial["config"]
-    fidelity = trial["fidelity"]
-    config_id = trial["config-id"]
+    config: dict = trial["config"]
+    fidelity = str(trial["fidelity"])
+    config_id = str(trial["config-id"])
 
-    data_dir = trial_info["data-dir"]
-    output_dir = trial_info["output-dir"]
+    data_dir: str = trial_info["data-dir"]
+    output_dir: str = trial_info["output-dir"]
 
     args = ["train", "--data-dir", data_dir, "--output", output_dir, "--experiment", str(config_id)]
+    args += ["--fidelity", fidelity]
     args += static_args
     for arg in trial_args:
         args += [f"--{arg}", str(trial_info[arg])]
 
     # DATA AUGMENTATIONS
-    match config.get("data_augmentation"):
-        case "auto_augment":
-            args += ["--aa", config["auto_augment"]]
-        case "trivial_augment":
+    match config.get("data-augmentation"):
+        case "auto-augment":
+            args += ["--aa", config["auto-augment"]]
+        case "trivial-augment":
             args += ["--ta"]
-        case "random_augment":
+        case "random-augment":
             args += ["--ra"]
-            args += ["--ra-num-ops", str(config["ra_num_ops"])]
-            args += ["--ra-magnitude", str(config["ra_magnitude"])]
+            args += ["--ra-num-ops", str(config["ra-num-ops"])]
+            args += ["--ra-magnitude", str(config["ra-magnitude"])]
 
     for k, v in config.items():
-        k = k.replace("_", "-")
         if k in hp_list:
-            args.append(f"--{k}")
-            args.append(str(v))
+            args += [f"--{k}", str(v)]
         elif k in num_hp_list:
             if v > 0:
                 args += [f"--{k}", str(v)]
         elif k in bool_hp_list:
-            if bool(v):
-                args.append(f"--{k}")
+            if v:
+                args += [f"--{k}"]
 
-    start = time.time()
-    process = subprocess.Popen(args)
-    try:
-        process.wait()
-    except KeyboardInterrupt:
-        process.terminate()
-    finally:
-        if process.poll() is None:
-            process.terminate()
-    end = time.time()
+    if os.path.exists(os.path.join(output_dir, str(config_id))):
+        resume_path = os.path.join(output_dir, str(config_id), "last.pth.tar")
+        args += ["--resume", resume_path]
 
+    report: dict = {}
     if process.returncode == 0:
-        status = True
         output_path = os.path.join(output_dir, str(config_id))
         df = pd.read_csv(os.path.join(output_path, "summary.csv"))
-        score = df["eval_top1"].values[-1] / 100
-        cost = end - start
+        report["status"] = True
+        report["score"] = df["eval_top1"].values[-1] / 100
+        report["cost"] = end - start
     else:
-        status = False
-        score = -1
-        cost = -1
+        report["status"] = False
+        report["score"] = 0.0
+        report["cost"] = float("inf")
 
-    return {
-        "config_id": config_id,
-        "fidelity": fidelity,
-        "status": status,
-        "score": score,
-        "cost": cost,
-    }
+    trial.update(report)
+    return trial