analysis sweepers + examples

examples/configs/branin_ablation_path.yaml

@@ -0,0 +1,25 @@
+
+defaults:
+  - _self_
+  - target_function: branin
+  - search_space: branin
+  - override hydra/sweeper: HyperAblationPath
+
+hydra:
+  sweeper:
+    n_trials: 100
+    sweeper_kwargs:
+      max_budget: 100
+      optimizer_kwargs:
+        source_config:
+          x0: -5.0
+          x1: 0.0
+        target_config:
+          x0: 10.0
+          x1: 5.0
+        run_source: True
+    search_space: ${search_space}
+  run:
+    dir: ./tmp/branin_ablation_path/
+  sweep:
+    dir: ./tmp/branin_ablation_path/

examples/configs/branin_grid.yaml

@@ -0,0 +1,19 @@
+
+defaults:
+  - _self_
+  - target_function: branin
+  - search_space: branin
+  - override hydra/sweeper: HyperGrid
+
+hydra:
+  sweeper:
+    n_trials: 20
+    sweeper_kwargs:
+      max_budget: 20
+      optimizer_kwargs:
+        max_grid_size: 20
+    search_space: ${search_space}
+  run:
+    dir: ./tmp/branin_grid/
+  sweep:
+    dir: ./tmp/branin_grid/

examples/configs/mlp_ablation_path.yaml

@@ -0,0 +1,35 @@
+
+defaults:
+  - _self_
+  - target_function: mlp
+  - search_space: mlp_no_conditions
+  - override hydra/sweeper: HyperAblationPath
+
+hydra:
+  sweeper:
+    n_trials: 100
+    sweeper_kwargs:
+      max_budget: 100
+      optimizer_kwargs:
+        source_config:
+          n_layer: 1
+          n_neurons: 8
+          activation: logistic
+          solver: lbfgs
+          batch_size: 30
+          learning_rate: constant
+          learning_rate_init: 1.0
+        target_config:
+          n_layer: 1
+          n_neurons: 1024
+          activation: logistic
+          solver: adam
+          batch_size: 300
+          learning_rate: constant
+          learning_rate_init: 0.0001
+        run_source: True
+    search_space: ${search_space}
+  run:
+    dir: ./tmp/mlp_ablation_path/
+  sweep:
+    dir: ./tmp/mlp_ablation_path/

examples/configs/mlp_grid.yaml

@@ -0,0 +1,19 @@
+
+defaults:
+  - _self_
+  - target_function: mlp
+  - search_space: mlp_no_conditions
+  - override hydra/sweeper: HyperGrid
+
+hydra:
+  sweeper:
+    n_trials: 100
+    sweeper_kwargs:
+      max_budget: 100
+      optimizer_kwargs:
+        configs_per_hp: 2
+    search_space: ${search_space}
+  run:
+    dir: ./tmp/mlp_grid/
+  sweep:
+    dir: ./tmp/mlp_grid/

examples/example_ablation_paths/branin/final_config.yaml

@@ -0,0 +1,19 @@
+x0: 3
+x1: 4
+seed: None
+budget: None
+spurious_var: 3.14
+search_space:
+  seed: 0
+  hyperparameters:
+    x0:
+      type: uniform_float
+      lower: -5
+      upper: 10
+      log: false
+    x1:
+      type: uniform_float
+      lower: 0
+      upper: 15
+      log: false
+      default_value: 2

examples/example_ablation_paths/branin/incumbent.csv

@@ -0,0 +1,3 @@
+config_id,performance,budget,budget_used,total_wallclock_time,total_optimization_time,x0,x1
+2,10.960889035651505,100,300,0.1647658348083496,9.012222290039062e-05,10.0,0.0
+3,5.93132298356619,100,400,0.22229909896850586,0.0033359527587890625,10.0,5.0

examples/example_ablation_paths/branin/runhistory.csv

@@ -0,0 +1,5 @@
+config_id,budget,performance,x0,x1
+0,100,308.12909601160663,-5.0,0.0
+1,100,161.25549717966388,-5.0,5.0
+2,100,10.960889035651505,10.0,0.0
+3,100,5.93132298356619,10.0,5.0

examples/example_ablation_paths/mlp/final_config.yaml

@@ -0,0 +1,54 @@
+learning_rate: constant
+learning_rate_init: 0.001
+batch_size: 200
+n_neurons: 10
+n_layer: 1
+solver: adam
+activation: tanh
+seed: 42
+epochs: 10
+search_space:
+  hyperparameters:
+    n_layer:
+      type: uniform_int
+      lower: 1
+      upper: 5
+      default: 1
+    n_neurons:
+      type: uniform_int
+      lower: 8
+      upper: 1024
+      log: true
+      default_value: 10
+    activation:
+      type: categorical
+      choices:
+      - logistic
+      - tanh
+      - relu
+      default_value: tanh
+    solver:
+      type: categorical
+      choices:
+      - lbfgs
+      - sgd
+      - adam
+      default_value: adam
+    batch_size:
+      type: uniform_int
+      lower: 30
+      upper: 300
+      default_value: 200
+    learning_rate:
+      type: categorical
+      choices:
+      - constant
+      - invscaling
+      - adaptive
+      default_value: constant
+    learning_rate_init:
+      type: uniform_float
+      lower: 0.0001
+      upper: 1
+      default_value: 0.001
+      log: true

examples/example_ablation_paths/mlp/incumbent.csv

@@ -0,0 +1,5 @@
+config_id,performance,budget,budget_used,total_wallclock_time,total_optimization_time,activation,batch_size,learning_rate,learning_rate_init,n_layer,n_neurons,solver
+2,0.06788919839059115,100,500,5.364800214767456,0.0035288333892822266,logistic,30,constant,1.0,1,1024,lbfgs
+2,0.06788919839059115,100,800,11.051582098007202,0.020783185958862305,logistic,30,constant,1.0,1,1024,lbfgs
+2,0.06788919839059115,100,1000,15.861778974533081,0.029172420501708984,logistic,30,constant,1.0,1,1024,lbfgs
+2,0.06788919839059115,100,1100,19.89451313018799,0.0342564582824707,logistic,30,constant,1.0,1,1024,lbfgs

examples/example_ablation_paths/mlp/runhistory.csv

@@ -0,0 +1,12 @@
+config_id,budget,performance,activation,batch_size,learning_rate,learning_rate_init,n_layer,n_neurons,solver
+0,100,0.4718539151965335,logistic,30,constant,1.0,1,8,lbfgs
+1,100,0.8881352522438873,logistic,30,constant,1.0,1,8,adam
+2,100,0.06788919839059115,logistic,30,constant,1.0,1,1024,lbfgs
+3,100,0.4718539151965335,logistic,30,constant,0.0001,1,8,lbfgs
+4,100,0.4718539151965335,logistic,300,constant,1.0,1,8,lbfgs
+5,100,0.736225626740947,logistic,30,constant,0.0001,1,8,adam
+6,100,0.06788919839059115,logistic,30,constant,0.0001,1,1024,lbfgs
+7,100,0.4718539151965335,logistic,300,constant,0.0001,1,8,lbfgs
+8,100,0.9103884246363355,logistic,300,constant,0.0001,1,8,adam
+9,100,0.06788919839059115,logistic,300,constant,0.0001,1,1024,lbfgs
+10,100,0.13633240482822662,logistic,300,constant,0.0001,1,1024,adam

examples/example_grids/branin/final_config.yaml

@@ -0,0 +1,19 @@
+x0: 3
+x1: 4
+seed: None
+budget: None
+spurious_var: 3.14
+search_space:
+  seed: 0
+  hyperparameters:
+    x0:
+      type: uniform_float
+      lower: -5
+      upper: 10
+      log: false
+    x1:
+      type: uniform_float
+      lower: 0
+      upper: 15
+      log: false
+      default_value: 2

examples/example_grids/branin/incumbent.csv

@@ -0,0 +1,11 @@
+config_id,performance,budget,budget_used,total_wallclock_time,total_optimization_time,x0,x1
+1,161.25549717966388,20,40,0.11734890937805176,0.0010728836059570312,-5.0,5.0
+3,17.508299515778166,20,80,0.22930479049682617,0.009371757507324219,-5.0,15.0
+3,17.508299515778166,20,120,0.33854174613952637,0.011746644973754883,-5.0,15.0
+3,17.508299515778166,20,160,0.48398494720458984,0.01403498649597168,-5.0,15.0
+8,14.341398295508888,20,200,0.6280927658081055,0.016147851943969727,5.0,0.0
+8,14.341398295508888,20,240,0.7760419845581055,0.01865983009338379,5.0,0.0
+13,5.93132298356619,20,280,0.8843858242034912,0.020876646041870117,10.0,5.0
+13,5.93132298356619,20,320,0.991621732711792,0.023010730743408203,10.0,5.0
+13,5.93132298356619,20,360,1.0989739894866943,0.024906635284423828,10.0,5.0
+13,5.93132298356619,20,400,1.232278823852539,0.027173519134521484,10.0,5.0

examples/example_grids/branin/runhistory.csv

@@ -0,0 +1,17 @@
+config_id,budget,performance,x0,x1
+0,20,308.12909601160663,-5.0,0.0
+1,20,161.25549717966388,-5.0,5.0
+2,20,64.381898347721,-5.0,10.0
+3,20,17.508299515778166,-5.0,15.0
+4,20,55.602112642270264,0.0,0.0
+5,20,20.602112642270264,0.0,5.0
+6,20,35.602112642270264,0.0,10.0
+7,20,100.60211264227026,0.0,15.0
+8,20,14.341398295508888,5.0,0.0
+9,20,26.622742555461393,5.0,5.0
+10,20,88.90408681541389,5.0,10.0
+11,20,201.1854310753664,5.0,15.0
+12,20,10.960889035651505,10.0,0.0
+13,20,5.93132298356619,10.0,5.0
+14,20,50.901756931480875,10.0,10.0
+15,20,145.87219087939556,10.0,15.0

examples/example_grids/mlp/final_config.yaml

@@ -0,0 +1,54 @@
+learning_rate: constant
+learning_rate_init: 0.001
+batch_size: 200
+n_neurons: 10
+n_layer: 1
+solver: adam
+activation: tanh
+seed: 42
+epochs: 10
+search_space:
+  hyperparameters:
+    n_layer:
+      type: uniform_int
+      lower: 1
+      upper: 5
+      default: 1
+    n_neurons:
+      type: uniform_int
+      lower: 8
+      upper: 1024
+      log: true
+      default_value: 10
+    activation:
+      type: categorical
+      choices:
+      - logistic
+      - tanh
+      - relu
+      default_value: tanh
+    solver:
+      type: categorical
+      choices:
+      - lbfgs
+      - sgd
+      - adam
+      default_value: adam
+    batch_size:
+      type: uniform_int
+      lower: 30
+      upper: 300
+      default_value: 200
+    learning_rate:
+      type: categorical
+      choices:
+      - constant
+      - invscaling
+      - adaptive
+      default_value: constant
+    learning_rate_init:
+      type: uniform_float
+      lower: 0.0001
+      upper: 1
+      default_value: 0.001
+      log: true

examples/example_grids/mlp/incumbent.csv

@@ -0,0 +1,11 @@
+config_id,performance,budget,budget_used,total_wallclock_time,total_optimization_time,activation,batch_size,learning_rate,learning_rate_init,n_layer,n_neurons,solver
+3,0.05508511296812135,100,1000,358.7331418991089,0.0015189647674560547,logistic,30,constant,0.0001,1,1024,adam
+3,0.05508511296812135,100,2000,704.0697720050812,0.02450704574584961,logistic,30,constant,0.0001,1,1024,adam
+3,0.05508511296812135,100,3000,1033.6736238002777,0.02901911735534668,logistic,30,constant,0.0001,1,1024,adam
+3,0.05508511296812135,100,4000,1437.3339347839355,0.03760504722595215,logistic,30,constant,0.0001,1,1024,adam
+3,0.05508511296812135,100,5000,1526.0074529647827,0.04636120796203613,logistic,30,constant,0.0001,1,1024,adam
+3,0.05508511296812135,100,6000,1628.795156955719,0.05406641960144043,logistic,30,constant,0.0001,1,1024,adam
+67,0.02893686165273901,100,7000,1719.8639659881592,0.07527446746826172,relu,30,constant,0.0001,1,1024,adam
+71,0.013354998452491329,100,8000,2463.8537600040436,0.0835425853729248,relu,30,constant,0.0001,5,1024,adam
+71,0.013354998452491329,100,9000,2834.903028011322,0.125138521194458,relu,30,constant,0.0001,5,1024,adam
+71,0.013354998452491329,100,10000,3205.357225894928,0.14641737937927246,relu,30,constant,0.0001,5,1024,adam