Adding DM3 with more noise but it also didn't work

Author: pizarrob

experiments/crazyflie/config_overrides/crazyflie_track.yaml

@@ -28,8 +28,8 @@ task_config:
   inertial_prop_randomization_info:
     M:
       distrib: 'uniform'
-      low: -0.0015
-      high: 0.0015
+      low: -0.0025
+      high: 0.0025
     Ixx:
       distrib: 'uniform'
       low: -0.1e-5
@@ -135,7 +135,7 @@ task_config:
   disturbances:
     observation:
       - disturbance_func: white_noise
-        std: 0.001
+        std: 0.003
     dynamics:
       - disturbance_func: white_noise
-        std: 0.1
+        std: 0.3

experiments/crazyflie/models/rl_models/ppo/mpsf_0.1_dm3/config.yaml

@@ -0,0 +1,246 @@
+algo: ppo
+algo_config:
+  activation: tanh
+  actor_lr: 0.001
+  clip_obs: 10
+  clip_param: 0.2
+  clip_reward: 10
+  critic_lr: 0.001
+  deque_size: 10
+  entropy_coef: 0.01
+  eval_batch_size: 10
+  eval_interval: 10000
+  eval_save_best: true
+  filter_train_actions: true
+  gae_lambda: 0.95
+  gamma: 0.99
+  hidden_dim: 128
+  log_interval: 10000
+  max_env_steps: 250000
+  max_grad_norm: 0.5
+  mini_batch_size: 256
+  norm_obs: false
+  norm_reward: false
+  num_checkpoints: 0
+  num_workers: 1
+  opt_epochs: 20
+  penalize_sf_diff: true
+  rollout_batch_size: 1
+  rollout_steps: 1000
+  save_interval: 0
+  sf_penalty: 0.1
+  target_kl: 0.01
+  tensorboard: false
+  training: true
+  use_clipped_value: false
+  use_gae: true
+  use_safe_reset: true
+device: cpu
+kv_overrides:
+- sf_config.cost_function=one_step_cost
+- algo_config.filter_train_actions=True
+- algo_config.penalize_sf_diff=True
+- algo_config.use_safe_reset=True
+- algo_config.sf_penalty=0.1
+- task_config.use_constraint_penalty=False
+output_dir: ./models/rl_models/ppo/mpsf_0.1_dm3
+overrides:
+- ./config_overrides/crazyflie_track.yaml
+- ./config_overrides/ppo_crazyflie.yaml
+- ./config_overrides/nl_mpsc.yaml
+restore: null
+safety_filter: nl_mpsc
+seed: null
+sf_config:
+  cost_function: one_step_cost
+  decay_factor: 0.85
+  horizon: 10
+  integration_algo: rk4
+  mpsc_cost_horizon: 5
+  n_samples: 600
+  prior_info:
+    prior_prop: null
+    prior_prop_rand_info: null
+    randomize_prior_prop: false
+  q_lin:
+  - 0.008
+  - 1.85
+  - 0.008
+  - 1.85
+  - 10
+  - 10
+  r_lin:
+  - 2
+  use_acados: true
+  use_terminal_set: false
+  warmstart: true
+tag: temp
+task: quadrotor
+task_config:
+  adversary_disturbance: null
+  adversary_disturbance_offset: 0.0
+  adversary_disturbance_scale: 0.01
+  camera_view:
+  - 5
+  - -40
+  - -40
+  - 0.5
+  - -1
+  - 0.5
+  constraint_penalty: -1
+  constraints:
+  - active_dims:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 6
+    - 7
+    constrained_variable: state
+    constraint_form: bounded_constraint
+    lower_bounds:
+    - -0.95
+    - -2
+    - -0.95
+    - -2
+    - -0.25
+    - -0.25
+    upper_bounds:
+    - 0.95
+    - 2
+    - 0.95
+    - 2
+    - 0.25
+    - 0.25
+  - constrained_variable: input
+    constraint_form: default_constraint
+  cost: quadratic
+  ctrl_freq: 500
+  disturbances:
+    dynamics:
+    - disturbance_func: white_noise
+      std: 0.3
+    observation:
+    - disturbance_func: white_noise
+      std: 0.003
+  done_on_out_of_bound: true
+  done_on_violation: false
+  episode_len_sec: 15
+  gui: false
+  inertial_prop:
+    Ixx: 1.4e-05
+    Iyy: 1.4e-05
+    Izz: 2.17e-05
+    M: 0.0345
+  inertial_prop_randomization_info:
+    Ixx:
+      distrib: uniform
+      high: 1.0e-06
+      low: -1.0e-06
+    Iyy:
+      distrib: uniform
+      high: 1.0e-06
+      low: -1.0e-06
+    Izz:
+      distrib: uniform
+      high: 1.0e-06
+      low: -1.0e-06
+    M:
+      distrib: uniform
+      high: 0.0025
+      low: -0.0025
+  info_in_reset: true
+  init_state:
+    init_p: 0
+    init_phi: 0
+    init_psi: 0
+    init_q: 0
+    init_r: 0
+    init_theta: 0
+    init_x: 0
+    init_x_dot: 0
+    init_y: 0
+    init_y_dot: 0
+    init_z: 1
+    init_z_dot: 0
+  init_state_randomization_info:
+    init_p:
+      distrib: uniform
+      high: 0.5
+      low: -0.5
+    init_phi:
+      distrib: uniform
+      high: 0.25
+      low: -0.25
+    init_psi:
+      distrib: uniform
+      high: 0
+      low: 0
+    init_q:
+      distrib: uniform
+      high: 0.5
+      low: -0.5
+    init_r:
+      distrib: uniform
+      high: 0
+      low: 0
+    init_theta:
+      distrib: uniform
+      high: 0.25
+      low: -0.25
+    init_x:
+      distrib: uniform
+      high: 0.95
+      low: -0.95
+    init_x_dot:
+      distrib: uniform
+      high: 2
+      low: -2
+    init_y:
+      distrib: uniform
+      high: 0.95
+      low: -0.95
+    init_y_dot:
+      distrib: uniform
+      high: 2
+      low: -2
+    init_z:
+      distrib: uniform
+      high: 1
+      low: 1
+    init_z_dot:
+      distrib: uniform
+      high: 0
+      low: 0
+  norm_act_scale: 0.1
+  normalized_rl_action_space: false
+  obs_goal_horizon: 0
+  physics: pyb
+  pyb_freq: 1000
+  quad_type: 3
+  randomized_inertial_prop: true
+  randomized_init: false
+  rew_act_weight: 0.0001
+  rew_exponential: true
+  rew_state_weight: 1.0
+  seed: 1337
+  task: traj_tracking
+  task_info:
+    num_cycles: 1
+    proj_normal:
+    - 0
+    - 1
+    - 1
+    proj_point:
+    - 0
+    - 0
+    - 0.5
+    trajectory_plane: xz
+    trajectory_position_offset:
+    - 0
+    - 1
+    trajectory_scale: 1
+    trajectory_type: figure8
+  use_constraint_penalty: false
+  verbose: false
+use_gpu: false

experiments/crazyflie/models/rl_models/ppo/mpsf_0.1_dm3/logs/loss/approx_kl.log

@@ -0,0 +1,26 @@
+step,loss/approx_kl
+10000,0.014540497004054487
+20000,0.012409154457661012
+30000,0.011346515376741688
+40000,0.035299180285073825
+50000,0.021245106402784583
+60000,0.011667467308385918
+70000,0.006656894968667377
+80000,0.014440136899550756
+90000,0.023396018807155396
+100000,0.014779321589351944
+110000,0.023984983928191164
+120000,0.020489891265363745
+130000,0.01915428274078295
+140000,0.01767941640767579
+150000,0.015961797760489088
+160000,0.025441439830077194
+170000,0.011703287883816907
+180000,0.017753030012439316
+190000,0.017511220268594724
+200000,0.017385097426207117
+210000,0.021932779786099373
+220000,0.01595768651653392
+230000,0.023145760580276453
+240000,0.011205980827799066
+250000,0.018960489249244955

experiments/crazyflie/models/rl_models/ppo/mpsf_0.1_dm3/logs/loss/entropy_loss.log

@@ -0,0 +1,26 @@
+step,loss/entropy_loss
+10000,-1.8106971601645152
+20000,-1.8322240412235256
+30000,-1.8304906249046329
+40000,-1.8484702567259472
+50000,-1.733134748538335
+60000,-1.7376533726851147
+70000,-1.6336932837963105
+80000,-1.59782941142718
+90000,-1.5392753720283507
+100000,-1.4729125042756397
+110000,-1.4312396665414173
+120000,-1.3802246312300364
+130000,-1.3919271488984426
+140000,-1.3630793670813244
+150000,-1.247284871339798
+160000,-1.2033760945002239
+170000,-1.1260549108187359
+180000,-1.1539814134438833
+190000,-1.0648897012074787
+200000,-1.0050092359383902
+210000,-0.9445812960465749
+220000,-0.8714521269003551
+230000,-0.8680322647094727
+240000,-0.7861478596925735
+250000,-0.8180636356274288

experiments/crazyflie/models/rl_models/ppo/mpsf_0.1_dm3/logs/loss/policy_loss.log

@@ -0,0 +1,26 @@
+step,loss/policy_loss
+10000,-0.005690815966606119
+20000,-0.01259742840586284
+30000,-0.009443554758569069
+40000,-0.007235572013100111
+50000,-0.015047529632433337
+60000,-0.017006347467995946
+70000,-0.002489459361173485
+80000,-0.012903842581236991
+90000,-0.003840118485633351
+100000,-0.007168129051580952
+110000,-0.0014811347991109526
+120000,-0.003636718594477153
+130000,-0.01774967541777613
+140000,-0.00125425776132927
+150000,-0.01088452067844718
+160000,-0.0027276332932849325
+170000,-0.013703564653057743
+180000,-0.009056110161266208
+190000,-0.010057948827489347
+200000,-0.010866868573943506
+210000,-0.00702074262882479
+220000,-0.007562065251478287
+230000,-0.00805554889624997
+240000,-0.014234422214343716
+250000,-0.011923532010163712

experiments/crazyflie/models/rl_models/ppo/mpsf_0.1_dm3/logs/loss/value_loss.log

@@ -0,0 +1,26 @@
+step,loss/value_loss
+10000,13.269518279808343
+20000,13.763680027678394
+30000,8.96380062877704
+40000,14.288173564770474
+50000,10.736329231750975
+60000,12.9411191784862
+70000,23.509398347812912
+80000,24.774437726823308
+90000,23.809120391281493
+100000,12.89694149895158
+110000,42.12703010948331
+120000,27.93424511378899
+130000,42.34400811920298
+140000,24.363797277094278
+150000,18.025186236420705
+160000,13.754035599705697
+170000,11.474913011699913
+180000,46.118299395552796
+190000,36.187337796267904
+200000,5.288323052674792
+210000,37.57964238106247
+220000,6.2835994539467315
+230000,27.778048256723473
+240000,7.67200679565516
+250000,34.50073091352491