merge develop

lightning_pose/utils/frame_selection.py

@@ -5,6 +5,8 @@
 from sklearn.decomposition import PCA
 from sklearn.cluster import KMeans
 import torch
+from tqdm import tqdm
+
 
 from lightning_pose.data.dali import video_pipe, LitDaliWrapper
 from lightning_pose.data.utils import count_frames
@@ -65,7 +67,7 @@ def select_frame_idxs(video_file: str, resize_dims: int = 64, n_clusters: int =
 
     # collect all data
     batches = []
-    for batch in iterator:
+    for batch in tqdm(iterator):
         # batches.append(batch)
         # take mean over color channel, remove spatial dims
         # result is shape (batch_size, height * width)

scripts/config_default.yaml

@@ -193,26 +193,24 @@ losses:
     # nan removal threshold value. 
     # (in prob; heatmaps with confidences below this value are removed)
     prob_threshold: 0.05
-
+  # loss = mse between model heatmap and idealized gaussian heatmap centered on softargmax
   unimodal_mse:
     # weight in front of unimodal_mse loss
     log_weight: 5.0
-
-    prob_threshold: 0.05 # we compute this loss only for heatmaps with confidence prob > prob_threshold
-
+    # we compute this loss only for heatmaps with confidence prob > prob_threshold
+    prob_threshold: 0.05
+  # loss = kl divergence between model heatmap and idealized gaussian heatmap centered on softargmax
   unimodal_kl:
     # weight in front of unimodal_mse loss
     log_weight: 5.0
-
-    prob_threshold: 0.05 # we compute this loss only for heatmaps with confidence prob > prob_threshold
-
+    # we compute this loss only for heatmaps with confidence prob > prob_threshold
+    prob_threshold: 0.05
+  # loss = jensen-shannon divergence between model heatmap and idealized gaussian heatmap centered on softargmax
   unimodal_js:
     # weight in front of unimodal_js loss
     log_weight: 5.0
-
-    prob_threshold: 0.05 # we compute this loss only for heatmaps with confidence prob > prob_threshold
-
-
+    # we compute this loss only for heatmaps with confidence prob > prob_threshold
+    prob_threshold: 0.05
 
 eval:
   # paths to the hydra config files in the output folder, OR absolute paths to such folders.
@@ -244,7 +242,7 @@ eval:
   # create labeled video from each video in test_videos_directory
   create_labeled_video: true
   # confidence threshold for plotting a vid
-  confidence_thresh_for_vid: 0.05
+  confidence_thresh_for_vid: 0.90
   # str with absolute path to the video file you want plotted with keypoints
   video_file_to_plot: ""
   # a list of strings, each points to a .csv file with predictions of a given model to the same video. will be combined with video_file_to_plot to make a visualization
@@ -257,3 +255,10 @@ callbacks:
     increase_factor: 0.01
     final_val: 1.0
     freeze_until_epoch: 0
+
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}

scripts/configs/config.yaml

@@ -188,6 +188,9 @@ losses:
         19.2,
         6.8,
       ]
+    # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
   # loss = mse loss between generated heatmap and ideal gaussian heatmap with same center as generated heatmap
   unimodal_mse:
     # weight in front of unimodal loss

scripts/configs_fish-implant/config_fish-implant.yaml

@@ -308,6 +308,9 @@ losses:
     # for epsilon insensitive rectification
     # (in pixels; diffs below this are not penalized)
     epsilon: 10.0
+    # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
 
 eval:
   # paths to the hydra config files in the output folder, OR absolute paths to such folders.

scripts/configs_fly/config_fly.yaml

@@ -150,6 +150,9 @@ losses:
     # for epsilon insensitive rectification
     # (in pixels; diffs below this are not penalized)
     epsilon: 20.0
+    # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
   # loss = mse loss between generated heatmap and ideal gaussian heatmap with
   # same center as generated heatmap
   unimodal_mse:

scripts/configs_hillman-fly/config_hillman-fly.yaml

@@ -173,6 +173,9 @@ losses:
     # for epsilon insensitive rectification
     # (in pixels; diffs below this are not penalized)
     epsilon: 20.0
+    # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
 
 eval:
   # paths to the hydra config files in the output folder, OR absolute paths to such folders.

scripts/configs_ibl-pupil/config_ibl-pupil.yaml

@@ -150,6 +150,9 @@ losses:
     # for epsilon insensitive rectification
     # (in pixels; diffs below this are not penalized)
     epsilon: 10.0
+      # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
 
 eval:
   # paths to the hydra config files in the output folder, OR absolute paths to such folders.

scripts/configs_mirror-fish/config_mirror-fish.yaml

@@ -174,6 +174,9 @@ losses:
     # for epsilon insensitive rectification
     # (in pixels; diffs below this are not penalized)
     epsilon: 10.0
+    # nan removal value. 
+    # (in prob; heatmaps with max prob values are removed)
+    prob_threshold: 0.05
 
 eval:
   # paths to the hydra config files in the output folder, OR absolute paths to such folders.