quic · WAKEYOU · Mar 21, 2021 · Mar 21, 2021 · Apr 6, 2021 · May 2, 2021
diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,4 @@
 __pycache__
 *.pyc
 videos
-dataset
+dataset
diff --git a/.travis.yml b/.travis.yml
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
diff --git a/LICENSE b/LICENSE
diff --git a/README.md b/README.md
diff --git a/docs/audio/countdown_sound.wav b/docs/audio/countdown_sound.wav
diff --git a/docs/audio/done_sound.wav b/docs/audio/done_sound.wav
diff --git a/docs/audio/exit_sound.wav b/docs/audio/exit_sound.wav
diff --git a/docs/gifs/action_recognition_1.gif b/docs/gifs/action_recognition_1.gif
diff --git a/docs/gifs/fitness_tracking.gif b/docs/gifs/fitness_tracking.gif
diff --git a/docs/gifs/fitness_tracking_1.gif b/docs/gifs/fitness_tracking_1.gif
diff --git a/docs/gifs/fitness_tracking_2.gif b/docs/gifs/fitness_tracking_2.gif
diff --git a/docs/gifs/gesture_recognition.gif b/docs/gifs/gesture_recognition.gif
diff --git a/docs/gifs/gesture_recognition_1.gif b/docs/gifs/gesture_recognition_1.gif
diff --git a/docs/gifs/hri_recognition.gif b/docs/gifs/hri_recognition.gif
diff --git a/docs/imgs/sense_core_logo.svg b/docs/imgs/sense_core_logo.svg
diff --git a/docs/wiki/home.md b/docs/wiki/home.md
diff --git a/examples/run_action_recognition.py b/examples/run_action_recognition.py
diff --git a/examples/run_calorie_estimation.py b/examples/run_calorie_estimation.py
diff --git a/examples/run_fitness_rep_counter.py b/examples/run_fitness_rep_counter.py
diff --git a/examples/run_fitness_tracker.py b/examples/run_fitness_tracker.py
diff --git a/examples/run_hri_recognition.py b/examples/run_hri_recognition.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+"""
+Real time detection of 30 actions.
+
+Usage:
+  run_action_recognition.py [--camera_id=CAMERA_ID]
+                            [--path_in=FILENAME]
+                            [--path_out=FILENAME]
+                            [--title=TITLE]
+                            [--model_name=NAME]
+                            [--model_version=VERSION]
+                            [--use_gpu]
+  run_action_recognition.py (-h | --help)
+
+Options:
+  --path_in=FILENAME         Video file to stream from
+  --path_out=FILENAME        Video file to stream to
+  --title=TITLE              This adds a title to the window display
+  --model_name=NAME          Name of the model to be used.
+  --model_version=VERSION    Version of the model to be used.
+  --use_gpu                  Whether to run inference on the GPU or not.
+"""
+from docopt import docopt
+
+import sense.display
+from sense.controller import Controller
+from sense.downstream_tasks.hri_recognition import INT2LAB
+from sense.downstream_tasks.hri_recognition import GESTURE_THRESHOLDS
+from sense.downstream_tasks.nn_utils import LogisticRegression
+from sense.downstream_tasks.nn_utils import Pipe
+from sense.downstream_tasks.postprocess import PostprocessClassificationOutput
+from sense.loading import update_backbone_weights
+from sense.loading import build_backbone_network
+from sense.loading import load_weights_from_resources
+from sense.loading import ModelConfig
+
+
+SUPPORTED_MODEL_CONFIGURATIONS = [
+    ModelConfig('StridedInflatedEfficientNet', 'pro', ['action_recognition']),
+    ModelConfig('StridedInflatedMobileNetV2', 'pro', ['action_recognition']),
+    ModelConfig('StridedInflatedEfficientNet', 'lite', ['action_recognition']),
+    ModelConfig('StridedInflatedMobileNetV2', 'lite', ['action_recognition']),
+]
+
+if __name__ == "__main__":
+    # Parse arguments
+    args = docopt(__doc__)
+    camera_id = int(args['--camera_id'] or 0)
+    path_in = args['--path_in'] or None
+    path_out = args['--path_out'] or None
+    title = args['--title'] or None
+    use_gpu = args['--use_gpu']
+
+    # Load backbone network according to config file
+    backbone_model_config = ModelConfig('StridedInflatedEfficientNet', 'pro', [])
+    backbone_weights = backbone_model_config.load_weights()['backbone']
+
+    # Load custom network
+    checkpoint_classifier = load_weights_from_resources('hri_recognition/fyp5024.checkpoint')
+
+    # Update original weights in case some intermediate layers have been finetuned
+    update_backbone_weights(backbone_weights, checkpoint_classifier)
+
+    # Create backbone network
+    backbone_network = build_backbone_network(backbone_model_config, backbone_weights)
+
+    hri_classifier = LogisticRegression(num_in=backbone_network.feature_dim,
+                                           num_out=len(INT2LAB))
+    hri_classifier.load_state_dict(checkpoint_classifier)
+    hri_classifier.eval()
+
+    # Concatenate backbone network and logistic regression
+    net = Pipe(backbone_network, hri_classifier)
+
+    postprocessor = [
+        PostprocessClassificationOutput(INT2LAB, smoothing=4)
+    ]
+
+    border_size = 30
+
+    display_ops = [
+        sense.display.DisplayFPS(expected_camera_fps=net.fps,
+                                 expected_inference_fps=net.fps / net.step_size),
+        sense.display.DisplayTopKClassificationOutputs(top_k=1, threshold=0.5),
+        sense.display.DisplayClassnameOverlay(thresholds=GESTURE_THRESHOLDS,
+                                              border_size_top=border_size if not title else border_size + 50),
+    ]
+
+    display_results = sense.display.DisplayResults(title=title, display_ops=display_ops)
+
+    # Run live inference
+    controller = Controller(
+        neural_network=net,
+        post_processors=postprocessor,
+        results_display=display_results,
+        callbacks=[],
+        camera_id=camera_id,
+        path_in=path_in,
+        path_out=path_out,
+        use_gpu=use_gpu
+    )
+    controller.run_inference()
diff --git a/requirements.txt b/requirements.txt
diff --git a/resources/action_recognition/sien_lite_logreg.ckpt b/resources/action_recognition/sien_lite_logreg.ckpt
diff --git a/resources/action_recognition/sien_logreg.ckpt b/resources/action_recognition/sien_logreg.ckpt
diff --git a/resources/action_recognition/simn_lite_logreg.ckpt b/resources/action_recognition/simn_lite_logreg.ckpt
diff --git a/resources/action_recognition/simn_logreg.ckpt b/resources/action_recognition/simn_logreg.ckpt
diff --git a/resources/calorie_estimation/sien_lite_met_converter.ckpt b/resources/calorie_estimation/sien_lite_met_converter.ckpt
diff --git a/resources/calorie_estimation/sien_met_converter.ckpt b/resources/calorie_estimation/sien_met_converter.ckpt
diff --git a/resources/calorie_estimation/simn_lite_met_converter.ckpt b/resources/calorie_estimation/simn_lite_met_converter.ckpt
diff --git a/resources/calorie_estimation/simn_met_converter.ckpt b/resources/calorie_estimation/simn_met_converter.ckpt
diff --git a/resources/fitness_activity_recognition/sien_lite_logreg.ckpt b/resources/fitness_activity_recognition/sien_lite_logreg.ckpt
diff --git a/resources/fitness_activity_recognition/sien_logreg.ckpt b/resources/fitness_activity_recognition/sien_logreg.ckpt
diff --git a/resources/fitness_activity_recognition/simn_lite_logreg.ckpt b/resources/fitness_activity_recognition/simn_lite_logreg.ckpt
diff --git a/resources/fitness_activity_recognition/simn_logreg.ckpt b/resources/fitness_activity_recognition/simn_logreg.ckpt
diff --git a/resources/hri_recognition/fyp5024.checkpoint b/resources/hri_recognition/fyp5024.checkpoint
diff --git a/resources/hri_recognition/fyp5024.png b/resources/hri_recognition/fyp5024.png
diff --git a/sense/__init__.py b/sense/__init__.py
diff --git a/sense/backbone_networks/__init__.py b/sense/backbone_networks/__init__.py
diff --git a/sense/backbone_networks/efficientnet.py b/sense/backbone_networks/efficientnet.py
diff --git a/sense/backbone_networks/mobilenet.py b/sense/backbone_networks/mobilenet.py
diff --git a/sense/camera.py b/sense/camera.py
diff --git a/sense/controller.py b/sense/controller.py
diff --git a/sense/display.py b/sense/display.py
diff --git a/sense/downstream_tasks/__init__.py b/sense/downstream_tasks/__init__.py
diff --git a/sense/downstream_tasks/action_recognition/__init__.py b/sense/downstream_tasks/action_recognition/__init__.py
diff --git a/sense/downstream_tasks/calorie_estimation/__init__.py b/sense/downstream_tasks/calorie_estimation/__init__.py
diff --git a/sense/downstream_tasks/calorie_estimation/calorie_accumulator.py b/sense/downstream_tasks/calorie_estimation/calorie_accumulator.py
diff --git a/sense/downstream_tasks/calorie_estimation/met_converter.py b/sense/downstream_tasks/calorie_estimation/met_converter.py
diff --git a/sense/downstream_tasks/fitness_activity_recognition/__init__.py b/sense/downstream_tasks/fitness_activity_recognition/__init__.py
diff --git a/sense/downstream_tasks/fitness_rep_counting/__init__.py b/sense/downstream_tasks/fitness_rep_counting/__init__.py
diff --git a/sense/downstream_tasks/hri_recognition/__init__.py b/sense/downstream_tasks/hri_recognition/__init__.py
@@ -0,0 +1,40 @@
+LAB2INT = {
+  "Come forward": 0,
+  "Do Nothing": 1,
+  "Doing something else": 2,
+  "Handwave": 3,
+  "Move backward": 4,
+  "Move to the left": 5,
+  "Move to the right": 6,
+  "Pause": 7,
+  "Pointing: Object": 8,
+  "Repeat: Rotate arm clockwise": 9,
+  "Resume": 10,
+  "Start": 11,
+  "Stop": 12,
+  "Thumbs down": 13,
+  "Thumbs up": 14,
+  "Undo: Rotate arm anti-clockwise": 15,
+  "Watch out": 16
+}
+
+INT2LAB = {value: key for key, value in LAB2INT.items()}
+
+GESTURE_THRESHOLDS = {
+    "Pause": 0.6,
+    "Move to the left": 0.6,
+    "Undo: Rotate arm anti-clockwise": 0.3, 
+    "Thumbs down": 0.6,  
+    "Watch out": 0.5, 
+    "Thumbs up": 0.9, 
+    "Resume": 0.7, 
+    "Repeat: Rotate arm clockwise": 0.3, 
+    "Come forward": 0.5, 
+    "Move to the right": 0.6, 
+    "Move backward": 0.8, 
+    "Handwave": 0.4, 
+    "Start": 0.5, 
+    "Pointing: Object": 0.5, 
+    "Stop": 0.95
+}
+
diff --git a/sense/downstream_tasks/nn_utils.py b/sense/downstream_tasks/nn_utils.py
diff --git a/sense/downstream_tasks/postprocess.py b/sense/downstream_tasks/postprocess.py
diff --git a/sense/engine.py b/sense/engine.py
diff --git a/sense/finetuning.py b/sense/finetuning.py
diff --git a/sense/loading.py b/sense/loading.py
@@ -61,7 +61,6 @@ def __init__(self, model_name: str, version: str, feature_converters: List[str])
         self.model_name = model_name
         self.version = version
         self.combined_model_name = f'{self.model_name}-{self.version}'
-
         self.feature_converters = feature_converters
 
     def check_weight_files(self):
@@ -117,17 +116,17 @@ def get_relevant_weights(
     :return:
         First available model config and dictionary of model weights
     """
+    for config in model_config_list:
+        print(config.feature_converters)
 
     # Filter out model configurations that don't match requested name, version and converter
     if requested_model_name:
         model_config_list = [config for config in model_config_list
                              if config.model_name == requested_model_name]
+        print(model_config_list)
     if requested_version:
         model_config_list = [config for config in model_config_list
                              if config.version == requested_version]
-    if requested_converter:
-        model_config_list = [config for config in model_config_list
-                             if requested_converter in config.feature_converters]
 
     # Check if not empty
     if not model_config_list:

diff --git a/sense/models.yml b/sense/models.yml
diff --git a/sense/utils.py b/sense/utils.py
diff --git a/setup.cfg b/setup.cfg
diff --git a/tests/resources/checkpoint_dir/best_classifier.checkpoint b/tests/resources/checkpoint_dir/best_classifier.checkpoint
diff --git a/tests/resources/checkpoint_dir/classifier.checkpoint b/tests/resources/checkpoint_dir/classifier.checkpoint
diff --git a/tests/resources/checkpoint_dir/label2int.json b/tests/resources/checkpoint_dir/label2int.json
diff --git a/tests/resources/test_video.mp4 b/tests/resources/test_video.mp4
diff --git a/tests/scripts/test_run_custom_classifier.sh b/tests/scripts/test_run_custom_classifier.sh
diff --git a/tests/test_display.py b/tests/test_display.py
diff --git a/tests/test_loading.py b/tests/test_loading.py
diff --git a/tools/__init__.py b/tools/__init__.py
diff --git a/tools/conversion/cfg/efficientnet.cfg b/tools/conversion/cfg/efficientnet.cfg
diff --git a/tools/conversion/cfg/logistic_regression.cfg b/tools/conversion/cfg/logistic_regression.cfg
diff --git a/tools/conversion/config_loader.py b/tools/conversion/config_loader.py
diff --git a/tools/conversion/convert_to_tflite.py b/tools/conversion/convert_to_tflite.py
diff --git a/tools/conversion/keras_converter.py b/tools/conversion/keras_converter.py
diff --git a/tools/conversion/keras_exporter.py b/tools/conversion/keras_exporter.py
diff --git a/tools/conversion/section_conversion.py b/tools/conversion/section_conversion.py
diff --git a/tools/conversion/weights_loader.py b/tools/conversion/weights_loader.py
diff --git a/tools/flip_video.py b/tools/flip_video.py
diff --git a/tools/run_custom_classifier.py b/tools/run_custom_classifier.py
diff --git a/tools/sense_studio/.gitignore b/tools/sense_studio/.gitignore
diff --git a/tools/sense_studio/__init__.py b/tools/sense_studio/__init__.py
diff --git a/tools/sense_studio/annotation.py b/tools/sense_studio/annotation.py
diff --git a/tools/sense_studio/sense_studio.py b/tools/sense_studio/sense_studio.py
diff --git a/tools/sense_studio/static/main.css b/tools/sense_studio/static/main.css
diff --git a/tools/sense_studio/static/main.js b/tools/sense_studio/static/main.js
diff --git a/tools/sense_studio/static/sense_favicon.png b/tools/sense_studio/static/sense_favicon.png
diff --git a/tools/sense_studio/static/sense_studio_logo.svg b/tools/sense_studio/static/sense_studio_logo.svg
diff --git a/tools/sense_studio/static/video_recording.js b/tools/sense_studio/static/video_recording.js
diff --git a/tools/sense_studio/templates/frame_annotation.html b/tools/sense_studio/templates/frame_annotation.html
diff --git a/tools/sense_studio/templates/navigation.html b/tools/sense_studio/templates/navigation.html
diff --git a/tools/sense_studio/templates/project_details.html b/tools/sense_studio/templates/project_details.html
diff --git a/tools/sense_studio/templates/projects_overview.html b/tools/sense_studio/templates/projects_overview.html
diff --git a/tools/sense_studio/templates/skeleton.html b/tools/sense_studio/templates/skeleton.html
diff --git a/tools/sense_studio/templates/video_list.html b/tools/sense_studio/templates/video_list.html
diff --git a/tools/sense_studio/templates/video_recording.html b/tools/sense_studio/templates/video_recording.html
diff --git a/tools/sense_studio/utils.py b/tools/sense_studio/utils.py
diff --git a/tools/sense_studio/video_recording.py b/tools/sense_studio/video_recording.py
diff --git a/tools/train_classifier.py b/tools/train_classifier.py
diff --git a/tools/video_capture.py b/tools/video_capture.py