added multimodal emotion

.github/workflows/python-package-macos.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.11.5"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Install FFMPEG
+      run: |
+        brew install ffmpeg
+    - name: Test with pytest
+      run: |
+        pytest

README.md

@@ -5,12 +5,13 @@ from vcab import Model, save_video_stream_predictions_v2
 video_path = "example.mp4"
 
 # Get video stream predictions
-predictions = Model().predict_stream(video_path=video_path)
+actions, emotions, autism, autism_percentage, video_output = Model().predict_stream_emotion(video_path=video_path)
 
 # Mask video with prediction
 save_video_stream_predictions_v2(
     video_path=video_path,
-    predictions=predictions, 
+    action_predictions=actions,
+    autism_predictions=autism, 
     output_path="example_output.mp4")
 ```
 

index.py

@@ -1,6 +1,6 @@
-from vcab import Model, save_video_stream_predictions_v2
+from vcab import Model
+from vcab.utils import save_video_stream_predictions_v3
 
-video_path = "videos/v_Headbanging_01.mp4"
-predictions = Model().predict_stream(video_path=video_path)
-save_video_stream_predictions_v2(
-    video_path=video_path, predictions=predictions, output_path="output/v_Headbanging_01.mp4")
+video_path = "videos/v_ArmFlapping_01.mp4"
+actions, emotions, autism, autism_percentage, video_output = Model().predict_stream_emotion(video_path=video_path)
+save_video_stream_predictions_v3(video_output, actions, autism, video_output)

requirements.txt

@@ -1,46 +1,84 @@
 aiohttp==3.9.3
 aiosignal==1.3.1
+antlr4-python3-runtime==4.9.3
 async-timeout==4.0.3
 attrs==23.2.0
 av==12.0.0
+beautifulsoup4==4.12.3
+certifi==2024.2.2 ; platform_system=="Windows"
+charset-normalizer==3.3.2
 colorama==0.4.6
-exceptiongroup==1.2.0
+contourpy==1.2.1
+cycler==0.12.1
+easydict==1.13
+exceptiongroup==1.2.1
 ffmpeg-python==0.2.0
 ffmpegcv==0.3.13
-filelock
+filelock==3.14.0
+fonttools==4.51.0
 frozenlist==1.4.1
 fsspec==2024.3.1
 future==1.0.0
 fvcore==0.1.5.post20221221
-idna
+gdown==5.1.0
+huggingface-hub==0.23.0
+idna==3.7
 iniconfig==2.0.0
+intel-openmp==2021.4.0 ; platform_system=="Windows"
 iopath==0.1.10
-Jinja2==3.1.3
+Jinja2==3.1.4
+joblib==1.4.2
+kiwisolver==1.4.5
 lightning-utilities==0.11.2
 MarkupSafe==2.1.5
+matplotlib==3.8.4
+MEGraphAU @ git+https://github.com/Andreas-UI/ME-GraphAU-Video.git@0fbbd31d4f2b3a36b6a2f30ce0e729f052db2370
+mkl==2021.4.0 ; platform_system=="Windows"
 mpmath==1.3.0
 multidict==6.0.5
-networkx
-numpy
-opencv-python
+networkx==3.3
+numpy==1.26.4
+omegaconf==2.3.0
+opencv-python==4.9.0.80
 packaging==24.0
+pandas==2.2.2
 parameterized==0.9.0
 pillow==10.3.0
-pluggy
+pluggy==1.5.0
 portalocker==2.8.2
-pytest
+psutil==5.9.8
+py-cpuinfo==9.0.0
+pyparsing==3.1.2
+PySocks==1.7.1
+pytest==8.2.0
+python-dateutil==2.9.0.post0
 pytorch-lightning==2.2.1
 pytorchvideo @ git+https://github.com/facebookresearch/pytorchvideo.git@1fadaef40dd393ca09680f55582399f4679fc9b7
+pytz==2024.1
 pywin32==306 ; platform_system=="Windows"
 PyYAML==6.0.1
+requests==2.31.0
+safetensors==0.4.3
+scikit-learn==1.4.2
+scipy==1.13.0
+seaborn==0.13.2
+six==1.16.0
+soupsieve==2.5
 sympy==1.12
 tabulate==0.9.0
+tbb==2021.12.0 ; platform_system=="Windows"
 termcolor==2.4.0
+thop==0.1.1.post2209072238
+threadpoolctl==3.5.0
+timm==0.9.16
 tomli==2.0.1
-torch==2.2.2
+torch==2.3.0
 torchmetrics==1.3.2
-torchvision==0.17.2
-tqdm
+torchvision==0.18.0
+tqdm==4.66.4
 typing_extensions==4.11.0
+tzdata==2024.1
+ultralytics==8.2.10
+urllib3==2.2.1
 yacs==0.1.8
 yarl==1.9.4