Merge pull request #1209 from luxonis/fix_depth_recording

Fix depth recording

apps/record/requirements.txt

@@ -1,6 +1,6 @@
 depthai==2.17.0.0
 depthai-sdk==1.2.4 # New SDK has Record module completely refactored. TODO port this app to new SDK
-numpy==1.23.4
+numpy==1.26.4
 av==9.2.0
 mcap==0.0.10
 mcap-ros1-support==0.0.8

apps/uvc/requirements.txt

@@ -1,3 +1,4 @@
 --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/
 depthai==2.19.1.0.dev+e88af9a9db3dc630a3011dd52d1f5700cf6bf9b8
-depthai-sdk==1.2.1
+depthai-sdk==1.2.1
+numpy==1.26.4

depthai_sdk/examples/recording/depth_video_record.py

@@ -0,0 +1,30 @@
+from depthai_sdk import OakCamera, RecordType
+import depthai as dai
+
+FPS = 30
+with OakCamera() as oak:
+    color = oak.create_camera('CAM_A', resolution='1080p', encode='mjpeg', fps=FPS)
+    color.config_color_camera(isp_scale=(2,3)) # 720P color frames, mjpeg
+
+    stereo = oak.create_stereo(resolution='720p', fps=FPS)
+    stereo.config_stereo(align=color, subpixel=True, lr_check=True)
+    stereo.node.setOutputSize(640, 360) # 720p, downscaled to 640x360 (decimation filter, median filtering)
+    # On-device post processing for stereo depth
+    config = stereo.node.initialConfig.get()
+    stereo.node.setPostProcessingHardwareResources(3, 3)
+    config.postProcessing.speckleFilter.enable = True
+    config.postProcessing.thresholdFilter.minRange = 400
+    config.postProcessing.thresholdFilter.maxRange = 10_000 # 10m
+    config.postProcessing.decimationFilter.decimationFactor = 2
+    config.postProcessing.decimationFilter.decimationMode = dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEDIAN
+    stereo.node.initialConfig.set(config)
+    """
+    Depth will get encoded on-host with FFV1 codec (it supports 16bit grey), and color will be
+    encoded on-device with MJPEG codec. FFV1 works well with .avi container, so depth video will be
+    saved as .avi, and color video will be saved as .mp4.
+    """
+    record_components = [stereo.out.depth, color.out.encoded]
+    oak.record(record_components, './', record_type=RecordType.VIDEO).configure_syncing(True, threshold_ms=500/30)
+    oak.start(blocking=True)
+
+

depthai_sdk/setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name='depthai-sdk',
-    version='1.15.0',
+    version='1.15.1',
     description='This package provides an abstraction of the DepthAI API library.',
     long_description=io.open("README.md", encoding="utf-8").read(),
     long_description_content_type="text/markdown",

depthai_sdk/src/depthai_sdk/__init__.py

@@ -10,7 +10,7 @@
 from depthai_sdk.utils import _create_config, get_config_field, _sentry_before_send
 from depthai_sdk.visualize import *
 
-__version__ = '1.15'
+__version__ = '1.15.1'
 
 
 def __import_sentry(sentry_dsn: str) -> None:

depthai_sdk/src/depthai_sdk/recorders/video_writers/video_writer.py

@@ -60,7 +60,7 @@ def _create_file(self, path_to_file: str, frame: dai.ImgFrame):
         options = {}
         if self._lossless:
             self._fourcc = 'rawvideo'
-        elif frame.getType() == dai.ImgFrame.Type.RAW16:
+        elif frame.getType() == dai.ImgFrame.Type.RAW16: # Depth map
             self._fourcc = 'ffv1'
             self._format = 'gray16le'
         else:  # Mono/Color, encode
@@ -74,6 +74,11 @@ def _create_file(self, path_to_file: str, frame: dai.ImgFrame):
         self._stream.codec_context.width = frame.getWidth()
         self._stream.codec_context.height = frame.getHeight()
 
+        if self._fourcc == 'ffv1':
+            self._stream.width = frame.getWidth()
+            self._stream.height = frame.getHeight()
+            self._stream.pix_fmt = 'gray16le' # Required for depth recording to work correctly
+
     def write(self, img_frame: dai.ImgFrame):
         if self._file is None:
             self.create_file(subfolder='', frame=img_frame)
@@ -90,7 +95,6 @@ def write(self, img_frame: dai.ImgFrame):
             video_format = 'gray16le'
         else:
             raise ValueError(f'Unsupported frame type: {img_frame.getType()}')
-
         video_frame = av.VideoFrame.from_ndarray(img_frame.getFrame(), format=video_format)
 
         ts = int((img_frame.getTimestampDevice() - self._start_ts).total_seconds() * 1e3)  # To milliseconds