Introduced support for LiDAR depth streaming. Use with the iOS app Re…

…cord3D 1.4 and newer.
marek-simonik · Sep 17, 2020 · 8127e38 · 8127e38
1 parent f6ba87b
commit 8127e38
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
 # [*Record3D – Point Cloud Animation and Streaming*](https://record3d.app/): the accompanying library 
 
+**2020/09/17 Update**: Introduced LiDAR support. To be used with Record3D 1.4 and newer.
+
 This project provides C++ and Python libraries for the [iOS Record3D app](https://record3d.app/) which allows you (among other features) to 
 live-stream RGB**D** video from iOS devices with TrueDepth camera to a computer via USB cable.
 
@@ -47,7 +49,7 @@ After running the following, you will find compiled static library in the `build
 ## Sample applications
 There is a Python (`demo-main.py`) and C++ (`src/DemoMain.cpp`) sample project that demonstrates how to use the library to receive and display RGBD stream.
 
-Before running the sample applications, connect your iOS device to your computer and open the Record3D iOS app. Go to the Settings tab and enable "USB Streaming mode". Note that the fr
+Before running the sample applications, connect your iOS device to your computer and open the Record3D iOS app. Go to the Settings tab and enable "USB Streaming mode".
 
 ### Python
 After installing the `record3d` library, run `python demo-main.py` and press the record button to start streaming RGBD data.

diff --git a/demo-main.py b/demo-main.py
@@ -52,8 +52,11 @@ def start_processing_stream(self):
             # You can now e.g. create point cloud by projecting the depth map using the intrinsic matrix.
 
             # Postprocess it
-            depth = cv2.flip(depth, 1)
-            rgb = cv2.flip(rgb, 1)
+            are_truedepth_camera_data_being_streamed = depth.shape[0] == 640
+            if are_truedepth_camera_data_being_streamed:
+                depth = cv2.flip(depth, 1)
+                rgb = cv2.flip(rgb, 1)
+
             rgb = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
 
             # Show the RGBD Stream

diff --git a/include/record3d/Record3DStream.h b/include/record3d/Record3DStream.h
@@ -94,8 +94,8 @@ namespace Record3D
     public:
         // Constants
         static constexpr uint16_t DEVICE_PORT{ 1337 }; /** Port on iDevice that we are listening to for RGBD stream. */
-        static constexpr uint32_t FRAME_WIDTH{ 480 }; /** Width of the RGB and Depth components of the RGBD stream. */
-        static constexpr uint32_t FRAME_HEIGHT{ 640 }; /** Height of the RGB and Depth components of the RGBD stream. */
+        static constexpr uint32_t MAXIMUM_FRAME_WIDTH{ 480 }; /** Maximum width of the RGB and Depth components of the RGBD stream. */
+        static constexpr uint32_t MAXIMUM_FRAME_HEIGHT{ 640 }; /** Maximum height of the RGB and Depth components of the RGBD stream. */
 
 #ifdef PYTHON_BINDINGS_BUILD
         /**
@@ -141,12 +141,16 @@ namespace Record3D
          */
         py::array_t<float> GetCurrentDepthFrame()
         {
-            auto result        = py::array_t<float>(FRAME_WIDTH * FRAME_HEIGHT);
+            size_t currentFrameWidth = currentFrameWidth_;
+            size_t currentFrameHeight = currentFrameHeight_;
+
+            size_t bufferSize  = currentFrameWidth * currentFrameHeight * sizeof(float);
+            auto result        = py::array_t<float>(currentFrameWidth * currentFrameHeight);
             auto result_buffer = result.request();
             float *result_ptr  = (float *) result_buffer.ptr;
 
-            std::memcpy(result_ptr, depthImageBuffer_.data(), depthImageBuffer_.size());
-            result.resize(std::vector<int>{static_cast<int>(FRAME_HEIGHT), static_cast<int>(FRAME_WIDTH)});
+            std::memcpy(result_ptr, depthImageBuffer_.data(), bufferSize);
+            result.resize(std::vector<int>{static_cast<int>(currentFrameHeight), static_cast<int>(currentFrameWidth)});
 
             return result;
         }
@@ -158,12 +162,17 @@ namespace Record3D
          */
         py::array_t<uint8_t> GetCurrentRGBFrame()
         {
-            auto result        = py::array_t<uint8_t>(FRAME_WIDTH * FRAME_HEIGHT * 3);
+            size_t currentFrameWidth = currentFrameWidth_;
+            size_t currentFrameHeight = currentFrameHeight_;
+
+            constexpr int numChannels = 3;
+            size_t bufferSize  = currentFrameWidth * currentFrameHeight * numChannels * sizeof(uint8_t);
+            auto result        = py::array_t<uint8_t>(bufferSize);
             auto result_buffer = result.request();
-            float *result_ptr  = (float *) result_buffer.ptr;
+            uint8_t *result_ptr  = (uint8_t *) result_buffer.ptr;
 
-            std::memcpy(result_ptr, RGBImageBuffer_.data(), RGBImageBuffer_.size());
-            result.resize(std::vector<int>{static_cast<int>(FRAME_HEIGHT), static_cast<int>(FRAME_WIDTH), 3});
+            std::memcpy(result_ptr, RGBImageBuffer_.data(), bufferSize);
+            result.resize(std::vector<int>{static_cast<int>(currentFrameHeight), static_cast<int>(currentFrameWidth), numChannels});
 
             return result;
         }
@@ -179,7 +188,10 @@ namespace Record3D
         }
 #endif
     private:
-        static constexpr size_t depthBufferSize_{ FRAME_WIDTH * FRAME_HEIGHT * sizeof( float ) }; /** Size in bytes of decompressed Depth frame. */
+        static constexpr size_t depthBufferSize_{MAXIMUM_FRAME_WIDTH * MAXIMUM_FRAME_HEIGHT * sizeof( float ) }; /** Size in bytes of decompressed Depth frame. */
+
+        size_t currentFrameWidth_{ MAXIMUM_FRAME_WIDTH };
+        size_t currentFrameHeight_{ MAXIMUM_FRAME_HEIGHT };
 
         uint8_t* compressedDepthBuffer_{ nullptr }; /** Preallocated buffer holding decompressed depth data. */
         uint8_t* lzfseScratchBuffer_{ nullptr }; /** Preallocated LZFSE scratch buffer. */

diff --git a/setup.py b/setup.py
@@ -67,7 +67,7 @@ def build_extension(self, ext):
 
 setup(
     name='record3d',
-    version='1.0.0',
+    version='1.2.0',
     license='lgpl-2.1',
     author='Marek Simonik',
     author_email='[email protected]',

diff --git a/src/DemoMain.cpp b/src/DemoMain.cpp
@@ -67,8 +67,14 @@ class Record3DDemoApp
 #ifdef HAS_OPENCV
                 // Postprocess images
                 cv::cvtColor( imgRGB_, imgRGB_, cv::COLOR_RGB2BGR );
-                cv::flip( imgRGB_, imgRGB_, 1 );
-                cv::flip( imgDepth_, imgDepth_, 1 );
+
+                // The TrueDepth camera is a selfie camera; we mirror the RGBD frame so it looks plausible.
+                bool areTrueDepthDataBeingStreamed = imgDepth_.rows == Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT && imgDepth_.cols == Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH;
+                if ( areTrueDepthDataBeingStreamed )
+                {
+                    cv::flip( imgRGB_, imgRGB_, 1 );
+                    cv::flip( imgDepth_, imgDepth_, 1 );
+                }
 
                 // Show images
                 cv::imshow( "RGB", imgRGB_ );
@@ -97,8 +103,22 @@ class Record3DDemoApp
                     Record3D::IntrinsicMatrixCoeffs $K)
     {
 #ifdef HAS_OPENCV
-        memcpy( imgRGB_.data, $rgbFrame.data(), $rgbFrame.size());
-        memcpy( imgDepth_.data, $depthFrame.data(), $depthFrame.size());
+        // When we switch between the TrueDepth and the LiDAR camera, the size frame size changes.
+        // Recreate the RGB and Depth images with fitting size.
+        if (    imgRGB_.rows != $frameHeight || imgRGB_.cols != $frameWidth
+             || imgDepth_.rows != $frameHeight || imgDepth_.cols != $frameWidth )
+        {
+            imgRGB_.release();
+            imgDepth_.release();
+
+            imgRGB_ = cv::Mat::zeros( $frameHeight, $frameWidth, CV_8UC3);
+            imgDepth_ = cv::Mat::zeros( $frameHeight, $frameWidth, CV_32F );
+        }
+
+        // The `BufferRGB` and `BufferDepth` may be larger than the actual payload, therefore the true frame size is computed.
+        constexpr int numRGBChannels = 3;
+        memcpy( imgRGB_.data, $rgbFrame.data(), $frameWidth * $frameHeight * numRGBChannels * sizeof(uint8_t));
+        memcpy( imgDepth_.data, $depthFrame.data(), $frameWidth * $frameHeight * sizeof(float));
 #endif
         mainThreadLock_.unlock();
     }
@@ -111,8 +131,8 @@ class Record3DDemoApp
 #endif
 
 #ifdef HAS_OPENCV
-    cv::Mat imgRGB_ = cv::Mat::zeros( Record3D::Record3DStream::FRAME_HEIGHT, Record3D::Record3DStream::FRAME_WIDTH, CV_8UC3);
-    cv::Mat imgDepth_ = cv::Mat::zeros( Record3D::Record3DStream::FRAME_HEIGHT, Record3D::Record3DStream::FRAME_WIDTH, CV_32F );
+    cv::Mat imgRGB_ = cv::Mat::zeros(Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT, Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH, CV_8UC3);;
+    cv::Mat imgDepth_ = cv::Mat::zeros(Record3D::Record3DStream::MAXIMUM_FRAME_HEIGHT, Record3D::Record3DStream::MAXIMUM_FRAME_WIDTH, CV_32F );;
 #endif
 };
 

diff --git a/src/Record3DStream.cpp b/src/Record3DStream.cpp
@@ -20,7 +20,7 @@ namespace Record3D
         depthImageBuffer_.resize( depthBufferSize_ );
 
         constexpr int numRGBChannels = 3;
-        RGBImageBuffer_.resize( Record3DStream::FRAME_WIDTH * Record3DStream::FRAME_HEIGHT * sizeof( uint8_t ) * numRGBChannels );
+        RGBImageBuffer_.resize(Record3DStream::MAXIMUM_FRAME_WIDTH * Record3DStream::MAXIMUM_FRAME_HEIGHT * sizeof( uint8_t ) * numRGBChannels );
     }
 
     Record3DStream::~Record3DStream()
@@ -107,6 +107,8 @@ namespace Record3D
 
     struct Record3DHeader
     {
+        uint32_t frameWidth;
+        uint32_t frameHeight;
         uint32_t rgbSize;
         uint32_t depthSize;
     };
@@ -153,7 +155,7 @@ namespace Record3D
             currSize = record3DHeader.rgbSize;
             int loadedWidth, loadedHeight, loadedChannels;
             uint8_t* rgbPixels = stbi_load_from_memory( rawMessageBuffer.data() + offset, currSize, &loadedWidth, &loadedHeight, &loadedChannels, STBI_rgb );
-            memcpy( RGBImageBuffer_.data(), rgbPixels, RGBImageBuffer_.size());
+            memcpy( RGBImageBuffer_.data(), rgbPixels, loadedWidth * loadedHeight * loadedChannels * sizeof(uint8_t));
             stbi_image_free( rgbPixels );
             offset += currSize;
 
@@ -163,10 +165,13 @@ namespace Record3D
 
             if ( onNewFrame )
             {
+                currentFrameWidth_ = record3DHeader.frameWidth;
+                currentFrameHeight_ = record3DHeader.frameHeight;
+
 #ifdef PYTHON_BINDINGS_BUILD
-                onNewFrame();
+                onNewFrame( );
 #else
-                onNewFrame( RGBImageBuffer_, depthImageBuffer_, FRAME_WIDTH, FRAME_HEIGHT, intrinsicMatrixCoeffs_ );
+                onNewFrame( RGBImageBuffer_, depthImageBuffer_, record3DHeader.frameWidth, record3DHeader.frameHeight, intrinsicMatrixCoeffs_ );
 #endif
             }
         }