From f70818feeb37d135194ef44755e1fd7ba243955e Mon Sep 17 00:00:00 2001 From: rapterjet2004 Date: Thu, 16 Oct 2025 11:25:53 -0500 Subject: [PATCH] Optimizing the VideoFrame.I420Buffer.toMat function Removing unused code Better error handling Signed-off-by: rapterjet2004 --- .../nextcloud/talk/activities/CallActivity.kt | 3 +- .../camera/BackgroundBlurFrameProcessor.kt | 167 ++++++++---------- .../talk/camera/ImageSegmenterHelper.kt | 45 +---- 3 files changed, 89 insertions(+), 126 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 3155fd9a77..1b1f7a969c 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -1,6 +1,7 @@ /* * Nextcloud Talk - Android Client * + * SPDX-FileCopyrightText: 2025 Julius Linus * SPDX-FileCopyrightText: 2023 Marcel Hibbe * SPDX-FileCopyrightText: 2022 Tim Krüger * SPDX-FileCopyrightText: 2017-2018 Mario Danic @@ -504,7 +505,7 @@ class CallActivity : CallBaseActivity() { val isOn = state == BackgroundBlurOn val processor = if (isOn) { - BackgroundBlurFrameProcessor(context, frontFacing) + BackgroundBlurFrameProcessor(context) } else { null } diff --git a/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt b/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt index 2ebb1b27a3..1c25249b00 100644 --- a/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt +++ b/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt @@ -22,9 +22,10 @@ import org.webrtc.JavaI420Buffer import org.webrtc.VideoFrame import org.webrtc.VideoProcessor import org.webrtc.VideoSink +import org.webrtc.YuvHelper import java.nio.ByteBuffer -class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Boolean) : +class BackgroundBlurFrameProcessor(val context: Context) : VideoProcessor, ImageSegmenterHelper.SegmenterListener { @@ -92,9 +93,13 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool val finalFrame = blurredMat.toVideoFrame(resultBundle.inferenceTime) + if (finalFrame == null) { + Log.e(TAG, "Frame was null") + } + sink?.onFrame(finalFrame) - finalFrame.release() + finalFrame?.release() } finally { frameMat.release() blurredMat.release() @@ -132,6 +137,7 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool try { // weirdly rotation is 270 degree in portrait and 180 degree in landscape, no idea why + // regardless if this behavior is device dependant, this calculation should correct the orientation val angle = ROT_360 - videoFrame.rotation.toDouble() rotationMat = Imgproc.getRotationMatrix2D( center, @@ -166,107 +172,90 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool this.sink = sink } - private fun Mat.toVideoFrame(time: Long): VideoFrame { - val i420Mat = Mat() - Imgproc.cvtColor(this, i420Mat, Imgproc.COLOR_RGBA2YUV_I420) - - // Get the raw bytes from the new I420 Mat - val i420ByteArray = ByteArray((i420Mat.total() * i420Mat.elemSize()).toInt()) - i420Mat.get(0, 0, i420ByteArray) - - val width = this.width() - val height = this.height() - - val yPlaneSize = width * height - val uvPlaneSize = (width / 2) * (height / 2) - - val yBuffer = ByteBuffer.allocateDirect(yPlaneSize) - val uBuffer = ByteBuffer.allocateDirect(uvPlaneSize) - val vBuffer = ByteBuffer.allocateDirect(uvPlaneSize) - - yBuffer.put(i420ByteArray, 0, yPlaneSize) - uBuffer.put(i420ByteArray, yPlaneSize, uvPlaneSize) - vBuffer.put(i420ByteArray, yPlaneSize + uvPlaneSize, uvPlaneSize) - - yBuffer.rewind() - uBuffer.rewind() - vBuffer.rewind() - - // Create the I420Buffer using the separate planes - val finalFrameBuffer = JavaI420Buffer.wrap( - width, - height, - yBuffer, - width, - uBuffer, - width / 2, - vBuffer, - width / 2, - null - ) + private fun Mat.toVideoFrame(time: Long): VideoFrame? = + runCatching { + val i420Mat = Mat() + Imgproc.cvtColor(this, i420Mat, Imgproc.COLOR_RGBA2YUV_I420) - i420Mat.release() - - return VideoFrame(finalFrameBuffer, 0, time) - } + // Get the raw bytes from the new I420 Mat to i420ByteArray + val i420ByteArray = ByteArray((i420Mat.total() * i420Mat.elemSize()).toInt()) + i420Mat.get(0, 0, i420ByteArray) - private fun VideoFrame.I420Buffer.toMat(): Mat? = - kotlin.runCatching { - val i420Buffer = this + val width = this.width() + val height = this.height() - val width = i420Buffer.width - val height = i420Buffer.height val yPlaneSize = width * height + val uvPlaneSize = (width / 2) * (height / 2) + + val yBuffer = ByteBuffer.allocateDirect(yPlaneSize) + val uBuffer = ByteBuffer.allocateDirect(uvPlaneSize) + val vBuffer = ByteBuffer.allocateDirect(uvPlaneSize) + + yBuffer.put(i420ByteArray, 0, yPlaneSize) + uBuffer.put(i420ByteArray, yPlaneSize, uvPlaneSize) + vBuffer.put(i420ByteArray, yPlaneSize + uvPlaneSize, uvPlaneSize) + + yBuffer.rewind() + uBuffer.rewind() + vBuffer.rewind() + + // Create the I420Buffer using the separate planes + val finalFrameBuffer = JavaI420Buffer.wrap( + width, + height, + yBuffer, + width, + uBuffer, + width / 2, + vBuffer, + width / 2, + null + ) - val nv21Height = (height * NV21_HEIGHT_MULTI).toInt() - val nv21Width = width - val nv21Size = nv21Height * nv21Width - val nv21Data = ByteArray(nv21Size) - - val dataY = i420Buffer.dataY - val dataU = i420Buffer.dataU - val dataV = i420Buffer.dataV + i420Mat.release() - val strideY = i420Buffer.strideY // Likely equal to the width, but not always, depending on mem alignment - val strideU = i420Buffer.strideU // U and V have identical dimens and strides - val strideV = i420Buffer.strideV + return VideoFrame(finalFrameBuffer, 0, time) + }.getOrElse { throwable -> + Log.e(TAG, "Error in Mat.toVideoFrame $throwable") - if (strideY == width) { - // Fast path: contiguous data - dataY.get(nv21Data, 0, yPlaneSize) - } else { - // Slow path: row-by-row copy - for (row in 0 until height) { - dataY.position(row * strideY) - dataY.get(nv21Data, row * width, width) - } - } + null + } - val vuPlaneOffset = width * height - for (row in 0 until height / 2) { - for (col in 0 until width / 2) { - // Get U and V values from their respective planes using row/col/stride - val v = dataV[row * strideV + col] - val u = dataU[row * strideU + col] - - // Put them into the NV21 buffer (V, then U) - val nv21Index = vuPlaneOffset + (row * width) + (col * 2) - nv21Data[nv21Index] = v - nv21Data[nv21Index + 1] = u - } - } + private fun VideoFrame.I420Buffer.toMat(): Mat? = + runCatching { + val chromaWidth = (width + 1) / 2 + val chromaHeight = (height + 1) / 2 + val minSize = width * height + chromaWidth * chromaHeight * 2 + + val nv12ByteBuffer = ByteBuffer.allocateDirect(minSize) + YuvHelper.I420ToNV12( + this.dataY, + this.strideY, + this.dataU, + this.strideU, + this.dataV, + this.strideV, + nv12ByteBuffer, + width, + height + ) val mat = Mat( - nv21Height, - nv21Width, + (height * NV21_HEIGHT_MULTI).toInt(), + width, CvType.CV_8UC1 // 8 bit unsigned 1 channel ) - mat.put(0, 0, nv21Data) - Imgproc.cvtColor(mat, mat, Imgproc.COLOR_YUV2RGBA_NV21) + mat.put(0, 0, nv12ByteBuffer.array()) - i420Buffer.release() + Imgproc.cvtColor(mat, mat, Imgproc.COLOR_YUV2RGBA_NV12) + + this.release() mat - }.getOrNull() + }.getOrElse { throwable -> + Log.e(TAG, "Error in VideoFrame.I420Buffer.toMat $throwable") + + null + } } diff --git a/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt b/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt index f2b9c0dc41..584e78c2f6 100644 --- a/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt +++ b/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt @@ -24,12 +24,7 @@ import org.opencv.core.Mat import org.opencv.core.Scalar import java.nio.ByteBuffer -class ImageSegmenterHelper( - var currentDelegate: Int = DELEGATE_CPU, - var runningMode: RunningMode = RunningMode.LIVE_STREAM, - val context: Context, - var imageSegmenterListener: SegmenterListener? = null -) { +class ImageSegmenterHelper(val context: Context, var imageSegmenterListener: SegmenterListener? = null) { private var imageSegmenter: ImageSegmenter? = null @@ -53,19 +48,11 @@ class ImageSegmenterHelper( * @throws IllegalStateException */ fun setupImageSegmenter() { - val baseOptionsBuilder = BaseOptions.builder() - when (currentDelegate) { - DELEGATE_CPU -> { - baseOptionsBuilder.setDelegate(Delegate.CPU) - } - - DELEGATE_GPU -> { - baseOptionsBuilder.setDelegate(Delegate.GPU) - } + val baseOptionsBuilder = BaseOptions.builder().apply { + setDelegate(Delegate.CPU) + setModelAssetPath(MODEL_SELFIE_SEGMENTER_PATH) } - baseOptionsBuilder.setModelAssetPath(MODEL_SELFIE_SEGMENTER_PATH) - if (imageSegmenterListener == null) { throw IllegalStateException("ImageSegmenterListener must be set.") } @@ -73,19 +60,14 @@ class ImageSegmenterHelper( runCatching { val baseOptions = baseOptionsBuilder.build() val optionsBuilder = ImageSegmenter.ImageSegmenterOptions.builder() - .setRunningMode(runningMode) + .setRunningMode(RunningMode.LIVE_STREAM) .setBaseOptions(baseOptions) .setOutputCategoryMask(true) .setOutputConfidenceMasks(false) + .setResultListener(this::returnSegmentationResult) + .setErrorListener(this::returnSegmentationHelperError) - if (runningMode == RunningMode.LIVE_STREAM) { - optionsBuilder - .setResultListener(this::returnSegmentationResult) - .setErrorListener(this::returnSegmentationHelperError) - } - - val options = optionsBuilder.build() - imageSegmenter = ImageSegmenter.createFromOptions(context, options) + imageSegmenter = ImageSegmenter.createFromOptions(context, optionsBuilder.build()) }.getOrElse { e -> when (e) { is IllegalStateException -> { @@ -114,12 +96,6 @@ class ImageSegmenterHelper( * @throws IllegalArgumentException */ fun segmentLiveStreamFrame(bitmap: Bitmap, videoFrameTimeStamp: Long) { - if (runningMode != RunningMode.LIVE_STREAM) { - throw IllegalArgumentException( - "Attempting to call segmentLiveStreamFrame while not using RunningMode.LIVE_STREAM" - ) - } - val mpImage = BitmapImageBuilder(bitmap).build() imageSegmenter?.segmentAsync(mpImage, videoFrameTimeStamp) @@ -146,6 +122,7 @@ class ImageSegmenterHelper( mat.put(0, 0, data) Core.bitwise_not(mat, mat) + Core.multiply(mat, Scalar(RGB_MAX), mat) imageSegmenterListener?.onResults( @@ -167,14 +144,10 @@ class ImageSegmenterHelper( data class ResultBundle(val mask: Mat, val inferenceTime: Long) companion object { - const val DELEGATE_CPU = 0 - const val DELEGATE_GPU = 1 // DO NOT USE THIS const val OTHER_ERROR = 0 const val GPU_ERROR = 1 - const val MODEL_SELFIE_SEGMENTER_PATH = "selfie_segmenter.tflite" const val RGB_MAX = 255.0 - private const val TAG = "ImageSegmenterHelper" }