Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
* Nextcloud Talk - Android Client
*
* SPDX-FileCopyrightText: 2025 Julius Linus <[email protected]>
* SPDX-FileCopyrightText: 2023 Marcel Hibbe <[email protected]>
* SPDX-FileCopyrightText: 2022 Tim Krüger <[email protected]>
* SPDX-FileCopyrightText: 2017-2018 Mario Danic <[email protected]>
Expand Down Expand Up @@ -506,7 +507,7 @@ class CallActivity : CallBaseActivity() {
val isOn = state == BackgroundBlurOn

val processor = if (isOn) {
BackgroundBlurFrameProcessor(context, frontFacing)
BackgroundBlurFrameProcessor(context)
} else {
null
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ import org.webrtc.JavaI420Buffer
import org.webrtc.VideoFrame
import org.webrtc.VideoProcessor
import org.webrtc.VideoSink
import org.webrtc.YuvHelper
import java.nio.ByteBuffer

class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Boolean) :
class BackgroundBlurFrameProcessor(val context: Context) :
VideoProcessor,
ImageSegmenterHelper.SegmenterListener {

Expand Down Expand Up @@ -92,9 +93,13 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool

val finalFrame = blurredMat.toVideoFrame(resultBundle.inferenceTime)

if (finalFrame == null) {
Log.e(TAG, "Frame was null")
}

sink?.onFrame(finalFrame)

finalFrame.release()
finalFrame?.release()
} finally {
frameMat.release()
blurredMat.release()
Expand Down Expand Up @@ -132,6 +137,7 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool

try {
// weirdly rotation is 270 degree in portrait and 180 degree in landscape, no idea why
// regardless if this behavior is device dependant, this calculation should correct the orientation
val angle = ROT_360 - videoFrame.rotation.toDouble()
rotationMat = Imgproc.getRotationMatrix2D(
center,
Expand Down Expand Up @@ -166,107 +172,90 @@ class BackgroundBlurFrameProcessor(val context: Context, val isFrontFacing: Bool
this.sink = sink
}

private fun Mat.toVideoFrame(time: Long): VideoFrame {
val i420Mat = Mat()
Imgproc.cvtColor(this, i420Mat, Imgproc.COLOR_RGBA2YUV_I420)

// Get the raw bytes from the new I420 Mat
val i420ByteArray = ByteArray((i420Mat.total() * i420Mat.elemSize()).toInt())
i420Mat.get(0, 0, i420ByteArray)

val width = this.width()
val height = this.height()

val yPlaneSize = width * height
val uvPlaneSize = (width / 2) * (height / 2)

val yBuffer = ByteBuffer.allocateDirect(yPlaneSize)
val uBuffer = ByteBuffer.allocateDirect(uvPlaneSize)
val vBuffer = ByteBuffer.allocateDirect(uvPlaneSize)

yBuffer.put(i420ByteArray, 0, yPlaneSize)
uBuffer.put(i420ByteArray, yPlaneSize, uvPlaneSize)
vBuffer.put(i420ByteArray, yPlaneSize + uvPlaneSize, uvPlaneSize)

yBuffer.rewind()
uBuffer.rewind()
vBuffer.rewind()

// Create the I420Buffer using the separate planes
val finalFrameBuffer = JavaI420Buffer.wrap(
width,
height,
yBuffer,
width,
uBuffer,
width / 2,
vBuffer,
width / 2,
null
)
private fun Mat.toVideoFrame(time: Long): VideoFrame? =
runCatching {
val i420Mat = Mat()
Imgproc.cvtColor(this, i420Mat, Imgproc.COLOR_RGBA2YUV_I420)

i420Mat.release()

return VideoFrame(finalFrameBuffer, 0, time)
}
// Get the raw bytes from the new I420 Mat to i420ByteArray
val i420ByteArray = ByteArray((i420Mat.total() * i420Mat.elemSize()).toInt())
i420Mat.get(0, 0, i420ByteArray)

private fun VideoFrame.I420Buffer.toMat(): Mat? =
kotlin.runCatching {
val i420Buffer = this
val width = this.width()
val height = this.height()

val width = i420Buffer.width
val height = i420Buffer.height
val yPlaneSize = width * height
val uvPlaneSize = (width / 2) * (height / 2)

val yBuffer = ByteBuffer.allocateDirect(yPlaneSize)
val uBuffer = ByteBuffer.allocateDirect(uvPlaneSize)
val vBuffer = ByteBuffer.allocateDirect(uvPlaneSize)

yBuffer.put(i420ByteArray, 0, yPlaneSize)
uBuffer.put(i420ByteArray, yPlaneSize, uvPlaneSize)
vBuffer.put(i420ByteArray, yPlaneSize + uvPlaneSize, uvPlaneSize)

yBuffer.rewind()
uBuffer.rewind()
vBuffer.rewind()

// Create the I420Buffer using the separate planes
val finalFrameBuffer = JavaI420Buffer.wrap(
width,
height,
yBuffer,
width,
uBuffer,
width / 2,
vBuffer,
width / 2,
null
)

val nv21Height = (height * NV21_HEIGHT_MULTI).toInt()
val nv21Width = width
val nv21Size = nv21Height * nv21Width
val nv21Data = ByteArray(nv21Size)

val dataY = i420Buffer.dataY
val dataU = i420Buffer.dataU
val dataV = i420Buffer.dataV
i420Mat.release()

val strideY = i420Buffer.strideY // Likely equal to the width, but not always, depending on mem alignment
val strideU = i420Buffer.strideU // U and V have identical dimens and strides
val strideV = i420Buffer.strideV
return VideoFrame(finalFrameBuffer, 0, time)
}.getOrElse { throwable ->
Log.e(TAG, "Error in Mat.toVideoFrame $throwable")

if (strideY == width) {
// Fast path: contiguous data
dataY.get(nv21Data, 0, yPlaneSize)
} else {
// Slow path: row-by-row copy
for (row in 0 until height) {
dataY.position(row * strideY)
dataY.get(nv21Data, row * width, width)
}
}
null
}

val vuPlaneOffset = width * height
for (row in 0 until height / 2) {
for (col in 0 until width / 2) {
// Get U and V values from their respective planes using row/col/stride
val v = dataV[row * strideV + col]
val u = dataU[row * strideU + col]

// Put them into the NV21 buffer (V, then U)
val nv21Index = vuPlaneOffset + (row * width) + (col * 2)
nv21Data[nv21Index] = v
nv21Data[nv21Index + 1] = u
}
}
private fun VideoFrame.I420Buffer.toMat(): Mat? =
runCatching {
val chromaWidth = (width + 1) / 2
val chromaHeight = (height + 1) / 2
val minSize = width * height + chromaWidth * chromaHeight * 2

val nv12ByteBuffer = ByteBuffer.allocateDirect(minSize)
YuvHelper.I420ToNV12(
this.dataY,
this.strideY,
this.dataU,
this.strideU,
this.dataV,
this.strideV,
nv12ByteBuffer,
width,
height
)

val mat = Mat(
nv21Height,
nv21Width,
(height * NV21_HEIGHT_MULTI).toInt(),
width,
CvType.CV_8UC1 // 8 bit unsigned 1 channel
)

mat.put(0, 0, nv21Data)
Imgproc.cvtColor(mat, mat, Imgproc.COLOR_YUV2RGBA_NV21)
mat.put(0, 0, nv12ByteBuffer.array())

i420Buffer.release()
Imgproc.cvtColor(mat, mat, Imgproc.COLOR_YUV2RGBA_NV12)

this.release()

mat
}.getOrNull()
}.getOrElse { throwable ->
Log.e(TAG, "Error in VideoFrame.I420Buffer.toMat $throwable")

null
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,7 @@ import org.opencv.core.Mat
import org.opencv.core.Scalar
import java.nio.ByteBuffer

class ImageSegmenterHelper(
var currentDelegate: Int = DELEGATE_CPU,
var runningMode: RunningMode = RunningMode.LIVE_STREAM,
val context: Context,
var imageSegmenterListener: SegmenterListener? = null
) {
class ImageSegmenterHelper(val context: Context, var imageSegmenterListener: SegmenterListener? = null) {

private var imageSegmenter: ImageSegmenter? = null

Expand All @@ -53,39 +48,26 @@ class ImageSegmenterHelper(
* @throws IllegalStateException
*/
fun setupImageSegmenter() {
val baseOptionsBuilder = BaseOptions.builder()
when (currentDelegate) {
DELEGATE_CPU -> {
baseOptionsBuilder.setDelegate(Delegate.CPU)
}

DELEGATE_GPU -> {
baseOptionsBuilder.setDelegate(Delegate.GPU)
}
val baseOptionsBuilder = BaseOptions.builder().apply {
setDelegate(Delegate.CPU)
setModelAssetPath(MODEL_SELFIE_SEGMENTER_PATH)
}

baseOptionsBuilder.setModelAssetPath(MODEL_SELFIE_SEGMENTER_PATH)

if (imageSegmenterListener == null) {
throw IllegalStateException("ImageSegmenterListener must be set.")
}

runCatching {
val baseOptions = baseOptionsBuilder.build()
val optionsBuilder = ImageSegmenter.ImageSegmenterOptions.builder()
.setRunningMode(runningMode)
.setRunningMode(RunningMode.LIVE_STREAM)
.setBaseOptions(baseOptions)
.setOutputCategoryMask(true)
.setOutputConfidenceMasks(false)
.setResultListener(this::returnSegmentationResult)
.setErrorListener(this::returnSegmentationHelperError)

if (runningMode == RunningMode.LIVE_STREAM) {
optionsBuilder
.setResultListener(this::returnSegmentationResult)
.setErrorListener(this::returnSegmentationHelperError)
}

val options = optionsBuilder.build()
imageSegmenter = ImageSegmenter.createFromOptions(context, options)
imageSegmenter = ImageSegmenter.createFromOptions(context, optionsBuilder.build())
}.getOrElse { e ->
when (e) {
is IllegalStateException -> {
Expand Down Expand Up @@ -114,12 +96,6 @@ class ImageSegmenterHelper(
* @throws IllegalArgumentException
*/
fun segmentLiveStreamFrame(bitmap: Bitmap, videoFrameTimeStamp: Long) {
if (runningMode != RunningMode.LIVE_STREAM) {
throw IllegalArgumentException(
"Attempting to call segmentLiveStreamFrame while not using RunningMode.LIVE_STREAM"
)
}

val mpImage = BitmapImageBuilder(bitmap).build()

imageSegmenter?.segmentAsync(mpImage, videoFrameTimeStamp)
Expand All @@ -146,6 +122,7 @@ class ImageSegmenterHelper(
mat.put(0, 0, data)

Core.bitwise_not(mat, mat)

Core.multiply(mat, Scalar(RGB_MAX), mat)

imageSegmenterListener?.onResults(
Expand All @@ -167,14 +144,10 @@ class ImageSegmenterHelper(
data class ResultBundle(val mask: Mat, val inferenceTime: Long)

companion object {
const val DELEGATE_CPU = 0
const val DELEGATE_GPU = 1 // DO NOT USE THIS
const val OTHER_ERROR = 0
const val GPU_ERROR = 1

const val MODEL_SELFIE_SEGMENTER_PATH = "selfie_segmenter.tflite"
const val RGB_MAX = 255.0

private const val TAG = "ImageSegmenterHelper"
}

Expand Down
Loading