diff --git a/README.md b/README.md index 9a67c38c8..f9f930c55 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,8 @@ To simplify integration, StreamPack provides an `PreviewView`. There are 2 types of streamers: -- Kotlin Coroutine based -- callback based +- Kotlin Coroutine based: streamer APIs use `suspend` functions and `Flow` +- callback based: streamer APIs use callbacks ```kotlin // For coroutine based @@ -147,6 +147,9 @@ val streamer = DefaultCameraStreamer(context = requireContext()) 4. Configures audio and video settings ```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + val audioConfig = AudioConfig( startBitrate = 128000, sampleRate = 44100, @@ -165,12 +168,15 @@ streamer.configure(audioConfig, videoConfig) 5. Inflates the camera preview with the streamer ```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + /** - * If the preview is in a PreviewView + * If the preview is a [PreviewView] */ preview.streamer = streamer /** - * If the preview is in a SurfaceView, a TextureView, or any View that can provide a Surface + * If the preview is in a SurfaceView, a TextureView, a Surface,... you can use: */ streamer.startPreview(preview) ``` @@ -178,6 +184,10 @@ streamer.startPreview(preview) 6. Starts the live streaming ```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + + val descriptor = UriMediaDescriptor("rtmps://serverip:1935/s/streamKey") // For RTMP/RTMPS. Uri also supports SRT url, file, content path,... /** @@ -192,6 +202,9 @@ streamer.startStream() 7. Stops and releases the streamer ```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + streamer.stopStream() streamer.close() // Disconnect from server or close the file streamer.stopPreview() // The StreamerSurfaceView will be automatically stop the preview @@ -265,50 +278,85 @@ You will also have to declare the `Service`, ``` +## Rotations + +To set the `Streamer` orientation, you can use the `targetRotation` setter: + +```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + +streamer.targetRotation = + Surface.ROTATION_90 // Or Surface.ROTATION_0, Surface.ROTATION_180, Surface.ROTATION_270 +``` + +StreamPack comes with a `RotationProvider` that fetches and listens the device rotation: the +`DeviceRotationProvider`. + +```kotlin +// Already instantiated streamer +val streamer = DefaultCameraStreamer(context = requireContext()) + +val listener = object : IRotationProvider.Listener { + override fun onOrientationChanged(rotation: Int) { + streamer.targetRotation = rotation + } +} +rotationProvider.addListener(listener) + +// Don't forget to remove the listener when you don't need it anymore +rotationProvider.removeListener(listener) +``` + +See the `demos` for a complete example. + ## Tips ### RTMP or SRT -RTMP and SRT are both live streaming protocols. SRT is a UDP-based modern protocol, it is reliable -and ultra low latency. RTMP is a TCP-based protocol, it is also reliable but it is only low latency. +RTMP and SRT are both live streaming protocols . SRT is a UDP - based modern protocol, it is +reliable +and ultra low latency . RTMP is a TCP - based protocol, it is also reliable but it is only low +latency . There are already a lot of comparison over the Internet, so here is a summary: SRT: -- Ultra low latency (< 1s) -- HEVC support through MPEG-TS RTMP: -- Low latency (2-3s) -- HEVC not officially support (specification has been aban by its creator) +-Ultra low latency(< 1 s) +-HEVC support through MPEG -TS RTMP : +-Low latency (2 - 3 s) +-HEVC not officially support (specification has been aban by its creator) -So, the main question is: "which protocol to use?" +So, the main question is : "which protocol to use?" It is easy: if your server has SRT support, use SRT otherwise use RTMP. ### Streamers Let's start with some definitions! `Streamers` are classes that represent a streaming pipeline: -capture, encode, mux and send. -They comes in multiple flavours: with different audio and video source. 3 types of base streamers -are available: +capture, encode, mux and send.They comes in multiple flavours: with different audio and video +source . 3 types of base streamers +are available : -- `DefaultCameraStreamer`: for streaming from camera -- `DefaultScreenRecorderStreamer`: for streaming from screen -- `DefaultAudioOnlyStreamer`: for streaming audio only +-`DefaultCameraStreamer`: for streaming from camera +-`DefaultScreenRecorderStreamer`: for streaming from screen +-`DefaultAudioOnlyStreamer`: for streaming audio only Since 3.0.0, the endpoint of a `Streamer` is inferred from the `MediaDescriptor` object passed to -the `open` or `startStream` methods. It is possible to limit the possibility of the endpoint by +the `open` or `startStream` methods.It is possible to limit the possibility of the endpoint by implementing your own `DynamicEndpoint.Factory` or passing a endpoint as the `Streamer` `endpoint` -parameter. - -To create a `Streamer` for a new source, you have to create a new `Streamer` class that inherits -from `DefaultStreamer`. +parameter.To create a `Streamer` for a new source, you have to create a new `Streamer` class that +inherits +from `DefaultStreamer` . ### Get device capabilities -Have you ever wonder: "What are the supported resolution of my cameras?" or "What is the supported -sample rate of my audio codecs?"? `Info` classes are made for this. All `Streamer` comes with a -specific `Info` object: +Have you ever wonder : "What are the supported resolution of my cameras?" or "What is the supported +sample rate of my audio codecs ?"? `Info` classes are made for this. All `Streamer` comes with a +specific `Info` object : + + ```kotlin -```kotlin val info = streamer.getInfo(MediaDescriptor("rtmps://serverip:1935/s/streamKey")) + ``` For static endpoint or an opened dynamic endpoint, you can directly get the info: diff --git a/core/build.gradle.kts b/core/build.gradle.kts index f36a55d1a..8d38d121a 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -19,6 +19,8 @@ dependencies { implementation(libs.androidx.core.ktx) implementation(libs.androidx.activity) implementation(libs.kotlinx.coroutines.android) + implementation(libs.androidx.window) + implementation(libs.androidx.concurrent.futures) testImplementation(libs.androidx.test.rules) testImplementation(libs.androidx.test.core.ktx) @@ -28,7 +30,6 @@ dependencies { testImplementation(libs.kotlinx.coroutines.test) testImplementation(libs.robolectric) - androidTestImplementation(libs.androidx.test.core.ktx) androidTestImplementation(libs.androidx.test.rules) androidTestImplementation(libs.androidx.junit) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/data/VideoConfig.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/data/VideoConfig.kt index edbad53a0..32972f947 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/data/VideoConfig.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/data/VideoConfig.kt @@ -33,12 +33,13 @@ import android.media.MediaFormat import android.media.MediaFormat.KEY_PRIORITY import android.os.Build import android.util.Size +import androidx.annotation.IntRange import io.github.thibaultbee.streampack.core.internal.encoders.mediacodec.MediaCodecHelper +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue import io.github.thibaultbee.streampack.core.internal.utils.av.video.DynamicRangeProfile -import io.github.thibaultbee.streampack.core.internal.utils.extensions.isDevicePortrait import io.github.thibaultbee.streampack.core.internal.utils.extensions.isVideo -import io.github.thibaultbee.streampack.core.internal.utils.extensions.landscapize -import io.github.thibaultbee.streampack.core.internal.utils.extensions.portraitize +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotateFromNaturalOrientation +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotationToDegrees import io.github.thibaultbee.streampack.core.streamers.DefaultStreamer import java.security.InvalidParameterException import kotlin.math.roundToInt @@ -153,20 +154,6 @@ class VideoConfig( */ val isHdr by lazy { dynamicRangeProfile != DynamicRangeProfile.sdr } - /** - * Get resolution according to device orientation - * - * @param context activity context - * @return oriented resolution - */ - fun getDeviceOrientedResolution(context: Context): Size { - return if (context.isDevicePortrait) { - resolution.portraitize - } else { - resolution.landscapize - } - } - /** * Get the media format from the video configuration * @@ -348,3 +335,24 @@ class VideoConfig( } } +/** + * Rotates video configuration to [rotation] from device natural orientation. + */ +fun VideoConfig.rotateFromNaturalOrientation(context: Context, @RotationValue rotation: Int) = + rotateDegreesFromNaturalOrientation(context, rotation.rotationToDegrees) + +/** + * Rotatse video configuration to [rotationDegrees] from device natural orientation. + */ +fun VideoConfig.rotateDegreesFromNaturalOrientation( + context: Context, + @IntRange(from = 0, to = 359) rotationDegrees: Int +): VideoConfig { + val newResolution = resolution.rotateFromNaturalOrientation(context, rotationDegrees) + return if (resolution != newResolution) { + copy(resolution = newResolution) + } else { + this + } +} + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/IEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/IEncoder.kt index 57f1bba3b..8414217c8 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/IEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/IEncoder.kt @@ -16,6 +16,7 @@ package io.github.thibaultbee.streampack.core.internal.encoders import android.view.Surface +import io.github.thibaultbee.streampack.core.data.Config import io.github.thibaultbee.streampack.core.internal.data.Frame import io.github.thibaultbee.streampack.core.internal.interfaces.Releaseable import io.github.thibaultbee.streampack.core.internal.interfaces.SuspendStreamable @@ -62,6 +63,12 @@ interface IEncoder { interface IEncoderInternal : SuspendStreamable, Releaseable, IEncoder { + + /** + * The encoder configuration + */ + val config: Config + interface IListener { /** * Calls when an encoder has an error. @@ -99,6 +106,11 @@ interface IEncoderInternal : SuspendStreamable, Releaseable, */ interface ISurfaceInput : IEncoderInput { + /** + * The surface where to write the frame + */ + val surface: Surface? + /** * The surface update listener */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/EncoderConfig.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/EncoderConfig.kt index 42ddd6c94..6e5e2c5b5 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/EncoderConfig.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/EncoderConfig.kt @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.github.thibaultbee.streampack.core.internal.encoders.mediacodec import android.media.MediaCodecInfo @@ -6,7 +21,6 @@ import android.os.Build import io.github.thibaultbee.streampack.core.data.AudioConfig import io.github.thibaultbee.streampack.core.data.Config import io.github.thibaultbee.streampack.core.data.VideoConfig -import io.github.thibaultbee.streampack.core.internal.orientation.ISourceOrientationProvider sealed class EncoderConfig(val config: T) { /** @@ -39,13 +53,12 @@ sealed class EncoderConfig(val config: T) { class VideoEncoderConfig( videoConfig: VideoConfig, - val useSurfaceMode: Boolean = true, - private val orientationProvider: ISourceOrientationProvider? = null + val useSurfaceMode: Boolean = true ) : EncoderConfig( videoConfig ) { override val isVideo = true - + override fun buildFormat(withProfileLevel: Boolean): MediaFormat { val format = config.getFormat(withProfileLevel) if (useSurfaceMode) { @@ -68,23 +81,12 @@ class VideoEncoderConfig( return format } - fun orientateFormat(format: MediaFormat) { - orientationProvider?.let { - it.getOrientedSize(config.resolution).apply { - // Override previous format - format.setInteger(MediaFormat.KEY_WIDTH, width) - format.setInteger(MediaFormat.KEY_HEIGHT, height) - } - } - } - override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is VideoEncoderConfig) return false if (!super.equals(other)) return false if (useSurfaceMode != other.useSurfaceMode) return false - if (orientationProvider != other.orientationProvider) return false return true } @@ -92,7 +94,6 @@ class VideoEncoderConfig( override fun hashCode(): Int { var result = super.hashCode() result = 31 * result + useSurfaceMode.hashCode() - result = 31 * result + (orientationProvider?.hashCode() ?: 0) result = 31 * result + isVideo.hashCode() return result } diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/MediaCodecEncoder.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/MediaCodecEncoder.kt index 06c128dfa..88adefac9 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/MediaCodecEncoder.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/encoders/mediacodec/MediaCodecEncoder.kt @@ -85,6 +85,7 @@ internal constructor( } } + override val config = encoderConfig.config private val encoderCallback = EncoderCallback() @@ -128,15 +129,6 @@ internal constructor( } override fun configure() { - /** - * This is a workaround because few Samsung devices (such as Samsung Galaxy J7 Prime does - * not find any encoder if the width and height are oriented to portrait. - * We defer orientation of width and height to here. - */ - if (encoderConfig is VideoEncoderConfig) { - encoderConfig.orientateFormat(format) - } - try { /** * Set encoder callback without handler. @@ -448,7 +440,8 @@ internal constructor( internal inner class SurfaceInput : IEncoderInternal.ISurfaceInput { private val obsoleteSurfaces = mutableListOf() - private var surface: Surface? = null + override var surface: Surface? = null + private set override var listener = object : IEncoderInternal.ISurfaceInput.OnSurfaceUpdateListener {} set(value) { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/CodecSurface.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/CodecSurface.kt deleted file mode 100644 index f059808a7..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/CodecSurface.kt +++ /dev/null @@ -1,194 +0,0 @@ -package io.github.thibaultbee.streampack.core.internal.gl - -import android.annotation.SuppressLint -import android.graphics.SurfaceTexture -import android.util.Size -import android.view.Surface -import io.github.thibaultbee.streampack.core.internal.orientation.ISourceOrientationListener -import io.github.thibaultbee.streampack.core.internal.orientation.ISourceOrientationProvider -import java.util.concurrent.Executors - -class CodecSurface( - private val orientationProvider: ISourceOrientationProvider? -) : - SurfaceTexture.OnFrameAvailableListener, ISourceOrientationListener { - private val executor = Executors.newSingleThreadExecutor() - - private var eglSurface: EglWindowSurface? = null - private var fullFrameRect: FullFrameRect? = null - private var textureId = -1 - - private var isRunning = false - private var surfaceTexture: SurfaceTexture? = null - private val stMatrix = FloatArray(16) - - private var _inputSurface: Surface? = null - - val input: Surface? - get() = _inputSurface - - /** - * If true, the encoder will use high bit depth (10 bits) for encoding. - */ - var useHighBitDepth = false - - var outputSurface: Surface? = null - set(value) { - /** - * When surface is called twice without the stopStream(). When configure() is - * called twice for example, - */ - executor.submit { - if (eglSurface != null) { - detachSurfaceTexture() - } - synchronized(this) { - value?.let { - initOrUpdateSurfaceTexture(it) - } - } - - }.get() // Wait till executor returns - field = value - } - - init { - orientationProvider?.addListener(this) - } - - private fun initOrUpdateSurfaceTexture(surface: Surface) { - eglSurface = ensureGlContext(EglWindowSurface(surface, useHighBitDepth)) { - val width = it.getWidth() - val height = it.getHeight() - val size = - orientationProvider?.getOrientedSize(Size(width, height)) ?: Size(width, height) - val orientation = orientationProvider?.orientation ?: 0 - fullFrameRect = FullFrameRect(Texture2DProgram()).apply { - textureId = createTextureObject() - setMVPMatrixAndViewPort( - orientation.toFloat(), - size, - orientationProvider?.mirroredVertically ?: false - ) - } - - val defaultBufferSize = - orientationProvider?.getDefaultBufferSize(size) ?: Size(width, height) - surfaceTexture = attachOrBuildSurfaceTexture(surfaceTexture).apply { - setDefaultBufferSize(defaultBufferSize.width, defaultBufferSize.height) - setOnFrameAvailableListener(this@CodecSurface) - } - } - } - - @SuppressLint("Recycle") - private fun attachOrBuildSurfaceTexture(surfaceTexture: SurfaceTexture?): SurfaceTexture { - return if (surfaceTexture == null) { - SurfaceTexture(textureId).apply { - _inputSurface = Surface(this) - } - } else { - surfaceTexture.attachToGLContext(textureId) - surfaceTexture - } - } - - private fun ensureGlContext( - surface: EglWindowSurface?, - action: (EglWindowSurface) -> Unit - ): EglWindowSurface? { - surface?.let { - it.makeCurrent() - action(it) - it.makeUnCurrent() - } - return surface - } - - override fun onOrientationChanged() { - executor.execute { - synchronized(this) { - ensureGlContext(eglSurface) { - val width = it.getWidth() - val height = it.getHeight() - - fullFrameRect?.setMVPMatrixAndViewPort( - (orientationProvider?.orientation ?: 0).toFloat(), - orientationProvider?.getOrientedSize(Size(width, height)) ?: Size( - width, - height - ), - orientationProvider?.mirroredVertically ?: false - ) - - /** - * Flushing spurious latest camera frames that block SurfaceTexture buffer - * to avoid having a misoriented frame. - */ - surfaceTexture?.updateTexImage() - surfaceTexture?.releaseTexImage() - } - } - } - } - - override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { - if (!isRunning) { - return - } - - executor.execute { - synchronized(this) { - eglSurface?.let { - it.makeCurrent() - surfaceTexture.updateTexImage() - surfaceTexture.getTransformMatrix(stMatrix) - - // Use the identity matrix for MVP so our 2x2 FULL_RECTANGLE covers the viewport. - fullFrameRect?.drawFrame(textureId, stMatrix) - it.setPresentationTime(surfaceTexture.timestamp) - it.swapBuffers() - surfaceTexture.releaseTexImage() - } - } - } - } - - fun startStream() { - // Flushing spurious latest camera frames that block SurfaceTexture buffer. - ensureGlContext(eglSurface) { - surfaceTexture?.updateTexImage() - } - isRunning = true - } - - private fun detachSurfaceTexture() { - ensureGlContext(eglSurface) { - surfaceTexture?.detachFromGLContext() - fullFrameRect?.release(true) - } - eglSurface?.release() - eglSurface = null - fullFrameRect = null - } - - fun stopStream() { - if (!isRunning) { - return - } - executor.submit { - synchronized(this) { - isRunning = false - detachSurfaceTexture() - } - }.get() - } - - fun release() { - orientationProvider?.removeListener(this) - stopStream() - surfaceTexture?.setOnFrameAvailableListener(null) - surfaceTexture?.release() - surfaceTexture = null - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/EglWindowSurface.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/EglWindowSurface.kt deleted file mode 100644 index 3d3fab9ee..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/EglWindowSurface.kt +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2018 Google Inc. All rights reserved. - * Copyright 2021 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.gl - -import android.opengl.EGL14 -import android.opengl.EGLConfig -import android.opengl.EGLContext -import android.opengl.EGLDisplay -import android.opengl.EGLExt -import android.opengl.EGLSurface -import android.view.Surface -import java.util.Objects - -/** - * Holds state associated with a Surface used for MediaCodec encoder input. - *

- * The constructor takes a Surface obtained from MediaCodec.createInputSurface(), and uses that - * to create an EGL window surface. Calls to eglSwapBuffers() cause a frame of data to be sent - * to the video encoder. - * - * (Contains mostly code borrowed from CameraX) - */ - -class EglWindowSurface(private val surface: Surface, useHighBitDepth: Boolean = false) { - private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY - private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT - private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE - private val configs = arrayOfNulls(1) - - companion object { - private const val EGL_RECORDABLE_ANDROID = 0x3142 - } - - init { - eglSetup(useHighBitDepth) - } - - /** - * Prepares EGL. We want a GLES 2.0 context and a surface that supports recording. - */ - private fun eglSetup(useHighBitDepth: Boolean) { - eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) - if (Objects.equals(eglDisplay, EGL14.EGL_NO_DISPLAY)) { - throw RuntimeException("unable to get EGL14 display") - } - val version = IntArray(2) - if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { - throw RuntimeException("unable to initialize EGL14") - } - - // Configure EGL for recordable and OpenGL ES 2.0. We want enough RGB bits - // to minimize artifacts from possible YUV conversion. - val eglColorSize = if (useHighBitDepth) 10 else 8 - val eglAlphaSize = if (useHighBitDepth) 2 else 0 - val recordable = if (useHighBitDepth) 0 else 1 - var attribList = intArrayOf( - EGL14.EGL_RED_SIZE, eglColorSize, - EGL14.EGL_GREEN_SIZE, eglColorSize, - EGL14.EGL_BLUE_SIZE, eglColorSize, - EGL14.EGL_ALPHA_SIZE, eglAlphaSize, - EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, - EGL_RECORDABLE_ANDROID, recordable, - EGL14.EGL_NONE - ) - val numConfigs = IntArray(1) - if (!EGL14.eglChooseConfig( - eglDisplay, attribList, 0, configs, 0, configs.size, - numConfigs, 0 - ) - ) { - throw RuntimeException("unable to find RGB888+recordable ES2 EGL config") - } - - // Configure context for OpenGL ES 2.0. - attribList = intArrayOf( - EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, - EGL14.EGL_NONE - ) - eglContext = EGL14.eglCreateContext( - eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, - attribList, 0 - ) - GlUtils.checkEglError("eglCreateContext") - - // Create a window surface, and attach it to the Surface we received. - createEGLSurface() - } - - private fun createEGLSurface() { - val surfaceAttribs = intArrayOf( - EGL14.EGL_NONE - ) - eglSurface = EGL14.eglCreateWindowSurface( - eglDisplay, configs[0], surface, - surfaceAttribs, 0 - ) - GlUtils.checkEglError("eglCreateWindowSurface") - } - - /** - * Discard all resources held by this class, notably the EGL context. Also releases the - * Surface that was passed to our constructor. - */ - fun release() { - if (!Objects.equals(eglDisplay, EGL14.EGL_NO_DISPLAY)) { - EGL14.eglDestroySurface(eglDisplay, eglSurface) - EGL14.eglDestroyContext(eglDisplay, eglContext) - EGL14.eglReleaseThread() - EGL14.eglTerminate(eglDisplay) - } - surface.release() - eglDisplay = EGL14.EGL_NO_DISPLAY - eglContext = EGL14.EGL_NO_CONTEXT - eglSurface = EGL14.EGL_NO_SURFACE - } - - /** - * Makes our EGL context and surface current. - */ - fun makeCurrent() { - if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { - throw RuntimeException("eglMakeCurrent failed") - } - } - - /** - * Makes our EGL context and surface not current. - */ - fun makeUnCurrent() { - if (!EGL14.eglMakeCurrent( - eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, - EGL14.EGL_NO_CONTEXT - ) - ) { - throw RuntimeException("eglMakeCurrent failed") - } - } - - /** - * Calls eglSwapBuffers. Use this to "publish" the current frame. - */ - fun swapBuffers(): Boolean { - return EGL14.eglSwapBuffers(eglDisplay, eglSurface) - } - - /** - * Queries the surface's width. - */ - fun getWidth(): Int { - val value = IntArray(1) - EGL14.eglQuerySurface(eglDisplay, eglSurface, EGL14.EGL_WIDTH, value, 0) - return value[0] - } - - /** - * Queries the surface's height. - */ - fun getHeight(): Int { - val value = IntArray(1) - EGL14.eglQuerySurface(eglDisplay, eglSurface, EGL14.EGL_HEIGHT, value, 0) - return value[0] - } - - /** - * Sends the presentation time stamp to EGL. Time is expressed in nanoseconds. - */ - fun setPresentationTime(nSecs: Long) { - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nSecs) - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/FullFrameRect.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/FullFrameRect.kt deleted file mode 100644 index 14da0aa4d..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/FullFrameRect.kt +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2018 Google Inc. All rights reserved. - * Copyright 2021 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.gl - -import android.opengl.GLES20 -import android.opengl.Matrix -import android.util.Size -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.nio.FloatBuffer - - -/** - * This class essentially represents a viewport-sized sprite that will be rendered with - * a texture, usually from an external source like the camera or video decoder. - * - * (Contains mostly code borrowed from graphika) - */ -class FullFrameRect(var program: Texture2DProgram) { - private val mvpMatrix = FloatArray(16) - - companion object { - /** - * A "full" square, extending from -1 to +1 in both dimensions. When the - * model/view/projection matrix is identity, this will exactly cover the viewport. - */ - private val FULL_RECTANGLE_COORDS = floatArrayOf( - -1.0f, -1.0f, // 0 bottom left - 1.0f, -1.0f, // 1 bottom right - -1.0f, 1.0f, // 2 top left - 1.0f, 1.0f - ) - - private val FULL_RECTANGLE_BUF: FloatBuffer = createFloatBuffer(FULL_RECTANGLE_COORDS) - - private val FULL_RECTANGLE_TEX_COORDS = floatArrayOf( - 0.0f, 0.0f, // 0 bottom left - 1.0f, 0.0f, // 1 bottom right - 0.0f, 1.0f, // 2 top left - 1.0f, 1.0f // 3 top right - ) - private val FULL_RECTANGLE_TEX_BUF: FloatBuffer = - createFloatBuffer(FULL_RECTANGLE_TEX_COORDS) - - /** - * Allocates a direct float buffer, and populates it with the float array data. - */ - private fun createFloatBuffer(coords: FloatArray): FloatBuffer { - // Allocate a direct ByteBuffer, using 4 bytes per float, and copy coords into it. - val bb: ByteBuffer = ByteBuffer.allocateDirect(coords.size * Float.SIZE_BYTES) - bb.order(ByteOrder.nativeOrder()) - val fb: FloatBuffer = bb.asFloatBuffer() - fb.put(coords) - fb.position(0) - return fb - } - } - - /** - * Releases resources. - * - * - * This must be called with the appropriate EGL context current (i.e. the one that was - * current when the constructor was called). If we're about to destroy the EGL context, - * there's no value in having the caller make it current just to do this cleanup, so you - * can pass a flag that will tell this function to skip any EGL-context-specific cleanup. - */ - fun release(doEglCleanup: Boolean) { - if (doEglCleanup) { - program.release() - } - } - - /** - * Changes the program. The previous program will be released. - * - * - * The appropriate EGL context must be current. - */ - fun changeProgram(program: Texture2DProgram) { - this.program.release() - this.program = program - } - - /** - * Creates a texture object suitable for use with drawFrame(). - */ - fun createTextureObject(): Int { - return program.createTextureObject() - } - - fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size, mirroredVertically: Boolean) { - Matrix.setIdentityM(mvpMatrix, 0) - Matrix.scaleM(mvpMatrix, 0, if (mirroredVertically) -1f else 1f, 1f, 0f) - Matrix.rotateM( - mvpMatrix, 0, - rotation, 0f, 0f, -1f - ) - GLES20.glViewport(0, 0, resolution.width, resolution.height) - } - - /** - * Draws a viewport-filling rect, texturing it with the specified texture object. - */ - fun drawFrame(textureId: Int, texMatrix: FloatArray) { - // Use the identity matrix for MVP so our 2x2 FULL_RECTANGLE covers the viewport. - program.draw( - mvpMatrix, FULL_RECTANGLE_BUF, 0, - 4, 2, 2 * Float.SIZE_BYTES, - texMatrix, FULL_RECTANGLE_TEX_BUF, textureId, 2 * Float.SIZE_BYTES - ) - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/GlUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/GlUtils.kt deleted file mode 100644 index 636dbad6f..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/GlUtils.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2021 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.gl - -import android.opengl.EGL14 -import android.opengl.GLES20 - - -object GlUtils { - /** - * Checks for EGL errors. Throws an exception if one is found. - */ - fun checkEglError(msg: String) { - var error: Int - if (EGL14.eglGetError().also { error = it } != EGL14.EGL_SUCCESS) { - throw RuntimeException(msg + ": EGL error: 0x" + Integer.toHexString(error)) - } - } - - /** - * Checks to see if a GLES error has been raised. - */ - fun checkGlError(op: String) { - val error = GLES20.glGetError() - if (error == GLES20.GL_OUT_OF_MEMORY) { - throw RuntimeException("$op GL_OUT_OF_MEMORY") - } - if (error != GLES20.GL_NO_ERROR && error != GLES20.GL_OUT_OF_MEMORY) { - val msg = op + ": glError 0x" + Integer.toHexString(error) - throw RuntimeException(msg) - } - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/Texture2DProgram.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/Texture2DProgram.kt deleted file mode 100644 index 36008afd9..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/gl/Texture2DProgram.kt +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright 2018 Google Inc. All rights reserved. - * Copyright 2021 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.gl - -import android.opengl.GLES11Ext -import android.opengl.GLES20 -import java.nio.FloatBuffer - -/** - * GL program and supporting functions for textured 2D shapes. - * - * (Contains mostly code borrowed from CameraX) - * - */ -class Texture2DProgram { - // Handles to the GL program and various components of it. - private val programHandle: Int - private val uMVPMatrixLoc: Int - private val uTexMatrixLoc: Int - private val aPositionLoc: Int - private val aTextureCoordLoc: Int - - init { - programHandle = createProgram(VERTEX_SHADER, FRAGMENT_SHADER_EXT) - if (programHandle == 0) { - throw RuntimeException("Unable to create program") - } - - // get locations of attributes and uniforms - aPositionLoc = GLES20.glGetAttribLocation(programHandle, "aPosition") - checkLocation(aPositionLoc, "aPosition") - aTextureCoordLoc = GLES20.glGetAttribLocation(programHandle, "aTextureCoord") - checkLocation(aTextureCoordLoc, "aTextureCoord") - uMVPMatrixLoc = GLES20.glGetUniformLocation(programHandle, "uMVPMatrix") - checkLocation(uMVPMatrixLoc, "uMVPMatrix") - uTexMatrixLoc = GLES20.glGetUniformLocation(programHandle, "uTexMatrix") - checkLocation(uTexMatrixLoc, "uTexMatrix") - } - - /** - * Releases the program. - * - * - * The appropriate EGL context must be current (i.e. the one that was used to create - * the program). - */ - fun release() { - GLES20.glDeleteProgram(programHandle) - } - - /** - * Creates a texture object suitable for use with this program. - *

- * On exit, the texture will be bound. - */ - fun createTextureObject(): Int { - val textureID = IntArray(1) - GLES20.glGenTextures(1, textureID, 0) - GLES20.glActiveTexture(GLES20.GL_TEXTURE0) - - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureID[0]) - GlUtils.checkGlError("glBindTexture mTextureID") - - GLES20.glTexParameterf( - GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, - GLES20.GL_LINEAR.toFloat() - ) - GLES20.glTexParameterf( - GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, - GLES20.GL_LINEAR.toFloat() - ) - GLES20.glTexParameteri( - GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, - GLES20.GL_CLAMP_TO_EDGE - ) - GLES20.glTexParameteri( - GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, - GLES20.GL_CLAMP_TO_EDGE - ) - GlUtils.checkGlError("glTexParameter") - return textureID[0] - } - - - /** - * Creates a new program from the supplied vertex and fragment shaders. - * - * @return A handle to the program, or 0 on failure. - */ - private fun createProgram(vertexSource: String?, fragmentSource: String?): Int { - val vertexShader: Int = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource) - if (vertexShader == 0) { - return 0 - } - val pixelShader: Int = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource) - if (pixelShader == 0) { - return 0 - } - val program = GLES20.glCreateProgram() - GlUtils.checkGlError("glCreateProgram") - if (program == 0) { - throw Exception("Could not create program") - } - GLES20.glAttachShader(program, vertexShader) - GlUtils.checkGlError("glAttachShader") - GLES20.glAttachShader(program, pixelShader) - GlUtils.checkGlError("glAttachShader") - GLES20.glLinkProgram(program) - val linkStatus = IntArray(1) - GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0) - if (linkStatus[0] != GLES20.GL_TRUE) { - val info = GLES20.glGetProgramInfoLog(program) - GLES20.glDeleteProgram(program) - throw Exception("Could not link program: $info") - } - return program - } - - - /** - * Compiles the provided shader source. - * - * @return A handle to the shader, or 0 on failure. - */ - private fun loadShader(shaderType: Int, source: String?): Int { - val shader = GLES20.glCreateShader(shaderType) - GlUtils.checkGlError("glCreateShader type=$shaderType") - GLES20.glShaderSource(shader, source) - GLES20.glCompileShader(shader) - val compiled = IntArray(1) - GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0) - if (compiled[0] == 0) { - val info = GLES20.glGetShaderInfoLog(shader) - GLES20.glDeleteShader(shader) - throw Exception("Could not compile shader $shaderType: $info") - } - return shader - } - - /** - * Checks to see if the location we obtained is valid. GLES returns -1 if a label - * could not be found, but does not set the GL error. - * - * - * Throws a RuntimeException if the location is invalid. - */ - private fun checkLocation(location: Int, label: String) { - if (location < 0) { - throw java.lang.RuntimeException("Unable to locate '$label' in program") - } - } - - /** - * Issues the draw call. Does the full setup on every call. - * - * @param mvpMatrix The 4x4 projection matrix. - * @param vertexBuffer Buffer with vertex position data. - * @param firstVertex Index of first vertex to use in vertexBuffer. - * @param vertexCount Number of vertices in vertexBuffer. - * @param coordsPerVertex The number of coordinates per vertex (e.g. x,y is 2). - * @param vertexStride Width, in bytes, of the position data for each vertex (often - * vertexCount * sizeof(float)). - * @param texMatrix A 4x4 transformation matrix for texture coords. (Primarily intended - * for use with SurfaceTexture.) - * @param texBuffer Buffer with vertex texture data. - * @param texStride Width, in bytes, of the texture data for each vertex. - */ - fun draw( - mvpMatrix: FloatArray, vertexBuffer: FloatBuffer, firstVertex: Int, - vertexCount: Int, coordsPerVertex: Int, vertexStride: Int, - texMatrix: FloatArray, texBuffer: FloatBuffer, textureId: Int, texStride: Int - ) { - GlUtils.checkGlError("draw start") - - // Select the program. - GLES20.glUseProgram(programHandle) - GlUtils.checkGlError("glUseProgram") - - // Set the texture. - GLES20.glActiveTexture(GLES20.GL_TEXTURE0) - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) - - // Copy the model / view / projection matrix over. - GLES20.glUniformMatrix4fv(uMVPMatrixLoc, 1, false, mvpMatrix, 0) - GlUtils.checkGlError("glUniformMatrix4fv") - - // Copy the texture transformation matrix over. - GLES20.glUniformMatrix4fv(uTexMatrixLoc, 1, false, texMatrix, 0) - GlUtils.checkGlError("glUniformMatrix4fv") - - // Enable the "aPosition" vertex attribute. - GLES20.glEnableVertexAttribArray(aPositionLoc) - GlUtils.checkGlError("glEnableVertexAttribArray") - - // Connect vertexBuffer to "aPosition". - GLES20.glVertexAttribPointer( - aPositionLoc, coordsPerVertex, - GLES20.GL_FLOAT, false, vertexStride, vertexBuffer - ) - GlUtils.checkGlError("glVertexAttribPointer") - - // Enable the "aTextureCoord" vertex attribute. - GLES20.glEnableVertexAttribArray(aTextureCoordLoc) - GlUtils.checkGlError("glEnableVertexAttribArray") - - // Connect texBuffer to "aTextureCoord". - GLES20.glVertexAttribPointer( - aTextureCoordLoc, 2, - GLES20.GL_FLOAT, false, texStride, texBuffer - ) - GlUtils.checkGlError("glVertexAttribPointer") - - // Draw the rect. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, firstVertex, vertexCount) - GlUtils.checkGlError("glDrawArrays") - - // Done -- disable vertex array, texture, and program. - GLES20.glDisableVertexAttribArray(aPositionLoc) - GLES20.glDisableVertexAttribArray(aTextureCoordLoc) - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0) - GLES20.glUseProgram(0) - } - - companion object { - // Simple vertex shader, used for all programs. - private const val VERTEX_SHADER = """uniform mat4 uMVPMatrix; - uniform mat4 uTexMatrix; - attribute vec4 aPosition; - attribute vec4 aTextureCoord; - varying vec2 vTextureCoord; - void main() { - gl_Position = uMVPMatrix * aPosition; - vTextureCoord = (uTexMatrix * aTextureCoord).xy; - } - """ - - // Simple fragment shader for use with external 2D textures (e.g. what we get from - // SurfaceTexture). - private const val FRAGMENT_SHADER_EXT = """#extension GL_OES_EGL_image_external : require - precision mediump float; - varying vec2 vTextureCoord; - uniform samplerExternalOES sTexture; - void main() { - gl_FragColor = texture2D(sTexture, vTextureCoord); - } - """ - } - -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/AbstractSourceOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/AbstractSourceOrientationProvider.kt deleted file mode 100644 index c8bedbeb1..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/AbstractSourceOrientationProvider.kt +++ /dev/null @@ -1,18 +0,0 @@ -package io.github.thibaultbee.streampack.core.internal.orientation - -abstract class AbstractSourceOrientationProvider : ISourceOrientationProvider { - protected val listeners = mutableSetOf() - - override val mirroredVertically = false - override fun addListener(listener: ISourceOrientationListener) { - listeners.add(listener) - } - - override fun removeListener(listener: ISourceOrientationListener) { - listeners.remove(listener) - } - - override fun removeAllListeners() { - listeners.clear() - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/ISourceOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/ISourceOrientationProvider.kt deleted file mode 100644 index c3d1048bb..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/orientation/ISourceOrientationProvider.kt +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2023 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.orientation - -import android.graphics.SurfaceTexture -import android.util.Size - -/** - * Interface to get the orientation of the capture surface. - * These information are used to rotate the frames in the codec surface if the source needs to be rotated. - * It might not be the case for certain sources. - */ -interface ISourceOrientationProvider { - /** - * Orientation in degrees of the surface. - * Expected values: 0, 90, 180, 270. - */ - val orientation: Int - - /** - * If true, the source is mirrored vertically. - * Example: should be true for a front camera. - */ - val mirroredVertically: Boolean - - /** - * Returns the size with the correct orientation. - * If orientation is portrait, it returns a portrait size. - * Example: - * - Size = 1920x1080, if orientation is portrait, it returns 1080x1920. - */ - fun getOrientedSize(size: Size): Size - - /** - * Returns the size for [SurfaceTexture.setDefaultBufferSize]. - * Override this method if the image is stretched. - */ - fun getDefaultBufferSize(size: Size) = size - - /** - * Adds a listener to be notified when the orientation changes. - * - * @param listener to add. - */ - fun addListener(listener: ISourceOrientationListener) - - /** - * Removes a listener. - * - * @param listener to remove. - */ - fun removeListener(listener: ISourceOrientationListener) - - /** - * Removes all registered listeners. - */ - fun removeAllListeners() -} - -/** - * Interface to be notified when the orientation changes. - */ -interface ISourceOrientationListener { - /** - * Called when the orientation changes. - * Only called if [ISourceOrientationProvider.mirroredVertically] changes for now. - */ - fun onOrientationChanged() -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ISurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ISurfaceProcessor.kt new file mode 100644 index 000000000..bb308306e --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ISurfaceProcessor.kt @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video + +import android.util.Size +import android.view.Surface +import io.github.thibaultbee.streampack.core.internal.interfaces.Releaseable +import io.github.thibaultbee.streampack.core.internal.processing.video.outputs.AbstractSurfaceOutput + +interface ISurfaceProcessor + +interface ISurfaceProcessorInternal : ISurfaceProcessor, Releaseable { + fun createInputSurface(surfaceSize: Size): Surface? + + fun removeInputSurface(surface: Surface) + + fun addOutputSurface(surfaceOutput: AbstractSurfaceOutput) + + fun removeOutputSurface(surfaceOutput: AbstractSurfaceOutput) + + fun removeOutputSurface(surface: Surface) + + fun removeAllOutputSurfaces() +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/OpenGlRenderer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/OpenGlRenderer.kt new file mode 100644 index 000000000..5ef669b86 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/OpenGlRenderer.kt @@ -0,0 +1,617 @@ +/* + * Copyright 2022 The Android Open Source Project + * Copyright 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video + +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLExt +import android.opengl.EGLSurface +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.util.Log +import android.util.Size +import android.view.Surface +import androidx.annotation.WorkerThread +import androidx.core.util.Pair +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.EMPTY_ATTRIBS +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.InputFormat +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.NO_OUTPUT_SURFACE +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.Program2D +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.SamplerShaderProgram +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.checkEglErrorOrLog +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.checkEglErrorOrThrow +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.checkGlErrorOrThrow +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.checkGlThreadOrThrow +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.checkInitializedOrThrow +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.chooseSurfaceAttrib +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.createPBufferSurface +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.createPrograms +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.createTexture +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.createWindowSurface +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.deleteFbo +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.deleteTexture +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.generateFbo +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.generateTexture +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.getSurfaceSize +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.glVersionNumber +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GraphicDeviceInfo +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.OutputSurface +import io.github.thibaultbee.streampack.core.internal.utils.av.video.DynamicRangeProfile +import io.github.thibaultbee.streampack.core.logger.Logger +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicBoolean +import javax.microedition.khronos.egl.EGL10 + +/** + * OpenGLRenderer renders texture image to the output surface. + * + * + * OpenGLRenderer's methods must run on the same thread, so called GL thread. The GL thread is + * locked as the thread running the [.init] method, otherwise an + * [IllegalStateException] will be thrown when other methods are called. + */ +@WorkerThread +class OpenGlRenderer { + protected val mInitialized: AtomicBoolean = AtomicBoolean(false) + protected val mOutputSurfaceMap: MutableMap = + HashMap() + protected var mGlThread: Thread? = null + protected var mEglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + protected var mEglContext: EGLContext = EGL14.EGL_NO_CONTEXT + protected var mSurfaceAttrib: IntArray = EMPTY_ATTRIBS + protected var mEglConfig: EGLConfig? = null + protected var mTempSurface: EGLSurface = EGL14.EGL_NO_SURFACE + protected var mCurrentSurface: Surface? = null + protected var mProgramHandles: Map = emptyMap() + protected var mCurrentProgram: Program2D? = null + protected var mCurrentInputformat: InputFormat = InputFormat.UNKNOWN + + private var mExternalTextureId = -1 + + /** + * Initializes the OpenGLRenderer + * + * + * This is equivalent to calling [.init] without providing any + * shader overrides. Default shaders will be used for the dynamic range specified. + */ + fun init(dynamicRange: DynamicRangeProfile): GraphicDeviceInfo { + return init(dynamicRange, emptyMap()) + } + + /** + * Initializes the OpenGLRenderer + * + * + * Initialization must be done before calling other methods, otherwise an + * [IllegalStateException] will be thrown. Following methods must run on the same + * thread as this method, so called GL thread, otherwise an [IllegalStateException] + * will be thrown. + * + * @param dynamicRange the dynamic range used to select default shaders. + * @param shaderOverrides specific shader overrides for fragment shaders + * per [InputFormat]. + * @return Info about the initialized graphics device. + * @throws IllegalStateException if the renderer is already initialized or failed to be + * initialized. + * @throws IllegalArgumentException if the ShaderProvider fails to create shader or provides + * invalid shader string. + */ + fun init( + dynamicRange: DynamicRangeProfile, + shaderOverrides: Map + ): GraphicDeviceInfo { + checkInitializedOrThrow(mInitialized, false) + val infoBuilder = GraphicDeviceInfo.Builder() + try { + var dynamicRangeCorrected = dynamicRange + if (dynamicRange.isHdr) { + val extensions = getExtensionsBeforeInitialized(dynamicRange) + val glExtensions = requireNotNull(extensions.first) + val eglExtensions = requireNotNull(extensions.second) + if (!glExtensions.contains("GL_EXT_YUV_target")) { + Logger.w(TAG, "Device does not support GL_EXT_YUV_target. Fallback to SDR.") + dynamicRangeCorrected = DynamicRangeProfile.sdr + } + mSurfaceAttrib = chooseSurfaceAttrib(eglExtensions, dynamicRangeCorrected) + infoBuilder.setGlExtensions(glExtensions) + infoBuilder.setEglExtensions(eglExtensions) + } + createEglContext(dynamicRangeCorrected, infoBuilder) + createTempSurface() + makeCurrent(mTempSurface) + infoBuilder.setGlVersion(glVersionNumber) + mProgramHandles = createPrograms(dynamicRangeCorrected, shaderOverrides) + mExternalTextureId = createTexture() + useAndConfigureProgramWithTexture(mExternalTextureId) + } catch (e: IllegalStateException) { + releaseInternal() + throw e + } catch (e: IllegalArgumentException) { + releaseInternal() + throw e + } + mGlThread = Thread.currentThread() + mInitialized.set(true) + return infoBuilder.build() + } + + /** + * Releases the OpenGLRenderer + * + * @throws IllegalStateException if the caller doesn't run on the GL thread. + */ + fun release() { + if (!mInitialized.getAndSet(false)) { + return + } + checkGlThreadOrThrow(mGlThread) + releaseInternal() + } + + /** + * Register the output surface. + * + * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run + * on the GL thread. + */ + fun registerOutputSurface(surface: Surface) { + checkInitializedOrThrow(mInitialized, true) + checkGlThreadOrThrow(mGlThread) + + if (!mOutputSurfaceMap.containsKey(surface)) { + mOutputSurfaceMap[surface] = NO_OUTPUT_SURFACE + } + } + + /** + * Unregister the output surface. + * + * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run + * on the GL thread. + */ + fun unregisterOutputSurface(surface: Surface) { + checkInitializedOrThrow(mInitialized, true) + checkGlThreadOrThrow(mGlThread) + + removeOutputSurfaceInternal(surface, true) + } + + val textureName: Int + /** + * Gets the texture name. + * + * @return the texture name + * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run + * on the GL thread. + */ + get() { + checkInitializedOrThrow(mInitialized, true) + checkGlThreadOrThrow(mGlThread) + + return mExternalTextureId + } + + /** + * Sets the input format. + * + * + * This will ensure the correct sampler is used for the input. + * + * @param inputFormat The input format for the input texture. + * @throws IllegalStateException if the renderer is not initialized or the caller doesn't run + * on the GL thread. + */ + fun setInputFormat(inputFormat: InputFormat) { + checkInitializedOrThrow(mInitialized, true) + checkGlThreadOrThrow(mGlThread) + + if (mCurrentInputformat !== inputFormat) { + mCurrentInputformat = inputFormat + useAndConfigureProgramWithTexture(mExternalTextureId) + } + } + + private fun activateExternalTexture(externalTextureId: Int) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + checkGlErrorOrThrow("glActiveTexture") + + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, externalTextureId) + checkGlErrorOrThrow("glBindTexture") + } + + /** + * Renders the texture image to the output surface. + * + * @throws IllegalStateException if the renderer is not initialized, the caller doesn't run + * on the GL thread or the surface is not registered by + * [.registerOutputSurface]. + */ + fun render( + timestampNs: Long, + textureTransform: FloatArray, + surface: Surface + ) { + checkInitializedOrThrow(mInitialized, true) + checkGlThreadOrThrow(mGlThread) + + var outputSurface: OutputSurface? = getOutSurfaceOrThrow(surface) + + // Workaround situations that out surface is failed to create or needs to be recreated. + if (outputSurface === NO_OUTPUT_SURFACE) { + outputSurface = createOutputSurfaceInternal(surface) + if (outputSurface == null) { + return + } + + mOutputSurfaceMap[surface] = outputSurface + } + + requireNotNull(outputSurface) + + // Set output surface. + if (surface !== mCurrentSurface) { + makeCurrent(outputSurface.eglSurface) + mCurrentSurface = surface + GLES20.glViewport(0, 0, outputSurface.width, outputSurface.height) + GLES20.glScissor(0, 0, outputSurface.width, outputSurface.height) + } + + // TODO(b/245855601): Upload the matrix to GPU when textureTransform is changed. + val program: Program2D = requireNotNull(mCurrentProgram) + if (program is SamplerShaderProgram) { + // Copy the texture transformation matrix over. + program.updateTextureMatrix(textureTransform) + } + + // Draw the rect. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*firstVertex=*/0, /*vertexCount=*/4) + checkGlErrorOrThrow("glDrawArrays") + + // Set timestamp + EGLExt.eglPresentationTimeANDROID(mEglDisplay, outputSurface.eglSurface, timestampNs) + + // Swap buffer + if (!EGL14.eglSwapBuffers(mEglDisplay, outputSurface.eglSurface)) { + Logger.w( + TAG, "Failed to swap buffers with EGL error: 0x" + Integer.toHexString( + EGL14.eglGetError() + ) + ) + removeOutputSurfaceInternal(surface, false) + } + } + + /** + * Takes a snapshot of the current external texture and stores it in the given byte buffer. + * + * + * The image is stored as RGBA with pixel stride of 4 bytes and row stride of width * 4 + * bytes. + * + * @param byteBuffer the byte buffer to store the snapshot. + * @param size the size of the output image. + * @param textureTransform the transformation matrix. + * See: [SurfaceOutput.updateTransformMatrix] + */ + private fun snapshot( + byteBuffer: ByteBuffer, size: Size, + textureTransform: FloatArray + ) { + check(byteBuffer.capacity() == size.width * size.height * 4) { + "ByteBuffer capacity is not equal to width * height * 4." + } + check(byteBuffer.isDirect) { "ByteBuffer is not direct." } + + // Create and initialize intermediate texture. + val texture: Int = generateTexture() + GLES20.glActiveTexture(GLES20.GL_TEXTURE1) + checkGlErrorOrThrow("glActiveTexture") + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture) + checkGlErrorOrThrow("glBindTexture") + // Configure the texture. + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGB, size.width, + size.height, 0, GLES20.GL_RGB, GLES20.GL_UNSIGNED_BYTE, null + ) + checkGlErrorOrThrow("glTexImage2D") + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR + ) + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR + ) + + // Create FBO. + val fbo: Int = generateFbo() + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo) + checkGlErrorOrThrow("glBindFramebuffer") + + // Attach the intermediate texture to the FBO + GLES20.glFramebufferTexture2D( + GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, + GLES20.GL_TEXTURE_2D, texture, 0 + ) + checkGlErrorOrThrow("glFramebufferTexture2D") + + // Bind external texture (camera output). + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + checkGlErrorOrThrow("glActiveTexture") + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mExternalTextureId) + checkGlErrorOrThrow("glBindTexture") + + // Set scissor and viewport. + mCurrentSurface = null + GLES20.glViewport(0, 0, size.width, size.height) + GLES20.glScissor(0, 0, size.width, size.height) + + val program: Program2D = requireNotNull(mCurrentProgram) + if (program is SamplerShaderProgram) { + // Upload transform matrix. + program.updateTextureMatrix(textureTransform) + } + + // Draw the external texture to the intermediate texture. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*firstVertex=*/0, /*vertexCount=*/4) + checkGlErrorOrThrow("glDrawArrays") + + // Read the pixels from the framebuffer + GLES20.glReadPixels( + 0, 0, size.width, size.height, GLES20.GL_RGBA, + GLES20.GL_UNSIGNED_BYTE, + byteBuffer + ) + checkGlErrorOrThrow("glReadPixels") + + // Clean up + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0) + deleteTexture(texture) + deleteFbo(fbo) + // Set the external texture to be active. + activateExternalTexture(mExternalTextureId) + } + + // Returns a pair of GL extension (first) and EGL extension (second) strings. + private fun getExtensionsBeforeInitialized( + dynamicRangeToInitialize: DynamicRangeProfile + ): Pair { + checkInitializedOrThrow(mInitialized, false) + try { + createEglContext(dynamicRangeToInitialize, /*infoBuilder=*/null) + createTempSurface() + makeCurrent(mTempSurface) + // eglMakeCurrent() has to be called before checking GL_EXTENSIONS. + val glExtensions = GLES20.glGetString(GLES20.GL_EXTENSIONS) + val eglExtensions = EGL14.eglQueryString(mEglDisplay, EGL14.EGL_EXTENSIONS) + return Pair( + glExtensions ?: "", eglExtensions ?: "" + ) + } catch (e: IllegalStateException) { + Logger.w(TAG, "Failed to get GL or EGL extensions: " + e.message, e) + return Pair("", "") + } finally { + releaseInternal() + } + } + + private fun createEglContext( + dynamicRange: DynamicRangeProfile, + infoBuilder: GraphicDeviceInfo.Builder? + ) { + mEglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + check(mEglDisplay != EGL14.EGL_NO_DISPLAY) { "Unable to get EGL14 display" } + val version = IntArray(2) + if (!EGL14.eglInitialize(mEglDisplay, version, 0, version, 1)) { + mEglDisplay = EGL14.EGL_NO_DISPLAY + throw IllegalStateException("Unable to initialize EGL14") + } + + infoBuilder?.setEglVersion(version[0].toString() + "." + version[1]) + + val rgbBits = if (dynamicRange.isHdr) 10 else 8 + val alphaBits = if (dynamicRange.isHdr) 2 else 8 + val renderType = if (dynamicRange.isHdr) + EGLExt.EGL_OPENGL_ES3_BIT_KHR + else + EGL14.EGL_OPENGL_ES2_BIT + // TODO(b/319277249): It will crash on older Samsung devices for HDR video 10-bit + // because EGLExt.EGL_RECORDABLE_ANDROID is only supported from OneUI 6.1. We need to + // check by GPU Driver version when new OS is release. + val recordableAndroid = + if (dynamicRange.isHdr) EGL10.EGL_DONT_CARE else EGL14.EGL_TRUE + val attribToChooseConfig = intArrayOf( + EGL14.EGL_RED_SIZE, rgbBits, + EGL14.EGL_GREEN_SIZE, rgbBits, + EGL14.EGL_BLUE_SIZE, rgbBits, + EGL14.EGL_ALPHA_SIZE, alphaBits, + EGL14.EGL_DEPTH_SIZE, 0, + EGL14.EGL_STENCIL_SIZE, 0, + EGL14.EGL_RENDERABLE_TYPE, renderType, + EGLExt.EGL_RECORDABLE_ANDROID, recordableAndroid, + EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT or EGL14.EGL_PBUFFER_BIT, + EGL14.EGL_NONE + ) + val configs = arrayOfNulls(1) + val numConfigs = IntArray(1) + check( + EGL14.eglChooseConfig( + mEglDisplay, attribToChooseConfig, 0, configs, 0, configs.size, + numConfigs, 0 + ) + ) { "Unable to find a suitable EGLConfig" } + val config = configs[0] + val attribToCreateContext = intArrayOf( + EGL14.EGL_CONTEXT_CLIENT_VERSION, if (dynamicRange.isHdr) 3 else 2, + EGL14.EGL_NONE + ) + val context = EGL14.eglCreateContext( + mEglDisplay, config, EGL14.EGL_NO_CONTEXT, + attribToCreateContext, 0 + ) + checkEglErrorOrThrow("eglCreateContext") + mEglConfig = config + mEglContext = context + + // Confirm with query. + val values = IntArray(1) + EGL14.eglQueryContext( + mEglDisplay, mEglContext, EGL14.EGL_CONTEXT_CLIENT_VERSION, values, + 0 + ) + Log.d(TAG, "EGLContext created, client version " + values[0]) + } + + private fun createTempSurface() { + mTempSurface = createPBufferSurface( + mEglDisplay, requireNotNull(mEglConfig), /*width=*/1, /*height=*/ + 1 + ) + } + + protected fun makeCurrent(eglSurface: EGLSurface) { + check( + EGL14.eglMakeCurrent( + mEglDisplay, + eglSurface, + eglSurface, + mEglContext + ) + ) { "eglMakeCurrent failed" } + } + + protected fun useAndConfigureProgramWithTexture(textureId: Int) { + val program = requireNotNull(mProgramHandles[mCurrentInputformat]) { + "Unable to configure program for input format: $mCurrentInputformat" + } + if (mCurrentProgram !== program) { + mCurrentProgram = program + program.use() + Log.d( + TAG, ("Using program for input format " + mCurrentInputformat + ": " + + mCurrentProgram) + ) + } + + // Activate the texture + activateExternalTexture(textureId) + } + + private fun releaseInternal() { + // Delete program + for (program in mProgramHandles.values) { + program.delete() + } + mProgramHandles = emptyMap() + mCurrentProgram = null + + if (mEglDisplay != EGL14.EGL_NO_DISPLAY) { + EGL14.eglMakeCurrent( + mEglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_CONTEXT + ) + + // Destroy EGLSurfaces + for (outputSurface in mOutputSurfaceMap.values) { + if (outputSurface.eglSurface != EGL14.EGL_NO_SURFACE) { + if (!EGL14.eglDestroySurface(mEglDisplay, outputSurface.eglSurface)) { + checkEglErrorOrLog("eglDestroySurface") + } + } + } + mOutputSurfaceMap.clear() + + // Destroy temp surface + if (mTempSurface != EGL14.EGL_NO_SURFACE) { + EGL14.eglDestroySurface(mEglDisplay, mTempSurface) + mTempSurface = EGL14.EGL_NO_SURFACE + } + + // Destroy EGLContext and terminate display + if (mEglContext != EGL14.EGL_NO_CONTEXT) { + EGL14.eglDestroyContext(mEglDisplay, mEglContext) + mEglContext = EGL14.EGL_NO_CONTEXT + } + EGL14.eglReleaseThread() + EGL14.eglTerminate(mEglDisplay) + mEglDisplay = EGL14.EGL_NO_DISPLAY + } + + // Reset other members + mEglConfig = null + mExternalTextureId = -1 + mCurrentInputformat = InputFormat.UNKNOWN + mCurrentSurface = null + mGlThread = null + } + + protected fun getOutSurfaceOrThrow(surface: Surface): OutputSurface { + check(mOutputSurfaceMap.containsKey(surface)) { + "The surface is not registered." + } + + return requireNotNull(mOutputSurfaceMap[surface]) + } + + protected fun createOutputSurfaceInternal(surface: Surface): OutputSurface? { + val eglSurface: EGLSurface + try { + eglSurface = createWindowSurface( + mEglDisplay, requireNotNull(mEglConfig), surface, + mSurfaceAttrib + ) + } catch (e: IllegalStateException) { + Logger.w(TAG, "Failed to create EGL surface: " + e.message, e) + return null + } catch (e: IllegalArgumentException) { + Logger.w(TAG, "Failed to create EGL surface: " + e.message, e) + return null + } + + val size: Size = getSurfaceSize(mEglDisplay, eglSurface) + return OutputSurface.of(eglSurface, size.width, size.height) + } + + protected fun removeOutputSurfaceInternal(surface: Surface, unregister: Boolean) { + // Unmake current surface. + if (mCurrentSurface === surface) { + mCurrentSurface = null + makeCurrent(mTempSurface) + } + + // Remove cached EGL surface. + val removedOutputSurface: OutputSurface = if (unregister) { + mOutputSurfaceMap.remove(surface)!! + } else { + mOutputSurfaceMap.put(surface, NO_OUTPUT_SURFACE)!! + } + + // Destroy EGL surface. + if (removedOutputSurface !== NO_OUTPUT_SURFACE) { + try { + EGL14.eglDestroySurface(mEglDisplay, removedOutputSurface.eglSurface) + } catch (e: RuntimeException) { + Logger.w(TAG, "Failed to destroy EGL surface: " + e.message, e) + } + } + } + + companion object { + private const val TAG = "OpenGlRenderer" + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ShaderProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ShaderProvider.kt new file mode 100644 index 000000000..8c8cd7c0a --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/ShaderProvider.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2022 The Android Open Source Project + * Copyright 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video + +/** + * A provider that supplies OpenGL shader code. + */ +interface ShaderProvider { + /** + * Creates the fragment shader code with the given variable names. + * + * + * The provider must use the variable names to construct the shader code, or it will fail + * to create the OpenGL program when the provider is used. For example: + *

`#extension GL_OES_EGL_image_external : require
+     * precision mediump float;
+     * uniform samplerExternalOES {$samplerVarName};
+     * varying vec2 {$fragCoordsVarName};
+     * void main() {
+     * vec4 sampleColor = texture2D({$samplerVarName}, {$fragCoordsVarName});
+     * gl_FragColor = vec4(
+     * sampleColor.r * 0.5 + sampleColor.g * 0.8 + sampleColor.b * 0.3,
+     * sampleColor.r * 0.4 + sampleColor.g * 0.7 + sampleColor.b * 0.2,
+     * sampleColor.r * 0.3 + sampleColor.g * 0.5 + sampleColor.b * 0.1,
+     * 1.0);
+     * }
+    `
* + * + * @param samplerVarName the variable name of the samplerExternalOES. + * @param fragCoordsVarName the variable name of the fragment coordinates. + * @return the shader code. Return null to use the default shader. + */ + fun createFragmentShader( + samplerVarName: String, + fragCoordsVarName: String + ): String? { + return null + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/SurfaceProcessor.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/SurfaceProcessor.kt new file mode 100644 index 000000000..3796ae45e --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/SurfaceProcessor.kt @@ -0,0 +1,221 @@ +package io.github.thibaultbee.streampack.core.internal.processing.video + +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.HandlerThread +import android.util.Size +import android.view.Surface +import androidx.concurrent.futures.CallbackToFutureAdapter +import io.github.thibaultbee.streampack.core.internal.processing.video.outputs.AbstractSurfaceOutput +import io.github.thibaultbee.streampack.core.internal.utils.av.video.DynamicRangeProfile +import io.github.thibaultbee.streampack.core.logger.Logger +import java.util.concurrent.Future +import java.util.concurrent.atomic.AtomicBoolean + + +class SurfaceProcessor( + val dynamicRangeProfile: DynamicRangeProfile +) : ISurfaceProcessorInternal, + SurfaceTexture.OnFrameAvailableListener { + private val renderer = OpenGlRenderer() + + private val isReleaseRequested = AtomicBoolean(false) + private var isReleased = false + + private val textureMatrix = FloatArray(16) + private val surfaceOutputMatrix = FloatArray(16) + + private val surfaceOutputs: MutableList = mutableListOf() + private val surfaceInputs: MutableList = mutableListOf() + + private val glThread = HandlerThread("GL Thread").apply { + start() + } + private val glHandler = Handler(glThread.looper) + + init { + val future = submitSafely { + renderer.init(dynamicRangeProfile) + } + try { + future.get() + } catch (e: Exception) { + release() + Logger.e(TAG, "Error while initializing renderer", e) + throw e + } + } + + override fun createInputSurface(surfaceSize: Size): Surface? { + if (isReleaseRequested.get()) { + return null + } + + val future = submitSafely { + val surfaceTexture = SurfaceTexture(renderer.textureName) + surfaceTexture.setDefaultBufferSize(surfaceSize.width, surfaceSize.height) + surfaceTexture.setOnFrameAvailableListener(this, glHandler) + SurfaceInput(Surface(surfaceTexture), surfaceTexture) + } + + val surfaceInput = future.get() + surfaceInputs.add(surfaceInput) + return surfaceInput.surface + } + + override fun removeInputSurface(surface: Surface) { + executeSafely { + val surfaceInput = surfaceInputs.find { it.surface == surface } + if (surfaceInput != null) { + val surfaceTexture = surfaceInput.surfaceTexture + surfaceTexture.setOnFrameAvailableListener(null) + surfaceTexture.release() + surface.release() + + surfaceInputs.remove(surfaceInput) + + checkReadyToRelease() + } else { + Logger.w(TAG, "Surface not found") + } + } + } + + override fun addOutputSurface(surfaceOutput: AbstractSurfaceOutput) { + if (isReleaseRequested.get()) { + return + } + + executeSafely { + if (!surfaceOutputs.contains(surfaceOutput)) { + renderer.registerOutputSurface(surfaceOutput.surface) + surfaceOutputs.add(surfaceOutput) + } else { + Logger.w(TAG, "Surface already added") + } + } + } + + override fun removeOutputSurface(surfaceOutput: AbstractSurfaceOutput) { + if (isReleaseRequested.get()) { + return + } + + executeSafely { + if (surfaceOutputs.contains(surfaceOutput)) { + renderer.unregisterOutputSurface(surfaceOutput.surface) + surfaceOutputs.remove(surfaceOutput) + } else { + Logger.w(TAG, "Surface not found") + } + } + } + + override fun removeOutputSurface(surface: Surface) { + val surfaceOutput = surfaceOutputs.firstOrNull { it.surface == surface } + if (surfaceOutput != null) { + removeOutputSurface(surfaceOutput) + } else { + Logger.w(TAG, "Surface not found") + } + } + + override fun removeAllOutputSurfaces() { + surfaceOutputs.forEach { + try { + removeOutputSurface(it) + } catch (e: Exception) { + Logger.w(TAG, "Error while removing output surface", e) + } + } + } + + override fun release() { + if (isReleaseRequested.getAndSet(true)) { + return + } + executeSafely(block = { + if (!isReleased) { + isReleased = true + + checkReadyToRelease() + } + }) + } + + private fun checkReadyToRelease() { + if (isReleased && surfaceInputs.isEmpty()) { + // Once release is called, we can stop sending frame to output surfaces. + surfaceOutputs.forEach { it.close() } + + surfaceOutputs.clear() + renderer.release() + glThread.quit() + } + } + + + private fun onFrameAvailableInternal(surfaceTexture: SurfaceTexture) { + if (isReleaseRequested.get()) { + return + } + + surfaceTexture.updateTexImage() + surfaceTexture.getTransformMatrix(textureMatrix) + + surfaceOutputs.forEach { + try { + it.updateTransformMatrix(surfaceOutputMatrix, textureMatrix) + renderer.render(surfaceTexture.timestamp, surfaceOutputMatrix, it.surface) + } catch (e: Exception) { + Logger.e(TAG, "Error while rendering frame", e) + } + } + } + + override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { + onFrameAvailableInternal(surfaceTexture) + } + + private fun executeSafely( + block: () -> Unit, + ) { + executeSafely(block, {}, {}) + } + + private fun executeSafely( + block: () -> T, + onSuccess: ((T) -> Unit), + onError: ((Throwable) -> Unit) + ) { + try { + glHandler.post { + if (isReleased) { + Logger.w(TAG, "SurfaceProcessor is released, block will not be executed") + onError(IllegalStateException("SurfaceProcessor is released")) + } else { + try { + onSuccess(block()) + } catch (t: Throwable) { + onError(t) + } + } + } + } catch (t: Throwable) { + Logger.e(TAG, "Error while executing block", t) + onError(t) + } + } + + private fun submitSafely(block: () -> T): Future { + return CallbackToFutureAdapter.getFuture { + executeSafely(block, { result -> it.set(result) }, { t -> it.setException(t) }) + } + } + + companion object { + private const val TAG = "SurfaceProcessor" + } + + private data class SurfaceInput(val surface: Surface, val surfaceTexture: SurfaceTexture) +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/AbstractSurfaceOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/AbstractSurfaceOutput.kt new file mode 100644 index 000000000..f8b03df26 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/AbstractSurfaceOutput.kt @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.outputs + +import android.graphics.Rect +import android.opengl.Matrix +import android.util.Size +import android.view.Surface +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils + + +open class AbstractSurfaceOutput( + override val surface: Surface, + final override val resolution: Size +) : ISurfaceOutput { + protected val lock = Any() + protected var isClosed = false + + // Full frame. Keep for future usage. + override val cropRect: Rect = Rect(0, 0, resolution.width, resolution.height) + + override fun updateTransformMatrix(output: FloatArray, input: FloatArray) { + Matrix.multiplyMM( + output, 0, input, 0, GLUtils.IDENTITY_MATRIX, 0 + ) + } + + override fun close() { + synchronized(lock) { + if (!isClosed) { + isClosed = true + } + } + } +} + +interface ISurfaceOutput { + val surface: Surface + val cropRect: Rect + val resolution: Size + + fun updateTransformMatrix(output: FloatArray, input: FloatArray) + + fun close() +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/SurfaceOutput.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/SurfaceOutput.kt new file mode 100644 index 000000000..3532d2ccf --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/outputs/SurfaceOutput.kt @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.outputs + +import android.graphics.RectF +import android.opengl.Matrix +import android.util.Size +import android.view.Surface +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.processing.video.source.ISourceInfoProvider +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.TransformUtils +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions.preRotate +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions.preVerticalFlip +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions.toRectF +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotate + +class SurfaceOutput( + surface: Surface, + resolution: Size, + private val transformationInfo: TransformationInfo +) : + AbstractSurfaceOutput(surface, resolution) { + + private val infoProvider: ISourceInfoProvider + get() = transformationInfo.infoProvider + + @IntRange(from = 0, to = 359) + val rotationDegrees = infoProvider.getRelativeRotationDegrees( + transformationInfo.targetRotation, + transformationInfo.needMirroring + ) + + @IntRange(from = 0, to = 359) + val sourceRotationDegrees = infoProvider.rotationDegrees + + private val additionalTransform = FloatArray(16) + private val invertedTextureTransform = FloatArray(16) + + init { + calculateAdditionalTransform( + additionalTransform, + invertedTextureTransform, + transformationInfo.infoProvider + ) + } + + override fun updateTransformMatrix(output: FloatArray, input: FloatArray) { + Matrix.multiplyMM( + output, 0, input, 0, additionalTransform, 0 + ) + } + + /** + * Calculates the additional GL transform and saves it to additionalTransform. + * + * + * The effect implementation needs to apply this value on top of texture transform obtained + * from [SurfaceTexture.getTransformMatrix]. + * + * + * The overall transformation (A * B) is a concatenation of 2 values: A) the texture + * transform (value of SurfaceTexture#getTransformMatrix), and B) CameraX's additional + * transform based on user config such as the ViewPort API and UseCase#targetRotation. To + * calculate B, we do it in 3 steps: + * + * 1. 1. Calculate A * B by using CameraX transformation value such as crop rect, relative + * rotation, and mirroring. It already contains the texture transform(A). + * 1. 2. Calculate A^-1 by predicating the texture transform(A) based on camera + * characteristics then inverting it. + * 1. 3. Calculate B by multiplying A^-1 * A * B. + * + */ + private fun calculateAdditionalTransform( + additionalTransform: FloatArray, + invertedTransform: FloatArray, + sourceInfoProvider: ISourceInfoProvider + ) { + Matrix.setIdentityM(additionalTransform, 0) + + // Step 1, calculate the overall transformation(A * B) with the following steps: + // - Flip compensate the GL coordinates v.s. image coordinates + // - Rotate the image based on the relative rotation + // - Mirror the image if needed + // - Apply the crop rect + + // Flipping for GL. + additionalTransform.preVerticalFlip(0.5f) + + // Rotation + additionalTransform.preRotate(rotationDegrees.toFloat(), 0.5f, 0.5f) + + // Mirroring + if (transformationInfo.needMirroring) { + Matrix.translateM(additionalTransform, 0, 1f, 0f, 0f) + Matrix.scaleM(additionalTransform, 0, -1f, 1f, 1f) + } + + // Crop + // Rotate the size and cropRect, and mirror the cropRect. + val rotatedSize = resolution.rotate( + rotationDegrees + ) + val imageTransform = TransformUtils.getRectToRect( + resolution.toRectF(), + rotatedSize.toRectF(), + rotationDegrees, + sourceInfoProvider.isMirror + ) + val rotatedCroppedRect = RectF(cropRect) + imageTransform.mapRect(rotatedCroppedRect) + // According to the rotated size and cropRect, compute the normalized offset and the scale + // of X and Y. + val offsetX: Float = rotatedCroppedRect.left / rotatedSize.width + val offsetY: Float = ((rotatedSize.height - rotatedCroppedRect.height() + - rotatedCroppedRect.top)) / rotatedSize.height + val scaleX: Float = rotatedCroppedRect.width() / rotatedSize.width + val scaleY: Float = rotatedCroppedRect.height() / rotatedSize.height + // Move to the new left-bottom position and apply the scale. + Matrix.translateM(additionalTransform, 0, offsetX, offsetY, 0f) + Matrix.scaleM(additionalTransform, 0, scaleX, scaleY, 1f) + + // Step 2: calculate the inverted texture transform: A^-1 + calculateInvertedTextureTransform(invertedTransform, sourceInfoProvider) + + // Step 3: calculate the additional transform: B = A^-1 * A * B + Matrix.multiplyMM( + additionalTransform, 0, invertedTransform, 0, + additionalTransform, 0 + ) + } + + /** + * Calculates the inverted texture transform and saves it to invertedTextureTransform. + * + * This method predicts the value of [SurfaceTexture.getTransformMatrix] based on + * camera characteristics then invert it. The result is used to remove the texture transform + * from overall transformation. + */ + private fun calculateInvertedTextureTransform( + invertedTextureTransform: FloatArray, + sourceInfoProvider: ISourceInfoProvider + ) { + Matrix.setIdentityM(invertedTextureTransform, 0) + + // Flip for GL. SurfaceTexture#getTransformMatrix always contains this flipping regardless + // of whether it has the camera transform. + invertedTextureTransform.preVerticalFlip(0.5f) + + // Applies the camera sensor orientation if the input surface contains camera transform. + // Rotation + invertedTextureTransform.preRotate( + sourceRotationDegrees.toFloat(), + 0.5f, + 0.5f + ) + + // Mirroring + if (sourceInfoProvider.isMirror) { + Matrix.translateM(invertedTextureTransform, 0, 1f, 0f, 0f) + Matrix.scaleM(invertedTextureTransform, 0, -1f, 1f, 1f) + } + + // Invert the matrix so it can be used to "undo" the SurfaceTexture#getTransformMatrix. + Matrix.invertM(invertedTextureTransform, 0, invertedTextureTransform, 0) + } + + data class TransformationInfo( + @RotationValue val targetRotation: Int, + val needMirroring: Boolean, + val infoProvider: ISourceInfoProvider + ) +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/AbstractSourceInfoProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/AbstractSourceInfoProvider.kt new file mode 100644 index 000000000..13d600cca --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/AbstractSourceInfoProvider.kt @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.source + +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue + +abstract class AbstractSourceInfoProvider : ISourceInfoProvider { + protected val listeners = mutableSetOf() + + override val isMirror = false + + @IntRange(from = 0, to = 359) + override val rotationDegrees = 0 + + @IntRange(from = 0, to = 359) + override fun getRelativeRotationDegrees( + @RotationValue targetRotation: Int, requiredMirroring: Boolean + ): Int { + return 0 + } + + override fun addListener(listener: ISourceInfoListener) { + listeners.add(listener) + } + + override fun removeListener(listener: ISourceInfoListener) { + listeners.remove(listener) + } + + override fun removeAllListeners() { + listeners.clear() + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/ISourceInfoProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/ISourceInfoProvider.kt new file mode 100644 index 000000000..2e95495a8 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/source/ISourceInfoProvider.kt @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.source + +import android.util.Size +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue + +/** + * Interface to get the orientation of the capture surface. + * These information are used to rotate the frames in the codec surface if the source needs to be rotated. + * It might not be the case for certain sources. + */ +interface ISourceInfoProvider { + /** + * Orientation in degrees of the source. + * For camera, it is the sensor orientation. + * To follow device rotation use, it is the current device rotation. + * Expected values: 0, 90, 180, 270. + */ + @get:IntRange(from = 0, to = 359) + val rotationDegrees: Int + + /** + * True if the source is natively mirrored. + */ + val isMirror: Boolean + + /** + * Calculates the relative rotation between the source and the destination. + * + * @param targetRotation rotation of the destination (Surface.ROTATION_0, Surface.ROTATION_90, Surface.ROTATION_180 or Surface.ROTATION_270). + * @param requiredMirroring true if the destination requires mirroring. + * @return relative rotation between the source and the destination. + */ + @IntRange(from = 0, to = 359) + fun getRelativeRotationDegrees( + @RotationValue targetRotation: Int, + requiredMirroring: Boolean + ): Int = 0 + + /** + * Gets the size of the surface to allocate to display the source. + */ + fun getSurfaceSize(size: Size, @RotationValue targetRotation: Int): Size = size + + /** + * Adds a listener to be notified when the orientation changes. + * + * @param listener to add. + */ + fun addListener(listener: ISourceInfoListener) + + /** + * Removes a listener. + * + * @param listener to remove. + */ + fun removeListener(listener: ISourceInfoListener) + + /** + * Removes all registered listeners. + */ + fun removeAllListeners() +} + +/** + * Interface to be notified when the orientation changes. + */ +interface ISourceInfoListener { + /** + * Called when the orientation changes. + */ + fun onInfoChanged() +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GLUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GLUtils.kt new file mode 100644 index 000000000..713df59b0 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GLUtils.kt @@ -0,0 +1,795 @@ +/* + * Copyright 2024 The Android Open Source Project + * Copyright 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils + +import android.media.MediaFormat +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLDisplay +import android.opengl.EGLSurface +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.opengl.GLES20.glUniformMatrix4fv +import android.opengl.Matrix +import android.util.Size +import android.view.Surface +import io.github.thibaultbee.streampack.core.internal.processing.video.ShaderProvider +import io.github.thibaultbee.streampack.core.internal.utils.av.video.DynamicRangeProfile +import io.github.thibaultbee.streampack.core.logger.Logger +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer +import java.util.Locale +import java.util.concurrent.atomic.AtomicBoolean +import java.util.regex.Pattern + +/** + * Utility class for OpenGL ES. + */ +object GLUtils { + /** + * Unknown version information. + */ + const val VERSION_UNKNOWN: String = "0.0" + + const val TAG: String = "GLUtils" + + val IDENTITY_MATRIX = create4x4IdentityMatrix() + + const val EGL_GL_COLORSPACE_KHR: Int = 0x309D + const val EGL_GL_COLORSPACE_BT2020_HLG_EXT: Int = 0x3540 + + const val VAR_TEXTURE_COORD: String = "vTextureCoord" + const val VAR_TEXTURE: String = "sTexture" + const val PIXEL_STRIDE: Int = 4 + val EMPTY_ATTRIBS: IntArray = intArrayOf(EGL14.EGL_NONE) + val HLG_SURFACE_ATTRIBS: IntArray = intArrayOf( + EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_HLG_EXT, + EGL14.EGL_NONE + ) + + val DEFAULT_VERTEX_SHADER: String = String.format( + Locale.US, + ("""uniform mat4 uTexMatrix; +uniform mat4 uTransMatrix; +attribute vec4 aPosition; +attribute vec4 aTextureCoord; +varying vec2 %s; +void main() { + gl_Position = uTransMatrix * aPosition; + %s = (uTexMatrix * aTextureCoord).xy; +} +"""), VAR_TEXTURE_COORD, VAR_TEXTURE_COORD + ) + + val HDR_VERTEX_SHADER: String = String.format( + Locale.US, + ("""#version 300 es +in vec4 aPosition; +in vec4 aTextureCoord; +uniform mat4 uTexMatrix; +uniform mat4 uTransMatrix; +out vec2 %s; +void main() { + gl_Position = uTransMatrix * aPosition; + %s = (uTexMatrix * aTextureCoord).xy; +} +"""), VAR_TEXTURE_COORD, VAR_TEXTURE_COORD + ) + + const val BLANK_VERTEX_SHADER: String = ("uniform mat4 uTransMatrix;\n" + + "attribute vec4 aPosition;\n" + + "void main() {\n" + + " gl_Position = uTransMatrix * aPosition;\n" + + "}\n") + + const val BLANK_FRAGMENT_SHADER: String = ("precision mediump float;\n" + + "uniform float uAlphaScale;\n" + + "void main() {\n" + + " gl_FragColor = vec4(0.0, 0.0, 0.0, uAlphaScale);\n" + + "}\n") + val VERTEX_COORDS: FloatArray = floatArrayOf( + -1.0f, -1.0f, // 0 bottom left + 1.0f, -1.0f, // 1 bottom right + -1.0f, 1.0f, // 2 top left + 1.0f, 1.0f, // 3 top right + ) + val TEX_COORDS: FloatArray = floatArrayOf( + 0.0f, 0.0f, // 0 bottom left + 1.0f, 0.0f, // 1 bottom right + 0.0f, 1.0f, // 2 top left + 1.0f, 1.0f // 3 top right + ) + const val SIZEOF_FLOAT: Int = 4 + val VERTEX_BUF: FloatBuffer = createFloatBuffer(VERTEX_COORDS) + val TEX_BUF: FloatBuffer = createFloatBuffer(TEX_COORDS) + val NO_OUTPUT_SURFACE: OutputSurface = OutputSurface.of(EGL14.EGL_NO_SURFACE, 0, 0) + private val SHADER_PROVIDER_DEFAULT: ShaderProvider = object : ShaderProvider { + override fun createFragmentShader( + samplerVarName: String, + fragCoordsVarName: String + ): String { + return String.format( + Locale.US, + ("""#extension GL_OES_EGL_image_external : require +precision mediump float; +varying vec2 %s; +uniform samplerExternalOES %s; +uniform float uAlphaScale; +void main() { + vec4 src = texture2D(%s, %s); + gl_FragColor = vec4(src.rgb, src.a * uAlphaScale); +} +"""), + fragCoordsVarName, samplerVarName, samplerVarName, fragCoordsVarName + ) + } + } + private val SHADER_PROVIDER_HDR_DEFAULT: ShaderProvider = object : ShaderProvider { + override fun createFragmentShader( + samplerVarName: String, + fragCoordsVarName: String + ): String { + return String.format( + Locale.US, + ("""#version 300 es +#extension GL_OES_EGL_image_external_essl3 : require +precision mediump float; +uniform samplerExternalOES %s; +uniform float uAlphaScale; +in vec2 %s; +out vec4 outColor; + +void main() { + vec4 src = texture(%s, %s); + outColor = vec4(src.rgb, src.a * uAlphaScale); +}"""), + samplerVarName, fragCoordsVarName, samplerVarName, fragCoordsVarName + ) + } + } + private val SHADER_PROVIDER_HDR_YUV: ShaderProvider = object : ShaderProvider { + override fun createFragmentShader( + samplerVarName: String, + fragCoordsVarName: String + ): String { + return String.format( + Locale.US, + ("""#version 300 es +#extension GL_EXT_YUV_target : require +precision mediump float; +uniform __samplerExternal2DY2YEXT %s; +uniform float uAlphaScale; +in vec2 %s; +out vec4 outColor; + +vec3 yuvToRgb(vec3 yuv) { + const vec3 yuvOffset = vec3(0.0625, 0.5, 0.5); + const mat3 yuvToRgbColorMat = mat3( + 1.1689f, 1.1689f, 1.1689f, + 0.0000f, -0.1881f, 2.1502f, + 1.6853f, -0.6530f, 0.0000f + ); + return clamp(yuvToRgbColorMat * (yuv - yuvOffset), 0.0, 1.0); +} + +void main() { + vec3 srcYuv = texture(%s, %s).xyz; + vec3 srcRgb = yuvToRgb(srcYuv); + outColor = vec4(srcRgb, uAlphaScale); +}"""), + samplerVarName, fragCoordsVarName, samplerVarName, fragCoordsVarName + ) + } + } + + /** + * Creates an [EGLSurface]. + */ + fun createWindowSurface( + eglDisplay: EGLDisplay, + eglConfig: EGLConfig, surface: Surface, surfaceAttrib: IntArray + ): EGLSurface { + // Create a window surface, and attach it to the Surface we received. + val eglSurface = EGL14.eglCreateWindowSurface( + eglDisplay, eglConfig, surface, + surfaceAttrib, /*offset=*/0 + ) + checkEglErrorOrThrow("eglCreateWindowSurface") + checkNotNull(eglSurface) { "surface was null" } + return eglSurface + } + + /** + * Creates the vertex or fragment shader. + */ + fun loadShader(shaderType: Int, source: String): Int { + val shader = GLES20.glCreateShader(shaderType) + checkGlErrorOrThrow( + "glCreateShader type=$shaderType" + ) + GLES20.glShaderSource(shader, source) + GLES20.glCompileShader(shader) + val compiled = IntArray(1) + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, /*offset=*/0) + if (compiled[0] == 0) { + Logger.w( + TAG, + "Could not compile shader: $source" + ) + val shaderLog = GLES20.glGetShaderInfoLog(shader) + GLES20.glDeleteShader(shader) + throw IllegalStateException( + "Could not compile shader type $shaderType:$shaderLog" + ) + } + return shader + } + + /** + * Queries the [EGLSurface] information. + */ + fun querySurface( + eglDisplay: EGLDisplay, eglSurface: EGLSurface, + what: Int + ): Int { + val value = IntArray(1) + EGL14.eglQuerySurface(eglDisplay, eglSurface, what, value, /*offset=*/0) + return value[0] + } + + /** + * Gets the size of [EGLSurface]. + */ + fun getSurfaceSize( + eglDisplay: EGLDisplay, + eglSurface: EGLSurface + ): Size { + val width = querySurface(eglDisplay, eglSurface, EGL14.EGL_WIDTH) + val height = querySurface(eglDisplay, eglSurface, EGL14.EGL_HEIGHT) + return Size(width, height) + } + + /** + * Creates a [FloatBuffer]. + */ + fun createFloatBuffer(coords: FloatArray): FloatBuffer { + val bb = ByteBuffer.allocateDirect(coords.size * SIZEOF_FLOAT) + bb.order(ByteOrder.nativeOrder()) + val fb = bb.asFloatBuffer() + fb.put(coords) + fb.position(0) + return fb + } + + /** + * Creates a new EGL pixel buffer surface. + */ + fun createPBufferSurface( + eglDisplay: EGLDisplay, + eglConfig: EGLConfig, width: Int, height: Int + ): EGLSurface { + val surfaceAttrib = intArrayOf( + EGL14.EGL_WIDTH, width, + EGL14.EGL_HEIGHT, height, + EGL14.EGL_NONE + ) + val eglSurface = EGL14.eglCreatePbufferSurface( + eglDisplay, eglConfig, surfaceAttrib, /*offset=*/ + 0 + ) + checkEglErrorOrThrow("eglCreatePbufferSurface") + checkNotNull(eglSurface) { "surface was null" } + return eglSurface + } + + /** + * Creates program objects based on shaders which are appropriate for each input format. + * + * + * Each [InputFormat] may have different sampler requirements based on the dynamic + * range. For that reason, we create a separate program for each input format, and will switch + * to the program when the input format changes so we correctly sample the input texture + * (or no-op, in some cases). + */ + fun createPrograms( + dynamicRange: DynamicRangeProfile, + shaderProviderOverrides: Map + ): Map { + val programs = HashMap() + for (inputFormat in InputFormat.entries) { + val shaderProviderOverride = shaderProviderOverrides[inputFormat] + var program: Program2D + if (shaderProviderOverride != null) { + // Always use the overridden shader provider if present + program = SamplerShaderProgram(dynamicRange, shaderProviderOverride) + } else if (inputFormat == InputFormat.YUV || inputFormat == InputFormat.DEFAULT) { + // Use a default sampler shader for DEFAULT or YUV + program = SamplerShaderProgram(dynamicRange, inputFormat) + } else { + check(inputFormat == InputFormat.UNKNOWN) { + "Unhandled input format: $inputFormat" + } + if (dynamicRange.isHdr) { + // InputFormat is UNKNOWN and we don't know if we need to use a + // YUV-specific sampler for HDR. Use a blank shader program. + program = BlankShaderProgram() + } else { + // If we're not rendering HDR content, we can use the default sampler shader + // program since it can handle both YUV and DEFAULT inputs when the format + // is UNKNOWN. + val defaultShaderProviderOverride = + shaderProviderOverrides[InputFormat.DEFAULT] + program = if (defaultShaderProviderOverride != null) { + SamplerShaderProgram( + dynamicRange, + defaultShaderProviderOverride + ) + } else { + SamplerShaderProgram(dynamicRange, InputFormat.DEFAULT) + } + } + } + Logger.d( + TAG, ("Shader program for input format " + inputFormat + " created: " + + program) + ) + programs[inputFormat] = program + } + return programs + } + + /** + * Creates a texture. + */ + fun createTexture(): Int { + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + checkGlErrorOrThrow("glGenTextures") + + val texId = textures[0] + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId) + checkGlErrorOrThrow( + "glBindTexture $texId" + ) + + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, + GLES20.GL_NEAREST + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, + GLES20.GL_LINEAR + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, + GLES20.GL_CLAMP_TO_EDGE + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, + GLES20.GL_CLAMP_TO_EDGE + ) + checkGlErrorOrThrow("glTexParameter") + return texId + } + + /** + * Creates a 4x4 identity matrix. + */ + fun create4x4IdentityMatrix(): FloatArray { + val matrix = FloatArray(16) + Matrix.setIdentityM(matrix, /* smOffset= */0) + return matrix + } + + /** + * Checks the location error. + */ + fun checkLocationOrThrow(location: Int, label: String) { + check(location >= 0) { "Unable to locate '$label' in program" } + } + + /** + * Checks the egl error and throw. + */ + fun checkEglErrorOrThrow(op: String) { + val error = EGL14.eglGetError() + check(error == EGL14.EGL_SUCCESS) { op + ": EGL error: 0x" + Integer.toHexString(error) } + } + + /** + * Checks the gl error and throw. + */ + fun checkGlErrorOrThrow(op: String) { + val error = GLES20.glGetError() + check(error == GLES20.GL_NO_ERROR) { op + ": GL error 0x" + Integer.toHexString(error) } + } + + /** + * Checks the egl error and log. + */ + fun checkEglErrorOrLog(op: String) { + try { + checkEglErrorOrThrow(op) + } catch (e: IllegalStateException) { + Logger.e(TAG, e.toString(), e) + } + } + + /** + * Checks the initialization status. + */ + fun checkInitializedOrThrow( + initialized: AtomicBoolean, + shouldInitialized: Boolean + ) { + val result = shouldInitialized == initialized.get() + val message = if (shouldInitialized) + "OpenGlRenderer is not initialized" + else + "OpenGlRenderer is already initialized" + check(result) { message } + } + + /** + * Checks the gl thread. + */ + fun checkGlThreadOrThrow(thread: Thread?) { + check(thread === Thread.currentThread()) { + "Method call must be called on the GL thread." + } + } + + val glVersionNumber: String + /** + * Gets the gl version number. + */ + get() { + // Logic adapted from CTS Egl14Utils: + // https://cs.android.com/android/platform/superproject/+/master:cts/tests/tests/opengl/src/android/opengl/cts/Egl14Utils.java;l=46;drc=1c705168ab5118c42e5831cd84871d51ff5176d1 + val glVersion = GLES20.glGetString(GLES20.GL_VERSION) + val pattern = Pattern.compile("OpenGL ES ([0-9]+)\\.([0-9]+).*") + val matcher = pattern.matcher(glVersion) + if (matcher.find()) { + val major = requireNotNull(matcher.group(1)) + val minor = requireNotNull(matcher.group(2)) + return "$major.$minor" + } + return VERSION_UNKNOWN + } + + /** + * Chooses the surface attributes for HDR 10bit. + */ + fun chooseSurfaceAttrib( + eglExtensions: String, + dynamicRange: DynamicRangeProfile + ): IntArray { + var attribs = EMPTY_ATTRIBS + if (dynamicRange.transferFunction == MediaFormat.COLOR_TRANSFER_HLG) { + if (eglExtensions.contains("EGL_EXT_gl_colorspace_bt2020_hlg")) { + attribs = HLG_SURFACE_ATTRIBS + } else { + Logger.w( + TAG, ("Dynamic range uses HLG encoding, but " + + "device does not support EGL_EXT_gl_colorspace_bt2020_hlg." + + "Fallback to default colorspace.") + ) + } + } + // TODO(b/303675500): Add path for PQ (EGL_EXT_gl_colorspace_bt2020_pq) output for + // HDR10/HDR10+ + return attribs + } + + /** + * Generates framebuffer object. + */ + fun generateFbo(): Int { + val fbos = IntArray(1) + GLES20.glGenFramebuffers(1, fbos, 0) + checkGlErrorOrThrow("glGenFramebuffers") + return fbos[0] + } + + /** + * Generates texture. + */ + fun generateTexture(): Int { + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + checkGlErrorOrThrow("glGenTextures") + return textures[0] + } + + /** + * Deletes texture. + */ + fun deleteTexture(texture: Int) { + val textures = intArrayOf(texture) + GLES20.glDeleteTextures(1, textures, 0) + checkGlErrorOrThrow("glDeleteTextures") + } + + /** + * Deletes framebuffer object. + */ + fun deleteFbo(fbo: Int) { + val fbos = intArrayOf(fbo) + GLES20.glDeleteFramebuffers(1, fbos, 0) + checkGlErrorOrThrow("glDeleteFramebuffers") + } + + private fun getFragmentShaderSource(shaderProvider: ShaderProvider): String { + // Throw IllegalArgumentException if the shader provider can not provide a valid + // fragment shader. + try { + val source = shaderProvider.createFragmentShader(VAR_TEXTURE, VAR_TEXTURE_COORD) + // A simple check to workaround custom shader doesn't contain required variable. + // See b/241193761. + require( + !(source == null || !source.contains(VAR_TEXTURE_COORD) || !source.contains( + VAR_TEXTURE + )) + ) { "Invalid fragment shader" } + return source + } catch (t: Throwable) { + if (t is IllegalArgumentException) { + throw t + } + throw IllegalArgumentException("Unable retrieve fragment shader source", t) + } + } + + enum class InputFormat { + /** + * Input texture format is unknown. + * + * + * When the input format is unknown, HDR content may require rendering blank frames + * since we are not sure what type of sampler can be used. For SDR content, it is + * typically safe to use samplerExternalOES since this can handle both RGB and YUV inputs + * for SDR content. + */ + UNKNOWN, + + /** + * Input texture format is the default format. + * + * + * The texture format may be RGB or YUV. For SDR content, using samplerExternalOES is + * safe since it will be able to convert YUV to RGB automatically within the shader. For + * HDR content, the input is expected to be RGB. + */ + DEFAULT, + + /** + * Input format is explicitly YUV. + * + * + * This needs to be specified for HDR content. Only __samplerExternal2DY2YEXT should be + * used for HDR YUV content as samplerExternalOES may not correctly convert to RGB. + */ + YUV + } + + abstract class Program2D protected constructor( + vertexShaderSource: String, + fragmentShaderSource: String + ) { + protected var mProgramHandle: Int = 0 + protected var mTransMatrixLoc: Int = -1 + protected var mAlphaScaleLoc: Int = -1 + protected var mPositionLoc: Int = -1 + + init { + var vertexShader = -1 + var fragmentShader = -1 + var program = -1 + try { + vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderSource) + fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderSource) + program = GLES20.glCreateProgram() + checkGlErrorOrThrow("glCreateProgram") + GLES20.glAttachShader(program, vertexShader) + checkGlErrorOrThrow("glAttachShader") + GLES20.glAttachShader(program, fragmentShader) + checkGlErrorOrThrow("glAttachShader") + GLES20.glLinkProgram(program) + val linkStatus = IntArray(1) + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, /*offset=*/0) + check(linkStatus[0] == GLES20.GL_TRUE) { + "Could not link program: " + GLES20.glGetProgramInfoLog( + program + ) + } + mProgramHandle = program + } catch (e: IllegalStateException) { + if (vertexShader != -1) { + GLES20.glDeleteShader(vertexShader) + } + if (fragmentShader != -1) { + GLES20.glDeleteShader(fragmentShader) + } + if (program != -1) { + GLES20.glDeleteProgram(program) + } + throw e + } catch (e: IllegalArgumentException) { + if (vertexShader != -1) { + GLES20.glDeleteShader(vertexShader) + } + if (fragmentShader != -1) { + GLES20.glDeleteShader(fragmentShader) + } + if (program != -1) { + GLES20.glDeleteProgram(program) + } + throw e + } + + loadLocations() + } + + /** + * Use this shader program + */ + open fun use() { + // Select the program. + GLES20.glUseProgram(mProgramHandle) + checkGlErrorOrThrow("glUseProgram") + + // Enable the "aPosition" vertex attribute. + GLES20.glEnableVertexAttribArray(mPositionLoc) + checkGlErrorOrThrow("glEnableVertexAttribArray") + + // Connect vertexBuffer to "aPosition". + val coordsPerVertex = 2 + val vertexStride = 0 + GLES20.glVertexAttribPointer( + mPositionLoc, coordsPerVertex, GLES20.GL_FLOAT, /*normalized=*/ + false, vertexStride, VERTEX_BUF + ) + checkGlErrorOrThrow("glVertexAttribPointer") + + // Set to default value for single camera case + updateTransformMatrix(create4x4IdentityMatrix()) + updateAlpha(1.0f) + } + + /** + * Updates the global transform matrix + */ + fun updateTransformMatrix(transformMat: FloatArray) { + glUniformMatrix4fv( + mTransMatrixLoc, /*count=*/ + 1, /*transpose=*/false, transformMat, /*offset=*/ + 0 + ) + checkGlErrorOrThrow("glUniformMatrix4fv") + } + + /** + * Updates the alpha of the drawn frame + */ + fun updateAlpha(alpha: Float) { + GLES20.glUniform1f(mAlphaScaleLoc, alpha) + checkGlErrorOrThrow("glUniform1f") + } + + /** + * Delete the shader program + * + * + * Once called, this program should no longer be used. + */ + fun delete() { + GLES20.glDeleteProgram(mProgramHandle) + } + + protected open fun loadLocations() { + mPositionLoc = GLES20.glGetAttribLocation(mProgramHandle, "aPosition") + checkLocationOrThrow(mPositionLoc, "aPosition") + mTransMatrixLoc = GLES20.glGetUniformLocation(mProgramHandle, "uTransMatrix") + checkLocationOrThrow(mTransMatrixLoc, "uTransMatrix") + mAlphaScaleLoc = GLES20.glGetUniformLocation(mProgramHandle, "uAlphaScale") + checkLocationOrThrow(mAlphaScaleLoc, "uAlphaScale") + } + } + + class SamplerShaderProgram( + dynamicRange: DynamicRangeProfile, + shaderProvider: ShaderProvider + ) : Program2D( + if (dynamicRange.isHdr) HDR_VERTEX_SHADER else DEFAULT_VERTEX_SHADER, + getFragmentShaderSource(shaderProvider) + ) { + private var mSamplerLoc = -1 + private var mTexMatrixLoc = -1 + private var mTexCoordLoc = -1 + + constructor( + dynamicRange: DynamicRangeProfile, + inputFormat: InputFormat + ) : this(dynamicRange, resolveDefaultShaderProvider(dynamicRange, inputFormat)) + + init { + loadLocations() + } + + override fun use() { + super.use() + // Initialize the sampler to the correct texture unit offset + GLES20.glUniform1i(mSamplerLoc, 0) + + // Enable the "aTextureCoord" vertex attribute. + GLES20.glEnableVertexAttribArray(mTexCoordLoc) + checkGlErrorOrThrow("glEnableVertexAttribArray") + + // Connect texBuffer to "aTextureCoord". + val coordsPerTex = 2 + val texStride = 0 + GLES20.glVertexAttribPointer( + mTexCoordLoc, coordsPerTex, GLES20.GL_FLOAT, /*normalized=*/ + false, texStride, TEX_BUF + ) + checkGlErrorOrThrow("glVertexAttribPointer") + } + + /** + * Updates the texture transform matrix + */ + fun updateTextureMatrix(textureMat: FloatArray) { + glUniformMatrix4fv( + mTexMatrixLoc, /*count=*/1, /*transpose=*/false, + textureMat, /*offset=*/0 + ) + checkGlErrorOrThrow("glUniformMatrix4fv") + } + + override fun loadLocations() { + super.loadLocations() + mSamplerLoc = GLES20.glGetUniformLocation(mProgramHandle, VAR_TEXTURE) + checkLocationOrThrow(mSamplerLoc, VAR_TEXTURE) + mTexCoordLoc = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord") + checkLocationOrThrow(mTexCoordLoc, "aTextureCoord") + mTexMatrixLoc = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix") + checkLocationOrThrow(mTexMatrixLoc, "uTexMatrix") + } + + companion object { + private fun resolveDefaultShaderProvider( + dynamicRange: DynamicRangeProfile, + inputFormat: InputFormat? + ): ShaderProvider { + if (dynamicRange.isHdr) { + check(inputFormat != InputFormat.UNKNOWN) { + "No default sampler shader available for $inputFormat" + } + if (inputFormat == InputFormat.YUV) { + return SHADER_PROVIDER_HDR_YUV + } + return SHADER_PROVIDER_HDR_DEFAULT + } else { + return SHADER_PROVIDER_DEFAULT + } + } + } + } + + class BlankShaderProgram : Program2D(BLANK_VERTEX_SHADER, BLANK_FRAGMENT_SHADER) +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GraphicDeviceInfo.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GraphicDeviceInfo.kt new file mode 100644 index 000000000..4eb80e473 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/GraphicDeviceInfo.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2024 The Android Open Source Project + * Copyright 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils + +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.GLUtils.VERSION_UNKNOWN + +/** + * Information about an initialized graphics device. + * + * + * This information can be used to determine which version or extensions of OpenGL and EGL + * are supported on the device to ensure the attached output surface will have expected + * characteristics. + */ +class GraphicDeviceInfo // Should not be instantiated directly +private constructor( + /** + * Returns the OpenGL version this graphics device has been initialized to. + * + * + * The version is in the form <major>.<minor>. + * + * + * Returns [GLUtils.VERSION_UNKNOWN] if version information can't be + * retrieved. + */ + val glVersion: String = VERSION_UNKNOWN, + + /** + * Returns the EGL version this graphics device has been initialized to. + * + * + * The version is in the form <major>.<minor>. + * + * + * Returns [GLUtils.VERSION_UNKNOWN] if version information can't be + * retrieved. + */ + val eglVersion: String = VERSION_UNKNOWN, + + /** + * Returns a space separated list of OpenGL extensions or an empty string if extensions + * could not be retrieved. + */ + val glExtensions: String = "", + + /** + * Returns a space separated list of EGL extensions or an empty string if extensions + * could not be retrieved. + */ + val eglExtensions: String = "" +) { + class Builder { + private var glVersion: String = VERSION_UNKNOWN + private var eglVersion: String = VERSION_UNKNOWN + private var glExtensions: String = "" + private var eglExtensions: String = "" + + fun setGlVersion(glVersion: String) = apply { this.glVersion = glVersion } + fun setEglVersion(eglVersion: String) = apply { this.eglVersion = eglVersion } + fun setGlExtensions(glExtensions: String) = apply { this.glExtensions = glExtensions } + fun setEglExtensions(eglExtensions: String) = apply { this.eglExtensions = eglExtensions } + + fun build() = GraphicDeviceInfo(glVersion, eglVersion, glExtensions, eglExtensions) + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/OutputSurface.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/OutputSurface.kt new file mode 100644 index 000000000..00528c93e --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/OutputSurface.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2024 The Android Open Source Project + * Copyright 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils + +import android.opengl.EGLSurface + +/** + * Wrapper for output [EGLSurface] in [io.github.thibaultbee.streampack.core.internal.processing.video.OpenGlRenderer]. + */ + +data class OutputSurface( + /** + * Gets [EGLSurface]. + */ + val eglSurface: EGLSurface, + + /** + * Gets [EGLSurface] width. + */ + val width: Int, + + /** + * Gets [EGLSurface] height. + */ + val height: Int +) { + + companion object { + /** + * Creates [OutputSurface]. + */ + fun of(eglSurface: EGLSurface, width: Int, height: Int): OutputSurface { + return OutputSurface(eglSurface, width, height) + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/TransformUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/TransformUtils.kt new file mode 100644 index 000000000..128e31a85 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/TransformUtils.kt @@ -0,0 +1,45 @@ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils + +import android.graphics.Matrix +import android.graphics.RectF +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions.normalized + + +object TransformUtils { + val NORMALIZED_RECT: RectF = RectF(-1f, -1f, 1f, 1f) + + /** + * Gets the transform from one [RectF] to another with rotation degrees and mirroring. + * + * + * Following is how the source is mapped to the target with a 90° rotation and a mirroring. + * The rect , b, c, d> is mapped to ', b', c', d'>. + * + *
+     * a----------b                           a'-----------d'
+     * |  source  |    -90° + mirroring ->    |            |
+     * d----------c                           |   target   |
+     *                                        |            |
+     *                                        b'-----------c'
+    
* + */ + fun getRectToRect( + source: RectF, + target: RectF, + @IntRange(from = 0, to = 359) rotationDegrees: Int, + mirroring: Boolean + ): Matrix { + // Map source to normalized space. + val matrix = Matrix() + matrix.setRectToRect(source, NORMALIZED_RECT, Matrix.ScaleToFit.FILL) + // Add rotation. + matrix.postRotate(rotationDegrees.toFloat()) + if (mirroring) { + matrix.postScale(-1f, 1f) + } + // Restore the normalized space to target's coordinates. + matrix.postConcat(target.normalized) + return matrix + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/FloatExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/FloatExtensions.kt new file mode 100644 index 000000000..5ce653382 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/FloatExtensions.kt @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions + +import android.opengl.Matrix + +/** + * Preconcats the matrix with the specified rotation. M' = M * R(degrees, px, py) + * + * + * The pivot point is the coordinate that should remain unchanged by the specified + * transformation. + * + * @param matrix the matrix to rotate + * @param degrees the rotation degrees + * @param px px of pivot point at (px, py) + * @param py py of pivot point at (px, py) + */ +fun FloatArray.preRotate(degrees: Float, px: Float, py: Float) { + normalize(px, py) + Matrix.rotateM(this, 0, degrees, 0f, 0f, 1f) + denormalize(px, py) +} + +/** + * Preconcats the matrix with a vertical flip operation. + * + * @param y the horizontal line to flip along + */ +fun FloatArray.preVerticalFlip(y: Float) { + normalize(0.toFloat(), y) + Matrix.scaleM(this, 0, 1f, -1f, 1f) + denormalize(0.toFloat(), y) +} + +private fun FloatArray.normalize(px: Float, py: Float) { + Matrix.translateM(this, 0, px, py, 0f) +} + +private fun FloatArray.denormalize(px: Float, py: Float) { + Matrix.translateM(this, 0, -px, -py, 0f) +} + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/IntExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/IntExtensions.kt new file mode 100644 index 000000000..d969795f8 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/IntExtensions.kt @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions + +val Int.is90or270: Boolean + get() { + if (this == 90 || this == 270) { + return true + } + if (this == 0 || this == 180) { + return false + } + throw IllegalArgumentException("Invalid rotation degrees: $this") + } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/RectFExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/RectFExtensions.kt new file mode 100644 index 000000000..6a3f25cf3 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/RectFExtensions.kt @@ -0,0 +1,15 @@ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions + +import android.graphics.Matrix +import android.graphics.RectF +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.TransformUtils.NORMALIZED_RECT + +/** + * Gets the transform from a normalized space (-1, -1) - (1, 1) to the given rect. + */ +val RectF.normalized: Matrix + get() { + val normalizedToBuffer = Matrix() + normalizedToBuffer.setRectToRect(NORMALIZED_RECT, this, Matrix.ScaleToFit.FILL) + return normalizedToBuffer + } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/SizeExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/SizeExtensions.kt new file mode 100644 index 000000000..82f2e5e5c --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/processing/video/utils/extensions/SizeExtensions.kt @@ -0,0 +1,26 @@ +package io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions + +import android.graphics.RectF +import android.util.Size + +/** + * Transforms size to a [RectF] with zero left and top. + */ +fun Size.toRectF(): RectF { + return toRectF(0, 0) +} + +/** + * Transforms a size to a [RectF] with given left and top. + */ +fun Size.toRectF(left: Int, top: Int): RectF { + return RectF( + left.toFloat(), + top.toFloat(), + (left + width).toFloat(), + (top + height).toFloat() + ) +} + + + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/IVideoSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/IVideoSource.kt index 5decd8562..dc3afff83 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/IVideoSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/IVideoSource.kt @@ -19,7 +19,7 @@ import io.github.thibaultbee.streampack.core.data.VideoConfig import io.github.thibaultbee.streampack.core.internal.interfaces.Configurable import io.github.thibaultbee.streampack.core.internal.interfaces.Releaseable import io.github.thibaultbee.streampack.core.internal.interfaces.SuspendStreamable -import io.github.thibaultbee.streampack.core.internal.orientation.ISourceOrientationProvider +import io.github.thibaultbee.streampack.core.internal.processing.video.source.ISourceInfoProvider import io.github.thibaultbee.streampack.core.internal.sources.IFrameSource import io.github.thibaultbee.streampack.core.internal.sources.ISurfaceSource @@ -41,7 +41,7 @@ interface IVideoSourceInternal : IFrameSource, ISurfaceSource, IVid * Orientation provider of the capture source. * It is used to orientate the frame according to the source orientation. */ - val orientationProvider: ISourceOrientationProvider + val infoProvider: ISourceInfoProvider } interface IVideoSource \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraInfoProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraInfoProvider.kt new file mode 100644 index 000000000..1571bd337 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraInfoProvider.kt @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.sources.video.camera + +import android.content.Context +import android.hardware.camera2.CameraCharacteristics +import android.util.Size +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.processing.video.source.AbstractSourceInfoProvider +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.landscapize +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotationToDegrees +import io.github.thibaultbee.streampack.core.utils.extensions.getCameraCharacteristics +import io.github.thibaultbee.streampack.core.utils.extensions.getFacingDirection + +class CameraInfoProvider(private val context: Context, initialCameraId: String) : + AbstractSourceInfoProvider() { + + var cameraId: String = initialCameraId + set(value) { + if (field == value) { + return + } + field = value + } + + override val rotationDegrees: Int + @IntRange(from = 0, to = 359) + get() { + val characteristics = context.getCameraCharacteristics(cameraId) + return characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 + } + + val isFrontFacing: Boolean + get() = context.getFacingDirection(cameraId) == CameraCharacteristics.LENS_FACING_FRONT + + override val isMirror = false + + @IntRange(from = 0, to = 359) + override fun getRelativeRotationDegrees( + @RotationValue targetRotation: Int, requiredMirroring: Boolean + ) = getSensorRotationDegrees(targetRotation) + + @IntRange(from = 0, to = 359) + private fun getSensorRotationDegrees(@RotationValue targetRotation: Int): Int { + val sensorOrientation = rotationDegrees + + val targetRotationDegrees = targetRotation.rotationToDegrees + + // Currently this assumes that a back-facing camera is always opposite to the screen. + // This may not be the case for all devices, so in the future we may need to handle that + // scenario. + val lensFacing = context.getFacingDirection(cameraId) + val isOppositeFacingScreen = CameraCharacteristics.LENS_FACING_BACK == lensFacing + return CameraOrientationUtils.getRelativeRotation( + targetRotationDegrees, sensorOrientation, isOppositeFacingScreen + ) + } + + override fun getSurfaceSize(size: Size, targetRotation: Int) = size.landscapize +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationProvider.kt deleted file mode 100644 index efcd41630..000000000 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationProvider.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2024 Thibault B. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.github.thibaultbee.streampack.core.internal.sources.video.camera - -import android.content.Context -import android.hardware.camera2.CameraCharacteristics -import android.util.Size -import android.view.Surface -import io.github.thibaultbee.streampack.core.internal.orientation.AbstractSourceOrientationProvider -import io.github.thibaultbee.streampack.core.internal.utils.extensions.deviceOrientation -import io.github.thibaultbee.streampack.core.internal.utils.extensions.isDevicePortrait -import io.github.thibaultbee.streampack.core.internal.utils.extensions.landscapize -import io.github.thibaultbee.streampack.core.internal.utils.extensions.portraitize -import io.github.thibaultbee.streampack.core.utils.extensions.cameras -import io.github.thibaultbee.streampack.core.utils.extensions.getFacingDirection -import kotlin.math.max -import kotlin.math.min - - -class CameraOrientationProvider(private val context: Context, initialCameraId: String) : - AbstractSourceOrientationProvider() { - private val isFrontFacingMap = - context.cameras.associateWith { (context.getFacingDirection(it) == CameraCharacteristics.LENS_FACING_FRONT) } - - var cameraId: String = initialCameraId - set(value) { - if (field == value) { - return - } - val orientationChanged = mirroredVertically != isFrontFacing(value) - field = value - if (orientationChanged) { - listeners.forEach { it.onOrientationChanged() } - } - } - - override val orientation: Int - get() = when (context.deviceOrientation) { - Surface.ROTATION_0 -> 0 - Surface.ROTATION_90 -> 270 - Surface.ROTATION_180 -> 180 - Surface.ROTATION_270 -> 90 - else -> 0 - } - - private fun isFrontFacing(cameraId: String): Boolean { - return isFrontFacingMap[cameraId] ?: false - } - - override val mirroredVertically: Boolean - get() = isFrontFacing(cameraId) - - override fun getOrientedSize(size: Size): Size { - return if (context.isDevicePortrait) { - size.portraitize - } else { - size.landscapize - } - } - - override fun getDefaultBufferSize(size: Size): Size { - return Size(max(size.width, size.height), min(size.width, size.height)) - } -} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationUtils.kt new file mode 100644 index 000000000..825e3abf7 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraOrientationUtils.kt @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.sources.video.camera + +object CameraOrientationUtils { + /** + * Calculates the delta between a source rotation and destination rotation. + * + *

A typical use of this method would be calculating the angular difference between the + * display orientation (destRotationDegrees) and camera sensor orientation + * (sourceRotationDegrees). + * + * @param destRotationDegrees The destination rotation relative to the device's natural + * rotation. + * @param sourceRotationDegrees The source rotation relative to the device's natural rotation. + * @param isOppositeFacing Whether the source and destination planes are facing opposite + * directions. + */ + fun getRelativeRotation( + destRotationDegrees: Int, sourceRotationDegrees: Int, isOppositeFacing: Boolean + ): Int { + return if (isOppositeFacing) { + (sourceRotationDegrees - destRotationDegrees + 360) % 360 + } else { + (sourceRotationDegrees + destRotationDegrees) % 360 + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSettings.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSettings.kt index e3abc9294..52bf4f448 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSettings.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSettings.kt @@ -26,10 +26,11 @@ import android.hardware.camera2.params.MeteringRectangle import android.os.Build import android.util.Range import android.util.Rational +import androidx.annotation.IntRange import androidx.annotation.RequiresApi import io.github.thibaultbee.streampack.core.internal.utils.* import io.github.thibaultbee.streampack.core.internal.utils.extensions.clamp -import io.github.thibaultbee.streampack.core.internal.utils.extensions.isDevicePortrait +import io.github.thibaultbee.streampack.core.internal.utils.extensions.isApplicationPortrait import io.github.thibaultbee.streampack.core.internal.utils.extensions.isNormalized import io.github.thibaultbee.streampack.core.internal.utils.extensions.normalize import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotate @@ -827,10 +828,11 @@ class FocusMetering( * @param surfaceRotationDegrees The current Surface orientation in degrees. * @return Relative rotation of the camera sensor output. */ + @IntRange(from = 0, to = 359) private fun getSensorRotationDegrees( context: Context, cameraId: String, - surfaceRotationDegrees: Int = 0 + @IntRange(from = 0, to = 359) surfaceRotationDegrees: Int = 0 ): Int { val characteristics = context.getCameraCharacteristics(cameraId) val sensorOrientationDegrees = @@ -853,9 +855,10 @@ class FocusMetering( ) } + @IntRange(from = 0, to = 359) private fun getRelativeRotationDegrees( - sourceRotationDegrees: Int, - destRotationDegrees: Int, + @IntRange(from = 0, to = 359) sourceRotationDegrees: Int, + @IntRange(from = 0, to = 359) destRotationDegrees: Int, isFacingFront: Boolean ): Int { return if (isFacingFront) { @@ -907,7 +910,7 @@ class FocusMetering( afPoints.map { normalizePoint(it, fovRect, relativeRotation) }, aePoints.map { normalizePoint(it, fovRect, relativeRotation) }, awbPoints.map { normalizePoint(it, fovRect, relativeRotation) }, - if (context.isDevicePortrait) { + if (context.isApplicationPortrait) { Rational(fovRect.height(), fovRect.width()) } else { Rational(fovRect.width(), fovRect.height()) diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSource.kt index 3ac974aea..c3d87ac34 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/camera/CameraSource.kt @@ -88,7 +88,7 @@ class CameraSource( } field = value - orientationProvider.cameraId = value + infoProvider.cameraId = value runBlocking { restartCamera() } @@ -100,7 +100,7 @@ class CameraSource( override val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId) override val hasOutputSurface = true override val hasFrames = false - override val orientationProvider = CameraOrientationProvider(context, cameraId) + override val infoProvider = CameraInfoProvider(context, cameraId) override fun getFrame(buffer: ByteBuffer): Frame { throw UnsupportedOperationException("Camera expects to run in Surface mode") diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/mediaprojection/MediaProjectionVideoSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/mediaprojection/MediaProjectionVideoSource.kt index 5aacf247c..6571648af 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/mediaprojection/MediaProjectionVideoSource.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/sources/video/mediaprojection/MediaProjectionVideoSource.kt @@ -22,28 +22,26 @@ import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.Handler import android.os.HandlerThread -import android.util.Size import android.view.Surface import androidx.activity.result.ActivityResult import io.github.thibaultbee.streampack.core.data.VideoConfig import io.github.thibaultbee.streampack.core.internal.data.Frame -import io.github.thibaultbee.streampack.core.internal.orientation.AbstractSourceOrientationProvider +import io.github.thibaultbee.streampack.core.internal.processing.video.source.AbstractSourceInfoProvider import io.github.thibaultbee.streampack.core.internal.sources.IMediaProjectionSource import io.github.thibaultbee.streampack.core.internal.sources.video.IVideoSourceInternal -import io.github.thibaultbee.streampack.core.internal.utils.extensions.isDevicePortrait -import io.github.thibaultbee.streampack.core.internal.utils.extensions.landscapize -import io.github.thibaultbee.streampack.core.internal.utils.extensions.portraitize +import io.github.thibaultbee.streampack.core.internal.utils.extensions.densityDpi +import io.github.thibaultbee.streampack.core.internal.utils.extensions.screenRect import io.github.thibaultbee.streampack.core.logger.Logger import java.nio.ByteBuffer class MediaProjectionVideoSource( - context: Context + private val context: Context ) : IVideoSourceInternal, IMediaProjectionSource { override var outputSurface: Surface? = null override val timestampOffset = 0L override val hasOutputSurface = true override val hasFrames = false - override val orientationProvider = ScreenSourceOrientationProvider(context) + override val infoProvider = ScreenSourceInfoProvider(context) override fun getFrame(buffer: ByteBuffer): Frame { throw UnsupportedOperationException("Screen source run in Surface mode") @@ -66,7 +64,7 @@ class MediaProjectionVideoSource( private val mediaProjectionManager = context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager private var virtualDisplay: VirtualDisplay? = null - private var videoConfig: VideoConfig? = null + private val virtualDisplayThread = HandlerThread("VirtualDisplayThread").apply { start() } private val virtualDisplayHandler = Handler(virtualDisplayThread.looper) private val virtualDisplayCallback = object : VirtualDisplay.Callback() { @@ -102,19 +100,16 @@ class MediaProjectionVideoSource( private const val VIRTUAL_DISPLAY_NAME = "StreamPackScreenSource" } - override fun configure(config: VideoConfig) { - videoConfig = config - } + override fun configure(config: VideoConfig) = Unit override suspend fun startStream() { - val videoConfig = requireNotNull(videoConfig) { "Video has not been configured!" } val activityResult = requireNotNull(activityResult) { "MediaProjection requires an activity result to be set" } isStoppedByUser = false + val screenRect = context.screenRect - val orientedSize = orientationProvider.getOrientedSize(videoConfig.resolution) mediaProjection = mediaProjectionManager.getMediaProjection( activityResult.resultCode, activityResult.data!! @@ -122,9 +117,9 @@ class MediaProjectionVideoSource( registerCallback(mediaProjectionCallback, virtualDisplayHandler) virtualDisplay = createVirtualDisplay( VIRTUAL_DISPLAY_NAME, - orientedSize.width, - orientedSize.height, - 320, + screenRect.width(), + screenRect.height(), + context.densityDpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, outputSurface, virtualDisplayCallback, @@ -152,18 +147,8 @@ class MediaProjectionVideoSource( } } - class ScreenSourceOrientationProvider(private val context: Context) : - AbstractSourceOrientationProvider() { - override val orientation = 0 - - override fun getOrientedSize(size: Size): Size { - return if (context.isDevicePortrait) { - size.portraitize - } else { - size.landscapize - } - } - } + class ScreenSourceInfoProvider(private val context: Context) : + AbstractSourceInfoProvider() interface Listener { fun onStop() diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/Annotations.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/Annotations.kt new file mode 100644 index 000000000..ebea72456 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/Annotations.kt @@ -0,0 +1,13 @@ +package io.github.thibaultbee.streampack.core.internal.utils + +import android.view.Surface +import androidx.annotation.IntDef + +/** + * Valid integer rotation values. + */ +@IntDef(Surface.ROTATION_0, Surface.ROTATION_90, Surface.ROTATION_180, Surface.ROTATION_270) +@Retention( + AnnotationRetention.SOURCE +) +annotation class RotationValue \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/OrientationUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/OrientationUtils.kt index f875c0a9c..e0daf6059 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/OrientationUtils.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/OrientationUtils.kt @@ -16,12 +16,14 @@ package io.github.thibaultbee.streampack.core.internal.utils import android.view.Surface +import androidx.annotation.IntRange object OrientationUtils { /** - * Returns the surface orientation in degrees from [Surface] orientation ([Surface.ROTATION_0], ...). + * Returns the surface orientation in degrees from [Surface] rotation ([Surface.ROTATION_0], ...). */ - fun getSurfaceOrientationDegrees(surfaceOrientation: Int): Int { + @IntRange(from = 0, to = 359) + fun getSurfaceRotationDegrees(surfaceOrientation: Int): Int { return when (surfaceOrientation) { Surface.ROTATION_0 -> 0 Surface.ROTATION_90 -> 90 diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/WindowUtils.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/WindowUtils.kt new file mode 100644 index 000000000..c6148e674 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/WindowUtils.kt @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.utils + +import android.content.Context +import androidx.window.layout.WindowMetricsCalculator + +object WindowUtils { + fun getScreenRect(context: Context) = + WindowMetricsCalculator.getOrCreate().computeMaximumWindowMetrics(context).bounds +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/ContextExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/ContextExtensions.kt index 5006a51a4..39132b048 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/ContextExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/ContextExtensions.kt @@ -18,46 +18,128 @@ package io.github.thibaultbee.streampack.core.internal.utils.extensions import android.content.Context import android.content.res.Configuration.ORIENTATION_LANDSCAPE import android.content.res.Configuration.ORIENTATION_PORTRAIT -import android.hardware.display.DisplayManager -import android.view.Display +import android.graphics.Rect +import android.util.Size import android.view.Surface -import io.github.thibaultbee.streampack.core.internal.utils.OrientationUtils +import androidx.annotation.IntRange +import androidx.core.content.ContextCompat +import androidx.core.view.DisplayCompat +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.WindowUtils +import io.github.thibaultbee.streampack.core.utils.extensions.is90Multiple /** - * Returns the device orientation in degrees from the natural orientation: portrait. + * Returns the device orientation in degrees from the natural orientation. * * @return the device orientation in degrees */ -val Context.deviceOrientationDegrees: Int - get() { - val displayManager = this.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager - return OrientationUtils.getSurfaceOrientationDegrees(displayManager.getDisplay(Display.DEFAULT_DISPLAY).rotation) - } +val Context.deviceRotationDegrees: Int + @IntRange( + from = 0, + to = 359 + ) + get() = deviceRotation.rotationToDegrees /** - * Returns the device orientation in degrees from the natural orientation: portrait. + * Returns the device orientation in degrees from the natural orientation. * * @return the device orientation as [Surface.ROTATION_0], [Surface.ROTATION_90], [Surface.ROTATION_180] or [Surface.ROTATION_270] */ -val Context.deviceOrientation: Int + +val Context.deviceRotation: Int + @RotationValue get() { - val displayManager = this.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager - return displayManager.getDisplay(Display.DEFAULT_DISPLAY).rotation + return ContextCompat.getDisplayOrDefault(this).rotation } /** - * Check if the device is in portrait. - * - * @return true if the device is in portrait, otherwise false + * Whether the device is in portrait orientation. */ val Context.isDevicePortrait: Boolean - get() = resources.configuration.orientation == ORIENTATION_PORTRAIT + get() = isRotationPortrait(deviceRotation) + +/** + * Whether the device is in landscape orientation. + */ +val Context.isDeviceLandscape: Boolean + get() = !isDevicePortrait + +/** + * Returns the device natural size in pixels. + */ +private val Context.naturalSize: Size + get() { + val display = ContextCompat.getDisplayOrDefault(this) + val mode = DisplayCompat.getMode(this, display) + return Size(mode.physicalWidth, mode.physicalHeight) + } + +/** + * Whether the natural orientation is portrait. + */ +val Context.isNaturalToPortrait: Boolean + get() = naturalSize.isPortrait +/** + * Whether the application is in landscape orientation. + */ +val Context.isNaturalToLandscape: Boolean + get() = !isNaturalToPortrait /** - * Check if the device is in landscape. + * Whether the application is in portrait. * - * @return true if the device is in landscape, otherwise false + * @return true if the application is in portrait, otherwise false */ -val Context.isDeviceLandscape: Boolean +val Context.isApplicationPortrait: Boolean + get() = resources.configuration.orientation == ORIENTATION_PORTRAIT + +/** + * Whether the application is in landscape. + * + * @return true if the application is in landscape, otherwise false + */ +val Context.isApplicationLandscape: Boolean get() = resources.configuration.orientation == ORIENTATION_LANDSCAPE + +/** + * Whether the rotation is portrait for this device. + */ +fun Context.isRotationDegreesPortrait( + @IntRange( + from = 0, + to = 359 + ) rotationDegrees: Int +): Boolean { + require(rotationDegrees.is90Multiple) { "Orientation must be a multiple of 90 but $rotationDegrees" } + return if (isNaturalToPortrait) { + rotationDegrees % 180 == 0 + } else { + rotationDegrees % 180 != 0 + } +} + +/** + * Whether the rotation is portrait for this device. + */ +fun Context.isRotationPortrait( + @RotationValue rotation: Int +): Boolean { + return if (isNaturalToPortrait) { + rotation == Surface.ROTATION_0 || rotation == Surface.ROTATION_180 + } else { + rotation == Surface.ROTATION_90 || rotation == Surface.ROTATION_270 + } +} + +/** + * Get the application pixel density + */ +val Context.densityDpi: Int + get() = resources.displayMetrics.densityDpi + +/** + * Get the screen rectangle in pixels + */ +val Context.screenRect: Rect + get() = WindowUtils.getScreenRect(this) \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/IntExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/IntExtensions.kt new file mode 100644 index 000000000..a33536e3c --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/IntExtensions.kt @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.utils.extensions + +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.utils.OrientationUtils + +/** + * Returns the rotation in degrees from [Int] rotation. + */ +internal val Int.rotationToDegrees: Int + @IntRange(from = 0, to = 359) + get() = OrientationUtils.getSurfaceRotationDegrees(this) \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/RectExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/RectExtensions.kt new file mode 100644 index 000000000..b385a598b --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/RectExtensions.kt @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2023 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.internal.utils.extensions + +import android.graphics.Rect +import android.util.Size + +/** + * Returns the [Size] of the [Rect]. + */ +val Rect.size: Size + get() = Size(width(), height()) + diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/SizeExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/SizeExtensions.kt index c6e0735b2..006ca01d4 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/SizeExtensions.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/internal/utils/extensions/SizeExtensions.kt @@ -15,13 +15,18 @@ */ package io.github.thibaultbee.streampack.core.internal.utils.extensions +import android.content.Context import android.util.Size +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.processing.video.utils.extensions.is90or270 +import io.github.thibaultbee.streampack.core.utils.extensions.is90Multiple +import io.github.thibaultbee.streampack.core.utils.extensions.within360 import kotlin.math.abs import kotlin.math.max import kotlin.math.min /** - * Get the size in landscape orientation: the largest dimension as width and the smallest as height. + * Gets the size in landscape orientation: the largest dimension as width and the smallest as height. * * @return the size in landscape orientation. */ @@ -29,7 +34,7 @@ val Size.landscapize: Size get() = Size(max(width, height), min(width, height)) /** - * Get the size in portrait orientation: the smallest dimension as width and the largest as height. + * Gets the size in portrait orientation: the smallest dimension as width and the largest as height. * * @return the size in portrait orientation. */ @@ -37,17 +42,52 @@ val Size.portraitize: Size get() = Size(min(width, height), max(width, height)) /** - * Check if the size is in portrait orientation. + * Whether the size is in portrait orientation. */ val Size.isPortrait: Boolean get() = width < height /** - * Check if the size is in landscape orientation. + * Whether the size is in landscape orientation. */ val Size.isLandscape: Boolean get() = !isPortrait +/** + * Reverses width and height for a [Size]. + * + * @param size the size to reverse + * @return reversed size + */ +val Size.reverse: Size + get() = Size(height, width) + +/** + * Rotates a [Size] according to the rotation degrees. + * + * @param rotationDegrees the rotation degrees + * @return rotated size + * @throws IllegalArgumentException if the rotation degrees is not a multiple of 90 + */ +fun Size.rotate(@IntRange(from = 0, to = 359) rotationDegrees: Int): Size { + require(rotationDegrees.is90Multiple) { "Invalid rotation degrees: $rotationDegrees" } + return if (rotationDegrees.within360.is90or270) reverse else this +} + +/** + * Rotates a [Size] according to device natural orientation. + */ +fun Size.rotateFromNaturalOrientation( + context: Context, + @IntRange(from = 0, to = 359) rotationDegrees: Int +): Size { + return if (context.isRotationDegreesPortrait(rotationDegrees)) { + portraitize + } else { + landscapize + } +} + /** * Find the closest size to the given size in a list of sizes. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultCameraStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultCameraStreamer.kt index ba64741a2..22772c387 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultCameraStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultCameraStreamer.kt @@ -19,6 +19,7 @@ import android.Manifest import android.content.Context import android.view.Surface import androidx.annotation.RequiresPermission +import androidx.annotation.RestrictTo.* import io.github.thibaultbee.streampack.core.data.mediadescriptor.MediaDescriptor import io.github.thibaultbee.streampack.core.internal.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.internal.endpoints.IEndpointInternal @@ -26,25 +27,31 @@ import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.internal.sources.audio.MicrophoneSource import io.github.thibaultbee.streampack.core.internal.sources.video.camera.CameraSource import io.github.thibaultbee.streampack.core.internal.sources.video.camera.ICameraSource +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.deviceRotation import io.github.thibaultbee.streampack.core.streamers.infos.CameraStreamerConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.interfaces.ICameraCoroutineStreamer + /** * A [DefaultStreamer] that sends microphone and camera frames. * * @param context application context * @param enableMicrophone [Boolean.true] to capture audio * @param internalEndpoint the [IEndpointInternal] implementation + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. */ fun DefaultCameraStreamer( context: Context, enableMicrophone: Boolean = true, - internalEndpoint: IEndpointInternal = DynamicEndpoint(context) + internalEndpoint: IEndpointInternal = DynamicEndpoint(context), + @RotationValue defaultRotation: Int = context.deviceRotation ) = DefaultCameraStreamer( context, if (enableMicrophone) MicrophoneSource() else null, - internalEndpoint + internalEndpoint, + defaultRotation ) /** @@ -53,16 +60,19 @@ fun DefaultCameraStreamer( * @param context application context * @param audioSourceInternal the audio source implementation * @param internalEndpoint the [IEndpointInternal] implementation + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. */ open class DefaultCameraStreamer( context: Context, audioSourceInternal: IAudioSourceInternal?, - internalEndpoint: IEndpointInternal = DynamicEndpoint(context) + internalEndpoint: IEndpointInternal = DynamicEndpoint(context), + @RotationValue defaultRotation: Int = context.deviceRotation ) : DefaultStreamer( context = context, audioSourceInternal = audioSourceInternal, videoSourceInternal = CameraSource(context), - endpointInternal = internalEndpoint + endpointInternal = internalEndpoint, + defaultRotation = defaultRotation ), ICameraCoroutineStreamer { private val cameraSource = videoSourceInternal as CameraSource @@ -92,6 +102,7 @@ open class DefaultCameraStreamer( @RequiresPermission(Manifest.permission.CAMERA) set(value) { videoSource.cameraId = value + updateTransformation() } /** @@ -117,6 +128,10 @@ open class DefaultCameraStreamer( return CameraStreamerConfigurationInfo(endpointInfo) } + override fun isMirroringRequired(): Boolean { + return cameraSource.infoProvider.isFrontFacing + } + /** * Sets a preview surface. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultScreenRecorderStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultScreenRecorderStreamer.kt index 95e350527..e43bcc44e 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultScreenRecorderStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultScreenRecorderStreamer.kt @@ -18,6 +18,7 @@ package io.github.thibaultbee.streampack.core.streamers import android.content.Context import android.content.Intent import android.media.projection.MediaProjectionManager +import android.view.Surface import androidx.activity.ComponentActivity import androidx.activity.result.ActivityResult import androidx.core.app.ActivityCompat @@ -27,6 +28,8 @@ import io.github.thibaultbee.streampack.core.internal.sources.IMediaProjectionSo import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.internal.sources.audio.MicrophoneSource import io.github.thibaultbee.streampack.core.internal.sources.video.mediaprojection.MediaProjectionVideoSource +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.deviceRotation /** * A [DefaultStreamer] that sends microphone and screen frames. @@ -34,15 +37,18 @@ import io.github.thibaultbee.streampack.core.internal.sources.video.mediaproject * @param context application context * @param enableMicrophone [Boolean.true] to capture audio * @param internalEndpoint the [IEndpointInternal] implementation + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. */ fun DefaultScreenRecorderStreamer( context: Context, enableMicrophone: Boolean = true, - internalEndpoint: IEndpointInternal = DynamicEndpoint(context) + internalEndpoint: IEndpointInternal = DynamicEndpoint(context), + @RotationValue defaultRotation: Int = context.deviceRotation ) = DefaultScreenRecorderStreamer( context, if (enableMicrophone) MicrophoneSource() else null, - internalEndpoint + internalEndpoint, + defaultRotation ) /** @@ -51,16 +57,19 @@ fun DefaultScreenRecorderStreamer( * @param context application context * @param audioSourceInternal the audio source implementation * @param internalEndpoint the [IEndpointInternal] implementation + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. */ open class DefaultScreenRecorderStreamer( context: Context, audioSourceInternal: IAudioSourceInternal?, - internalEndpoint: IEndpointInternal = DynamicEndpoint(context) + internalEndpoint: IEndpointInternal = DynamicEndpoint(context), + @RotationValue defaultRotation: Int = context.deviceRotation ) : DefaultStreamer( context = context, audioSourceInternal = audioSourceInternal, videoSourceInternal = MediaProjectionVideoSource(context), - endpointInternal = internalEndpoint + endpointInternal = internalEndpoint, + defaultRotation = defaultRotation ) { private val mediaProjectionVideoSource = (videoSourceInternal as MediaProjectionVideoSource).apply { diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultStreamer.kt index 74cecea78..ef3f66865 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/DefaultStreamer.kt @@ -17,12 +17,14 @@ package io.github.thibaultbee.streampack.core.streamers import android.Manifest import android.content.Context +import android.util.Size import android.view.Surface import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.data.AudioConfig import io.github.thibaultbee.streampack.core.data.Config import io.github.thibaultbee.streampack.core.data.VideoConfig import io.github.thibaultbee.streampack.core.data.mediadescriptor.MediaDescriptor +import io.github.thibaultbee.streampack.core.data.rotateFromNaturalOrientation import io.github.thibaultbee.streampack.core.internal.data.Frame import io.github.thibaultbee.streampack.core.internal.encoders.IEncoder import io.github.thibaultbee.streampack.core.internal.encoders.IEncoderInternal @@ -32,11 +34,16 @@ import io.github.thibaultbee.streampack.core.internal.encoders.mediacodec.VideoE import io.github.thibaultbee.streampack.core.internal.endpoints.DynamicEndpoint import io.github.thibaultbee.streampack.core.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.internal.endpoints.IEndpointInternal -import io.github.thibaultbee.streampack.core.internal.gl.CodecSurface +import io.github.thibaultbee.streampack.core.internal.processing.video.SurfaceProcessor +import io.github.thibaultbee.streampack.core.internal.processing.video.outputs.AbstractSurfaceOutput +import io.github.thibaultbee.streampack.core.internal.processing.video.outputs.SurfaceOutput +import io.github.thibaultbee.streampack.core.internal.processing.video.source.ISourceInfoProvider import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSourceInternal import io.github.thibaultbee.streampack.core.internal.sources.video.IVideoSource import io.github.thibaultbee.streampack.core.internal.sources.video.IVideoSourceInternal +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.deviceRotation import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -58,12 +65,14 @@ import java.util.concurrent.Executors * @param videoSourceInternal the video source implementation * @param audioSourceInternal the audio source implementation * @param endpointInternal the [IEndpointInternal] implementation + * @param defaultRotation the default rotation in [Surface] rotation ([Surface.ROTATION_0], ...). By default, it is the current device orientation. */ open class DefaultStreamer( protected val context: Context, protected val audioSourceInternal: IAudioSourceInternal?, protected val videoSourceInternal: IVideoSourceInternal?, - protected val endpointInternal: IEndpointInternal = DynamicEndpoint(context) + protected val endpointInternal: IEndpointInternal = DynamicEndpoint(context), + @RotationValue defaultRotation: Int = context.deviceRotation ) : ICoroutineStreamer { private val dispatcher: CoroutineDispatcher = Executors.newSingleThreadExecutor().asCoroutineDispatcher() @@ -86,43 +95,41 @@ open class DefaultStreamer( override val videoConfig: VideoConfig? get() = _videoConfig - private val sourceOrientationProvider = videoSourceInternal?.orientationProvider + protected val sourceInfoProvider = videoSourceInternal?.infoProvider - private val audioEncoderListener = - object : IEncoderInternal.IListener { - override fun onError(t: Throwable) { - onStreamError(t) - } + private val audioEncoderListener = object : IEncoderInternal.IListener { + override fun onError(t: Throwable) { + onStreamError(t) + } - override fun onOutputFrame(frame: Frame) { - audioStreamId?.let { - runBlocking { - this@DefaultStreamer.endpointInternal.write(frame, it) - } + override fun onOutputFrame(frame: Frame) { + audioStreamId?.let { + runBlocking { + this@DefaultStreamer.endpointInternal.write(frame, it) } } } + } - private val videoEncoderListener = - object : IEncoderInternal.IListener { - override fun onError(t: Throwable) { - onStreamError(t) - } + private val videoEncoderListener = object : IEncoderInternal.IListener { + override fun onError(t: Throwable) { + onStreamError(t) + } - override fun onOutputFrame(frame: Frame) { - videoStreamId?.let { - frame.pts += videoSourceInternal!!.timestampOffset - frame.dts = if (frame.dts != null) { - frame.dts!! + videoSourceInternal.timestampOffset - } else { - null - } - runBlocking { - this@DefaultStreamer.endpointInternal.write(frame, it) - } + override fun onOutputFrame(frame: Frame) { + videoStreamId?.let { + frame.pts += videoSourceInternal!!.timestampOffset + frame.dts = if (frame.dts != null) { + frame.dts!! + videoSourceInternal.timestampOffset + } else { + null + } + runBlocking { + this@DefaultStreamer.endpointInternal.write(frame, it) } } } + } /** * Manages error on stream. @@ -179,9 +186,7 @@ open class DefaultStreamer( override val videoEncoder: IEncoder? get() = videoEncoderInternal - - private val codecSurface = - if (videoSourceInternal?.hasOutputSurface == true) CodecSurface(sourceOrientationProvider) else null + private var surfaceProcessor: SurfaceProcessor? = null // ENDPOINT @@ -215,6 +220,34 @@ open class DefaultStreamer( override val info: IConfigurationInfo get() = StreamerConfigurationInfo(endpoint.info) + /** + * The target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + @RotationValue + private var _targetRotation = defaultRotation + + /** + * Keep the target rotation if it can't be applied immediately. + * It will be applied when the stream is stopped. + */ + @RotationValue + private var pendingTargetRotation: Int? = null + + /** + * The target rotation in [Surface] rotation ([Surface.ROTATION_0], ...) + */ + override var targetRotation: Int + @RotationValue get() = _targetRotation + set(@RotationValue newTargetRotation) { + if (isStreaming.value) { + Logger.w(TAG, "Can't change rotation while streaming") + pendingTargetRotation = newTargetRotation + return + } + + setTargetRotationInternal(newTargetRotation) + } + /** * Gets configuration information from [MediaDescriptor]. * @@ -259,62 +292,145 @@ open class DefaultStreamer( audioSourceInternal.configure(audioConfig) audioEncoderInternal?.release() - audioEncoderInternal = - MediaCodecEncoder( - AudioEncoderConfig( - audioConfig - ), - listener = audioEncoderListener - ).apply { - if (input is MediaCodecEncoder.ByteBufferInput) { - input.listener = - object : - IEncoderInternal.IByteBufferInput.OnFrameRequestedListener { - override fun onFrameRequested(buffer: ByteBuffer): Frame { - return audioSourceInternal.getFrame(buffer) - } + audioEncoderInternal = MediaCodecEncoder( + AudioEncoderConfig( + audioConfig + ), listener = audioEncoderListener + ).apply { + if (input is MediaCodecEncoder.ByteBufferInput) { + input.listener = + object : IEncoderInternal.IByteBufferInput.OnFrameRequestedListener { + override fun onFrameRequested(buffer: ByteBuffer): Frame { + return audioSourceInternal.getFrame(buffer) } - } else { - throw UnsupportedOperationException("Audio encoder only support ByteBuffer mode") - } - configure() + } + } else { + throw UnsupportedOperationException("Audio encoder only support ByteBuffer mode") } + configure() + } } catch (t: Throwable) { release() throw t } } - private fun buildVideoEncoder( - videoConfig: VideoConfig, - videoSource: IVideoSourceInternal - ): MediaCodecEncoder { + /** + * Creates a surface output for the given surface. + * + * Use it for additional processing. + * + * @param surface the encoder surface + * @param resolution the resolution of the surface + * @param infoProvider the source info provider for internal processing + */ + protected open fun buildSurfaceOutput( + surface: Surface, resolution: Size, infoProvider: ISourceInfoProvider + ): AbstractSurfaceOutput { + return SurfaceOutput( + surface, resolution, SurfaceOutput.TransformationInfo( + targetRotation, isMirroringRequired(), infoProvider + ) + ) + } + + /** + * Whether the output surface needs to be mirrored. + */ + protected open fun isMirroringRequired(): Boolean { + return false + } + + /** + * Updates the transformation of the surface output. + * To be called when the source info provider or [isMirroringRequired] is updated. + */ + protected fun updateTransformation() { + val sourceInfoProvider = requireNotNull(sourceInfoProvider) { + "Source info provider must not be null" + } + val videoConfig = requireNotNull(videoConfig) { "Video config must not be null" } + + val videoEncoder = requireNotNull(videoEncoderInternal) { "Video encoder must not be null" } + val input = videoEncoder.input as MediaCodecEncoder.SurfaceInput + + val surface = requireNotNull(input.surface) { "Surface must not be null" } + updateTransformation(surface, videoConfig.resolution, sourceInfoProvider) + } + + /** + * Updates the transformation of the surface output. + */ + protected open fun updateTransformation( + surface: Surface, resolution: Size, infoProvider: ISourceInfoProvider + ) { + Logger.i(TAG, "Updating transformation") + surfaceProcessor?.removeOutputSurface(surface) + surfaceProcessor?.addOutputSurface( + buildSurfaceOutput( + surface, resolution, infoProvider + ) + ) + } + + private fun buildOrUpdateSurfaceProcessor( + videoConfig: VideoConfig, videoSource: IVideoSourceInternal + ): SurfaceProcessor { + val surfaceProcessorLocal = surfaceProcessor + return if (surfaceProcessorLocal == null) { + SurfaceProcessor(videoConfig.dynamicRangeProfile) + } else { + videoSource.outputSurface?.let { + surfaceProcessorLocal.removeInputSurface(it) + } + if (surfaceProcessorLocal.dynamicRangeProfile.isHdr != videoConfig.isHdr) { + surfaceProcessorLocal.removeAllOutputSurfaces() + surfaceProcessorLocal.release() + SurfaceProcessor(videoConfig.dynamicRangeProfile) + } else { + surfaceProcessorLocal + } + }.apply { + videoSource.outputSurface = createInputSurface( + videoSource.infoProvider.getSurfaceSize( + videoConfig.resolution, targetRotation + ) + ) + } + } + + private fun buildAndConfigureVideoEncoder( + videoConfig: VideoConfig, videoSource: IVideoSourceInternal + ): IEncoderInternal { val videoEncoder = MediaCodecEncoder( VideoEncoderConfig( - videoConfig, - videoSource.hasOutputSurface, - videoSource.orientationProvider + videoConfig, videoSource.hasOutputSurface ), listener = videoEncoderListener ) when (videoEncoder.input) { is MediaCodecEncoder.SurfaceInput -> { - codecSurface!!.useHighBitDepth = videoConfig.isHdr + surfaceProcessor = buildOrUpdateSurfaceProcessor(videoConfig, videoSource) + videoEncoder.input.listener = - object : - IEncoderInternal.ISurfaceInput.OnSurfaceUpdateListener { + object : IEncoderInternal.ISurfaceInput.OnSurfaceUpdateListener { override fun onSurfaceUpdated(surface: Surface) { + val surfaceProcessor = requireNotNull(surfaceProcessor) { + "Surface processor must not be null" + } Logger.d(TAG, "Updating with new encoder surface input") - codecSurface.outputSurface = surface - videoSource.outputSurface = codecSurface.input + surfaceProcessor.addOutputSurface( + buildSurfaceOutput( + surface, videoConfig.resolution, videoSource.infoProvider + ) + ) } } } is MediaCodecEncoder.ByteBufferInput -> { videoEncoder.input.listener = - object : - IEncoderInternal.IByteBufferInput.OnFrameRequestedListener { + object : IEncoderInternal.IByteBufferInput.OnFrameRequestedListener { override fun onFrameRequested(buffer: ByteBuffer): Frame { return videoSource.getFrame(buffer) } @@ -325,9 +441,34 @@ open class DefaultStreamer( throw UnsupportedOperationException("Unknown input type") } } + + videoEncoder.configure() + return videoEncoder } + private fun buildAndConfigureVideoEncoderIfNeeded( + videoConfig: VideoConfig, + videoSource: IVideoSourceInternal, + @RotationValue targetRotation: Int + ): IEncoderInternal { + val rotatedVideoConfig = videoConfig.rotateFromNaturalOrientation(context, targetRotation) + + // Release codec instance + videoEncoderInternal?.let { encoder -> + val input = encoder.input + if (input is MediaCodecEncoder.SurfaceInput) { + input.surface?.let { surface -> + surfaceProcessor?.removeOutputSurface(surface) + } + } + encoder.release() + } + + // Prepare new codec instance + return buildAndConfigureVideoEncoder(rotatedVideoConfig, videoSource) + } + /** * Configures video settings. * It is the first method to call after a [DefaultStreamer] instantiation. @@ -356,10 +497,9 @@ open class DefaultStreamer( try { videoSourceInternal.configure(videoConfig) - videoEncoderInternal?.release() - videoEncoderInternal = buildVideoEncoder(videoConfig, videoSourceInternal).apply { - configure() - } + videoEncoderInternal = buildAndConfigureVideoEncoderIfNeeded( + videoConfig, videoSourceInternal, targetRotation + ) } catch (t: Throwable) { release() throw t @@ -403,10 +543,8 @@ open class DefaultStreamer( * If sourceOrientationProvider is not null, we need to get oriented size. * For example, the [FlvMuxer] `onMetaData` event needs to know the oriented size. */ - if (sourceOrientationProvider != null) { - val orientedSize = - sourceOrientationProvider.getOrientedSize(videoConfig.resolution) - videoConfig.copy(resolution = orientedSize) + if (sourceInfoProvider != null) { + videoConfig.rotateFromNaturalOrientation(context, targetRotation) } else { videoConfig } @@ -432,7 +570,6 @@ open class DefaultStreamer( audioEncoderInternal?.startStream() videoSourceInternal?.startStream() - codecSurface?.startStream() videoEncoderInternal?.startStream() bitrateRegulatorController?.start() @@ -456,6 +593,19 @@ open class DefaultStreamer( stopStreamInternal() } + private fun resetVideoEncoder() { + val previousVideoEncoder = videoEncoderInternal + pendingTargetRotation?.let { + setTargetRotationInternal(it) + } + pendingTargetRotation = null + + // Only reset if the encoder is the same. Otherwise, it is already configured. + if (previousVideoEncoder == videoEncoderInternal) { + videoEncoderInternal?.reset() + } + } + /** * Stops audio/video and reset stream implementation. * @@ -465,7 +615,7 @@ open class DefaultStreamer( stopStreamImpl() audioEncoderInternal?.reset() - videoEncoderInternal?.reset() + resetVideoEncoder() _isStreaming.emit(false) } @@ -481,7 +631,6 @@ open class DefaultStreamer( // Sources audioSourceInternal?.stopStream() videoSourceInternal?.stopStream() - codecSurface?.stopStream() // Encoders try { @@ -509,7 +658,8 @@ open class DefaultStreamer( // Sources audioSourceInternal?.release() videoSourceInternal?.release() - codecSurface?.release() + surfaceProcessor?.removeAllOutputSurfaces() + surfaceProcessor?.release() // Encoders audioEncoderInternal?.release() @@ -528,13 +678,12 @@ open class DefaultStreamer( */ override fun addBitrateRegulatorController(controllerFactory: IBitrateRegulatorController.Factory) { bitrateRegulatorController?.stop() - bitrateRegulatorController = - controllerFactory.newBitrateRegulatorController(this).apply { - if (isStreaming.value) { - this.start() - } - Logger.d(TAG, "Bitrate regulator controller added: ${this.javaClass.simpleName}") + bitrateRegulatorController = controllerFactory.newBitrateRegulatorController(this).apply { + if (isStreaming.value) { + this.start() } + Logger.d(TAG, "Bitrate regulator controller added: ${this.javaClass.simpleName}") + } } @@ -547,7 +696,37 @@ open class DefaultStreamer( Logger.d(TAG, "Bitrate regulator controller removed") } + private fun setTargetRotationInternal(@RotationValue newTargetRotation: Int) { + if (shouldUpdateRotation(newTargetRotation)) { + sendTransformation() + } + } + + private fun sendTransformation() { + if (hasVideo) { + val videoConfig = videoConfig + if (videoConfig != null) { + videoSourceInternal?.configure(videoConfig) + videoEncoderInternal = buildAndConfigureVideoEncoderIfNeeded( + videoConfig, requireNotNull(videoSourceInternal), targetRotation + ) + } + } + } + + /** + * @return true if the target rotation has changed + */ + private fun shouldUpdateRotation(@RotationValue newTargetRotation: Int): Boolean { + return if (targetRotation != newTargetRotation) { + _targetRotation = newTargetRotation + true + } else { + false + } + } + companion object { - private const val TAG = "DefaultStreamer" + const val TAG = "DefaultStreamer" } } \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/callbacks/DefaultCallbackStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/callbacks/DefaultCallbackStreamer.kt index 93c65001c..7edc24510 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/callbacks/DefaultCallbackStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/callbacks/DefaultCallbackStreamer.kt @@ -9,6 +9,7 @@ import io.github.thibaultbee.streampack.core.internal.encoders.IEncoder import io.github.thibaultbee.streampack.core.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.internal.sources.video.IVideoSource +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo import io.github.thibaultbee.streampack.core.streamers.interfaces.ICallbackStreamer @@ -59,6 +60,13 @@ open class DefaultCallbackStreamer(val streamer: ICoroutineStreamer) : ICallback override val isStreaming: Boolean get() = streamer.isStreaming.value + override var targetRotation: Int + @RotationValue + get() = streamer.targetRotation + set(@RotationValue value) { + streamer.targetRotation = value + } + init { coroutineScope.launch { streamer.throwable.filterNotNull().filter { !it.isClosedException }.collect { e -> diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/interfaces/IStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/interfaces/IStreamer.kt index 3b2f63c1b..24ef73efc 100644 --- a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/interfaces/IStreamer.kt +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/interfaces/IStreamer.kt @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.core.streamers.interfaces import android.Manifest import android.net.Uri +import androidx.annotation.IntRange import androidx.annotation.RequiresPermission import io.github.thibaultbee.streampack.core.data.AudioConfig import io.github.thibaultbee.streampack.core.data.VideoConfig @@ -26,6 +27,8 @@ import io.github.thibaultbee.streampack.core.internal.encoders.IEncoder import io.github.thibaultbee.streampack.core.internal.endpoints.IEndpoint import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSource import io.github.thibaultbee.streampack.core.internal.sources.video.IVideoSource +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotationToDegrees import io.github.thibaultbee.streampack.core.regulator.controllers.IBitrateRegulatorController import io.github.thibaultbee.streampack.core.streamers.DefaultStreamer import io.github.thibaultbee.streampack.core.streamers.infos.IConfigurationInfo @@ -79,6 +82,12 @@ interface IStreamer { */ val info: IConfigurationInfo + /** + * The rotation in one the [Surface] rotations from the device natural orientation. + */ + @RotationValue + var targetRotation: Int + /** * Gets configuration information */ @@ -145,6 +154,13 @@ interface IStreamer { fun removeBitrateRegulatorController() } +/** + * Returns the rotation in degrees from [Int] rotation. + */ +val IStreamer.targetRotationDegrees: Int + @IntRange(from = 0, to = 359) + get() = targetRotation.rotationToDegrees + /** * A Streamer based on coroutines. */ diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/DeviceRotationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/DeviceRotationProvider.kt new file mode 100644 index 000000000..d35f75448 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/DeviceRotationProvider.kt @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.streamers.orientation + +import android.content.Context +import android.view.OrientationEventListener +import android.view.Surface +import io.github.thibaultbee.streampack.core.internal.utils.extensions.deviceRotation +import io.github.thibaultbee.streampack.core.utils.extensions.clamp90 + + +class DeviceRotationProvider(val context: Context) : RotationProvider() { + private val lock = Any() + private var _rotation = context.deviceRotation + + private val eventListener by lazy { + object : OrientationEventListener(context) { + override fun onOrientationChanged(orientation: Int) { + if (orientation == ORIENTATION_UNKNOWN) { + return + } + + val newRotation = orientationToSurfaceRotation(orientation.clamp90) + + if (_rotation != newRotation) { + _rotation = newRotation + + synchronized(lock) { + listeners.forEach { it.onOrientationChanged(newRotation) } + } + } + } + } + } + + override val rotation: Int + get() = _rotation + + override fun addListener(listener: IRotationProvider.Listener) { + synchronized(lock) { + super.addListener(listener) + eventListener.enable() + } + } + + override fun removeListener(listener: IRotationProvider.Listener) { + synchronized(lock) { + super.removeListener(listener) + if (listeners.isEmpty()) { + eventListener.disable() + } + } + } + + companion object { + /** + * Converts orientation degrees to [Surface] rotation. + */ + fun orientationToSurfaceRotation(rotationDegrees: Int): Int { + return if (rotationDegrees >= 315 || rotationDegrees < 45) { + Surface.ROTATION_0 + } else if (rotationDegrees >= 225) { + Surface.ROTATION_90 + } else if (rotationDegrees >= 135) { + Surface.ROTATION_180 + } else { + Surface.ROTATION_270 + } + } + + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/RotationProvider.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/RotationProvider.kt new file mode 100644 index 000000000..50f917fe0 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/streamers/orientation/RotationProvider.kt @@ -0,0 +1,39 @@ +package io.github.thibaultbee.streampack.core.streamers.orientation + +import androidx.annotation.IntRange +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue +import io.github.thibaultbee.streampack.core.internal.utils.extensions.rotationToDegrees + +val IRotationProvider.rotationDegrees: Int + @IntRange(from = 0, to = 359) + get() = rotation.rotationToDegrees + +interface IRotationProvider { + /** + * The rotation in one the [Surface] rotations from the device natural orientation. + */ + @get:RotationValue + val rotation: Int + + fun addListener(listener: Listener) + fun removeListener(listener: Listener) + + interface Listener { + /** + * @param rotation The rotation in one the [Surface] rotations from the device natural orientation. + */ + fun onOrientationChanged(@RotationValue rotation: Int) + } +} + +abstract class RotationProvider : IRotationProvider { + protected val listeners = mutableSetOf() + + override fun addListener(listener: IRotationProvider.Listener) { + listeners.add(listener) + } + + override fun removeListener(listener: IRotationProvider.Listener) { + listeners.remove(listener) + } +} \ No newline at end of file diff --git a/core/src/main/java/io/github/thibaultbee/streampack/core/utils/extensions/IntExtensions.kt b/core/src/main/java/io/github/thibaultbee/streampack/core/utils/extensions/IntExtensions.kt new file mode 100644 index 000000000..c015d7494 --- /dev/null +++ b/core/src/main/java/io/github/thibaultbee/streampack/core/utils/extensions/IntExtensions.kt @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thibault B. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.github.thibaultbee.streampack.core.utils.extensions + +/** + * Whether the integer is a multiple of 90. + */ +internal val Int.is90Multiple: Boolean + get() = this % 90 == 0 + +/** + * Clamps the integer to the nearest multiple of 90. + */ +internal val Int.clamp90: Int + get() = (this + 45) / 90 * 90 + +/** + * Converts the integer to a value within 360 degrees. + */ +internal val Int.within360: Int + get() { + return (this % 360 + 360) % 360 + } + diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt index 822be3870..064527127 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/ui/main/PreviewViewModel.kt @@ -32,6 +32,7 @@ import io.github.thibaultbee.streampack.app.utils.ObservableViewModel import io.github.thibaultbee.streampack.app.utils.StreamerManager import io.github.thibaultbee.streampack.app.utils.isEmpty import io.github.thibaultbee.streampack.core.streamers.observers.StreamerLifeCycleObserver +import io.github.thibaultbee.streampack.core.streamers.orientation.DeviceRotationProvider import io.github.thibaultbee.streampack.core.utils.extensions.isClosedException import io.github.thibaultbee.streampack.core.utils.extensions.isFrameRateSupported import io.github.thibaultbee.streampack.ui.views.PreviewView @@ -46,6 +47,8 @@ class PreviewViewModel(application: Application) : ObservableViewModel() { Configuration(application) ) + private val rotationProvider = DeviceRotationProvider(application) + val streamerLifeCycleObserver: StreamerLifeCycleObserver get() = streamerManager.streamerLifeCycleObserver @@ -80,6 +83,7 @@ class PreviewViewModel(application: Application) : ObservableViewModel() { Log.i(TAG, "Streamer is streaming: $it") } } + rotationProvider.addListener(streamerManager) } fun inflateStreamerView(view: PreviewView) { @@ -284,6 +288,7 @@ class PreviewViewModel(application: Application) : ObservableViewModel() { override fun onCleared() { super.onCleared() try { + rotationProvider.removeListener(streamerManager) streamerManager.release() } catch (t: Throwable) { Log.e(TAG, "streamer.release failed", t) diff --git a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt index ad36f2001..1b6980cc6 100644 --- a/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt +++ b/demos/camera/src/main/java/io/github/thibaultbee/streampack/app/utils/StreamerManager.kt @@ -31,11 +31,13 @@ import io.github.thibaultbee.streampack.core.data.VideoConfig import io.github.thibaultbee.streampack.core.data.mediadescriptor.UriMediaDescriptor import io.github.thibaultbee.streampack.core.internal.endpoints.composites.muxers.ts.data.TSServiceInfo import io.github.thibaultbee.streampack.core.internal.sources.video.camera.CameraSettings +import io.github.thibaultbee.streampack.core.internal.utils.RotationValue import io.github.thibaultbee.streampack.core.streamers.DefaultCameraStreamer import io.github.thibaultbee.streampack.core.streamers.interfaces.ICameraStreamer import io.github.thibaultbee.streampack.core.streamers.interfaces.ICoroutineStreamer import io.github.thibaultbee.streampack.core.streamers.interfaces.startStream import io.github.thibaultbee.streampack.core.streamers.observers.StreamerLifeCycleObserver +import io.github.thibaultbee.streampack.core.streamers.orientation.IRotationProvider import io.github.thibaultbee.streampack.core.utils.extensions.backCameras import io.github.thibaultbee.streampack.core.utils.extensions.frontCameras import io.github.thibaultbee.streampack.core.utils.extensions.isBackCamera @@ -48,7 +50,7 @@ import kotlinx.coroutines.flow.StateFlow class StreamerManager( private val context: Context, private val configuration: Configuration -) { +) : IRotationProvider.Listener { private val streamer: ICoroutineStreamer = DefaultCameraStreamer(context, configuration.audio.enable) @@ -92,6 +94,15 @@ class StreamerManager( Range(configuration.audio.bitrate, configuration.audio.bitrate) ) + /** + * Updates rotation value in streamer + * + * @param rotation the rotation in one the [Surface] rotations. + */ + override fun onOrientationChanged(@RotationValue rotation: Int) { + streamer.targetRotation = rotation + } + @RequiresPermission(Manifest.permission.RECORD_AUDIO) fun configureStreamer() { val videoConfig = VideoConfig( diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c79fde75f..0f2e0223e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,6 @@ [versions] agp = "8.7.2" +concurrentFutures = "1.2.0" guava = "33.3.1-android" videoApiClient = "1.5.7" androidxActivity = "1.9.3" @@ -25,6 +26,7 @@ robolectric = "4.14-beta-1" rtmpdroid = "1.2.1" srtdroid = "1.8.4" junitKtx = "1.2.1" +window = "1.3.0" [libraries] guava = { module = "com.google.guava:guava", version.ref = "guava" } @@ -34,6 +36,7 @@ android-material = { module = "com.google.android.material:material", version.re androidx-activity = { module = "androidx.activity:activity", version.ref = "androidxActivity" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidxAppcompat" } androidx-camera-viewfinder-view = { module = "androidx.camera.viewfinder:viewfinder-view", version.ref = "androidxCamera" } +androidx-concurrent-futures = { module = "androidx.concurrent:concurrent-futures", version.ref = "concurrentFutures" } androidx-constraintlayout = { module = "androidx.constraintlayout:constraintlayout", version.ref = "androidxConstraintlayout" } androidx-core-ktx = { module = "androidx.core:core-ktx", version.ref = "androidxCore" } androidx-databinding-common = { module = "androidx.databinding:databinding-common", version.ref = "androidxDatabinding" } @@ -56,6 +59,7 @@ mockk = { module = "io.mockk:mockk", version.ref = "mockk" } robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectric" } rtmpdroid = { module = "video.api:rtmpdroid", version.ref = "rtmpdroid" } srtdroid-ktx = { module = "io.github.thibaultbee.srtdroid:srtdroid-ktx", version.ref = "srtdroid" } +androidx-window = { group = "androidx.window", name = "window", version.ref = "window" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/services/src/main/java/io/github/thibaultbee/streampack/services/DefaultScreenRecorderService.kt b/services/src/main/java/io/github/thibaultbee/streampack/services/DefaultScreenRecorderService.kt index 1519ef382..250a9a749 100644 --- a/services/src/main/java/io/github/thibaultbee/streampack/services/DefaultScreenRecorderService.kt +++ b/services/src/main/java/io/github/thibaultbee/streampack/services/DefaultScreenRecorderService.kt @@ -37,6 +37,8 @@ import androidx.lifecycle.lifecycleScope import io.github.thibaultbee.streampack.core.internal.utils.extensions.rootCause import io.github.thibaultbee.streampack.core.logger.Logger import io.github.thibaultbee.streampack.core.streamers.DefaultScreenRecorderStreamer +import io.github.thibaultbee.streampack.core.streamers.orientation.DeviceRotationProvider +import io.github.thibaultbee.streampack.core.streamers.orientation.IRotationProvider import io.github.thibaultbee.streampack.services.utils.NotificationUtils import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.launch @@ -72,8 +74,12 @@ abstract class DefaultScreenRecorderService( @StringRes protected val channelNameResourceId: Int = R.string.default_channel_name, @StringRes protected val channelDescriptionResourceId: Int = 0, @DrawableRes protected val notificationIconResourceId: Int = R.drawable.ic_baseline_linked_camera_24 -) : LifecycleService() { +) : LifecycleService(), IRotationProvider.Listener { protected var streamer: DefaultScreenRecorderStreamer? = null + private set + + protected open val rotationProvider: IRotationProvider by lazy { DeviceRotationProvider(this) } + private val binder = ScreenRecorderServiceBinder() private val notificationUtils: NotificationUtils by lazy { NotificationUtils(this, channelId, notificationId) @@ -81,10 +87,14 @@ abstract class DefaultScreenRecorderService( override fun onCreate() { super.onCreate() + + rotationProvider.addListener(this) + notificationUtils.createNotificationChannel( channelNameResourceId, channelDescriptionResourceId ) + ServiceCompat.startForeground( this, notificationId, @@ -162,10 +172,16 @@ abstract class DefaultScreenRecorderService( ) } + override fun onOrientationChanged(rotation: Int) { + streamer?.targetRotation = rotation + } + override fun onDestroy() { super.onDestroy() notificationUtils.cancel() + rotationProvider.removeListener(this) + runBlocking { streamer?.stopStream() streamer?.close() diff --git a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt b/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt index 8bc765e70..ae8517868 100644 --- a/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt +++ b/ui/src/main/java/io/github/thibaultbee/streampack/ui/views/PreviewView.kt @@ -207,7 +207,7 @@ class PreviewView @JvmOverloads constructor( it.videoSource.settings.focusMetering.onTap( PointF(x, y), Rect(this.x.toInt(), this.y.toInt(), width, height), - OrientationUtils.getSurfaceOrientationDegrees(display.rotation) + OrientationUtils.getSurfaceRotationDegrees(display.rotation) ) } catch (t: Throwable) { Logger.e(TAG, "Failed to focus at $x, $y", t)