diff --git a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt index 9073f899f..eb792dfb3 100644 --- a/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt +++ b/app/src/main/java/com/pedro/streamer/rotation/CameraFragment.kt @@ -142,12 +142,12 @@ class CameraFragment: Fragment(), ConnectChecker { if (!folder.exists()) folder.mkdir() val sdf = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()) recordPath = "${folder.absolutePath}/${sdf.format(Date())}.mp4" + bRecord.setImageResource(R.drawable.pause_icon) genericStream.startRecord(recordPath) { status -> if (status == RecordController.Status.RECORDING) { bRecord.setImageResource(R.drawable.stop_icon) } } - bRecord.setImageResource(R.drawable.pause_icon) } else { genericStream.stopRecord() bRecord.setImageResource(R.drawable.record_icon) diff --git a/common/src/main/java/com/pedro/common/Extensions.kt b/common/src/main/java/com/pedro/common/Extensions.kt index c972a6976..5d549aceb 100644 --- a/common/src/main/java/com/pedro/common/Extensions.kt +++ b/common/src/main/java/com/pedro/common/Extensions.kt @@ -72,7 +72,7 @@ fun ByteBuffer.removeInfo(info: MediaFrame.Info): ByteBuffer { try { position(info.offset) limit(info.size) - } catch (ignored: Exception) { } + } catch (_: Exception) { } return slice() } diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index 2552be7b1..ab5351246 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -325,15 +325,15 @@ private boolean sendSPSandPPS(MediaFormat mediaFormat) { ByteBuffer bufferInfo = mediaFormat.getByteBuffer("csd-0"); //we need an av1ConfigurationRecord with sequenceObu to work if (bufferInfo != null && bufferInfo.remaining() > 4) { - oldSps = bufferInfo; - getVideoData.onVideoInfo(bufferInfo, null, null); + oldSps = bufferInfo.duplicate(); + getVideoData.onVideoInfo(oldSps, null, null); return true; } //H265 } else if (type.equals(CodecUtil.H265_MIME)) { ByteBuffer bufferInfo = mediaFormat.getByteBuffer("csd-0"); if (bufferInfo != null) { - List byteBufferList = extractVpsSpsPpsFromH265(bufferInfo); + List byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(bufferInfo.duplicate()); oldSps = byteBufferList.get(1); oldPps = byteBufferList.get(2); oldVps = byteBufferList.get(0); @@ -345,8 +345,8 @@ private boolean sendSPSandPPS(MediaFormat mediaFormat) { ByteBuffer sps = mediaFormat.getByteBuffer("csd-0"); ByteBuffer pps = mediaFormat.getByteBuffer("csd-1"); if (sps != null && pps != null) { - oldSps = sps; - oldPps = pps; + oldSps = sps.duplicate(); + oldPps = pps.duplicate(); oldVps = null; getVideoData.onVideoInfo(oldSps, oldPps, oldVps); return true; @@ -393,108 +393,6 @@ protected MediaCodecInfo chooseEncoder(String mime) { return null; } - /** - * decode sps and pps if the encoder never call to MediaCodec.INFO_OUTPUT_FORMAT_CHANGED - */ - private Pair decodeSpsPpsFromBuffer(ByteBuffer outputBuffer, int length) { - byte[] csd = new byte[length]; - outputBuffer.get(csd, 0, length); - outputBuffer.rewind(); - int i = 0; - int spsIndex = -1; - int ppsIndex = -1; - while (i < length - 4) { - if (csd[i] == 0 && csd[i + 1] == 0 && csd[i + 2] == 0 && csd[i + 3] == 1) { - if (spsIndex == -1) { - spsIndex = i; - } else { - ppsIndex = i; - break; - } - } - i++; - } - if (spsIndex != -1 && ppsIndex != -1) { - byte[] sps = new byte[ppsIndex]; - System.arraycopy(csd, spsIndex, sps, 0, ppsIndex); - byte[] pps = new byte[length - ppsIndex]; - System.arraycopy(csd, ppsIndex, pps, 0, length - ppsIndex); - return new Pair<>(ByteBuffer.wrap(sps), ByteBuffer.wrap(pps)); - } - return null; - } - - /** - * You need find 0 0 0 1 byte sequence that is the initiation of vps, sps and pps - * buffers. - * - * @param csd0byteBuffer get in mediacodec case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED - * @return list with vps, sps and pps - */ - private List extractVpsSpsPpsFromH265(ByteBuffer csd0byteBuffer) { - List byteBufferList = new ArrayList<>(); - int vpsPosition = -1; - int spsPosition = -1; - int ppsPosition = -1; - int contBufferInitiation = 0; - int length = csd0byteBuffer.remaining(); - byte[] csdArray = new byte[length]; - csd0byteBuffer.get(csdArray, 0, length); - csd0byteBuffer.rewind(); - for (int i = 0; i < csdArray.length; i++) { - if (contBufferInitiation == 3 && csdArray[i] == 1) { - if (vpsPosition == -1) { - vpsPosition = i - 3; - } else if (spsPosition == -1) { - spsPosition = i - 3; - } else { - ppsPosition = i - 3; - } - } - if (csdArray[i] == 0) { - contBufferInitiation++; - } else { - contBufferInitiation = 0; - } - } - byte[] vps = new byte[spsPosition]; - byte[] sps = new byte[ppsPosition - spsPosition]; - byte[] pps = new byte[csdArray.length - ppsPosition]; - for (int i = 0; i < csdArray.length; i++) { - if (i < spsPosition) { - vps[i] = csdArray[i]; - } else if (i < ppsPosition) { - sps[i - spsPosition] = csdArray[i]; - } else { - pps[i - ppsPosition] = csdArray[i]; - } - } - byteBufferList.add(ByteBuffer.wrap(vps)); - byteBufferList.add(ByteBuffer.wrap(sps)); - byteBufferList.add(ByteBuffer.wrap(pps)); - return byteBufferList; - } - - /** - * - * @param buffer key frame - * @return av1 ObuSequence - */ - private ByteBuffer extractObuSequence(ByteBuffer buffer, MediaCodec.BufferInfo bufferInfo) { - //we can only extract info from keyframes - if (bufferInfo.flags != MediaCodec.BUFFER_FLAG_KEY_FRAME) return null; - byte[] av1Data = new byte[buffer.remaining()]; - buffer.get(av1Data); - Av1Parser av1Parser = new Av1Parser(); - List obuList = av1Parser.getObus(av1Data); - for (Obu obu: obuList) { - if (av1Parser.getObuType(obu.getHeader()[0]) == ObuType.SEQUENCE_HEADER) { - return ByteBuffer.wrap(obu.getFullData()); - } - } - return null; - } - @Override protected Frame getInputFrame() throws InterruptedException { Frame frame = queue.take(); @@ -534,7 +432,7 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.B fixTimeStamp(bufferInfo); if (!spsPpsSetted && type.equals(CodecUtil.H264_MIME)) { Log.i(TAG, "formatChanged not called, doing manual sps/pps extraction..."); - Pair buffers = decodeSpsPpsFromBuffer(byteBuffer.duplicate(), bufferInfo.size); + Pair buffers = VideoEncoderHelper.decodeSpsPpsFromBuffer(byteBuffer.duplicate(), bufferInfo.size); if (buffers != null) { Log.i(TAG, "manual sps/pps extraction success"); oldSps = buffers.first; @@ -547,7 +445,7 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.B } } else if (!spsPpsSetted && type.equals(CodecUtil.H265_MIME)) { Log.i(TAG, "formatChanged not called, doing manual vps/sps/pps extraction..."); - List byteBufferList = extractVpsSpsPpsFromH265(byteBuffer.duplicate()); + List byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(byteBuffer.duplicate()); if (byteBufferList.size() == 3) { Log.i(TAG, "manual vps/sps/pps extraction success"); oldSps = byteBufferList.get(1); @@ -560,7 +458,7 @@ protected void checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.B } } else if (!spsPpsSetted && type.equals(CodecUtil.AV1_MIME)) { Log.i(TAG, "formatChanged not called, doing manual av1 extraction..."); - ByteBuffer obuSequence = extractObuSequence(byteBuffer.duplicate(), bufferInfo); + ByteBuffer obuSequence = VideoEncoderHelper.extractObuSequence(byteBuffer.duplicate(), bufferInfo); if (obuSequence != null) { oldSps = obuSequence; getVideoData.onVideoInfo(obuSequence, null, null); diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoderHelper.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoderHelper.java new file mode 100644 index 000000000..9aeb23163 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoderHelper.java @@ -0,0 +1,138 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.video; + +import android.media.MediaCodec; +import android.util.Pair; + +import com.pedro.common.av1.Av1Parser; +import com.pedro.common.av1.Obu; +import com.pedro.common.av1.ObuType; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Created by pedro on 3/12/25. + */ +public class VideoEncoderHelper { + /** + * decode sps and pps if the encoder never call to MediaCodec.INFO_OUTPUT_FORMAT_CHANGED + */ + public static Pair decodeSpsPpsFromBuffer(ByteBuffer outputBuffer, int length) { + byte[] csd = new byte[length]; + outputBuffer.get(csd, 0, length); + outputBuffer.rewind(); + int i = 0; + int spsIndex = -1; + int ppsIndex = -1; + while (i < length - 4) { + if (csd[i] == 0 && csd[i + 1] == 0 && csd[i + 2] == 0 && csd[i + 3] == 1) { + if (spsIndex == -1) { + spsIndex = i; + } else { + ppsIndex = i; + break; + } + } + i++; + } + if (spsIndex != -1 && ppsIndex != -1) { + byte[] sps = new byte[ppsIndex]; + System.arraycopy(csd, spsIndex, sps, 0, ppsIndex); + byte[] pps = new byte[length - ppsIndex]; + System.arraycopy(csd, ppsIndex, pps, 0, length - ppsIndex); + return new Pair<>(ByteBuffer.wrap(sps), ByteBuffer.wrap(pps)); + } + return null; + } + + /** + * You need find 0 0 0 1 byte sequence that is the initiation of vps, sps and pps + * buffers. + * + * @param csd0byteBuffer get in mediacodec case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED + * @return list with vps, sps and pps + */ + public static List extractVpsSpsPpsFromH265(ByteBuffer csd0byteBuffer) { + List byteBufferList = new ArrayList<>(); + int vpsPosition = -1; + int spsPosition = -1; + int ppsPosition = -1; + int contBufferInitiation = 0; + int length = csd0byteBuffer.remaining(); + byte[] csdArray = new byte[length]; + csd0byteBuffer.get(csdArray, 0, length); + csd0byteBuffer.rewind(); + for (int i = 0; i < csdArray.length; i++) { + if (contBufferInitiation == 3 && csdArray[i] == 1) { + if (vpsPosition == -1) { + vpsPosition = i - 3; + } else if (spsPosition == -1) { + spsPosition = i - 3; + } else { + ppsPosition = i - 3; + } + } + if (csdArray[i] == 0) { + contBufferInitiation++; + } else { + contBufferInitiation = 0; + } + } + if (vpsPosition == -1 || spsPosition == -1 || ppsPosition == -1) return byteBufferList; + byte[] vps = new byte[spsPosition]; + byte[] sps = new byte[ppsPosition - spsPosition]; + byte[] pps = new byte[csdArray.length - ppsPosition]; + for (int i = 0; i < csdArray.length; i++) { + if (i < spsPosition) { + vps[i] = csdArray[i]; + } else if (i < ppsPosition) { + sps[i - spsPosition] = csdArray[i]; + } else { + pps[i - ppsPosition] = csdArray[i]; + } + } + byteBufferList.add(ByteBuffer.wrap(vps)); + byteBufferList.add(ByteBuffer.wrap(sps)); + byteBufferList.add(ByteBuffer.wrap(pps)); + return byteBufferList; + } + + /** + * + * @param buffer key frame + * @return av1 ObuSequence + */ + public static ByteBuffer extractObuSequence(ByteBuffer buffer, MediaCodec.BufferInfo bufferInfo) { + //we can only extract info from keyframes + if (bufferInfo.flags != MediaCodec.BUFFER_FLAG_KEY_FRAME) return null; + byte[] av1Data = new byte[buffer.remaining()]; + buffer.get(av1Data); + Av1Parser av1Parser = new Av1Parser(); + List obuList = av1Parser.getObus(av1Data); + for (Obu obu: obuList) { + if (av1Parser.getObuType(obu.getHeader()[0]) == ObuType.SEQUENCE_HEADER) { + return ByteBuffer.wrap(obu.getFullData()); + } + } + return null; + } +} diff --git a/library/src/main/java/com/pedro/library/base/Camera1Base.java b/library/src/main/java/com/pedro/library/base/Camera1Base.java index 8a7cac34d..d975227b0 100644 --- a/library/src/main/java/com/pedro/library/base/Camera1Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera1Base.java @@ -952,7 +952,10 @@ public RecordController.Status getRecordStatus() { protected abstract void getVideoDataImp(ByteBuffer videoBuffer, MediaCodec.BufferInfo info); public void setRecordController(BaseRecordController recordController) { - if (!isRecording()) this.recordController = recordController; + if (!isRecording()) { + recordController.updateInfo(this.recordController); + this.recordController = recordController; + } } private final GetCameraData getCameraData = frame -> { diff --git a/library/src/main/java/com/pedro/library/base/Camera2Base.java b/library/src/main/java/com/pedro/library/base/Camera2Base.java index 15664e7ca..110b5f281 100644 --- a/library/src/main/java/com/pedro/library/base/Camera2Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera2Base.java @@ -1048,7 +1048,10 @@ public boolean isOnPreview() { protected abstract void getVideoDataImp(ByteBuffer videoBuffer, MediaCodec.BufferInfo info); public void setRecordController(BaseRecordController recordController) { - if (!isRecording()) this.recordController = recordController; + if (!isRecording()) { + recordController.updateInfo(this.recordController); + this.recordController = recordController; + } } private final GetMicrophoneData getMicrophoneData = frame -> { diff --git a/library/src/main/java/com/pedro/library/base/DisplayBase.java b/library/src/main/java/com/pedro/library/base/DisplayBase.java index ef2f39e0c..d30e55491 100644 --- a/library/src/main/java/com/pedro/library/base/DisplayBase.java +++ b/library/src/main/java/com/pedro/library/base/DisplayBase.java @@ -573,7 +573,10 @@ public RecordController.Status getRecordStatus() { protected abstract void getVideoDataImp(ByteBuffer videoBuffer, MediaCodec.BufferInfo info); public void setRecordController(BaseRecordController recordController) { - if (!isRecording()) this.recordController = recordController; + if (!isRecording()) { + recordController.updateInfo(this.recordController); + this.recordController = recordController; + } } private final GetMicrophoneData getMicrophoneData = frame -> { diff --git a/library/src/main/java/com/pedro/library/base/FromFileBase.java b/library/src/main/java/com/pedro/library/base/FromFileBase.java index acd489403..824866b22 100644 --- a/library/src/main/java/com/pedro/library/base/FromFileBase.java +++ b/library/src/main/java/com/pedro/library/base/FromFileBase.java @@ -729,7 +729,10 @@ public void setAudioExtractor(Extractor extractor) { protected abstract void getAudioDataImp(ByteBuffer audioBuffer, MediaCodec.BufferInfo info); public void setRecordController(BaseRecordController recordController) { - if (!isRecording()) this.recordController = recordController; + if (!isRecording()) { + recordController.updateInfo(this.recordController); + this.recordController = recordController; + } } private final GetMicrophoneData getMicrophoneData = frame -> { diff --git a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java index 79e525e80..1886d22fe 100644 --- a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java +++ b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java @@ -284,7 +284,10 @@ public boolean resetAudioEncoder() { protected abstract void getAudioDataImp(ByteBuffer audioBuffer, MediaCodec.BufferInfo info); public void setRecordController(BaseRecordController recordController) { - if (!isRecording()) this.recordController = recordController; + if (!isRecording()) { + recordController.updateInfo(this.recordController); + this.recordController = recordController; + } } private final GetMicrophoneData getMicrophoneData = frame -> { diff --git a/library/src/main/java/com/pedro/library/base/StreamBase.kt b/library/src/main/java/com/pedro/library/base/StreamBase.kt index c9b93d3cd..c8f07762f 100644 --- a/library/src/main/java/com/pedro/library/base/StreamBase.kt +++ b/library/src/main/java/com/pedro/library/base/StreamBase.kt @@ -483,7 +483,10 @@ abstract class StreamBase( * This method allow record in other format or even create your custom implementation and record in a new format. */ fun setRecordController(recordController: BaseRecordController) { - if (!isRecording) this.recordController = recordController + if (!isRecording) { + recordController.updateInfo(this.recordController) + this.recordController = recordController + } } /** @@ -593,8 +596,8 @@ abstract class StreamBase( override fun getVideoData(videoBuffer: ByteBuffer, info: MediaCodec.BufferInfo) { fpsListener.calculateFps() - getVideoDataImp(videoBuffer, info) if (!differentRecordResolution) recordController.recordVideo(videoBuffer, info) + getVideoDataImp(videoBuffer, info) } override fun onVideoFormat(mediaFormat: MediaFormat) { @@ -613,7 +616,6 @@ abstract class StreamBase( } override fun onVideoFormat(mediaFormat: MediaFormat) { - val isOnlyVideo = audioSource is NoAudioSource recordController.setVideoFormat(mediaFormat) } } diff --git a/library/src/main/java/com/pedro/library/base/recording/BaseRecordController.java b/library/src/main/java/com/pedro/library/base/recording/BaseRecordController.java index c31997a7e..5ec6506f2 100644 --- a/library/src/main/java/com/pedro/library/base/recording/BaseRecordController.java +++ b/library/src/main/java/com/pedro/library/base/recording/BaseRecordController.java @@ -43,6 +43,11 @@ public abstract class BaseRecordController implements RecordController { protected RecordTracks tracks = RecordTracks.ALL; protected RequestKeyFrame requestKeyFrame = null; + public void updateInfo(BaseRecordController recordController) { + videoCodec = recordController.videoCodec; + audioCodec = recordController.audioCodec; + } + public void setRequestKeyFrame(RequestKeyFrame requestKeyFrame) { this.requestKeyFrame = requestKeyFrame; } diff --git a/library/src/main/java/com/pedro/library/util/FlvMuxerRecordController.kt b/library/src/main/java/com/pedro/library/util/FlvMuxerRecordController.kt index 8223bf646..79356aeab 100644 --- a/library/src/main/java/com/pedro/library/util/FlvMuxerRecordController.kt +++ b/library/src/main/java/com/pedro/library/util/FlvMuxerRecordController.kt @@ -2,6 +2,7 @@ package com.pedro.library.util import android.media.MediaCodec import android.media.MediaFormat +import android.util.Log import com.pedro.common.AudioCodec import com.pedro.common.BitrateManager import com.pedro.common.VideoCodec @@ -11,6 +12,7 @@ import com.pedro.common.toMediaFrameInfo import com.pedro.common.toUInt24 import com.pedro.common.toUInt32 import com.pedro.common.trySend +import com.pedro.encoder.video.VideoEncoderHelper import com.pedro.library.base.recording.BaseRecordController import com.pedro.library.base.recording.RecordController import com.pedro.library.base.recording.RecordController.RecordTracks @@ -22,8 +24,11 @@ import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.AudioFormat import com.pedro.rtmp.flv.audio.packet.AacPacket import com.pedro.rtmp.flv.audio.packet.G711Packet +import com.pedro.rtmp.flv.audio.packet.OpusPacket import com.pedro.rtmp.flv.video.VideoFormat +import com.pedro.rtmp.flv.video.packet.Av1Packet import com.pedro.rtmp.flv.video.packet.H264Packet +import com.pedro.rtmp.flv.video.packet.H265Packet import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -35,7 +40,6 @@ import kotlinx.coroutines.runInterruptible import java.io.ByteArrayOutputStream import java.io.FileDescriptor import java.io.FileOutputStream -import java.io.IOException import java.io.OutputStream import java.nio.ByteBuffer import java.util.concurrent.LinkedBlockingQueue @@ -43,8 +47,8 @@ import java.util.concurrent.LinkedBlockingQueue class FlvMuxerRecordController: BaseRecordController() { private var outputStream: OutputStream? = null - private var videoPacket = H264Packet() - private var audioPacket: BasePacket = AacPacket() + private var videoPacket: BasePacket? = null + private var audioPacket: BasePacket? = null private val queue = LinkedBlockingQueue(200) private var job: Job? = null //metadata config @@ -53,6 +57,7 @@ class FlvMuxerRecordController: BaseRecordController() { private var fps = 0 private var sampleRate = 0 private var isStereo = true + private var sendInfo = false override fun startRecord(path: String, listener: RecordController.Listener?, tracks: RecordTracks) { this.tracks = tracks @@ -67,14 +72,15 @@ class FlvMuxerRecordController: BaseRecordController() { } private fun start(listener: RecordController.Listener?) { - if (audioCodec == AudioCodec.OPUS) throw IOException("Unsupported AudioCodec: " + audioCodec.name) - if (videoCodec != VideoCodec.H264) throw IOException("Unsupported VideoCodec: " + videoCodec.name) - when (audioCodec) { - AudioCodec.G711 -> audioPacket = G711Packet() - AudioCodec.AAC -> audioPacket = AacPacket().apply { - sendAudioInfo(sampleRate, isStereo) - } - else -> {} + audioPacket = when (audioCodec) { + AudioCodec.G711 -> G711Packet() + AudioCodec.AAC -> AacPacket().apply { sendAudioInfo(sampleRate, isStereo) } + AudioCodec.OPUS -> OpusPacket().apply { sendAudioInfo(sampleRate, isStereo) } + } + videoPacket = when (videoCodec) { + VideoCodec.H264 -> H264Packet() + VideoCodec.H265 -> H265Packet() + VideoCodec.AV1 -> Av1Packet() } this.listener = listener status = RecordController.Status.STARTED @@ -91,19 +97,19 @@ class FlvMuxerRecordController: BaseRecordController() { writeFlvFileMetadata(it) } catch (_: Exception) {} } - status = RecordController.Status.RECORDING + if (tracks == RecordTracks.AUDIO) status = RecordController.Status.RECORDING listener?.onStatusChange(status) job = CoroutineScope(Dispatchers.IO).launch { while (isActive) { val mediaFrame = runInterruptible { queue.take() } when (mediaFrame.type) { MediaFrame.Type.VIDEO -> { - videoPacket.createFlvPacket(mediaFrame) { packet -> + videoPacket?.createFlvPacket(mediaFrame) { packet -> outputStream?.let { writeFlvPacket(it, packet) } } } MediaFrame.Type.AUDIO -> { - audioPacket.createFlvPacket(mediaFrame) { packet -> + audioPacket?.createFlvPacket(mediaFrame) { packet -> outputStream?.let { writeFlvPacket(it, packet) } } } @@ -119,21 +125,26 @@ class FlvMuxerRecordController: BaseRecordController() { pauseTime = 0 startTs = 0 queue.clear() - videoPacket.reset(false) - audioPacket.reset(false) + videoPacket?.reset(false) + audioPacket?.reset(false) try { outputStream?.close() } catch (_: Exception) { } finally { outputStream = null } requestKeyFrame = null + sendInfo = false if (listener != null) listener.onStatusChange(status) } override fun recordVideo(videoBuffer: ByteBuffer, videoInfo: MediaCodec.BufferInfo) { - if (status == RecordController.Status.RECORDING && tracks != RecordTracks.AUDIO) { - val frame = MediaFrame(videoBuffer.clone(), videoInfo.toMediaFrameInfo(), MediaFrame.Type.VIDEO) - queue.trySend(frame) + if (tracks != RecordTracks.AUDIO) { + if (status == RecordController.Status.STARTED) { + getVideoInfo(videoBuffer, videoInfo) + } else if (status == RecordController.Status.RECORDING) { + val frame = MediaFrame(videoBuffer.clone(), videoInfo.toMediaFrameInfo(), MediaFrame.Type.VIDEO) + queue.trySend(frame) + } } } @@ -144,6 +155,61 @@ class FlvMuxerRecordController: BaseRecordController() { } } + private fun getVideoInfo(buffer: ByteBuffer, info: MediaCodec.BufferInfo) { + if (info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME || isKeyFrame(buffer)) { + if (!sendInfo) { + when (videoPacket) { + is H264Packet -> { + val buffers = + VideoEncoderHelper.decodeSpsPpsFromBuffer(buffer.duplicate(), info.size) + if (buffers != null) { + Log.i(TAG, "manual sps/pps extraction success") + val oldSps = buffers.first + val oldPps = buffers.second + (videoPacket as H264Packet).sendVideoInfo(oldSps, oldPps) + sendInfo = true + } else { + Log.e(TAG, "manual sps/pps extraction failed") + } + } + is H265Packet -> { + val byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(buffer.duplicate()) + if (byteBufferList.size == 3) { + Log.i(TAG, "manual vps/sps/pps extraction success") + val oldSps = byteBufferList[1] + val oldPps = byteBufferList[2] + val oldVps = byteBufferList[0] + (videoPacket as H265Packet).sendVideoInfo(oldSps, oldPps, oldVps) + sendInfo = true + } else { + Log.e(TAG, "manual vps/sps/pps extraction failed") + } + } + is Av1Packet -> { + val obuSequence = VideoEncoderHelper.extractObuSequence(buffer.duplicate(), info) + if (obuSequence != null) { + (videoPacket as Av1Packet).sendVideoInfo(obuSequence) + sendInfo = true + } else { + Log.e(TAG, "manual av1 extraction failed") + } + } + else -> { + Log.e(TAG, "Unsupported codec: ${videoPacket?.javaClass?.name ?: "null"}") + } + } + } + if (sendInfo && status == RecordController.Status.STARTED) { + requestKeyFrame = null + status = RecordController.Status.RECORDING + listener?.onStatusChange(status) + } + } else if (requestKeyFrame != null) { + requestKeyFrame.onRequestKeyFrame() + requestKeyFrame = null + } + } + override fun setVideoFormat(videoFormat: MediaFormat) { val width = videoFormat.getInteger(MediaFormat.KEY_WIDTH) val height = videoFormat.getInteger(MediaFormat.KEY_HEIGHT) @@ -151,9 +217,41 @@ class FlvMuxerRecordController: BaseRecordController() { this.width = width this.height = height this.fps = fps - val sps = videoFormat.getByteBuffer("csd-0") - val pps = videoFormat.getByteBuffer("csd-1") - if (sps != null && pps != null) videoPacket.sendVideoInfo(sps, pps) + when (videoPacket) { + is H264Packet -> { + val sps = videoFormat.getByteBuffer("csd-0") + val pps = videoFormat.getByteBuffer("csd-1") + if (sps != null && pps != null) { + (videoPacket as H264Packet).sendVideoInfo(sps.duplicate(), pps.duplicate()) + sendInfo = true + } + } + is H265Packet -> { + val bufferInfo = videoFormat.getByteBuffer("csd-0") + if (bufferInfo != null) { + val byteBufferList = VideoEncoderHelper.extractVpsSpsPpsFromH265(bufferInfo.duplicate()) + if (byteBufferList.size == 3) { + val sps = byteBufferList[1] + val pps = byteBufferList[2] + val vps = byteBufferList[0] + (videoPacket as H265Packet).sendVideoInfo(sps, pps, vps) + sendInfo = true + } + } + } + is Av1Packet -> { + val bufferInfo = videoFormat.getByteBuffer("csd-0") + if (bufferInfo != null && bufferInfo.remaining() > 4) { + (videoPacket as Av1Packet).sendVideoInfo(bufferInfo.duplicate()) + sendInfo = true + } + } + } + if (sendInfo && status == RecordController.Status.STARTED) { + requestKeyFrame = null + status = RecordController.Status.RECORDING + listener?.onStatusChange(status) + } } override fun setAudioFormat(audioFormat: MediaFormat) { @@ -194,7 +292,7 @@ class FlvMuxerRecordController: BaseRecordController() { val audioCodecValue = when (audioCodec) { AudioCodec.AAC -> AudioFormat.AAC.value AudioCodec.G711 -> AudioFormat.G711_A.value - else -> throw IllegalArgumentException("unsupported null codec") + AudioCodec.OPUS -> AudioFormat.OPUS.value } info.setProperty("audiocodecid", audioCodecValue.toDouble()) info.setProperty("audiosamplerate", sampleRate.toDouble()) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFormat.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFormat.kt index c41379ff0..d6fed009e 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFormat.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFormat.kt @@ -22,6 +22,13 @@ package com.pedro.rtmp.flv.audio */ enum class AudioFormat(val value: Int) { PCM(0), ADPCM(1), MP3(2), PCM_LE(3), NELLYMOSER_16K(4), - NELLYMOSER_8K(5), NELLYMOSER(6), G711_A(7), G711_MU(8), RESERVED(9), - AAC(10), SPEEX(11), MP3_8K(14), DEVICE_SPECIFIC(15) + NELLYMOSER_8K(5), NELLYMOSER(6), G711_A(7), G711_MU(8), EX_HEADER(9), + AAC(10), SPEEX(11), MP3_8K(14), DEVICE_SPECIFIC(15), + //fourCC extension + AC3(1633889587), // { "a", "c", "-", "3" } + EAC3(1700998451), // { "e", "c", "-", "3" } + OPUS(1332770163), // { "O", "p", "u", "s" } + MP3_CC(778924083), // { ".", "m", "p", "3" } + FLAC(1716281667), // { "f", "L", "a", "C" } + AAC_CC(1836069985), // { "m", "p", "4", "a" } } \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFourCCPacketType.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFourCCPacketType.kt new file mode 100644 index 000000000..29d1d88ac --- /dev/null +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/AudioFourCCPacketType.kt @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2024 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.rtmp.flv.audio + +/** + * Created by pedro on 14/08/23. + * + */ +enum class AudioFourCCPacketType(val value: Int) { + SEQUENCE_START(0), + CODED_FRAMES(1), + SEQUENCE_END(2), + CODED_FRAMES_X(3), + METADATA(4), + MULTITRACK(5), + RESERVED(6), + MOD_EX(7) +} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AudioSpecificConfig.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AacAudioSpecificConfig.kt similarity index 91% rename from rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AudioSpecificConfig.kt rename to rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AacAudioSpecificConfig.kt index 002cd3889..783cb3198 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AudioSpecificConfig.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/AacAudioSpecificConfig.kt @@ -23,7 +23,7 @@ import com.pedro.common.AudioUtils * * ISO 14496-3 */ -class AudioSpecificConfig(private val type: Int, private val sampleRate: Int, private val channels: Int) { +class AacAudioSpecificConfig(private val type: Int, private val sampleRate: Int, private val channels: Int) { val size = 9 diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/OpusAudioSpecificConfig.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/OpusAudioSpecificConfig.kt new file mode 100644 index 000000000..162a57823 --- /dev/null +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/config/OpusAudioSpecificConfig.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2025 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.rtmp.flv.audio.config + +/** + * Created by pedro on 04/12/25. + * + * RFC 7845 section-5.1.1 + */ +class OpusAudioSpecificConfig(private val sampleRate: Int, private val channels: Int) { + + val size = 19 + + fun write(buffer: ByteArray, offset: Int) { + buffer[offset] = 'O'.code.toByte() + buffer[offset + 1] = 'p'.code.toByte() + buffer[offset + 2] = 'u'.code.toByte() + buffer[offset + 3] = 's'.code.toByte() + buffer[offset + 4] = 'H'.code.toByte() + buffer[offset + 5] = 'e'.code.toByte() + buffer[offset + 6] = 'a'.code.toByte() + buffer[offset + 7] = 'd'.code.toByte() + buffer[offset + 8] = 0x01 //version 1 + buffer[offset + 9] = channels.toByte() + val preSkip = 3840 //this is the recommended value by the RFC + buffer[offset + 10] = (preSkip shr 8).toByte() + buffer[offset + 11] = preSkip.toByte() + buffer[offset + 12] = (sampleRate shr 24).toByte() + buffer[offset + 13] = (sampleRate shr 16).toByte() + buffer[offset + 14] = (sampleRate shr 8).toByte() + buffer[offset + 15] = sampleRate.toByte() + val outputGain = 0 + buffer[offset + 16] = (outputGain shr 8).toByte() + buffer[offset + 17] = outputGain.toByte() + val mappingFamily = 0 + buffer[offset + 18] = mappingFamily.toByte() + } +} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt index 74c774a0d..b1fef739b 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/AacPacket.kt @@ -26,7 +26,7 @@ import com.pedro.rtmp.flv.audio.AudioObjectType import com.pedro.rtmp.flv.audio.AudioSize import com.pedro.rtmp.flv.audio.AudioSoundRate import com.pedro.rtmp.flv.audio.AudioSoundType -import com.pedro.rtmp.flv.audio.config.AudioSpecificConfig +import com.pedro.rtmp.flv.audio.config.AacAudioSpecificConfig import kotlin.experimental.or /** @@ -74,7 +74,7 @@ class AacPacket: BasePacket() { header[0] = type or (audioSize.value shl 1).toByte() or (soundRate.value shl 2).toByte() or (AudioFormat.AAC.value shl 4).toByte() val buffer: ByteArray if (!configSend) { - val config = AudioSpecificConfig(objectType.value, sampleRate, if (isStereo) 2 else 1) + val config = AacAudioSpecificConfig(objectType.value, sampleRate, if (isStereo) 2 else 1) buffer = ByteArray(config.size + header.size) header[1] = Type.SEQUENCE.mark config.write(buffer, header.size) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/OpusPacket.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/OpusPacket.kt new file mode 100644 index 000000000..04fce2f94 --- /dev/null +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/audio/packet/OpusPacket.kt @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2025 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.rtmp.flv.audio.packet + +import com.pedro.common.frame.MediaFrame +import com.pedro.common.removeInfo +import com.pedro.rtmp.flv.BasePacket +import com.pedro.rtmp.flv.FlvPacket +import com.pedro.rtmp.flv.FlvType +import com.pedro.rtmp.flv.audio.AudioFormat +import com.pedro.rtmp.flv.audio.AudioFourCCPacketType +import com.pedro.rtmp.flv.audio.config.OpusAudioSpecificConfig + +/** + * Created by pedro on 8/04/21. + */ +class OpusPacket: BasePacket() { + + private val header = ByteArray(5) + //first time we need send audio config + private var configSend = false + + private var sampleRate = 48000 + private var isStereo = true + + fun sendAudioInfo(sampleRate: Int, isStereo: Boolean) { + this.sampleRate = sampleRate + this.isStereo = isStereo + } + + override suspend fun createFlvPacket( + mediaFrame: MediaFrame, + callback: suspend (FlvPacket) -> Unit + ) { + val fixedBuffer = mediaFrame.data.removeInfo(mediaFrame.info) + val ts = mediaFrame.info.timestamp / 1000 + + //header is 5 bytes length: + //mark first byte as extended header and packet type + //4 bytes codec type + val codec = AudioFormat.OPUS.value // { "O", "p", "u", "s" } + header[1] = (codec shr 24).toByte() + header[2] = (codec shr 16).toByte() + header[3] = (codec shr 8).toByte() + header[4] = codec.toByte() + + val buffer: ByteArray + if (!configSend) { + header[0] = ((AudioFormat.EX_HEADER.value shl 4) or (AudioFourCCPacketType.SEQUENCE_START.value and 0x0F)).toByte() + val config = OpusAudioSpecificConfig(sampleRate, if (isStereo) 2 else 1) + buffer = ByteArray(config.size + header.size) + config.write(buffer, header.size) + configSend = true + } else { + header[0] = ((AudioFormat.EX_HEADER.value shl 4) or (AudioFourCCPacketType.CODED_FRAMES.value and 0x0F)).toByte() + buffer = ByteArray(fixedBuffer.remaining() + header.size) + fixedBuffer.get(buffer, header.size, fixedBuffer.remaining()) + } + System.arraycopy(header, 0, buffer, 0, header.size) + callback(FlvPacket(buffer, ts, buffer.size, FlvType.AUDIO)) + } + + override fun reset(resetInfo: Boolean) { + configSend = false + } +} \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFormat.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFormat.kt index 8c8e36aa9..42978a310 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFormat.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFormat.kt @@ -25,5 +25,7 @@ enum class VideoFormat(val value: Int) { //fourCC extension HEVC(1752589105), // { "h", "v", "c", "1" } AV1(1635135537), // { "a", "v", "0", "1" } - VP9(1987063865) // { "v", "p", "0", "9" } + VP9(1987063865), // { "v", "p", "0", "9" } + VP8(1987063864), // { "v", "p", "0", "8" } + AVC_CC(1635148593), // { "a", "v", "c", "1" } } \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/FourCCPacketType.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFourCCPacketType.kt similarity index 87% rename from rtmp/src/main/java/com/pedro/rtmp/flv/video/FourCCPacketType.kt rename to rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFourCCPacketType.kt index 8a3d12b2a..28ed75b29 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/FourCCPacketType.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/VideoFourCCPacketType.kt @@ -20,11 +20,13 @@ package com.pedro.rtmp.flv.video * Created by pedro on 14/08/23. * */ -enum class FourCCPacketType(val value: Int) { +enum class VideoFourCCPacketType(val value: Int) { SEQUENCE_START(0), CODED_FRAMES(1), SEQUENCE_END(2), CODED_FRAMES_X(3), METADATA(4), - MPEG_2_TS_SEQUENCE_START(5) + MPEG_2_TS_SEQUENCE_START(5), + MULTITRACK(6), + MOD_EX(7) } \ No newline at end of file diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt index 28df900d4..a0f9f5012 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/Av1Packet.kt @@ -25,7 +25,7 @@ import com.pedro.common.toByteArray import com.pedro.rtmp.flv.BasePacket import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType -import com.pedro.rtmp.flv.video.FourCCPacketType +import com.pedro.rtmp.flv.video.VideoFourCCPacketType import com.pedro.rtmp.flv.video.VideoDataType import com.pedro.rtmp.flv.video.VideoFormat import com.pedro.rtmp.flv.video.config.VideoSpecificConfigAV1 @@ -56,11 +56,9 @@ class Av1Packet: BasePacket() { var fixedBuffer = mediaFrame.data.duplicate().removeInfo(mediaFrame.info) val ts = mediaFrame.info.timestamp / 1000 - //header is 8 bytes length: + //header is 5 bytes length: //mark first byte as extended header (0b10000000) //4 bits data type, 4 bits packet type - //4 bytes extended codec type (in this case av01) - //3 bytes CompositionTime, the cts. val codec = VideoFormat.AV1.value // { "a", "v", "0", "1" } header[1] = (codec shr 24).toByte() header[2] = (codec shr 16).toByte() @@ -69,7 +67,7 @@ class Av1Packet: BasePacket() { var buffer: ByteArray if (!configSend) { - header[0] = (0b10000000 or (VideoDataType.KEYFRAME.value shl 4) or FourCCPacketType.SEQUENCE_START.value).toByte() + header[0] = (0b10000000 or (VideoDataType.KEYFRAME.value shl 4) or VideoFourCCPacketType.SEQUENCE_START.value).toByte() val obuSequence = this.obuSequence if (obuSequence != null) { val config = VideoSpecificConfigAV1(obuSequence) @@ -95,7 +93,7 @@ class Av1Packet: BasePacket() { buffer = ByteArray(header.size + size) val nalType = if (mediaFrame.info.isKeyFrame) VideoDataType.KEYFRAME.value else VideoDataType.INTER_FRAME.value - header[0] = (0b10000000 or (nalType shl 4) or FourCCPacketType.CODED_FRAMES.value).toByte() + header[0] = (0b10000000 or (nalType shl 4) or VideoFourCCPacketType.CODED_FRAMES.value).toByte() fixedBuffer.get(buffer, header.size, size) System.arraycopy(header, 0, buffer, 0, header.size) diff --git a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt index d2db95a40..4d4de5b62 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/flv/video/packet/H265Packet.kt @@ -22,7 +22,7 @@ import com.pedro.common.removeInfo import com.pedro.rtmp.flv.BasePacket import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType -import com.pedro.rtmp.flv.video.FourCCPacketType +import com.pedro.rtmp.flv.video.VideoFourCCPacketType import com.pedro.rtmp.flv.video.VideoDataType import com.pedro.rtmp.flv.video.VideoFormat import com.pedro.rtmp.flv.video.VideoNalType @@ -85,11 +85,10 @@ class H265Packet: BasePacket() { header[6] = (cts shr 8).toByte() header[7] = cts.toByte() - val packets = mutableListOf() var buffer: ByteArray if (!configSend) { //avoid send cts on sequence start - header[0] = (0b10000000 or (VideoDataType.KEYFRAME.value shl 4) or FourCCPacketType.SEQUENCE_START.value).toByte() + header[0] = (0b10000000 or (VideoDataType.KEYFRAME.value shl 4) or VideoFourCCPacketType.SEQUENCE_START.value).toByte() val sps = this.sps val pps = this.pps val vps = this.vps @@ -121,7 +120,7 @@ class H265Packet: BasePacket() { // we don't need send it because we already do it in video config return } - header[0] = (0b10000000 or (nalType shl 4) or FourCCPacketType.CODED_FRAMES.value).toByte() + header[0] = (0b10000000 or (nalType shl 4) or VideoFourCCPacketType.CODED_FRAMES.value).toByte() writeNaluSize(buffer, header.size, size) validBuffer.get(buffer, header.size + naluSize, size) diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt index 0a10dec25..62fbc3972 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/CommandsManagerAmf0.kt @@ -42,18 +42,17 @@ class CommandsManagerAmf0: CommandsManager() { connectInfo.setProperty("app", appName + auth) connectInfo.setProperty("flashVer", flashVersion) connectInfo.setProperty("tcUrl", tcUrl + auth) + val list = mutableListOf() if (!videoDisabled) { - if (videoCodec == VideoCodec.H265) { - val list = mutableListOf() - list.add(AmfString("hvc1")) - val array = AmfStrictArray(list) - connectInfo.setProperty("fourCcList", array) - } else if (videoCodec == VideoCodec.AV1) { - val list = mutableListOf() - list.add(AmfString("av01")) - val array = AmfStrictArray(list) - connectInfo.setProperty("fourCcList", array) - } + if (videoCodec == VideoCodec.H265) list.add(AmfString("hvc1")) + else if (videoCodec == VideoCodec.AV1) list.add(AmfString("av01")) + } + if (!audioDisabled) { + if (audioCodec == AudioCodec.OPUS) list.add(AmfString("Opus")) + } + if (list.isNotEmpty()) { + val array = AmfStrictArray(list) + connectInfo.setProperty("fourCcList", array) } connectInfo.setProperty("objectEncoding", 0.0) @@ -123,7 +122,7 @@ class CommandsManagerAmf0: CommandsManager() { val codecValue = when (audioCodec) { AudioCodec.G711 -> AudioFormat.G711_A.value AudioCodec.AAC -> AudioFormat.AAC.value - AudioCodec.OPUS -> throw IllegalArgumentException("Unsupported codec: ${audioCodec.name}") + AudioCodec.OPUS -> AudioFormat.OPUS.value } amfEcmaArray.setProperty("audiocodecid", codecValue.toDouble()) amfEcmaArray.setProperty("audiosamplerate", sampleRate.toDouble()) diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt index cb958aa8a..ae4da360c 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpClient.kt @@ -125,10 +125,7 @@ class RtmpClient(private val connectChecker: ConnectChecker) { fun setAudioCodec(audioCodec: AudioCodec) { if (!isStreaming) { - commandsManager.audioCodec = when (audioCodec) { - AudioCodec.OPUS -> throw IllegalArgumentException("Unsupported codec: ${audioCodec.name}") - else -> audioCodec - } + commandsManager.audioCodec = audioCodec } } diff --git a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt index 5aa5d8b2f..c64513ca6 100644 --- a/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt +++ b/rtmp/src/main/java/com/pedro/rtmp/rtmp/RtmpSender.kt @@ -29,6 +29,7 @@ import com.pedro.rtmp.flv.FlvPacket import com.pedro.rtmp.flv.FlvType import com.pedro.rtmp.flv.audio.packet.AacPacket import com.pedro.rtmp.flv.audio.packet.G711Packet +import com.pedro.rtmp.flv.audio.packet.OpusPacket import com.pedro.rtmp.flv.video.packet.Av1Packet import com.pedro.rtmp.flv.video.packet.H264Packet import com.pedro.rtmp.flv.video.packet.H265Packet @@ -69,9 +70,7 @@ class RtmpSender( audioPacket = when (commandsManager.audioCodec) { AudioCodec.G711 -> G711Packet().apply { sendAudioInfo() } AudioCodec.AAC -> AacPacket().apply { sendAudioInfo(sampleRate, isStereo) } - AudioCodec.OPUS -> { - throw IllegalArgumentException("Unsupported codec: ${commandsManager.audioCodec.name}") - } + AudioCodec.OPUS -> OpusPacket().apply { sendAudioInfo(sampleRate, isStereo) } } } diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt index d6ac2ce61..03a0cf35c 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AacPacketTest.kt @@ -30,7 +30,7 @@ import java.nio.ByteBuffer class AacPacketTest { @Test - fun `GIVEN a aac buffer WHEN call create a aac packet 2 times THEN return config and expected buffer`() = runTest { + fun `GIVEN an AAC buffer WHEN call create an AAC packet 2 times THEN return config and expected buffer`() = runTest { val timestamp = 123456789L val buffer = ByteArray(256) { 0x00 } val info = MediaFrame.Info(0, buffer.size, timestamp, false) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AudioConfigTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AudioConfigTest.kt index 8ad55c6de..63507fe2a 100644 --- a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AudioConfigTest.kt +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/AudioConfigTest.kt @@ -16,20 +16,33 @@ package com.pedro.rtmp.flv.audio -import com.pedro.rtmp.flv.audio.config.AudioSpecificConfig +import com.pedro.rtmp.flv.audio.config.AacAudioSpecificConfig +import com.pedro.rtmp.flv.audio.config.OpusAudioSpecificConfig import org.junit.Assert.assertArrayEquals import org.junit.Test class AudioConfigTest { @Test - fun `GIVEN sps and pps WHEN create a video config for sequence packet THEN return a bytearray with the config`() { + fun `GIVEN objectType, sampleRate and channels WHEN create an AAC audio config for sequence packet THEN return a bytearray with the config`() { val sampleRate = 44100 val isStereo = true val objectType = AudioObjectType.AAC_LC val expectedConfig = byteArrayOf(18, 16, -1, -15, 80, -128, 1, 63, -4) - val config = AudioSpecificConfig(objectType.value, sampleRate, if (isStereo) 2 else 1) + val config = AacAudioSpecificConfig(objectType.value, sampleRate, if (isStereo) 2 else 1) + val data = ByteArray(config.size) + config.write(data, 0) + assertArrayEquals(expectedConfig, data) + } + + @Test + fun `GIVEN sampleRate and channels WHEN create an Opus audio config for sequence packet THEN return a bytearray with the config`() { + val sampleRate = 48000 + val isStereo = true + val expectedConfig = byteArrayOf(79, 112, 117, 115, 72, 101, 97, 100, 1, 2, 15, 0, 0, 0, -69, -128, 0, 0, 0) + + val config = OpusAudioSpecificConfig(sampleRate, if (isStereo) 2 else 1) val data = ByteArray(config.size) config.write(data, 0) assertArrayEquals(expectedConfig, data) diff --git a/rtmp/src/test/java/com/pedro/rtmp/flv/audio/OpusPacketTest.kt b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/OpusPacketTest.kt new file mode 100644 index 000000000..5a0bdfb48 --- /dev/null +++ b/rtmp/src/test/java/com/pedro/rtmp/flv/audio/OpusPacketTest.kt @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 pedroSG94. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedro.rtmp.flv.audio + +import com.pedro.common.frame.MediaFrame +import com.pedro.rtmp.flv.FlvType +import com.pedro.rtmp.flv.audio.packet.OpusPacket +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Test +import java.nio.ByteBuffer + +/** + * Created by pedro on 9/9/23. + */ +class OpusPacketTest { + + @Test + fun `GIVEN an Opus buffer WHEN call create an Opus packet 2 times THEN return config and expected buffer`() = runTest { + val timestamp = 123456789L + val buffer = ByteArray(256) { 0x00 } + val info = MediaFrame.Info(0, buffer.size, timestamp, false) + val mediaFrame = MediaFrame(ByteBuffer.wrap(buffer), info, MediaFrame.Type.AUDIO) + val opusPacket = OpusPacket() + opusPacket.sendAudioInfo(48000, true) + opusPacket.createFlvPacket(mediaFrame) { flvPacket -> + assertEquals(FlvType.AUDIO, flvPacket.type) + assertEquals(((AudioFormat.EX_HEADER.value shl 4) or (AudioFourCCPacketType.SEQUENCE_START.value and 0x0F)).toByte(), flvPacket.buffer[0]) + assertEquals(5 + 19, flvPacket.length) + } + opusPacket.createFlvPacket(mediaFrame) { flvPacket -> + assertEquals(FlvType.AUDIO, flvPacket.type) + assertEquals(((AudioFormat.EX_HEADER.value shl 4) or (AudioFourCCPacketType.CODED_FRAMES.value and 0x0F)).toByte(), flvPacket.buffer[0]) + assertEquals(5 + buffer.size, flvPacket.length) + } + } +} \ No newline at end of file