Skip to content

Commit

Permalink
feat(*): add new audio effects
Browse files Browse the repository at this point in the history
  • Loading branch information
@ThibaultBee
ThibaultBee committed Nov 17, 2024
1 parent 8adacbe commit dde6db4
Show file tree
Hide file tree
Showing 10 changed files with 348 additions and 105 deletions.
138 changes: 138 additions & 0 deletions ...ithub/thibaultbee/streampack/core/internal/sources/audio/audiorecord/AudioRecordEffect.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* Copyright (C) 2021 Thibault B.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.github.thibaultbee.streampack.core.internal.sources.audio.audiorecord

import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.AudioEffect
import android.media.audiofx.AutomaticGainControl
import android.media.audiofx.BassBoost
import android.media.audiofx.NoiseSuppressor
import android.os.Build
import java.util.UUID

/**
* Audio effect interface
*/
internal class AudioRecordEffect {
internal interface IFactory {
fun build(audioSessionId: Int): AudioEffect?
}

companion object {
private const val TAG = "AudioRecordEffect"

fun getSupportedEffects(): List<UUID> {
val descriptors = AudioEffect.queryEffects()
return descriptors.map { it.type }
}
}

internal abstract class Factory<T : AudioEffect> : IFactory {
protected abstract val name: String

protected abstract fun isAvailable(): Boolean
protected abstract fun create(audioSessionId: Int): T?
protected fun applyConfigurationFrom(audioEffect: T) = Unit

override fun build(audioSessionId: Int): T {
if (!isAvailable()) {
throw IllegalStateException("$name is not available")
}

val audioEffect = try {
create(audioSessionId)
} catch (t: Throwable) {
throw Exception("Failed to create $name", t)
}

requireNotNull(audioEffect) {
"Failed to create $name"
}

val result = audioEffect.setEnabled(true)
if (result != AudioEffect.SUCCESS) {
audioEffect.release()
throw Exception("Failed to enable $name")
}

return audioEffect
}

companion object {
fun getFactoryForEffectType(effectType: UUID): Factory<*> {
return when (effectType) {
AudioEffect.EFFECT_TYPE_AEC -> AcousticEchoCancelerFactory()
AudioEffect.EFFECT_TYPE_AGC -> AutomaticGainControlFactory()
AudioEffect.EFFECT_TYPE_BASS_BOOST -> BassBoostFactory()
AudioEffect.EFFECT_TYPE_NS -> NoiseSuppressorFactory()
else -> throw IllegalArgumentException("Unknown effect type: $effectType")
}
}

fun isValidUUID(uuid: UUID): Boolean {
return uuid == AudioEffect.EFFECT_TYPE_AEC ||
uuid == AudioEffect.EFFECT_TYPE_AGC ||
uuid == AudioEffect.EFFECT_TYPE_BASS_BOOST ||
((Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) &&
uuid == AudioEffect.EFFECT_TYPE_DYNAMICS_PROCESSING) ||
uuid == AudioEffect.EFFECT_TYPE_ENV_REVERB ||
uuid == AudioEffect.EFFECT_TYPE_EQUALIZER ||
((Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) &&
uuid == AudioEffect.EFFECT_TYPE_HAPTIC_GENERATOR) ||
uuid == AudioEffect.EFFECT_TYPE_LOUDNESS_ENHANCER ||
uuid == AudioEffect.EFFECT_TYPE_NS ||
uuid == AudioEffect.EFFECT_TYPE_PRESET_REVERB ||
uuid == AudioEffect.EFFECT_TYPE_VIRTUALIZER
}
}
}

internal class AcousticEchoCancelerFactory : Factory<AcousticEchoCanceler>() {
override val name: String = "Acoustic echo canceler"

override fun isAvailable(): Boolean = AcousticEchoCanceler.isAvailable()

override fun create(audioSessionId: Int): AcousticEchoCanceler? =
AcousticEchoCanceler.create(audioSessionId)
}

internal class AutomaticGainControlFactory : Factory<AutomaticGainControl>() {
override val name: String = "Automatic gain control"

override fun isAvailable(): Boolean = AutomaticGainControl.isAvailable()

override fun create(audioSessionId: Int): AutomaticGainControl? =
AutomaticGainControl.create(audioSessionId)
}

internal class BassBoostFactory(private val priority: Int = 0) : Factory<BassBoost>() {
override val name: String = "Bass boost"

override fun isAvailable(): Boolean =
getSupportedEffects().contains(AudioEffect.EFFECT_TYPE_BASS_BOOST)

override fun create(audioSessionId: Int) = BassBoost(priority, audioSessionId)
}

internal class NoiseSuppressorFactory : Factory<NoiseSuppressor>() {
override val name: String = "Noise suppressor"

override fun isAvailable(): Boolean = NoiseSuppressor.isAvailable()

override fun create(audioSessionId: Int): NoiseSuppressor? =
NoiseSuppressor.create(audioSessionId)
}
}
181 changes: 103 additions & 78 deletions ...ternal/sources/audio/AudioRecordSource.kt → ...es/audio/audiorecord/AudioRecordSource.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,37 +13,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.github.thibaultbee.streampack.core.internal.sources.audio
package io.github.thibaultbee.streampack.core.internal.sources.audio.audiorecord

import android.Manifest
import android.media.AudioRecord
import android.media.AudioTimestamp
import android.media.MediaFormat
import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.NoiseSuppressor
import android.media.audiofx.AudioEffect
import android.os.Build
import androidx.annotation.RequiresPermission
import io.github.thibaultbee.streampack.core.data.AudioConfig
import io.github.thibaultbee.streampack.core.internal.data.Frame
import io.github.thibaultbee.streampack.core.internal.sources.IFrameSource
import io.github.thibaultbee.streampack.core.internal.sources.audio.IAudioSourceInternal
import io.github.thibaultbee.streampack.core.internal.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.getFactoryForEffectType
import io.github.thibaultbee.streampack.core.internal.sources.audio.audiorecord.AudioRecordEffect.Factory.Companion.isValidUUID
import io.github.thibaultbee.streampack.core.internal.utils.TimeUtils
import io.github.thibaultbee.streampack.core.internal.utils.extensions.type
import io.github.thibaultbee.streampack.core.logger.Logger
import java.nio.ByteBuffer
import java.util.UUID

/**
* The [AudioRecordSource] class is an implementation of [IAudioSourceInternal] that captures audio
* from [AudioRecord].
*
* @param enableAcousticEchoCanceler [Boolean.true] to enable AcousticEchoCanceler
* @param enableNoiseSuppressor [Boolean.true] to enable NoiseSuppressor
* @param defaultEffects [List] of default effects type to enable. Default is [AudioEffect.EFFECT_TYPE_AEC] and [AudioEffect.EFFECT_TYPE_NS].
*/
sealed class AudioRecordSource(
private val enableAcousticEchoCanceler: Boolean = true,
private val enableNoiseSuppressor: Boolean = true
) : IAudioSourceInternal, IFrameSource<AudioConfig> {
sealed class AudioRecordSource : IAudioSourceInternal, IAudioRecordSourceSource {
private var audioRecord: AudioRecord? = null

private var processor: EffectProcessor? = null
private var processor = EffectProcessor()

private var mutedByteArray: ByteArray? = null
override var isMuted: Boolean = false
Expand Down Expand Up @@ -87,11 +86,7 @@ sealed class AudioRecordSource(
mutedByteArray = ByteArray(bufferSize)

audioRecord = buildAudioRecord(config, bufferSize).also {
processor = EffectProcessor(
enableAcousticEchoCanceler,
enableNoiseSuppressor,
it.audioSessionId
)
processor.audioSessionId = it.audioSessionId

if (it.state != AudioRecord.STATE_INITIALIZED) {
throw IllegalArgumentException("Failed to initialized audio source with config: $config")
Expand All @@ -117,12 +112,11 @@ sealed class AudioRecordSource(
override fun release() {
mutedByteArray = null

processor.clear()

// Release audio record
audioRecord?.release()
audioRecord = null

processor?.release()
processor = null
}

private fun getTimestamp(audioRecord: AudioRecord): Long {
Expand Down Expand Up @@ -182,6 +176,36 @@ sealed class AudioRecordSource(
else -> "Unknown audio record error: $audioRecordError"
}

/**
* Adds and enables an effect to the audio source.
*
* Get supported effects with [getSupportedEffectTypes].
*/
override fun addEffect(effectType: UUID): AudioEffect {
return processor.add(effectType)
}

/**
* Removes an effect from the audio source.
*/
override fun removeEffect(effectType: UUID) {
processor.remove(effectType)
}

/**
* Gets an effect from the audio source.
*/
override fun getEffect(effectType: UUID): AudioEffect {
return processor.get(effectType)
}

/**
* Gets all effects from the audio source.
*/
override fun getEffects(): List<AudioEffect> {
return processor.getAll().toList()
}

companion object {
private const val TAG = "AudioSource"

Expand All @@ -191,81 +215,82 @@ sealed class AudioRecordSource(
MediaFormat.MIMETYPE_AUDIO_RAW
)
}

/**
* Get supported effects.
*
* @return [List] of supported effects.
* @see AudioEffect
*/
fun getSupportedEffectTypes(): List<UUID> {
return AudioRecordEffect.getSupportedEffects()
}
}

private class EffectProcessor(
enableAcousticEchoCanceler: Boolean,
enableNoiseSuppressor: Boolean,
audioSessionId: Int
) {
private val acousticEchoCanceler =
if (enableAcousticEchoCanceler) initAcousticEchoCanceler(audioSessionId) else null

private val noiseSuppressor =
if (enableNoiseSuppressor) initNoiseSuppressor(audioSessionId) else null
private class EffectProcessor {
private val audioEffects: MutableSet<AudioEffect> = mutableSetOf()

var audioSessionId: Int = AudioRecord.ERROR_BAD_VALUE
set(value) {
require(value != AudioRecord.ERROR_BAD_VALUE) { "Invalid audio session id" }

fun release() {
acousticEchoCanceler?.release()
noiseSuppressor?.release()
}
// Adds previous effects to new audio session
val previousEffectTypes = audioEffects.map { it.type }
clear()

companion object {
private fun initNoiseSuppressor(audioSessionId: Int): NoiseSuppressor? {
if (!NoiseSuppressor.isAvailable()) {
Logger.w(TAG, "Noise suppressor is not available")
return null
}
previousEffectTypes.forEach { add(it) }
field = value
}

val noiseSuppressor = try {
NoiseSuppressor.create(audioSessionId)
} catch (t: Throwable) {
Logger.e(TAG, "Failed to create noise suppressor", t)
return null
}
fun get(effectType: UUID): AudioEffect {
require(isValidUUID(effectType)) { "Unknown effect type: $effectType" }

if (noiseSuppressor == null) {
Logger.w(TAG, "Failed to create noise suppressor")
return null
}
return audioEffects.firstOrNull { it.type == effectType }
?: throw IllegalArgumentException("Effect $effectType not enabled")
}

val result = noiseSuppressor.setEnabled(true)
if (result != NoiseSuppressor.SUCCESS) {
noiseSuppressor.release()
Logger.w(TAG, "Failed to enable noise suppressor")
return null
}
fun getAll(): List<AudioEffect> {
return audioEffects.toList()
}

return noiseSuppressor
fun add(effectType: UUID): AudioEffect {
require(isValidUUID(effectType)) { "Unknown effect type: $effectType" }

val previousEffect = audioEffects.firstOrNull { it.type == effectType }
if (previousEffect != null) {
Logger.w(TAG, "Effect ${previousEffect.descriptor.name} already enabled")
return previousEffect
}

private fun initAcousticEchoCanceler(audioSessionId: Int): AcousticEchoCanceler? {
if (!AcousticEchoCanceler.isAvailable()) {
Logger.w(TAG, "Acoustic echo canceler is not available")
return null
}
val factory = getFactoryForEffectType(effectType)
factory.build(audioSessionId).let {
audioEffects.add(it)
return it
}
}

val acousticEchoCanceler = try {
AcousticEchoCanceler.create(audioSessionId)
} catch (t: Throwable) {
Logger.e(TAG, "Failed to create acoustic echo canceler", t)
return null
}
fun remove(effectType: UUID) {
require(isValidUUID(effectType)) { "Unknown effect type: $effectType" }

if (acousticEchoCanceler == null) {
Logger.w(TAG, "Failed to create acoustic echo canceler")
return null
}
val effect = audioEffects.firstOrNull { it.descriptor.type == effectType }
if (effect != null) {
effect.release()
audioEffects.remove(effect)
}
}

val result = acousticEchoCanceler.setEnabled(true)
if (result != AcousticEchoCanceler.SUCCESS) {
acousticEchoCanceler.release()
Logger.w(TAG, "Failed to enable acoustic echo canceler")
return null
}
fun release() {
audioEffects.forEach { it.release() }
}

return acousticEchoCanceler
}
fun clear() {
release()
audioEffects.clear()
}

companion object {
private const val TAG = "EffectProcessor"
}
}
}
Loading

0 comments on commit dde6db4

Please sign in to comment.