Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for multi-channel audio streaming #1273

Merged
merged 2 commits into from
Aug 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions HaishinKit.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

/* Begin PBXBuildFile section */
035AFA042263868E009DD0BB /* RTMPStreamTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 035AFA032263868E009DD0BB /* RTMPStreamTests.swift */; };
1A21630C73D792376BBCFC7D /* AVAudioFormat+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */; };
1A216A40B7F375BC52D193CC /* AVAudioFormat+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */; };
1A216F07B0BD8E05C8ECC8F1 /* AVAudioFormat+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */; };
2901A4EE1D437170002BBD23 /* MediaLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2901A4ED1D437170002BBD23 /* MediaLink.swift */; };
2901A4EF1D437662002BBD23 /* MediaLink.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2901A4ED1D437170002BBD23 /* MediaLink.swift */; };
290686031DFDB7A7008EB7ED /* RTMPConnectionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 290686021DFDB7A6008EB7ED /* RTMPConnectionTests.swift */; };
Expand Down Expand Up @@ -593,6 +596,7 @@

/* Begin PBXFileReference section */
035AFA032263868E009DD0BB /* RTMPStreamTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RTMPStreamTests.swift; sourceTree = "<group>"; };
1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVAudioFormat+Extension.swift"; sourceTree = "<group>"; };
2901A4ED1D437170002BBD23 /* MediaLink.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaLink.swift; sourceTree = "<group>"; };
290686021DFDB7A6008EB7ED /* RTMPConnectionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RTMPConnectionTests.swift; sourceTree = "<group>"; };
290EA88E1DFB616000053022 /* Foundation+ExtensionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Foundation+ExtensionTests.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -1345,6 +1349,7 @@
BC6FC9212961B3D800A746EE /* vImage_CGImageFormat+Extension.swift */,
BC83A4722403D83B006BDE06 /* VTCompressionSession+Extension.swift */,
BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */,
1A2166D3A449D813866FE9D9 /* AVAudioFormat+Extension.swift */,
);
path = Extension;
sourceTree = "<group>";
Expand Down Expand Up @@ -2037,6 +2042,7 @@
29B8767A1CD70ACE00FC07DA /* M3U.swift in Sources */,
29B876901CD70AFE00FC07DA /* IOAudioUnit.swift in Sources */,
29B876771CD70ACE00FC07DA /* HTTPResponse.swift in Sources */,
1A216F07B0BD8E05C8ECC8F1 /* AVAudioFormat+Extension.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -2205,6 +2211,7 @@
BC4914AF28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift in Sources */,
2901A4EF1D437662002BBD23 /* MediaLink.swift in Sources */,
BC7C56BC299E595000C41A9B /* VideoCodecSettings.swift in Sources */,
1A21630C73D792376BBCFC7D /* AVAudioFormat+Extension.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -2361,6 +2368,7 @@
29EB3DF61ED0577C001CAE8B /* CMSampleBuffer+Extension.swift in Sources */,
29EB3E281ED05A0C001CAE8B /* RTMPTSocket.swift in Sources */,
BC11024C2925147300D48035 /* IOCaptureUnit.swift in Sources */,
1A216A40B7F375BC52D193CC /* AVAudioFormat+Extension.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
33 changes: 32 additions & 1 deletion Sources/Codec/AudioCodec.swift
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,38 @@ public class AudioCodec {
guard inSourceFormat.mBitsPerChannel == 16 else {
return nil
}
return AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: inSourceFormat.mSampleRate, channels: inSourceFormat.mChannelsPerFrame, interleaved: true)
if let layout = Self.makeChannelLayout(inSourceFormat.mChannelsPerFrame) {
return .init(commonFormat: .pcmFormatInt16, sampleRate: inSourceFormat.mSampleRate, interleaved: true,
channelLayout: layout)
}
return AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: inSourceFormat.mSampleRate,
channels: inSourceFormat.mChannelsPerFrame, interleaved: true)
}
if let layout = Self.makeChannelLayout(inSourceFormat.mChannelsPerFrame) {
return .init(streamDescription: &inSourceFormat, channelLayout: layout)
}
return .init(streamDescription: &inSourceFormat)
}

static func makeChannelLayout(_ numberOfChannels: UInt32) -> AVAudioChannelLayout? {
guard numberOfChannels > 2 else { return nil }

return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_DiscreteInOrder | numberOfChannels)
}

/// Creates a channel map for specific input and output format
/// - Examples:
/// - Input channel count is 4 and 2, result: [0, 1, -1, -1]
/// - Input channel count is 2 and 2, result: [0, 1]
static func makeChannelMap(from fromFormat: AVAudioFormat, to toFormat: AVAudioFormat) -> [NSNumber] {
let inChannels = Int(fromFormat.channelCount)
let outChannels = Int(toFormat.channelCount)
let channelIndexes = Array(0...inChannels - 1)
return channelIndexes
.prefix(outChannels)
.map { NSNumber(value: $0) }
+ Array(repeating: -1, count: max(0, inChannels - outChannels))
}

/// Specifies the delegate.
public weak var delegate: (any AudioCodecDelegate)?
Expand Down Expand Up @@ -164,7 +192,10 @@ public class AudioCodec {
let outputFormat = settings.format.makeAudioFormat(inSourceFormat) else {
return nil
}
logger.debug("inputFormat: \(inputFormat)")
logger.debug("outputFormat: \(outputFormat)")
let converter = AVAudioConverter(from: inputFormat, to: outputFormat)
converter?.channelMap = Self.makeChannelMap(from: inputFormat, to: outputFormat)
settings.apply(converter, oldValue: nil)
if converter == nil {
delegate?.audioCodec(self, errorOccurred: .failedToCreate(from: inputFormat, to: outputFormat))
Expand Down
11 changes: 7 additions & 4 deletions Sources/Codec/AudioCodecSettings.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ import Foundation

/// The AudioCodecSettings class specifying audio compression settings.
public struct AudioCodecSettings: Codable {
/// The defualt value.
/// The default value.
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😅

public static let `default` = AudioCodecSettings()


/// Maximum number of channels supported by the system
public static let maximumNumberOfChannels: UInt32 = 2

/// The type of the AudioCodec supports format.
public enum Format: Codable {
/// The AAC format.
Expand Down Expand Up @@ -98,7 +101,7 @@ public struct AudioCodecSettings: Codable {
mBytesPerPacket: bytesPerPacket,
mFramesPerPacket: framesPerPacket,
mBytesPerFrame: bytesPerFrame,
mChannelsPerFrame: inSourceFormat.mChannelsPerFrame,
mChannelsPerFrame: min(inSourceFormat.mChannelsPerFrame, AudioCodecSettings.maximumNumberOfChannels),
mBitsPerChannel: bitsPerChannel,
mReserved: 0
)
Expand All @@ -107,7 +110,7 @@ public struct AudioCodecSettings: Codable {
return AVAudioFormat(
commonFormat: .pcmFormatFloat32,
sampleRate: inSourceFormat.mSampleRate,
channels: inSourceFormat.mChannelsPerFrame,
channels: min(inSourceFormat.mChannelsPerFrame, AudioCodecSettings.maximumNumberOfChannels),
interleaved: true
)
}
Expand Down
122 changes: 122 additions & 0 deletions Sources/Extension/AVAudioFormat+Extension.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
//
// Created by Lev Sokolov on 2023-08-24.
// Copyright (c) 2023 Shogo Endo. All rights reserved.
//

import AVFoundation

extension AVAudioFormat {
public override var description: String {
var descriptionParts: [String] = []

descriptionParts.append("Sample Rate: \(sampleRate) Hz")
descriptionParts.append("Channels: \(channelCount)")

if let channelLayout = channelLayout {
descriptionParts.append("Channel Layout: \(channelLayout.layout.readableDescription)")
}

descriptionParts.append("Format: \(commonFormat.readableDescription)")
descriptionParts.append(isInterleaved ? "Interleaved" : "Non-interleaved")
descriptionParts.append(isStandard ? "Standard" : "Non-standard")

if let audioFormatID = audioFormatID {
descriptionParts.append("AudioFormatID: \(audioFormatID.audioFormatIDDescription) (\(audioFormatID))")
}

descriptionParts.append("Settings: \(settings)")

return descriptionParts.joined(separator: ", ")
}

var audioFormatID: AudioFormatID? {
guard let formatIDValue = settings[AVFormatIDKey] as? NSNumber else {
return nil
}
return AudioFormatID(formatIDValue.uint32Value)
}
}


extension UnsafePointer<AudioChannelLayout> {
var readableDescription: String {
let layout = pointee
let channelTag = layout.mChannelLayoutTag
let bitmap = layout.mChannelBitmap
let numberChannelDescriptions = layout.mNumberChannelDescriptions
let channelDescriptions = channelDescriptions
let channelLabels = channelDescriptions.map { $0.mChannelLabel }
return "tag: \(channelTag), bitmap: \(bitmap), channels: \(numberChannelDescriptions), channelLabels: \(channelLabels)"
}

var channelDescriptions: [AudioChannelDescription] {
var mutablePointee = UnsafeMutablePointer(mutating: self).pointee
let numberOfDescriptions = Int(mutablePointee.mNumberChannelDescriptions)
return withUnsafePointer(to: &mutablePointee.mChannelDescriptions) { start in
let descriptionsPointer = UnsafeBufferPointer<AudioChannelDescription>(start: start, count: numberOfDescriptions)
return (0..<numberOfDescriptions).map {
descriptionsPointer[$0]
}
}
}
}

extension AVAudioCommonFormat {
public var readableDescription: String {
switch self {
case .pcmFormatFloat32: return "float32"
case .pcmFormatFloat64: return "float64"
case .pcmFormatInt16: return "int16"
case .pcmFormatInt32: return "int32"
case .otherFormat: return "other"
@unknown default: return "unknown"
}
}
}

extension AudioFormatID {
public var audioFormatIDDescription: String {
switch self {
case kAudioFormatAC3: return "kAudioFormatAC3"
case kAudioFormatAES3: return "kAudioFormatAES3"
case kAudioFormatALaw: return "kAudioFormatALaw"
case kAudioFormatAMR: return "kAudioFormatAMR"
case kAudioFormatAMR_WB: return "kAudioFormatAMR_WB"
case kAudioFormatAppleIMA4: return "kAudioFormatAppleIMA4"
case kAudioFormatAppleLossless: return "kAudioFormatAppleLossless"
case kAudioFormatAudible: return "kAudioFormatAudible"
case kAudioFormatDVIIntelIMA: return "kAudioFormatDVIIntelIMA"
case kAudioFormatEnhancedAC3: return "kAudioFormatEnhancedAC3"
case kAudioFormatFLAC: return "kAudioFormatFLAC"
case kAudioFormatLinearPCM: return "kAudioFormatLinearPCM"
case kAudioFormatMACE3: return "kAudioFormatMACE3"
case kAudioFormatMACE6: return "kAudioFormatMACE6"
case kAudioFormatMIDIStream: return "kAudioFormatMIDIStream"
case kAudioFormatMPEG4AAC: return "kAudioFormatMPEG4AAC"
case kAudioFormatMPEG4AAC_ELD: return "kAudioFormatMPEG4AAC_ELD"
case kAudioFormatMPEG4AAC_ELD_SBR: return "kAudioFormatMPEG4AAC_ELD_SBR"
case kAudioFormatMPEG4AAC_ELD_V2: return "kAudioFormatMPEG4AAC_ELD_V2"
case kAudioFormatMPEG4AAC_HE: return "kAudioFormatMPEG4AAC_HE"
case kAudioFormatMPEG4AAC_HE_V2: return "kAudioFormatMPEG4AAC_HE_V2"
case kAudioFormatMPEG4AAC_LD: return "kAudioFormatMPEG4AAC_LD"
case kAudioFormatMPEG4AAC_Spatial: return "kAudioFormatMPEG4AAC_Spatial"
case kAudioFormatMPEG4CELP: return "kAudioFormatMPEG4CELP"
case kAudioFormatMPEG4HVXC: return "kAudioFormatMPEG4HVXC"
case kAudioFormatMPEG4TwinVQ: return "kAudioFormatMPEG4TwinVQ"
case kAudioFormatMPEGD_USAC: return "kAudioFormatMPEGD_USAC"
case kAudioFormatMPEGLayer1: return "kAudioFormatMPEGLayer1"
case kAudioFormatMPEGLayer2: return "kAudioFormatMPEGLayer2"
case kAudioFormatMPEGLayer3: return "kAudioFormatMPEGLayer3"
case kAudioFormatMicrosoftGSM: return "kAudioFormatMicrosoftGSM"
case kAudioFormatOpus: return "kAudioFormatOpus"
case kAudioFormatParameterValueStream: return "kAudioFormatParameterValueStream"
case kAudioFormatQDesign: return "kAudioFormatQDesign"
case kAudioFormatQDesign2: return "kAudioFormatQDesign2"
case kAudioFormatQUALCOMM: return "kAudioFormatQUALCOMM"
case kAudioFormatTimeCode: return "kAudioFormatTimeCode"
case kAudioFormatULaw: return "kAudioFormatULaw"
case kAudioFormatiLBC: return "kAudioFormatiLBC"
default: return "unknown"
}
}
}