Issue - 42 WhisperKit support simulator fixed (#52)

* Issue - 42 WhisperKit support simulator fixed * Issue - 42 enhancement done * Setup AVAudioSession, which fixes crash in simulators * Add availability checks for model compute options --------- Co-authored-by: ZachNagengast <[email protected]>
argmaxinc · Mar 8, 2024 · bfa357e · bfa357e
1 parent ccdd77d
commit bfa357e
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 3 deletions.
diff --git a/...s/WhisperAX/WhisperAX.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/...s/WhisperAX/WhisperAX.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -1,4 +1,5 @@
 {
+  "originHash" : "cd17206b47bb810af9459722192530e3838d8e6629a970988e32a432aaa05f6e",
   "pins" : [
     {
       "identity" : "networkimage",
@@ -37,5 +38,5 @@
       }
     }
   ],
-  "version" : 2
+  "version" : 3
 }
diff --git a/Examples/WhisperAX/WhisperAXWatchApp/WhisperAXExampleView.swift b/Examples/WhisperAX/WhisperAXWatchApp/WhisperAXExampleView.swift
@@ -73,7 +73,7 @@ struct WhisperAXWatchView: View {
 
     var body: some View {
         NavigationSplitView {
-            if WhisperKit.deviceName().hasPrefix("Watch7") {
+            if WhisperKit.deviceName().hasPrefix("Watch7") || WhisperKit.isRunningOnSimulator {
                 modelSelectorView
                     .navigationTitle("WhisperAX")
                     .navigationBarTitleDisplayMode(.automatic)

diff --git a/Sources/WhisperKit/Core/AudioProcessor.swift b/Sources/WhisperKit/Core/AudioProcessor.swift
@@ -487,7 +487,27 @@ public extension AudioProcessor {
         }
     }
     #endif
-
+
+    /// Attempts to setup the shared audio session if available on the device's OS
+    func setupAudioSessionForDevice() throws {
+        #if !os(macOS) // AVAudioSession is not available on macOS
+
+        #if !os(watchOS) // watchOS does not support .defaultToSpeaker
+        let options: AVAudioSession.CategoryOptions = [.defaultToSpeaker, .allowBluetooth]
+        #else
+        let options: AVAudioSession.CategoryOptions = .mixWithOthers
+        #endif
+
+        let audioSession = AVAudioSession.sharedInstance()
+        do {
+            try audioSession.setCategory(.playAndRecord, options: options)
+            try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
+        } catch let error as NSError {
+            throw WhisperError.audioProcessingFailed("Failed to set up audio session: \(error)")
+        }
+        #endif
+    }
+
     func setupEngine(inputDeviceID: DeviceID? = nil) throws -> AVAudioEngine {
         let audioEngine = AVAudioEngine()
         let inputNode = audioEngine.inputNode
@@ -546,6 +566,8 @@ public extension AudioProcessor {
     func startRecordingLive(inputDeviceID: DeviceID? = nil, callback: (([Float]) -> Void)? = nil) throws {
         audioSamples = []
         audioEnergy = []
+
+        try? setupAudioSessionForDevice()
 
         audioEngine = try setupEngine(inputDeviceID: inputDeviceID)
 

diff --git a/Sources/WhisperKit/Core/Models.swift b/Sources/WhisperKit/Core/Models.swift
@@ -126,6 +126,7 @@ public enum ModelState: CustomStringConvertible {
     }
 }
 
+@available(macOS 13, iOS 16, watchOS 10, visionOS 1, *)
 public struct ModelComputeOptions {
     public var melCompute: MLComputeUnits
     public var audioEncoderCompute: MLComputeUnits
@@ -138,6 +139,13 @@ public struct ModelComputeOptions {
         textDecoderCompute: MLComputeUnits = .cpuAndNeuralEngine,
         prefillCompute: MLComputeUnits = .cpuOnly
     ) {
+        if WhisperKit.isRunningOnSimulator {
+            self.melCompute = .cpuOnly
+            self.audioEncoderCompute = .cpuOnly
+            self.textDecoderCompute = .cpuOnly
+            self.prefillCompute = .cpuOnly
+            return
+        }
         self.melCompute = melCompute
         self.audioEncoderCompute = audioEncoderCompute
         self.textDecoderCompute = textDecoderCompute

diff --git a/Sources/WhisperKit/Core/Utils.swift b/Sources/WhisperKit/Core/Utils.swift
@@ -238,6 +238,17 @@ extension Process {
 }
 #endif
 
+@available(macOS 13, iOS 16, watchOS 10, visionOS 1, *)
+public extension WhisperKit {
+    static var isRunningOnSimulator: Bool {
+        #if targetEnvironment(simulator)
+        return true
+        #else
+        return false
+        #endif
+    }
+}
+
 public func resolveAbsolutePath(_ inputPath: String) -> String {
     let fileManager = FileManager.default