diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift index 413dc24..36ed130 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift @@ -17,10 +17,11 @@ class AudioPlayerStream { private var pcmBuffers = [AVAudioPCMBuffer]() public var isPlaying = false + public var isStopped = false init(sampleRate: Double) throws { let audioSession = AVAudioSession.sharedInstance() - try audioSession.setCategory(.playback, mode: .default) + try audioSession.setCategory(.playAndRecord, options: [.mixWithOthers, .allowBluetooth]) try audioSession.setActive(true) let format = AVAudioFormat( @@ -38,7 +39,10 @@ class AudioPlayerStream { try engine.start() } - func playStreamPCM(_ pcmData: [Int16], completion: @escaping (Bool) -> Void) { + func playStreamPCM(_ pcmData: [Int16]) throws { + if isStopped { + return + } let audioBuffer = AVAudioPCMBuffer( pcmFormat: playerNode.outputFormat(forBus: 0), frameCapacity: AVAudioFrameCount(pcmData.count))! @@ -56,32 +60,40 @@ class AudioPlayerStream { } pcmBuffers.append(audioBuffer) + + if !engine.isRunning { + try engine.start() + } if !isPlaying { - playNextPCMBuffer(completion: completion) - } else { - completion(true) + playNextPCMBuffer() } } - private func playNextPCMBuffer(completion: @escaping (Bool) -> Void) { + private func playNextPCMBuffer() { + if isStopped { + return + } guard let pcmData = pcmBuffers.first else { isPlaying = false - completion(false) return } pcmBuffers.removeFirst() playerNode.scheduleBuffer(pcmData) { [weak self] in - self?.playNextPCMBuffer(completion: completion) + self?.playNextPCMBuffer() } playerNode.play() isPlaying = true - completion(true) + } + + func resetAudioPlayer() { + isStopped = false + isPlaying = false } func stopStreamPCM() { - playerNode.stop() - engine.stop() + isStopped = true + pcmBuffers.removeAll() } } diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift index 1229fa9..b0133e5 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift @@ -192,7 +192,7 @@ You can download directly to your device or airdrop from a Mac. private func streamCallback(completion: String) { DispatchQueue.main.async { [self] in - if self.stopPhrases.contains(completion) { + if self.stopPhrases.contains(completion) || chatState != .GENERATE { return } @@ -222,23 +222,33 @@ You can download directly to your device or airdrop from a Mac. streamCallback: streamCallback) try dialog!.addLLMResponse(content: result.completion) + + DispatchQueue.main.async { [self] in + if result.endpoint == .interrupted { + statusText = "Listening..." + chatText.append(Message(speaker: "You:", msg: "")) + chatState = .STT + + promptText = "" + enableGenerateButton = true + } else { + statusText = ViewModel.statusTextDefault + chatState = .WAKEWORD + + promptText = "" + enableGenerateButton = true + } + } } catch { DispatchQueue.main.async { [self] in errorMessage = "\(error.localizedDescription)" } } - - DispatchQueue.main.async { [self] in - statusText = ViewModel.statusTextDefault - chatState = .WAKEWORD - - promptText = "" - enableGenerateButton = true - } } DispatchQueue.global(qos: .userInitiated).async { [self] in do { + audioStream!.resetAudioPlayer() let orcaStream = try self.orca!.streamOpen() var warmup = true @@ -262,32 +272,24 @@ You can download directly to your device or airdrop from a Mac. if warmup { warmupBuffer.append(contentsOf: pcm!) if warmupBuffer.count >= (1 * orca!.sampleRate!) { - audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in - if !isPlaying { - self.startAudioRecording() - } - }) + try audioStream!.playStreamPCM(pcm!) warmupBuffer.removeAll() warmup = false } } else { - audioStream!.playStreamPCM(pcm!, completion: {_ in }) + try audioStream!.playStreamPCM(pcm!) } } } } if !warmupBuffer.isEmpty { - audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in - if !isPlaying { - self.startAudioRecording() - } - }) + try audioStream!.playStreamPCM(warmupBuffer) } let pcm = try orcaStream.flush() if pcm != nil { - audioStream!.playStreamPCM(pcm!, completion: {_ in}) + try audioStream!.playStreamPCM(pcm!) } orcaStream.close() } catch { @@ -298,6 +300,17 @@ You can download directly to your device or airdrop from a Mac. } } + public func interrupt() { + do { + audioStream!.stopStreamPCM() + try picollm?.interrupt() + } catch { + DispatchQueue.main.async { [self] in + errorMessage = "\(error.localizedDescription)" + } + } + } + public func clearText() { promptText = "" chatText.removeAll() @@ -305,18 +318,22 @@ You can download directly to your device or airdrop from a Mac. private func audioCallback(frame: [Int16]) { do { - if audioStream?.isPlaying ?? false { - return - } if chatState == .WAKEWORD { - let keyword = try self.porcupine!.process(pcm: frame) - if keyword != -1 { + let keywordIndex = try self.porcupine!.process(pcm: frame) + if keywordIndex == 0 { DispatchQueue.main.async { [self] in statusText = "Listening..." chatText.append(Message(speaker: "You:", msg: "")) chatState = .STT } } + } else if chatState == .GENERATE { + let keywordIndex = try self.porcupine!.process(pcm: frame) + if keywordIndex == 0 { + DispatchQueue.main.async { [self] in + self.interrupt() + } + } } else if chatState == .STT { var (transcription, endpoint) = try self.cheetah!.process(frame) if endpoint { @@ -329,9 +346,8 @@ You can download directly to your device or airdrop from a Mac. } if endpoint { DispatchQueue.main.async { [self] in - statusText = "Generating..." + statusText = "Generating, Say `Picovoice` to interrupt" chatState = .GENERATE - stopAudioRecording() self.generate() } } diff --git a/recipes/llm-voice-assistant/ios/Podfile b/recipes/llm-voice-assistant/ios/Podfile index 8f63717..2c4d85f 100644 --- a/recipes/llm-voice-assistant/ios/Podfile +++ b/recipes/llm-voice-assistant/ios/Podfile @@ -4,7 +4,7 @@ platform :ios, '16.0' target 'LLMVoiceAssistantDemo' do pod 'Porcupine-iOS', '~> 3.0.1' pod 'Cheetah-iOS', '~> 2.0.0' - pod 'picoLLM-iOS', '~> 1.0.0' + pod 'picoLLM-iOS', '~> 1.1.0' pod 'Orca-iOS', '~> 1.0.0' pod 'ios-voice-processor', '~> 1.1.0' end diff --git a/recipes/llm-voice-assistant/ios/Podfile.lock b/recipes/llm-voice-assistant/ios/Podfile.lock index 89bed65..2beb050 100644 --- a/recipes/llm-voice-assistant/ios/Podfile.lock +++ b/recipes/llm-voice-assistant/ios/Podfile.lock @@ -2,7 +2,7 @@ PODS: - Cheetah-iOS (2.0.0) - ios-voice-processor (1.1.0) - Orca-iOS (1.0.0) - - picoLLM-iOS (1.0.0) + - picoLLM-iOS (1.1.0) - Porcupine-iOS (3.0.1): - ios-voice-processor (~> 1.1.0) @@ -10,7 +10,7 @@ DEPENDENCIES: - Cheetah-iOS (~> 2.0.0) - ios-voice-processor (~> 1.1.0) - Orca-iOS (~> 1.0.0) - - picoLLM-iOS (~> 1.0.0) + - picoLLM-iOS (~> 1.1.0) - Porcupine-iOS (~> 3.0.1) SPEC REPOS: @@ -25,9 +25,9 @@ SPEC CHECKSUMS: Cheetah-iOS: d98a5edcbf3b74dda6027aeac6a8c0f5997a47a2 ios-voice-processor: 8e32d7f980a06d392d128ef1cd19cf6ddcaca3c1 Orca-iOS: d50a0dbbf596f20c6c2e2f727f20f72ac012aa0e - picoLLM-iOS: 02cdb501b4beb74a9c1dea29d5cf461d65ea4a6c + picoLLM-iOS: dc03cd7e992c702ff34c667f9a35dd9a8084c061 Porcupine-iOS: 6d69509fa587f3ac0be1adfefb48e0c6ce029fff -PODFILE CHECKSUM: 64580b5dbb7bc16cb10af3dc7da63609228fe397 +PODFILE CHECKSUM: 1cc2ff3bc3e1abc97fbf7a3b3910e8d2b805f8b7 COCOAPODS: 1.15.2