From d7cbfe67b32747dd35c1cbe3fcb4e38b8e4f179a Mon Sep 17 00:00:00 2001 From: Ian Lavery Date: Fri, 13 Sep 2024 13:19:21 -0700 Subject: [PATCH 1/4] ios demo --- .../AudioPlayerStream.swift | 28 +++++--- .../ios/LLMVoiceAssistantDemo/ViewModel.swift | 71 ++++++++++++------- recipes/llm-voice-assistant/ios/Podfile | 2 +- recipes/llm-voice-assistant/ios/Podfile.lock | 15 ++-- 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift index 413dc24..8f589fb 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift @@ -17,10 +17,11 @@ class AudioPlayerStream { private var pcmBuffers = [AVAudioPCMBuffer]() public var isPlaying = false + public var isStopped = false init(sampleRate: Double) throws { let audioSession = AVAudioSession.sharedInstance() - try audioSession.setCategory(.playback, mode: .default) + try audioSession.setCategory(.playAndRecord, options: [.mixWithOthers, .allowBluetooth]) try audioSession.setActive(true) let format = AVAudioFormat( @@ -38,7 +39,10 @@ class AudioPlayerStream { try engine.start() } - func playStreamPCM(_ pcmData: [Int16], completion: @escaping (Bool) -> Void) { + func playStreamPCM(_ pcmData: [Int16]) { + if isStopped { + return + } let audioBuffer = AVAudioPCMBuffer( pcmFormat: playerNode.outputFormat(forBus: 0), frameCapacity: AVAudioFrameCount(pcmData.count))! @@ -57,31 +61,35 @@ class AudioPlayerStream { pcmBuffers.append(audioBuffer) if !isPlaying { - playNextPCMBuffer(completion: completion) - } else { - completion(true) + playNextPCMBuffer() } } - private func playNextPCMBuffer(completion: @escaping (Bool) -> Void) { + private func playNextPCMBuffer() { + if isStopped { + return + } guard let pcmData = pcmBuffers.first else { isPlaying = false - completion(false) return } pcmBuffers.removeFirst() playerNode.scheduleBuffer(pcmData) { [weak self] in - self?.playNextPCMBuffer(completion: completion) + self?.playNextPCMBuffer() } playerNode.play() isPlaying = true - completion(true) + } + + func resetAudioPlayer() { + isStopped = false + isPlaying = false } func stopStreamPCM() { playerNode.stop() - engine.stop() + isStopped = true } } diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift index 1229fa9..86b6020 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift @@ -192,7 +192,7 @@ You can download directly to your device or airdrop from a Mac. private func streamCallback(completion: String) { DispatchQueue.main.async { [self] in - if self.stopPhrases.contains(completion) { + if self.stopPhrases.contains(completion) || chatState != .GENERATE { return } @@ -222,23 +222,33 @@ You can download directly to your device or airdrop from a Mac. streamCallback: streamCallback) try dialog!.addLLMResponse(content: result.completion) + + DispatchQueue.main.async { [self] in + if result.endpoint == .interrupted { + statusText = "Listening..." + chatText.append(Message(speaker: "You:", msg: "")) + chatState = .STT + + promptText = "" + enableGenerateButton = true + } else { + statusText = ViewModel.statusTextDefault + chatState = .WAKEWORD + + promptText = "" + enableGenerateButton = true + } + } } catch { DispatchQueue.main.async { [self] in errorMessage = "\(error.localizedDescription)" } } - - DispatchQueue.main.async { [self] in - statusText = ViewModel.statusTextDefault - chatState = .WAKEWORD - - promptText = "" - enableGenerateButton = true - } } DispatchQueue.global(qos: .userInitiated).async { [self] in do { + audioStream!.resetAudioPlayer(); let orcaStream = try self.orca!.streamOpen() var warmup = true @@ -262,32 +272,24 @@ You can download directly to your device or airdrop from a Mac. if warmup { warmupBuffer.append(contentsOf: pcm!) if warmupBuffer.count >= (1 * orca!.sampleRate!) { - audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in - if !isPlaying { - self.startAudioRecording() - } - }) + audioStream!.playStreamPCM(pcm!) warmupBuffer.removeAll() warmup = false } } else { - audioStream!.playStreamPCM(pcm!, completion: {_ in }) + audioStream!.playStreamPCM(pcm!) } } } } if !warmupBuffer.isEmpty { - audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in - if !isPlaying { - self.startAudioRecording() - } - }) + audioStream!.playStreamPCM(warmupBuffer) } let pcm = try orcaStream.flush() if pcm != nil { - audioStream!.playStreamPCM(pcm!, completion: {_ in}) + audioStream!.playStreamPCM(pcm!) } orcaStream.close() } catch { @@ -297,6 +299,17 @@ You can download directly to your device or airdrop from a Mac. } } } + + public func interrupt() { + do { + audioStream!.stopStreamPCM(); + try picollm?.interrupt(); + } catch { + DispatchQueue.main.async { [self] in + errorMessage = "\(error.localizedDescription)" + } + } + } public func clearText() { promptText = "" @@ -305,18 +318,23 @@ You can download directly to your device or airdrop from a Mac. private func audioCallback(frame: [Int16]) { do { - if audioStream?.isPlaying ?? false { - return - } if chatState == .WAKEWORD { - let keyword = try self.porcupine!.process(pcm: frame) - if keyword != -1 { + let keywordIndex = try self.porcupine!.process(pcm: frame) + if keywordIndex == 0 { DispatchQueue.main.async { [self] in statusText = "Listening..." chatText.append(Message(speaker: "You:", msg: "")) chatState = .STT } } + } + else if chatState == .GENERATE { + let keywordIndex = try self.porcupine!.process(pcm: frame) + if keywordIndex == 0 { + DispatchQueue.main.async { [self] in + self.interrupt(); + } + } } else if chatState == .STT { var (transcription, endpoint) = try self.cheetah!.process(frame) if endpoint { @@ -331,7 +349,6 @@ You can download directly to your device or airdrop from a Mac. DispatchQueue.main.async { [self] in statusText = "Generating..." chatState = .GENERATE - stopAudioRecording() self.generate() } } diff --git a/recipes/llm-voice-assistant/ios/Podfile b/recipes/llm-voice-assistant/ios/Podfile index 8f63717..5f87367 100644 --- a/recipes/llm-voice-assistant/ios/Podfile +++ b/recipes/llm-voice-assistant/ios/Podfile @@ -4,7 +4,7 @@ platform :ios, '16.0' target 'LLMVoiceAssistantDemo' do pod 'Porcupine-iOS', '~> 3.0.1' pod 'Cheetah-iOS', '~> 2.0.0' - pod 'picoLLM-iOS', '~> 1.0.0' + pod 'picoLLM-iOS', :podspec => 'https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec' pod 'Orca-iOS', '~> 1.0.0' pod 'ios-voice-processor', '~> 1.1.0' end diff --git a/recipes/llm-voice-assistant/ios/Podfile.lock b/recipes/llm-voice-assistant/ios/Podfile.lock index 89bed65..78c4ad7 100644 --- a/recipes/llm-voice-assistant/ios/Podfile.lock +++ b/recipes/llm-voice-assistant/ios/Podfile.lock @@ -2,7 +2,7 @@ PODS: - Cheetah-iOS (2.0.0) - ios-voice-processor (1.1.0) - Orca-iOS (1.0.0) - - picoLLM-iOS (1.0.0) + - picoLLM-iOS (1.1.0) - Porcupine-iOS (3.0.1): - ios-voice-processor (~> 1.1.0) @@ -10,7 +10,7 @@ DEPENDENCIES: - Cheetah-iOS (~> 2.0.0) - ios-voice-processor (~> 1.1.0) - Orca-iOS (~> 1.0.0) - - picoLLM-iOS (~> 1.0.0) + - picoLLM-iOS (from `https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec`) - Porcupine-iOS (~> 3.0.1) SPEC REPOS: @@ -18,16 +18,19 @@ SPEC REPOS: - Cheetah-iOS - ios-voice-processor - Orca-iOS - - picoLLM-iOS - Porcupine-iOS +EXTERNAL SOURCES: + picoLLM-iOS: + :podspec: https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec + SPEC CHECKSUMS: Cheetah-iOS: d98a5edcbf3b74dda6027aeac6a8c0f5997a47a2 ios-voice-processor: 8e32d7f980a06d392d128ef1cd19cf6ddcaca3c1 Orca-iOS: d50a0dbbf596f20c6c2e2f727f20f72ac012aa0e - picoLLM-iOS: 02cdb501b4beb74a9c1dea29d5cf461d65ea4a6c + picoLLM-iOS: dc03cd7e992c702ff34c667f9a35dd9a8084c061 Porcupine-iOS: 6d69509fa587f3ac0be1adfefb48e0c6ce029fff -PODFILE CHECKSUM: 64580b5dbb7bc16cb10af3dc7da63609228fe397 +PODFILE CHECKSUM: c97e17229c09fc9f63ec858e2a40078a71550321 -COCOAPODS: 1.15.2 +COCOAPODS: 1.11.3 From 98fa7cd218d849aaa90afb0fcf83a46d7d6eb2d4 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Wed, 2 Oct 2024 11:47:32 -0700 Subject: [PATCH 2/4] v1.1 picollm changes --- .../LLMVoiceAssistantDemo/AudioPlayerStream.swift | 9 +++++++-- .../ios/LLMVoiceAssistantDemo/ViewModel.swift | 12 ++++++------ recipes/llm-voice-assistant/ios/Podfile | 2 +- recipes/llm-voice-assistant/ios/Podfile.lock | 11 ++++------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift index 8f589fb..80ff46c 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift @@ -39,7 +39,7 @@ class AudioPlayerStream { try engine.start() } - func playStreamPCM(_ pcmData: [Int16]) { + func playStreamPCM(_ pcmData: [Int16]) throws { if isStopped { return } @@ -60,6 +60,11 @@ class AudioPlayerStream { } pcmBuffers.append(audioBuffer) + + if !engine.isRunning { + try engine.start() + return + } if !isPlaying { playNextPCMBuffer() } @@ -89,7 +94,7 @@ class AudioPlayerStream { } func stopStreamPCM() { - playerNode.stop() isStopped = true + pcmBuffers.removeAll() } } diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift index 86b6020..3c47502 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift @@ -220,7 +220,7 @@ You can download directly to your device or airdrop from a Mac. prompt: dialog!.prompt(), completionTokenLimit: 128, streamCallback: streamCallback) - + try dialog!.addLLMResponse(content: result.completion) DispatchQueue.main.async { [self] in @@ -272,24 +272,24 @@ You can download directly to your device or airdrop from a Mac. if warmup { warmupBuffer.append(contentsOf: pcm!) if warmupBuffer.count >= (1 * orca!.sampleRate!) { - audioStream!.playStreamPCM(pcm!) + try audioStream!.playStreamPCM(pcm!) warmupBuffer.removeAll() warmup = false } } else { - audioStream!.playStreamPCM(pcm!) + try audioStream!.playStreamPCM(pcm!) } } } } if !warmupBuffer.isEmpty { - audioStream!.playStreamPCM(warmupBuffer) + try audioStream!.playStreamPCM(warmupBuffer) } let pcm = try orcaStream.flush() if pcm != nil { - audioStream!.playStreamPCM(pcm!) + try audioStream!.playStreamPCM(pcm!) } orcaStream.close() } catch { @@ -347,7 +347,7 @@ You can download directly to your device or airdrop from a Mac. } if endpoint { DispatchQueue.main.async { [self] in - statusText = "Generating..." + statusText = "Generating, Say `Picovoice` to interrupt" chatState = .GENERATE self.generate() } diff --git a/recipes/llm-voice-assistant/ios/Podfile b/recipes/llm-voice-assistant/ios/Podfile index 5f87367..2c4d85f 100644 --- a/recipes/llm-voice-assistant/ios/Podfile +++ b/recipes/llm-voice-assistant/ios/Podfile @@ -4,7 +4,7 @@ platform :ios, '16.0' target 'LLMVoiceAssistantDemo' do pod 'Porcupine-iOS', '~> 3.0.1' pod 'Cheetah-iOS', '~> 2.0.0' - pod 'picoLLM-iOS', :podspec => 'https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec' + pod 'picoLLM-iOS', '~> 1.1.0' pod 'Orca-iOS', '~> 1.0.0' pod 'ios-voice-processor', '~> 1.1.0' end diff --git a/recipes/llm-voice-assistant/ios/Podfile.lock b/recipes/llm-voice-assistant/ios/Podfile.lock index 78c4ad7..2beb050 100644 --- a/recipes/llm-voice-assistant/ios/Podfile.lock +++ b/recipes/llm-voice-assistant/ios/Podfile.lock @@ -10,7 +10,7 @@ DEPENDENCIES: - Cheetah-iOS (~> 2.0.0) - ios-voice-processor (~> 1.1.0) - Orca-iOS (~> 1.0.0) - - picoLLM-iOS (from `https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec`) + - picoLLM-iOS (~> 1.1.0) - Porcupine-iOS (~> 3.0.1) SPEC REPOS: @@ -18,12 +18,9 @@ SPEC REPOS: - Cheetah-iOS - ios-voice-processor - Orca-iOS + - picoLLM-iOS - Porcupine-iOS -EXTERNAL SOURCES: - picoLLM-iOS: - :podspec: https://raw.githubusercontent.com/Picovoice/picollm/v1.1-ios/binding/ios/picoLLM-iOS.podspec - SPEC CHECKSUMS: Cheetah-iOS: d98a5edcbf3b74dda6027aeac6a8c0f5997a47a2 ios-voice-processor: 8e32d7f980a06d392d128ef1cd19cf6ddcaca3c1 @@ -31,6 +28,6 @@ SPEC CHECKSUMS: picoLLM-iOS: dc03cd7e992c702ff34c667f9a35dd9a8084c061 Porcupine-iOS: 6d69509fa587f3ac0be1adfefb48e0c6ce029fff -PODFILE CHECKSUM: c97e17229c09fc9f63ec858e2a40078a71550321 +PODFILE CHECKSUM: 1cc2ff3bc3e1abc97fbf7a3b3910e8d2b805f8b7 -COCOAPODS: 1.11.3 +COCOAPODS: 1.15.2 From 8b42514c875956b9ae6121eafed001878e68b374 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Wed, 2 Oct 2024 11:49:11 -0700 Subject: [PATCH 3/4] remove return --- .../ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift index 80ff46c..4d4b93c 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift @@ -63,7 +63,6 @@ class AudioPlayerStream { if !engine.isRunning { try engine.start() - return } if !isPlaying { playNextPCMBuffer() From 35d380d5ac5aab10882fcaab8204849513166d3f Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Wed, 2 Oct 2024 11:55:26 -0700 Subject: [PATCH 4/4] fix lint --- .../AudioPlayerStream.swift | 2 +- .../ios/LLMVoiceAssistantDemo/ViewModel.swift | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift index 4d4b93c..36ed130 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift @@ -60,7 +60,7 @@ class AudioPlayerStream { } pcmBuffers.append(audioBuffer) - + if !engine.isRunning { try engine.start() } diff --git a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift index 3c47502..b0133e5 100644 --- a/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift +++ b/recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift @@ -220,15 +220,15 @@ You can download directly to your device or airdrop from a Mac. prompt: dialog!.prompt(), completionTokenLimit: 128, streamCallback: streamCallback) - + try dialog!.addLLMResponse(content: result.completion) - + DispatchQueue.main.async { [self] in if result.endpoint == .interrupted { statusText = "Listening..." chatText.append(Message(speaker: "You:", msg: "")) chatState = .STT - + promptText = "" enableGenerateButton = true } else { @@ -248,7 +248,7 @@ You can download directly to your device or airdrop from a Mac. DispatchQueue.global(qos: .userInitiated).async { [self] in do { - audioStream!.resetAudioPlayer(); + audioStream!.resetAudioPlayer() let orcaStream = try self.orca!.streamOpen() var warmup = true @@ -299,11 +299,11 @@ You can download directly to your device or airdrop from a Mac. } } } - + public func interrupt() { do { - audioStream!.stopStreamPCM(); - try picollm?.interrupt(); + audioStream!.stopStreamPCM() + try picollm?.interrupt() } catch { DispatchQueue.main.async { [self] in errorMessage = "\(error.localizedDescription)" @@ -327,12 +327,11 @@ You can download directly to your device or airdrop from a Mac. chatState = .STT } } - } - else if chatState == .GENERATE { + } else if chatState == .GENERATE { let keywordIndex = try self.porcupine!.process(pcm: frame) if keywordIndex == 0 { DispatchQueue.main.async { [self] in - self.interrupt(); + self.interrupt() } } } else if chatState == .STT {