Merge pull request #23 from Picovoice/v1.1-ios

v1.1 ios

recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/AudioPlayerStream.swift

@@ -17,10 +17,11 @@ class AudioPlayerStream {
 
     private var pcmBuffers = [AVAudioPCMBuffer]()
     public var isPlaying = false
+    public var isStopped = false
 
     init(sampleRate: Double) throws {
         let audioSession = AVAudioSession.sharedInstance()
-        try audioSession.setCategory(.playback, mode: .default)
+        try audioSession.setCategory(.playAndRecord, options: [.mixWithOthers, .allowBluetooth])
         try audioSession.setActive(true)
 
         let format = AVAudioFormat(
@@ -38,7 +39,10 @@ class AudioPlayerStream {
         try engine.start()
     }
 
-    func playStreamPCM(_ pcmData: [Int16], completion: @escaping (Bool) -> Void) {
+    func playStreamPCM(_ pcmData: [Int16]) throws {
+        if isStopped {
+            return
+        }
         let audioBuffer = AVAudioPCMBuffer(
             pcmFormat: playerNode.outputFormat(forBus: 0), frameCapacity: AVAudioFrameCount(pcmData.count))!
 
@@ -56,32 +60,40 @@ class AudioPlayerStream {
         }
 
         pcmBuffers.append(audioBuffer)
+
+        if !engine.isRunning {
+            try engine.start()
+        }
         if !isPlaying {
-            playNextPCMBuffer(completion: completion)
-        } else {
-            completion(true)
+            playNextPCMBuffer()
         }
     }
 
-    private func playNextPCMBuffer(completion: @escaping (Bool) -> Void) {
+    private func playNextPCMBuffer() {
+        if isStopped {
+            return
+        }
         guard let pcmData = pcmBuffers.first else {
             isPlaying = false
-            completion(false)
             return
         }
         pcmBuffers.removeFirst()
 
         playerNode.scheduleBuffer(pcmData) { [weak self] in
-            self?.playNextPCMBuffer(completion: completion)
+            self?.playNextPCMBuffer()
         }
 
         playerNode.play()
         isPlaying = true
-        completion(true)
+    }
+
+    func resetAudioPlayer() {
+        isStopped = false
+        isPlaying = false
     }
 
     func stopStreamPCM() {
-        playerNode.stop()
-        engine.stop()
+        isStopped = true
+        pcmBuffers.removeAll()
     }
 }

recipes/llm-voice-assistant/ios/LLMVoiceAssistantDemo/ViewModel.swift

@@ -192,7 +192,7 @@ You can download directly to your device or airdrop from a Mac.
 
     private func streamCallback(completion: String) {
         DispatchQueue.main.async { [self] in
-            if self.stopPhrases.contains(completion) {
+            if self.stopPhrases.contains(completion) || chatState != .GENERATE {
                 return
             }
 
@@ -222,23 +222,33 @@ You can download directly to your device or airdrop from a Mac.
                     streamCallback: streamCallback)
 
                 try dialog!.addLLMResponse(content: result.completion)
+
+                DispatchQueue.main.async { [self] in
+                    if result.endpoint == .interrupted {
+                        statusText = "Listening..."
+                        chatText.append(Message(speaker: "You:", msg: ""))
+                        chatState = .STT
+
+                        promptText = ""
+                        enableGenerateButton = true
+                    } else {
+                        statusText = ViewModel.statusTextDefault
+                        chatState = .WAKEWORD
+
+                        promptText = ""
+                        enableGenerateButton = true
+                    }
+                }
             } catch {
                 DispatchQueue.main.async { [self] in
                     errorMessage = "\(error.localizedDescription)"
                 }
             }
-
-            DispatchQueue.main.async { [self] in
-                statusText = ViewModel.statusTextDefault
-                chatState = .WAKEWORD
-
-                promptText = ""
-                enableGenerateButton = true
-            }
         }
 
         DispatchQueue.global(qos: .userInitiated).async { [self] in
             do {
+                audioStream!.resetAudioPlayer()
                 let orcaStream = try self.orca!.streamOpen()
 
                 var warmup = true
@@ -262,32 +272,24 @@ You can download directly to your device or airdrop from a Mac.
                             if warmup {
                                 warmupBuffer.append(contentsOf: pcm!)
                                 if warmupBuffer.count >= (1 * orca!.sampleRate!) {
-                                    audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in
-                                        if !isPlaying {
-                                            self.startAudioRecording()
-                                        }
-                                    })
+                                    try audioStream!.playStreamPCM(pcm!)
                                     warmupBuffer.removeAll()
                                     warmup = false
                                 }
                             } else {
-                                audioStream!.playStreamPCM(pcm!, completion: {_ in })
+                                try audioStream!.playStreamPCM(pcm!)
                             }
                         }
                     }
                 }
 
                 if !warmupBuffer.isEmpty {
-                    audioStream!.playStreamPCM(warmupBuffer, completion: { isPlaying in
-                        if !isPlaying {
-                            self.startAudioRecording()
-                        }
-                    })
+                    try audioStream!.playStreamPCM(warmupBuffer)
                 }
 
                 let pcm = try orcaStream.flush()
                 if pcm != nil {
-                    audioStream!.playStreamPCM(pcm!, completion: {_ in})
+                    try audioStream!.playStreamPCM(pcm!)
                 }
                 orcaStream.close()
             } catch {
@@ -298,25 +300,40 @@ You can download directly to your device or airdrop from a Mac.
         }
     }
 
+    public func interrupt() {
+        do {
+            audioStream!.stopStreamPCM()
+            try picollm?.interrupt()
+        } catch {
+            DispatchQueue.main.async { [self] in
+                errorMessage = "\(error.localizedDescription)"
+            }
+        }
+    }
+
     public func clearText() {
         promptText = ""
         chatText.removeAll()
     }
 
     private func audioCallback(frame: [Int16]) {
         do {
-            if audioStream?.isPlaying ?? false {
-                return
-            }
             if chatState == .WAKEWORD {
-                let keyword = try self.porcupine!.process(pcm: frame)
-                if keyword != -1 {
+                let keywordIndex = try self.porcupine!.process(pcm: frame)
+                if keywordIndex == 0 {
                     DispatchQueue.main.async { [self] in
                         statusText = "Listening..."
                         chatText.append(Message(speaker: "You:", msg: ""))
                         chatState = .STT
                     }
                 }
+            } else if chatState == .GENERATE {
+                let keywordIndex = try self.porcupine!.process(pcm: frame)
+                if keywordIndex == 0 {
+                    DispatchQueue.main.async { [self] in
+                        self.interrupt()
+                    }
+                }
             } else if chatState == .STT {
                 var (transcription, endpoint) = try self.cheetah!.process(frame)
                 if endpoint {
@@ -329,9 +346,8 @@ You can download directly to your device or airdrop from a Mac.
                 }
                 if endpoint {
                     DispatchQueue.main.async { [self] in
-                        statusText = "Generating..."
+                        statusText = "Generating, Say `Picovoice` to interrupt"
                         chatState = .GENERATE
-                        stopAudioRecording()
                         self.generate()
                     }
                 }

recipes/llm-voice-assistant/ios/Podfile

@@ -4,7 +4,7 @@ platform :ios, '16.0'
 target 'LLMVoiceAssistantDemo' do
   pod 'Porcupine-iOS', '~> 3.0.1'
   pod 'Cheetah-iOS', '~> 2.0.0'
-  pod 'picoLLM-iOS', '~> 1.0.0'
+  pod 'picoLLM-iOS', '~> 1.1.0'
   pod 'Orca-iOS', '~> 1.0.0'
   pod 'ios-voice-processor', '~> 1.1.0'
 end

recipes/llm-voice-assistant/ios/Podfile.lock

@@ -2,15 +2,15 @@ PODS:
   - Cheetah-iOS (2.0.0)
   - ios-voice-processor (1.1.0)
   - Orca-iOS (1.0.0)
-  - picoLLM-iOS (1.0.0)
+  - picoLLM-iOS (1.1.0)
   - Porcupine-iOS (3.0.1):
     - ios-voice-processor (~> 1.1.0)
 
 DEPENDENCIES:
   - Cheetah-iOS (~> 2.0.0)
   - ios-voice-processor (~> 1.1.0)
   - Orca-iOS (~> 1.0.0)
-  - picoLLM-iOS (~> 1.0.0)
+  - picoLLM-iOS (~> 1.1.0)
   - Porcupine-iOS (~> 3.0.1)
 
 SPEC REPOS:
@@ -25,9 +25,9 @@ SPEC CHECKSUMS:
   Cheetah-iOS: d98a5edcbf3b74dda6027aeac6a8c0f5997a47a2
   ios-voice-processor: 8e32d7f980a06d392d128ef1cd19cf6ddcaca3c1
   Orca-iOS: d50a0dbbf596f20c6c2e2f727f20f72ac012aa0e
-  picoLLM-iOS: 02cdb501b4beb74a9c1dea29d5cf461d65ea4a6c
+  picoLLM-iOS: dc03cd7e992c702ff34c667f9a35dd9a8084c061
   Porcupine-iOS: 6d69509fa587f3ac0be1adfefb48e0c6ce029fff
 
-PODFILE CHECKSUM: 64580b5dbb7bc16cb10af3dc7da63609228fe397
+PODFILE CHECKSUM: 1cc2ff3bc3e1abc97fbf7a3b3910e8d2b805f8b7
 
 COCOAPODS: 1.15.2