send audio delta

Author: richiejp

core/http/endpoints/openai/realtime.go

@@ -895,6 +895,7 @@ func generateResponse(config *config.ModelConfig, evaluator *templates.Evaluator
 	opts.Messages = protoMessages
 	opts.UseTokenizerTemplate = config.TemplateConfig.UseTokenizerTemplate
 
+	// TODO: We can use the PredictStream method, but then can we stream the results of that to TTS?
 	reply, err := session.ModelInterface.Predict(context.TODO(), &opts)
 	if err != nil {
 		sendError(c, "inference_failed", fmt.Sprintf("backend error: %v", err), "", item.ID)
@@ -956,18 +957,34 @@ func generateResponse(config *config.ModelConfig, evaluator *templates.Evaluator
 		sendError(c, "tts_error", fmt.Sprintf("Failed to read TTS audio: %v", err), "", item.ID)
 		return
 	}
+	audioString := base64.StdEncoding.EncodeToString(audioBytes)
 
-	// For some reason OpenAI doesn't send the audio now according to the docs. The user must request it with conversation.item.retrieve.
-	// This (almost) makes sense when not using a real any-to-any model because we can send the transcript before the audio is ready.
-	// However we don't do that for now, to keep things simple we only send the done event once we have the audio ready
+	sendEvent(c, types.ResponseAudioDeltaEvent{
+		ServerEventBase: types.ServerEventBase{
+			Type: types.ServerEventTypeResponseAudioDelta,
+		},
+		ItemID: item.ID,
+		// TODO: OutputIndex and ContentIndex
+		Delta: audioString,
+	})
+	sendEvent(c, types.ResponseAudioDoneEvent{
+		ServerEventBase: types.ServerEventBase{
+			Type: types.ServerEventTypeResponseAudioDone,
+		},
+		ItemID: item.ID,
+		// TODO: Indexs
+	})
+
+	// OpenAI does not send the audio as part of the conversation.
+	// It's sent as audio deltas or the user can request it with conversation.item.retrieve.
 	conv.Lock.Lock()
 	doneEvent := types.ConversationItemDoneEvent{
 		ServerEventBase: types.ServerEventBase{
-			Type: types.ServerEventTypeConversationItemAdded,
+			Type: types.ServerEventTypeConversationItemDone,
 		},
 		Item: item,
 	}
-	item.Content[0].Audio = base64.StdEncoding.EncodeToString(audioBytes)
+	item.Content[0].Audio = audioString
 	conv.Lock.Unlock()
 
 	sendEvent(c, doneEvent)