From c7f37e078f33bbc90c3bc431a91acfea4dfde629 Mon Sep 17 00:00:00 2001 From: CrushN8r Date: Wed, 31 Jul 2024 19:38:17 -0400 Subject: [PATCH] KGSim Code Updates Lots of code improvements and streamlining --- .gitignore | 1 + chipper/pkg/wirepod/ttr/kgsim.go | 229 ++++---- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 539 +++++++++--------- chipper/pkg/wirepod/ttr/special_characters.go | 114 ++++ 4 files changed, 497 insertions(+), 386 deletions(-) create mode 100644 chipper/pkg/wirepod/ttr/special_characters.go diff --git a/.gitignore b/.gitignore index ecfea84f..d4beb135 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ /whisper.cpp .DS_Store /chipper/openaiChats.json +*.txt diff --git a/chipper/pkg/wirepod/ttr/kgsim.go b/chipper/pkg/wirepod/ttr/kgsim.go index 82e65a6d..76f119f3 100644 --- a/chipper/pkg/wirepod/ttr/kgsim.go +++ b/chipper/pkg/wirepod/ttr/kgsim.go @@ -1,18 +1,15 @@ package wirepod_ttr +// kgsim.go + import ( "context" "errors" "fmt" "io" "log" - "regexp" "strings" "time" - "unicode" - - "golang.org/x/text/transform" - "golang.org/x/text/unicode/norm" "github.com/fforchino/vector-go-sdk/pkg/vector" "github.com/fforchino/vector-go-sdk/pkg/vectorpb" @@ -21,6 +18,36 @@ import ( "github.com/sashabaranov/go-openai" ) +/* + +const ( + esn1 = "abcd1234" + esn2 = "abcd5678" +) + +// You can then use esn1 and esn2 in your robot initialization +var robot1, robot2 *vector.Vector +var err error + +func initRobots() { + robot1, err = vector.New(vector.WithSerialNo(esn1), vector.WithToken("yourToken1"), vector.WithTarget("yourTarget1")) + if err != nil { + logger.Println("Error initializing robot 1:", err) + } + + robot2, err = vector.New(vector.WithSerialNo(esn2), vector.WithToken("yourToken2"), vector.WithTarget("yourTarget2")) + if err != nil { + logger.Println("Error initializing robot 2:", err) + } +} + +*/ + +// +// Chats Section +// + +// get chat func GetChat(esn string) vars.RememberedChat { for _, chat := range vars.RememberedChats { if chat.ESN == esn { @@ -32,24 +59,27 @@ func GetChat(esn string) vars.RememberedChat { } } +// place chat func PlaceChat(chat vars.RememberedChat) { for i, achat := range vars.RememberedChats { if achat.ESN == chat.ESN { vars.RememberedChats[i] = chat + vars.SaveChats() return } } vars.RememberedChats = append(vars.RememberedChats, chat) + vars.SaveChats() } -// remember last 16 lines of chat +// remember last # lines of chat func Remember(user, ai openai.ChatCompletionMessage, esn string) { chatAppend := []openai.ChatCompletionMessage{ user, ai, } currentChat := GetChat(esn) - if len(currentChat.Chats) == 16 { + if len(currentChat.Chats) == 16 { // lines of chat var newChat vars.RememberedChat newChat.ESN = currentChat.ESN for i, chat := range currentChat.Chats { @@ -65,97 +95,55 @@ func Remember(user, ai openai.ChatCompletionMessage, esn string) { PlaceChat(currentChat) } -func isMn(r rune) bool { - return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks -} +// +// AI Request section +// -func removeSpecialCharacters(str string) string { - - // these two lines create a transformation that decomposes characters, removes non-spacing marks (like diacritics), and then recomposes the characters, effectively removing special characters - t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) - result, _, _ := transform.String(t, str) - - // Define the regular expression to match special characters - re := regexp.MustCompile(`[&^*#@]`) - - // Replace special characters with an empty string - result = removeEmojis(re.ReplaceAllString(result, "")) - - // Replace special characters with ASCII - // * COPY/PASTE TO ADD MORE CHARACTERS: - // result = strings.ReplaceAll(result, "", "") - result = strings.ReplaceAll(result, "‘", "'") - result = strings.ReplaceAll(result, "’", "'") - result = strings.ReplaceAll(result, "“", "\"") - result = strings.ReplaceAll(result, "”", "\"") - result = strings.ReplaceAll(result, "—", "-") - result = strings.ReplaceAll(result, "–", "-") - result = strings.ReplaceAll(result, "…", "...") - result = strings.ReplaceAll(result, "\u00A0", " ") - result = strings.ReplaceAll(result, "•", "*") - result = strings.ReplaceAll(result, "¼", "1/4") - result = strings.ReplaceAll(result, "½", "1/2") - result = strings.ReplaceAll(result, "¾", "3/4") - result = strings.ReplaceAll(result, "×", "x") - result = strings.ReplaceAll(result, "÷", "/") - result = strings.ReplaceAll(result, "ç", "c") - result = strings.ReplaceAll(result, "©", "(c)") - result = strings.ReplaceAll(result, "®", "(r)") - result = strings.ReplaceAll(result, "™", "(tm)") - result = strings.ReplaceAll(result, "@", "(a)") - result = strings.ReplaceAll(result, " AI ", " A. I. ") - return result -} - -func removeEmojis(input string) string { - // a mess, but it works! - re := regexp.MustCompile(`[\x{1F600}-\x{1F64F}]|[\x{1F300}-\x{1F5FF}]|[\x{1F680}-\x{1F6FF}]|[\x{1F1E0}-\x{1F1FF}]|[\x{2600}-\x{26FF}]|[\x{2700}-\x{27BF}]|[\x{1F900}-\x{1F9FF}]|[\x{1F004}]|[\x{1F0CF}]|[\x{1F18E}]|[\x{1F191}-\x{1F251}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]|[\x{1F004}-\x{1F0CF}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]`) - result := re.ReplaceAllString(input, "") - return result +// New function to determine the model +func determineModel(gpt3tryagain bool) string { + if gpt3tryagain { + return openai.GPT3Dot5Turbo + } + if vars.APIConfig.Knowledge.Provider == "openai" { + model := openai.GPT4o + logger.Println("Using " + model) + return model + } + logger.Println("Using " + vars.APIConfig.Knowledge.Model) + return vars.APIConfig.Knowledge.Model } +// Streamlined CreateAIReq function func CreateAIReq(transcribedText, esn string, gpt3tryagain, isKG bool) openai.ChatCompletionRequest { defaultPrompt := "You are a helpful, animated robot called Vector. Keep the response concise yet informative." - - var nChat []openai.ChatCompletionMessage - - smsg := openai.ChatCompletionMessage{ - Role: openai.ChatMessageRoleSystem, - } - if strings.TrimSpace(vars.APIConfig.Knowledge.OpenAIPrompt) != "" { - smsg.Content = strings.TrimSpace(vars.APIConfig.Knowledge.OpenAIPrompt) - } else { - smsg.Content = defaultPrompt + systemMessageContent := strings.TrimSpace(vars.APIConfig.Knowledge.OpenAIPrompt) + if systemMessageContent == "" { + systemMessageContent = defaultPrompt } - var model string + model := determineModel(gpt3tryagain) - if gpt3tryagain { - model = openai.GPT3Dot5Turbo - } else if vars.APIConfig.Knowledge.Provider == "openai" { - model = openai.GPT4o - logger.Println("Using " + model) - } else { - logger.Println("Using " + vars.APIConfig.Knowledge.Model) - model = vars.APIConfig.Knowledge.Model + systemMessage := openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleSystem, + Content: CreatePrompt(systemMessageContent, model, isKG), } - smsg.Content = CreatePrompt(smsg.Content, model, isKG) + nChat := []openai.ChatCompletionMessage{systemMessage} - nChat = append(nChat, smsg) if vars.APIConfig.Knowledge.SaveChat { rchat := GetChat(esn) - logger.Println("Using remembered chats, length of " + fmt.Sprint(len(rchat.Chats)) + " messages") + logger.Println("Using remembered chats, length: " + fmt.Sprint(len(rchat.Chats))) nChat = append(nChat, rchat.Chats...) } + nChat = append(nChat, openai.ChatCompletionMessage{ Role: openai.ChatMessageRoleUser, Content: transcribedText, }) - aireq := openai.ChatCompletionRequest{ + return openai.ChatCompletionRequest{ Model: model, - MaxTokens: 2048, + MaxTokens: 4095, Temperature: 1, TopP: 1, FrequencyPenalty: 0, @@ -163,9 +151,13 @@ func CreateAIReq(transcribedText, esn string, gpt3tryagain, isKG bool) openai.Ch Messages: nChat, Stream: true, } - return aireq } +// +// KGSim section +// + +// streaming KGSim func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bool) (string, error) { start := make(chan bool) stop := make(chan bool) @@ -178,19 +170,19 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo var guid string var target string for _, bot := range vars.BotInfo.Robots { - if esn == bot.Esn { - guid = bot.GUID - target = bot.IPAddress + ":443" - matched = true - break - } + if esn == bot.Esn { + guid = bot.GUID + target = bot.IPAddress + ":443" + matched = true + break + } } if matched { - var err error - robot, err = vector.New(vector.WithSerialNo(esn), vector.WithToken(guid), vector.WithTarget(target)) - if err != nil { - return err.Error(), err - } + var err error + robot, err = vector.New(vector.WithSerialNo(esn), vector.WithToken(guid), vector.WithTarget(target)) + if err != nil { + return err.Error(), err + } } _, err := robot.Conn.BatteryState(context.Background(), &vectorpb.BatteryStateRequest{}) if err != nil { @@ -214,9 +206,9 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo } }() } - var fullRespText string + var accumulatedResponseText string var fullfullRespText string - var fullRespSlice []string + var responseChunks []string var isDone bool var c *openai.Client if vars.APIConfig.Knowledge.Provider == "together" { @@ -275,7 +267,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo response, err := stream.Recv() if errors.Is(err, io.EOF) { // prevents a crash - if len(fullRespSlice) == 0 { + if len(responseChunks) == 0 { logger.Println("LLM returned no response") successIntent <- false if isKG { @@ -290,9 +282,9 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo break } isDone = true - // if fullRespSlice != fullRespText, add that missing bit to fullRespSlice - newStr := fullRespSlice[0] - for i, str := range fullRespSlice { + // if responseChunks != accumulatedResponseText, add that missing bit to responseChunks + newStr := responseChunks[0] + for i, str := range responseChunks { if i == 0 { continue } @@ -300,8 +292,8 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo } if strings.TrimSpace(newStr) != strings.TrimSpace(fullfullRespText) { logger.Println("LLM debug: there is content after the last punctuation mark") - extraBit := strings.TrimPrefix(fullRespText, newStr) - fullRespSlice = append(fullRespSlice, extraBit) + extraBit := strings.TrimPrefix(accumulatedResponseText, newStr) + responseChunks = append(responseChunks, extraBit) } if vars.APIConfig.Knowledge.SaveChat { Remember(openai.ChatCompletionMessage{ @@ -314,7 +306,7 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo }, esn) } - logger.LogUI("LLM response for " + esn + ": " + newStr) + logger.LogUI("LLM response for " + esn + ": " + newStr + "\n\n") logger.Println("LLM stream finished") return } @@ -324,26 +316,26 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo return } - fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) - fullRespText = fullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) - if strings.Contains(fullRespText, "...") || strings.Contains(fullRespText, ".'") || strings.Contains(fullRespText, ".\"") || strings.Contains(fullRespText, ".") || strings.Contains(fullRespText, "?") || strings.Contains(fullRespText, "!") { + fullfullRespText += response.Choices[0].Delta.Content + accumulatedResponseText += response.Choices[0].Delta.Content + if strings.Contains(accumulatedResponseText, "...") || strings.Contains(accumulatedResponseText, ".'") || strings.Contains(accumulatedResponseText, ".\"") || strings.Contains(accumulatedResponseText, ".") || strings.Contains(accumulatedResponseText, "?") || strings.Contains(accumulatedResponseText, "!") { var sepStr string - if strings.Contains(fullRespText, "...") { + if strings.Contains(accumulatedResponseText, "...") { sepStr = "..." - } else if strings.Contains(fullRespText, ".'") { + } else if strings.Contains(accumulatedResponseText, ".'") { sepStr = ".'" - } else if strings.Contains(fullRespText, ".\"") { + } else if strings.Contains(accumulatedResponseText, ".\"") { sepStr = ".\"" - } else if strings.Contains(fullRespText, ".") { + } else if strings.Contains(accumulatedResponseText, ".") { sepStr = "." - } else if strings.Contains(fullRespText, "?") { + } else if strings.Contains(accumulatedResponseText, "?") { sepStr = "?" - } else if strings.Contains(fullRespText, "!") { + } else if strings.Contains(accumulatedResponseText, "!") { sepStr = "!" } - splitResp := strings.Split(strings.TrimSpace(fullRespText), sepStr) - fullRespSlice = append(fullRespSlice, strings.TrimSpace(splitResp[0])+sepStr) - fullRespText = splitResp[1] + splitResp := strings.Split(strings.TrimSpace(accumulatedResponseText), sepStr) + responseChunks = append(responseChunks, strings.TrimSpace(splitResp[0])+sepStr) + accumulatedResponseText = splitResp[1] select { case successIntent <- true: default: @@ -425,12 +417,12 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo var disconnect bool numInResp := 0 for { - respSlice := fullRespSlice + respSlice := responseChunks if len(respSlice)-1 < numInResp { if !isDone { logger.Println("Waiting for more content from LLM...") for range speakReady { - respSlice = fullRespSlice + respSlice = responseChunks break } } else { @@ -442,8 +434,8 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo } logger.Println(respSlice[numInResp]) acts := GetActionsFromString(respSlice[numInResp]) - nChat[len(nChat)-1].Content = fullRespText - disconnect = PerformActions(nChat, acts, robot, stopStop) + nChat[len(nChat)-1].Content = accumulatedResponseText + disconnect = PerformActions(nChat, acts, esn, stopStop) if disconnect { break } @@ -476,6 +468,11 @@ func StreamingKGSim(req interface{}, esn string, transcribedText string, isKG bo return "", nil } +// +// KGSim +// + +// KGSim func KGSim(esn string, textToSay string) error { ctx := context.Background() matched := false @@ -618,3 +615,7 @@ func KGSim(esn string, textToSay string) error { }() return nil } +// +// kgsim.go - END +// + diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 299ef1a2..63021dd9 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -1,5 +1,9 @@ package wirepod_ttr +// +// kgsim_cmds.go +// + import ( "context" "encoding/base64" @@ -17,74 +21,35 @@ import ( "github.com/sashabaranov/go-openai" ) +// +// data section +// + const ( - // arg: text to say - // not a command - ActionSayText = 0 - // arg: animation name + ActionSayText = 0 ActionPlayAnimation = 1 - // arg: animation name ActionPlayAnimationWI = 2 - // arg: now - ActionGetImage = 3 - ActionNewRequest = 4 - // arg: sound file - ActionPlaySound = 4 + ActionGetImage = 3 + ActionNewRequest = 4 + ActionPlaySound = 5 ) var animationMap [][2]string = [][2]string{ - //"happy, veryHappy, sad, verySad, angry, dartingEyes, confused, thinking, celebrate" - { - "happy", - "anim_onboarding_reacttoface_happy_01", - }, - { - "veryHappy", - "anim_blackjack_victorwin_01", - }, - { - "sad", - "anim_feedback_meanwords_01", - }, - { - "verySad", - "anim_feedback_meanwords_01", - }, - { - "angry", - "anim_rtpickup_loop_10", - }, - { - "frustrated", - "anim_feedback_shutup_01", - }, - { - "dartingEyes", - "anim_observing_self_absorbed_01", - }, - { - "confused", - "anim_meetvictor_lookface_timeout_01", - }, - { - "thinking", - "anim_explorer_scan_short_04", - }, - { - "celebrate", - "anim_pounce_success_03", - }, - { - "love", - "anim_feedback_iloveyou_02", - }, + {"happy", "anim_onboarding_reacttoface_happy_01"}, + {"veryHappy", "anim_blackjack_victorwin_01"}, + {"sad", "anim_feedback_meanwords_01"}, + {"verySad", "anim_feedback_meanwords_01"}, + {"angry", "anim_rtpickup_loop_10"}, + {"frustrated", "anim_feedback_shutup_01"}, + {"dartingEyes", "anim_observing_self_absorbed_01"}, + {"confused", "anim_meetvictor_lookface_timeout_01"}, + {"thinking", "anim_explorer_scan_short_04"}, + {"celebrate", "anim_pounce_success_03"}, + {"love", "anim_feedback_iloveyou_02"}, } var soundMap [][2]string = [][2]string{ - { - "drumroll", - "sounds/drumroll.wav", - }, + {"drumroll", "sounds/drumroll.wav"}, } type RobotAction struct { @@ -100,46 +65,53 @@ type LLMCommand struct { SupportedModels []string } -// create function which parses from LLM and makes a struct of RobotActions +var ( + playAnimationWIDescription = "Enhances storytelling by playing an animation on the robot without interrupting speech. This should be used frequently to animate responses and engage the audience. Choose from parameters like happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, or love to complement the dialogue and maintain context." + playAnimationDescription = "Interrupts speech to play an animation on the robot. Only use this when directed explicitly to play an animation, as it halts ongoing speech. Parameters include happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, or love for expressing emotions and reactions." + getImageDescription = "Retrieves an image from the robot's camera and displays it in the next message. Use this command to conclude a sentence or response when prompted by the user or when describing visual content, such as what the robot sees, with options for front or lookingUp perspectives. If asked 'What do you see in front of you?' or similar, default to taking a photo. Inform the user of your action before using the command." + newVoiceRequestDescription = "Starts a new voice command from the robot. Use this if you want more input from the user/if you want to carry out a conversation. You are the only one who can end it in this case. This goes at the end of your response, if you use it." + // var playSoundDescription = "Plays a sound on the robot." + LLMCommandsParamChoices = "happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, love" +) var ValidLLMCommands []LLMCommand = []LLMCommand{ { Command: "playAnimationWI", - Description: "Plays an animation on the robot without interrupting speech. This should be used FAR more than the playAnimation command. This is great for storytelling and making any normal response animated. Don't put two of these right next to each other. Use this MANY times. The param choices are the only choices you have. You can't create any.", - ParamChoices: "happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, love", + Description: playAnimationWIDescription, + ParamChoices: LLMCommandsParamChoices, Action: ActionPlayAnimationWI, SupportedModels: []string{"all"}, }, { Command: "playAnimation", - Description: "Plays an animation on the robot. This will interrupt speech. Only use this if you are directed to play an animaion.", - ParamChoices: "happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, love", + Description: playAnimationDescription, + ParamChoices: LLMCommandsParamChoices, Action: ActionPlayAnimation, SupportedModels: []string{"all"}, }, { - Command: "getImage", - Description: "Gets an image from the robot's camera and places it in the next message. If you want to do this, tell the user what you are about to do THEN use the command. This command should END a sentence. Your response will be stopped when this command is recognized. If a user says something like 'what do you see', you should assume that you need to take a new photo. Do NOT automatically assume that you are analyzing a previous photo.", - // not impl yet + Command: "getImage", + Description: getImageDescription, ParamChoices: "front, lookingUp", Action: ActionGetImage, SupportedModels: []string{openai.GPT4o}, }, { Command: "newVoiceRequest", - Description: "Starts a new voice command from the robot. Use this if you want more input from the user after your response/if you want to carry out a conversation. Below this, there should be a NOTE telling you whether you are in conversation mode or not. If you are, DONT BE AFRAID TO USE THIS COMMAND! This goes at the end of your response, if you use it.", + Description: newVoiceRequestDescription, ParamChoices: "now", Action: ActionNewRequest, SupportedModels: []string{"all"}, }, - // { - // Command: "playSound", - // Description: "Plays a sound on the robot.", - // ParamChoices: "drumroll", - // Action: ActionPlaySound, - // }, +/* { + Command: "playSound", + Description: playSoundDescription, + ParamChoices: "drumroll", + Action: ActionPlaySound, + }, */ } +// is AI model supported func ModelIsSupported(cmd LLMCommand, model string) bool { for _, str := range cmd.SupportedModels { if str == "all" || str == model { @@ -149,21 +121,31 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { return false } +var ( + promptForVector = " Format responses to communicate as an Anki Vector robot. User input may contain errors due to unreliable software. Evaluate for grammatical errors, missing words, or nonsensical phrases. Consider the overall context of the conversation to resolve ambiguities. If input is determined unintelligible or irrelevant, request clarification. Provide appropriate longer or shorter responses accordingly. " + + promptForVectorAI = " Integrate this format {{playAnimationWI||actionParameter}} into your responses, with the following command parameters: happy, veryHappy, sad, verySad, angry, frustrated, dartingEyes, confused, thinking, celebrate, and love. Add commas liberally in sentences to create pauses, for more natural-sounding speech breaks, and which allow for seamless embedding of commands. To enhance context, strategically place 'playAnimationWI' commands after commas, at the beginning of sentences, and one concluding command at the end of the finished LLM response. Select actions and parameters that amplify the expressiveness of the robot's phrases, aiming for engaging audience response. Examples: 1. Honestly, {{playAnimationWI||frustrated}} I never thought I'd ask, but, {{playAnimationWI||thinking}} where did I leave my keys? 2. You know, {{playAnimationWI||happy}} when everything goes right, {{playAnimationWI||veryHappy}} it feels like magic! 3. {{playAnimationWI||confused}} Sometimes, I think, {{playAnimationWI||thinking}} why do we even have Mondays? They're just like, {{playAnimationWI||frustrated}} a cruel joke, right? 4. Like, {{playAnimationWI||celebrate}} when I finish my tasks, {{playAnimationWI||excited}} I feel like celebrating, but, {{playAnimationWI||angry}} it's just Tuesday! 5. But overall, {{playAnimationWI||love}} I really do adore these odd days, {{playAnimationWI||excited}} they just add flavor to life! {{playAnimationWI||celebrate}} " + + conversationMode0 = "\n\nNOTE: You are NOT in 'conversation' mode. Avoid asking the user questions, and hence, from using 'newVoiceRequest'." + + conversationMode1 = "\n\nNOTE: You're now in 'conversation' mode. Near the end of your response, if you decide to ask the user a question, you MUST use 'newVoiceRequest'. Otherwise do not use 'newVoiceRequest' if you decide to end the conversation." +) + +// add to orig prompt func CreatePrompt(origPrompt string, model string, isKG bool) string { - prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." + prompt := origPrompt + promptForVector if vars.APIConfig.Knowledge.CommandsEnable { - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" + prompt += promptForVectorAI for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { - promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices - prompt = prompt + promptAppendage + prompt += fmt.Sprintf("\n\nCommand Name: %s\nDescription: %s\nParameter choices: %s", cmd.Command, cmd.Description, cmd.ParamChoices) } } if isKG && vars.APIConfig.Knowledge.SaveChat { - promptAppentage := "\n\nNOTE: You are in 'conversation' mode. If you ask the user a question near the end of your response, you MUST use newVoiceRequest. If you decide you want to end the conversation, you should not use it." + promptAppentage := conversationMode1 prompt = prompt + promptAppentage } else { - promptAppentage := "\n\nNOTE: You are NOT in 'conversation' mode. Refrain from asking the user any questions and from using newVoiceRequest." + promptAppentage := conversationMode0 prompt = prompt + promptAppentage } } @@ -173,15 +155,11 @@ func CreatePrompt(origPrompt string, model string, isKG bool) string { return prompt } +// parse actions from string func GetActionsFromString(input string) []RobotAction { splitInput := strings.Split(input, "{{") if len(splitInput) == 1 { - return []RobotAction{ - { - Action: ActionSayText, - Parameter: input, - }, - } + return []RobotAction{{Action: ActionSayText, Parameter: input}} } var actions []RobotAction for _, spl := range splitInput { @@ -189,12 +167,7 @@ func GetActionsFromString(input string) []RobotAction { continue } if !strings.Contains(spl, "}}") { - // sayText - action := RobotAction{ - Action: ActionSayText, - Parameter: strings.TrimSpace(spl), - } - actions = append(actions, action) + actions = append(actions, RobotAction{Action: ActionSayText, Parameter: strings.TrimSpace(spl)}) continue } @@ -206,76 +179,106 @@ func GetActionsFromString(input string) []RobotAction { actions = append(actions, action) } if len(strings.Split(spl, "}}")) != 1 { - action := RobotAction{ - Action: ActionSayText, - Parameter: strings.TrimSpace(strings.Split(spl, "}}")[1]), - } + action := RobotAction{Action: ActionSayText, Parameter: strings.TrimSpace(strings.Split(spl, "}}")[1])} actions = append(actions, action) } } return actions } +// commands to robot actions func CmdParamToAction(cmd, param string) RobotAction { for _, command := range ValidLLMCommands { if cmd == command.Command { - return RobotAction{ - Action: command.Action, - Parameter: param, - } + return RobotAction{Action: command.Action, Parameter: param} } } logger.Println("LLM tried to do a command which doesn't exist: " + cmd + " (param: " + param + ")") - return RobotAction{ - Action: -1, - } + return RobotAction{Action: -1} } -func DoPlayAnimation(animation string, robot *vector.Vector) error { + +// Do 'Play Animation' +func DoPlayAnimation(animation string, esn string) error { + // Find the robot instance associated with the given esn. + var robot *vector.Vector + for _, r := range vars.BotInfo.Robots { + if r.Esn == esn { + guid := r.GUID + target := r.IPAddress + ":443" + var err error + robot, err = vector.New(vector.WithSerialNo(esn), vector.WithToken(guid), vector.WithTarget(target)) + if err != nil { + return err + } + break + } + } + + if robot == nil { + logger.Println("No robot found with ESN: " + esn) + return errors.New("no robot found with the given ESN") + } + for _, animThing := range animationMap { if animation == animThing[0] { - StartAnim_Queue(robot.Cfg.SerialNo) - robot.Conn.PlayAnimation( - context.Background(), - &vectorpb.PlayAnimationRequest{ - Animation: &vectorpb.Animation{ - Name: animThing[1], - }, - Loops: 1, - }, - ) - StopAnim_Queue(robot.Cfg.SerialNo) + StartAnim_Queue(esn) + robot.Conn.PlayAnimation(context.Background(), &vectorpb.PlayAnimationRequest{ + Animation: &vectorpb.Animation{Name: animThing[1]}, + Loops: 1, + }) + StopAnim_Queue(esn) return nil } } + logger.Println("Animation provided by LLM doesn't exist: " + animation) return nil } -func DoPlayAnimationWI(animation string, robot *vector.Vector) error { + +// Do 'Play AnimationWI' +func DoPlayAnimationWI(animation string, esn string) error { + // Find the robot instance associated with the given esn. + var robot *vector.Vector + for _, r := range vars.BotInfo.Robots { + if r.Esn == esn { + guid := r.GUID + target := r.IPAddress + ":443" + var err error + robot, err = vector.New(vector.WithSerialNo(esn), vector.WithToken(guid), vector.WithTarget(target)) + if err != nil { + return err + } + break + } + } + + if robot == nil { + logger.Println("No robot found with ESN: " + esn) + return errors.New("no robot found with the given ESN") + } + for _, animThing := range animationMap { if animation == animThing[0] { go func() { - StartAnim_Queue(robot.Cfg.SerialNo) - robot.Conn.PlayAnimation( - context.Background(), - &vectorpb.PlayAnimationRequest{ - Animation: &vectorpb.Animation{ - Name: animThing[1], - }, - Loops: 1, - }, - ) - StopAnim_Queue(robot.Cfg.SerialNo) + StartAnim_Queue(esn) + robot.Conn.PlayAnimation(context.Background(), &vectorpb.PlayAnimationRequest{ + Animation: &vectorpb.Animation{Name: animThing[1]}, + Loops: 1, + }) + StopAnim_Queue(esn) }() return nil } } + logger.Println("Animation provided by LLM doesn't exist: " + animation) return nil } -func DoPlaySound(sound string, robot *vector.Vector) error { +// Do 'Play Sound' +func DoPlaySound(sound string, esn string) error { for _, soundThing := range soundMap { if sound == soundThing[0] { logger.Println("Would play sound") @@ -285,23 +288,23 @@ func DoPlaySound(sound string, robot *vector.Vector) error { return nil } +// Do 'Say Text' func DoSayText(input string, robot *vector.Vector) error { - - // just before vector speaks - removeSpecialCharacters(input) + cleanedInput, err := removeSpecialCharacters(input) + if err != nil { + logger.Println("Error removing special characters:", err) + return err + } if (vars.APIConfig.STT.Language != "en-US" && vars.APIConfig.Knowledge.Provider == "openai") || vars.APIConfig.Knowledge.OpenAIVoiceWithEnglish { - err := DoSayText_OpenAI(robot, input) - return err + return DoSayText_OpenAI(robot, cleanedInput) } - robot.Conn.SayText( - context.Background(), - &vectorpb.SayTextRequest{ - Text: input, - UseVectorVoice: true, - DurationScalar: 0.95, - }, - ) + + robot.Conn.SayText(context.Background(), &vectorpb.SayTextRequest{ + Text: cleanedInput, + UseVectorVoice: true, + DurationScalar: 0.95, + }) return nil } @@ -326,17 +329,12 @@ func getOpenAIVoice(voice string) openai.SpeechVoice { return voiceMap[voice] } -// TODO +// Do 'Say Text for OpenAI' func DoSayText_OpenAI(robot *vector.Vector, input string) error { if strings.TrimSpace(input) == "" { return nil } openaiVoice := getOpenAIVoice(vars.APIConfig.Knowledge.OpenAIPrompt) - // if vars.APIConfig.Knowledge.OpenAIVoice == "" { - // openaiVoice = openai.VoiceFable - // } else { - // openaiVoice = getOpenAIVoice(vars.APIConfig.Knowledge.OpenAIPrompt) - // } oc := openai.NewClient(vars.APIConfig.Knowledge.Key) resp, err := oc.CreateSpeech(context.Background(), openai.CreateSpeechRequest{ Model: openai.TTSModel1, @@ -361,7 +359,6 @@ func DoSayText_OpenAI(robot *vector.Vector, input string) error { }, }, }) - //time.Sleep(time.Millisecond * 30) audioChunks := downsample24kTo16k(speechBytes) var chunksToDetermineLength []byte @@ -390,7 +387,9 @@ func DoSayText_OpenAI(robot *vector.Vector, input string) error { return nil } -func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector.Vector, stopStop chan bool) { + +// Do 'Get Image' +func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector.Vector, stopStop chan bool, esn string) { stopImaging := false go func() { for range stopStop { @@ -399,71 +398,45 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector } }() logger.Println("Get image here...") - // get image - robot.Conn.EnableMirrorMode(context.Background(), &vectorpb.EnableMirrorModeRequest{ - Enable: true, - }) + robot.Conn.EnableMirrorMode(context.Background(), &vectorpb.EnableMirrorModeRequest{Enable: true}) for i := 3; i > 0; i-- { if stopImaging { return } time.Sleep(time.Millisecond * 300) - robot.Conn.SayText( - context.Background(), - &vectorpb.SayTextRequest{ - Text: fmt.Sprint(i), - UseVectorVoice: true, - DurationScalar: 1.05, - }, - ) + robot.Conn.SayText(context.Background(), &vectorpb.SayTextRequest{ + Text: fmt.Sprint(i), + UseVectorVoice: true, + DurationScalar: 1.05, + }) if stopImaging { return } } - resp, _ := robot.Conn.CaptureSingleImage( - context.Background(), - &vectorpb.CaptureSingleImageRequest{ - EnableHighResolution: true, - }, - ) - robot.Conn.EnableMirrorMode( - context.Background(), - &vectorpb.EnableMirrorModeRequest{ - Enable: false, - }, - ) + resp, _ := robot.Conn.CaptureSingleImage(context.Background(), &vectorpb.CaptureSingleImageRequest{EnableHighResolution: true}) + robot.Conn.EnableMirrorMode(context.Background(), &vectorpb.EnableMirrorModeRequest{Enable: false}) go func() { - robot.Conn.PlayAnimation( - context.Background(), - &vectorpb.PlayAnimationRequest{ - Animation: &vectorpb.Animation{ - Name: "anim_photo_shutter_01", - }, - Loops: 1, - }, - ) + robot.Conn.PlayAnimation(context.Background(), &vectorpb.PlayAnimationRequest{ + Animation: &vectorpb.Animation{Name: "anim_photo_shutter_01"}, + Loops: 1, + }) }() - // encode to base64 reqBase64 := base64.StdEncoding.EncodeToString(resp.Data) - // add image to messages msgs = append(msgs, openai.ChatCompletionMessage{ Role: openai.ChatMessageRoleUser, - MultiContent: []openai.ChatMessagePart{ - { - Type: openai.ChatMessagePartTypeImageURL, - ImageURL: &openai.ChatMessageImageURL{ - URL: fmt.Sprintf("data:image/jpeg;base64,%s", reqBase64), - Detail: openai.ImageURLDetailLow, - }, + MultiContent: []openai.ChatMessagePart{{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: fmt.Sprintf("data:image/jpeg;base64,%s", reqBase64), + Detail: openai.ImageURLDetailLow, }, - }, + }}, }) - // recreate openai - var fullRespText string - var fullfullRespText string - var fullRespSlice []string + var accumulatedResponseText string + var completeResponseText string + var responseChunks []string var isDone bool var c *openai.Client if vars.APIConfig.Knowledge.Provider == "together" { @@ -481,7 +454,7 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector speakReady := make(chan string) aireq := openai.ChatCompletionRequest{ - MaxTokens: 2048, + MaxTokens: 4095, Temperature: 1, TopP: 1, FrequencyPenalty: 0, @@ -502,14 +475,12 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector stream, err := c.CreateChatCompletionStream(ctx, aireq) if err != nil { if strings.Contains(err.Error(), "does not exist") && vars.APIConfig.Knowledge.Provider == "openai" { - logger.Println("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") - logger.LogUI("GPT-4 model cannot be accessed with this API key. You likely need to add more than $5 dollars of funds to your OpenAI account.") + logger.Println("GPT-4 model cannot be accessed with this API key.") aireq.Model = openai.GPT3Dot5Turbo logger.Println("Falling back to " + aireq.Model) - logger.LogUI("Falling back to " + aireq.Model) stream, err = c.CreateChatCompletionStream(ctx, aireq) if err != nil { - logger.Println("OpenAI still not returning a response even after falling back. Erroring.") + logger.Println("OpenAI still not returning a response even after falling back.") return } } else { @@ -517,7 +488,6 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector return } } - //defer stream.Close() fmt.Println("LLM stream response: ") go func() { @@ -525,17 +495,17 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector response, err := stream.Recv() if errors.Is(err, io.EOF) { isDone = true - newStr := fullRespSlice[0] - for i, str := range fullRespSlice { + newStr := responseChunks[0] + for i, str := range responseChunks { if i == 0 { continue } newStr = newStr + " " + str } - if strings.TrimSpace(newStr) != strings.TrimSpace(fullfullRespText) { + if strings.TrimSpace(newStr) != strings.TrimSpace(completeResponseText) { logger.Println("LLM debug: there is content after the last punctuation mark") - extraBit := strings.TrimPrefix(fullRespText, newStr) - fullRespSlice = append(fullRespSlice, extraBit) + extraBit := strings.TrimPrefix(accumulatedResponseText, newStr) + responseChunks = append(responseChunks, extraBit) } if vars.APIConfig.Knowledge.SaveChat { Remember(msgs[len(msgs)-1], @@ -543,9 +513,9 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector Role: openai.ChatMessageRoleAssistant, Content: newStr, }, - robot.Cfg.SerialNo) + esn) } - logger.LogUI("LLM response for " + robot.Cfg.SerialNo + ": " + newStr) + logger.LogUI("LLM response for " + esn + ": " + newStr + "\n\n") logger.Println("LLM stream finished") return } @@ -554,26 +524,26 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector logger.Println("Stream error: " + err.Error()) return } - fullfullRespText = fullfullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) - fullRespText = fullRespText + removeSpecialCharacters(response.Choices[0].Delta.Content) - if strings.Contains(fullRespText, "...") || strings.Contains(fullRespText, ".'") || strings.Contains(fullRespText, ".\"") || strings.Contains(fullRespText, ".") || strings.Contains(fullRespText, "?") || strings.Contains(fullRespText, "!") { + completeResponseText += response.Choices[0].Delta.Content + accumulatedResponseText += response.Choices[0].Delta.Content + if strings.Contains(accumulatedResponseText, "...") || strings.Contains(accumulatedResponseText, ".'") || strings.Contains(accumulatedResponseText, ".\"") || strings.Contains(accumulatedResponseText, ".") || strings.Contains(accumulatedResponseText, "?") || strings.Contains(accumulatedResponseText, "!") { var sepStr string - if strings.Contains(fullRespText, "...") { + if strings.Contains(accumulatedResponseText, "...") { sepStr = "..." - } else if strings.Contains(fullRespText, ".'") { + } else if strings.Contains(accumulatedResponseText, ".'") { sepStr = ".'" - } else if strings.Contains(fullRespText, ".\"") { + } else if strings.Contains(accumulatedResponseText, ".\"") { sepStr = ".\"" - } else if strings.Contains(fullRespText, ".") { + } else if strings.Contains(accumulatedResponseText, ".") { sepStr = "." - } else if strings.Contains(fullRespText, "?") { + } else if strings.Contains(accumulatedResponseText, "?") { sepStr = "?" - } else if strings.Contains(fullRespText, "!") { + } else if strings.Contains(accumulatedResponseText, "!") { sepStr = "!" } - splitResp := strings.Split(strings.TrimSpace(fullRespText), sepStr) - fullRespSlice = append(fullRespSlice, strings.TrimSpace(splitResp[0])+sepStr) - fullRespText = splitResp[1] + splitResp := strings.Split(strings.TrimSpace(accumulatedResponseText), sepStr) + responseChunks = append(responseChunks, strings.TrimSpace(splitResp[0])+sepStr) + accumulatedResponseText = splitResp[1] select { case speakReady <- strings.TrimSpace(splitResp[0]) + sepStr: default: @@ -586,12 +556,12 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector if stopImaging { return } - respSlice := fullRespSlice + respSlice := responseChunks if len(respSlice)-1 < numInResp { if !isDone { logger.Println("Waiting for more content from LLM...") for range speakReady { - respSlice = fullRespSlice + respSlice = responseChunks break } } else { @@ -600,7 +570,7 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector } logger.Println(respSlice[numInResp]) acts := GetActionsFromString(respSlice[numInResp]) - PerformActions(msgs, acts, robot, stopStop) + PerformActions(msgs, acts, esn, stopStop) // Directly using esn here numInResp = numInResp + 1 if stopImaging { return @@ -608,95 +578,120 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector } } +// Do 'New Request' func DoNewRequest(robot *vector.Vector) { time.Sleep(time.Second / 3) robot.Conn.AppIntent(context.Background(), &vectorpb.AppIntentRequest{Intent: "knowledge_question"}) } -func PerformActions(msgs []openai.ChatCompletionMessage, actions []RobotAction, robot *vector.Vector, stopStop chan bool) bool { - // assuming we have behavior control already +// perform the actions +func PerformActions(msgs []openai.ChatCompletionMessage, actions []RobotAction, esn string, stopStop chan bool) bool { stopPerforming := false + var robot *vector.Vector + + // Find robot instance associated with ESN + for _, r := range vars.BotInfo.Robots { + if r.Esn == esn { + guid := r.GUID + target := r.IPAddress + ":443" + var err error + robot, err = vector.New(vector.WithSerialNo(esn), vector.WithToken(guid), vector.WithTarget(target)) + if err != nil { + logger.Println("Error fetching robot:", err) + return false + } + break + } + } + + if robot == nil { + logger.Println("No robot found with ESN: " + esn) + return false + } + go func() { for range stopStop { stopPerforming = true } }() + for _, action := range actions { if stopPerforming { return false } switch { case action.Action == ActionSayText: - DoSayText(action.Parameter, robot) + if err := DoSayText(action.Parameter, robot); err != nil { + logger.Println("Error in DoSayText:", err) + return false + } case action.Action == ActionPlayAnimation: - DoPlayAnimation(action.Parameter, robot) + if err := DoPlayAnimation(action.Parameter, esn); err != nil { + logger.Println("Error in DoPlayAnimation:", err) + } case action.Action == ActionPlayAnimationWI: - DoPlayAnimationWI(action.Parameter, robot) + if err := DoPlayAnimationWI(action.Parameter, esn); err != nil { + logger.Println("Error in DoPlayAnimationWI:", err) + } case action.Action == ActionNewRequest: go DoNewRequest(robot) return true case action.Action == ActionGetImage: - DoGetImage(msgs, action.Parameter, robot, stopStop) + DoGetImage(msgs, action.Parameter, robot, stopStop, esn) // Changing the ESN type return true case action.Action == ActionPlaySound: - DoPlaySound(action.Parameter, robot) + DoPlaySound(action.Parameter, esn) } } - WaitForAnim_Queue(robot.Cfg.SerialNo) + WaitForAnim_Queue(esn) return false } +// +// Animation Queues Section +// +type AnimationQueue struct { + ESN string + AnimDone chan struct{} + AnimCurrentlyPlaying bool +} + +var AnimationQueues = make(map[string]*AnimationQueue) + +// wait for the queue func WaitForAnim_Queue(esn string) { - for i, q := range AnimationQueues { - if q.ESN == esn { - if q.AnimCurrentlyPlaying { - for range AnimationQueues[i].AnimDone { - break - } - return - } - } + if q, exists := AnimationQueues[esn]; exists && q.AnimCurrentlyPlaying { + <-q.AnimDone // Blocks until the animation is done } } +// start the queue func StartAnim_Queue(esn string) { - // if animation is already playing, just wait for it to be done - for i, q := range AnimationQueues { - if q.ESN == esn { - if q.AnimCurrentlyPlaying { - for range AnimationQueues[i].AnimDone { - logger.Println("(waiting for animation to be done...)") - break - } - } else { - AnimationQueues[i].AnimCurrentlyPlaying = true - } - return + if q, exists := AnimationQueues[esn]; exists { + if q.AnimCurrentlyPlaying { + logger.Println("(waiting for animation to be done...)") + <-q.AnimDone // Blocks until the animation is done + } + q.AnimCurrentlyPlaying = true + } else { + AnimationQueues[esn] = &AnimationQueue{ + ESN: esn, + AnimDone: make(chan struct{}), + AnimCurrentlyPlaying: true, } } - var aq AnimationQueue - aq.AnimCurrentlyPlaying = true - aq.AnimDone = make(chan bool) - aq.ESN = esn - AnimationQueues = append(AnimationQueues, aq) } +// stop the queue func StopAnim_Queue(esn string) { - for i, q := range AnimationQueues { - if q.ESN == esn { - AnimationQueues[i].AnimCurrentlyPlaying = false - select { - case AnimationQueues[i].AnimDone <- true: - default: - } - } + if q, exists := AnimationQueues[esn]; exists { + q.AnimCurrentlyPlaying = false + close(q.AnimDone) // Close the channel to signal that the animation is done + delete(AnimationQueues, esn) // Optionally remove the queue after stopping } } -type AnimationQueue struct { - ESN string - AnimDone chan bool - AnimCurrentlyPlaying bool -} +// +// kgsim_cmds.go - END +// -var AnimationQueues []AnimationQueue diff --git a/chipper/pkg/wirepod/ttr/special_characters.go b/chipper/pkg/wirepod/ttr/special_characters.go new file mode 100644 index 00000000..a9c0ee71 --- /dev/null +++ b/chipper/pkg/wirepod/ttr/special_characters.go @@ -0,0 +1,114 @@ +package wirepod_ttr + +// +//special_characters.go +// + +import ( + "regexp" + "strings" + "unicode" + + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" + + "github.com/kercre123/wire-pod/chipper/pkg/logger" +) + + +// +// Special Character Handling section +// + +// special characters replacements +var specialCharactersReplacements = strings.NewReplacer( + "‘", "'", "’", "'", "“", "\"", "”", "\"", + "—", " - ", "–", " - ", "…", ",", + "\u00A0", " ", " ", " ", + "¼", "1/4", "½", "1/2", "¾", "3/4", "×", "x", "@", "(a)", + "facepalm", "face-palm", "chatbot", "chat-bot", "bedsheet", "bed-sheet", " **", " ,", + "&", "and", "^", "caret", "#", "hashtag", + "%20", " ", "%21", "!", "%22", "\"", "%23", "#", "%24", "$", "%25", "%", + "%26", "&", "%27", "'", "%28", "(", "%29", ")", "%2A", "*", "%2B", "+", + "%2C", ",", "%2D", "-", "%2E", ".", "%2F", "/", "%3A", ":", "%3B", ";", + "%3C", "<", "%3D", "=", "%3E", ">", "%3F", "?", "%40", "@", "%5B", "[", + "%5C", "\\", "%5D", "]", "%5E", "^", "%5F", "_", "%60", "", "%7B", "{", + "%7C", "|", "%7D", "}", "%7E", "~", + "±", "+/-", "÷", "/", "√", "sqrt", "∞", "infinity", + "≈", "~", "≠", "!=", "≡", "==", "≤", "<=", "≥", ">=", "°", "deg", + "π", "pi", "∆", "delta", "∑", "sum", "∏", "prod", + "€", "EUR", "£", "GBP", "¥", "JPY", "₹", "INR", "$", "USD", + "©", "(c)", "®", "(r)", "™", "(tm)", "†", "+", "‡", "++", "§", "SS", + "•", "*", "‣", "*", "◦", "*", + "₀", "0", "₁", "1", "₂", "2", "₃", "3", "₄", "4", + "₅", "5", "₆", "6", "₇", "7", "₈", "8", "₉", "9", +) + +// emoji patterns +var emojiPattern = `[\x{1F600}-\x{1F64F}]|[\x{1F300}-\x{1F5FF}]|[\x{1F680}-\x{1F6FF}]|[\x{1F1E0}-\x{1F1FF}]|[\x{2600}-\x{26FF}]|[\x{2700}-\x{27BF}]|[\x{1F900}-\x{1F9FF}]|[\x{1F004}]|[\x{1F0CF}]|[\x{1F18E}]|[\x{1F191}-\x{1F251}]|[\x{2B50}]` + +// phonetic map for Vector +var phoneticReplacements = map[string]string{ + "AIs": "AY-EYES", "AI": "AY-EYE", "CrushN8r": "Crushinayter", + "SpaceX": "Space-X", "H2O": "H-2-O", "Communicate": "Cum-Uni-Kate", + "Aha": "Ah-ha", "A-ha": "Ah-ha", +} + +// normalize Text +func normalizeText(str string) (string, error) { + t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) + normalized, _, err := transform.String(t, str) + return normalized, err +} + +// checks if a rune is a non-spacing mark +func isMn(r rune) bool { + return unicode.Is(unicode.Mn, r) +} + +// case-insensitively replaces words/phrases +func replaceWords(str string) string { + for find, replace := range phoneticReplacements { + regex := regexp.MustCompile(`\b(?i)` + regexp.QuoteMeta(find) + `\b`) + str = regex.ReplaceAllString(str, replace) + } + return str +} + +// remove emojis using emojiPattern +func removeEmojis(str string) string { + return regexp.MustCompile(emojiPattern).ReplaceAllString(str, "") +} + +// ensures decimals are pronounced correctly +func replaceDecimals(str string) string { + decimalPattern := regexp.MustCompile(`(\d+)\.(\d+)`) + return decimalPattern.ReplaceAllStringFunc(str, func(match string) string { + parts := decimalPattern.FindStringSubmatch(match) + if len(parts) == 3 { + return parts[1] + " point " + parts[2] + } + logger.LogUI("\n\n"+"replaceDecimals() match = " + match+"\n\n") + return match + }) +} + +// removeSpecialCharacters main removal and replacements +func removeSpecialCharacters(str string) (string, error) { + normalized, err := normalizeText(str) + if err != nil { + return str, err // Returning original string for clarity on error + } + + result := specialCharactersReplacements.Replace(normalized) + result = replaceDecimals(result) + result = replaceWords(result) + result = removeEmojis(result) + + return result, nil +} + +// +// //special_characters.go - END +// +