From 061aa982d492d33b49704e71a116ac7d7d8c056a Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Tue, 12 Nov 2024 23:21:34 -0800 Subject: [PATCH 1/3] Supporting opensource multimodal LLMs such as Llama3.1 11B or Llama3.1 90B --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 936573e1..9412d27a 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -123,7 +123,7 @@ var ValidLLMCommands []LLMCommand = []LLMCommand{ // not impl yet ParamChoices: "front, lookingUp", Action: ActionGetImage, - SupportedModels: []string{openai.GPT4o, openai.GPT4oMini}, + SupportedModels: []string{"all"}, }, { Command: "newVoiceRequest", @@ -152,7 +152,7 @@ func ModelIsSupported(cmd LLMCommand, model string) bool { func CreatePrompt(origPrompt string, model string, isKG bool) string { prompt := origPrompt + "\n\n" + "Keep in mind, user input comes from speech-to-text software, so respond accordingly. No special characters, especially these: & ^ * # @ - . No lists. No formatting." if vars.APIConfig.Knowledge.CommandsEnable { - prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" + prompt = prompt + "\n\n" + "You are running ON an Anki Vector robot. You have a set of commands. If you include an emoji, I will make you start over. If you want to use a command but it doesn't exist or your desired parameter isn't in the list, avoid using the command. The format is {{command||parameter}}. You can embed these in sentences. Example: \"User: How are you feeling? | Response: \"{{playAnimationWI||sad}} I'm feeling sad...\". Square brackets ([]) are not valid.\n\nUse the playAnimation or playAnimationWI commands if you want to express emotion! You are very animated and good at following instructions. Animation takes precendence over words. You are to include many animations in your response.\n\nHere is every valid command:" for _, cmd := range ValidLLMCommands { if ModelIsSupported(cmd, model) { promptAppendage := "\n\nCommand Name: " + cmd.Command + "\nDescription: " + cmd.Description + "\nParameter choices: " + cmd.ParamChoices @@ -476,7 +476,11 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector c = openai.NewClientWithConfig(conf) } else if vars.APIConfig.Knowledge.Provider == "openai" { c = openai.NewClient(vars.APIConfig.Knowledge.Key) - } + } else { + conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key) + conf.BaseURL = vars.APIConfig.Knowledge.Endpoint + c = openai.NewClientWithConfig(conf) + } ctx := context.Background() speakReady := make(chan string) From 3178a5299a743e6fff67bd1343866a6311735af5 Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Fri, 15 Nov 2024 23:01:46 -0800 Subject: [PATCH 2/3] Fixing code review on adding condition to check for custom provider --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index 9412d27a..dd06c5b0 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -476,7 +476,7 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector c = openai.NewClientWithConfig(conf) } else if vars.APIConfig.Knowledge.Provider == "openai" { c = openai.NewClient(vars.APIConfig.Knowledge.Key) - } else { + } else if vars.APIConfig.Knowledge.Provider == "custom" { conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key) conf.BaseURL = vars.APIConfig.Knowledge.Endpoint c = openai.NewClientWithConfig(conf) From 011a01f2977018691cc792bc48e04e44548477ba Mon Sep 17 00:00:00 2001 From: Amitabha Banerjee Date: Fri, 15 Nov 2024 23:05:13 -0800 Subject: [PATCH 3/3] Alligning braces --- chipper/pkg/wirepod/ttr/kgsim_cmds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chipper/pkg/wirepod/ttr/kgsim_cmds.go b/chipper/pkg/wirepod/ttr/kgsim_cmds.go index dd06c5b0..70783b02 100644 --- a/chipper/pkg/wirepod/ttr/kgsim_cmds.go +++ b/chipper/pkg/wirepod/ttr/kgsim_cmds.go @@ -480,7 +480,7 @@ func DoGetImage(msgs []openai.ChatCompletionMessage, param string, robot *vector conf := openai.DefaultConfig(vars.APIConfig.Knowledge.Key) conf.BaseURL = vars.APIConfig.Knowledge.Endpoint c = openai.NewClientWithConfig(conf) - } + } ctx := context.Background() speakReady := make(chan string)