-
Notifications
You must be signed in to change notification settings - Fork 132
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(ai-model): support coze platfrom's model (#50)
* chore: optimize prompt structure * feat(ai-model): add coze platform * feat(ai-model): add coze platform * chore: add unit test * feat(ai-model): add extract unit test * chore: add unit test * chore: add unit test * chore: optimize plan prompt * chore: optimize plan prompt * chore: fix lint error * chore: fix lint error * chore: fix unit test * chore: ignore cache file
- Loading branch information
Showing
41 changed files
with
949 additions
and
1,236 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| import assert from 'node:assert'; | ||
| import type { PlanningAIResponse, PlanningAction, UIContext } from '@/types'; | ||
| import { AIActionType, type AIArgs, callAiFn } from '../common'; | ||
| import { describeUserPage } from '../prompt/util'; | ||
| import { systemPromptToTaskPlanning } from './planning'; | ||
|
|
||
| export async function plan( | ||
| userPrompt: string, | ||
| opts: { | ||
| context: UIContext; | ||
| callAI?: typeof callAiFn<PlanningAIResponse>; | ||
| }, | ||
| useModel?: 'coze' | 'openAI', | ||
| ): Promise<{ plans: PlanningAction[] }> { | ||
| const { callAI, context } = opts || {}; | ||
| const { screenshotBase64 } = context; | ||
| const { description: pageDescription } = await describeUserPage(context); | ||
| let planFromAI: PlanningAIResponse | undefined; | ||
|
|
||
| const systemPrompt = systemPromptToTaskPlanning(); | ||
| const msgs: AIArgs = [ | ||
| { role: 'system', content: systemPrompt }, | ||
| { | ||
| role: 'user', | ||
| content: [ | ||
| { | ||
| type: 'image_url', | ||
| image_url: { | ||
| url: screenshotBase64, | ||
| detail: 'high', | ||
| }, | ||
| }, | ||
| { | ||
| type: 'text', | ||
| text: ` | ||
| pageDescription: ${pageDescription} | ||
| `, | ||
| }, | ||
| { | ||
| type: 'text', | ||
| text: ` | ||
| Here is the description of the task. Just go ahead: | ||
| ===================================== | ||
| ${userPrompt} | ||
| ===================================== | ||
| `, | ||
| }, | ||
| ], | ||
| }, | ||
| ]; | ||
|
|
||
| if (callAI) { | ||
| planFromAI = await callAI({ | ||
| msgs, | ||
| AIActionType: AIActionType.PLAN, | ||
| useModel, | ||
| }); | ||
| } else { | ||
| planFromAI = await callAiFn({ | ||
| msgs, | ||
| AIActionType: AIActionType.PLAN, | ||
| useModel, | ||
| }); | ||
| } | ||
|
|
||
| const actions = planFromAI?.actions || []; | ||
|
|
||
| assert(planFromAI, "can't get planFromAI"); | ||
| assert(actions && actions.length > 0, 'no actions in ai plan'); | ||
|
|
||
| if (planFromAI.error) { | ||
| throw new Error(planFromAI.error); | ||
| } | ||
|
|
||
| actions.forEach((task) => { | ||
| if (task.type === 'Error') { | ||
| throw new Error(task.thought); | ||
| } | ||
| }); | ||
|
|
||
| return { plans: actions }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| import type { | ||
| ChatCompletionSystemMessageParam, | ||
| ChatCompletionUserMessageParam, | ||
| } from 'openai/resources'; | ||
| import { | ||
| COZE_AI_ACTION_BOT_ID, | ||
| COZE_AI_ASSERT_BOT_ID, | ||
| COZE_EXTRACT_INFO_BOT_ID, | ||
| COZE_INSPECT_ELEMENT_BOT_ID, | ||
| callCozeAi, | ||
| transfromOpenAiArgsToCoze, | ||
| useCozeModel, | ||
| } from './coze'; | ||
| import { callToGetJSONObject, useOpenAIModel } from './openai'; | ||
|
|
||
| export type AIArgs = [ | ||
| ChatCompletionSystemMessageParam, | ||
| ChatCompletionUserMessageParam, | ||
| ]; | ||
|
|
||
| export enum AIActionType { | ||
| ASSERT = 0, | ||
| INSPECT_ELEMENT = 1, | ||
| EXTRACT_DATA = 2, | ||
| PLAN = 3, | ||
| } | ||
|
|
||
| export async function callAiFn<T>(options: { | ||
| msgs: AIArgs; | ||
| AIActionType: AIActionType; | ||
| useModel?: 'openAI' | 'coze'; | ||
| }) { | ||
| const { useModel, msgs, AIActionType: AIActionTypeValue } = options; | ||
| if (useOpenAIModel(useModel)) { | ||
| const parseResult = await callToGetJSONObject<T>(msgs); | ||
| return parseResult; | ||
| } | ||
|
|
||
| if (useCozeModel(useModel)) { | ||
| let botId = ''; | ||
| switch (AIActionTypeValue) { | ||
| case AIActionType.ASSERT: | ||
| botId = COZE_AI_ASSERT_BOT_ID; | ||
| break; | ||
| case AIActionType.EXTRACT_DATA: | ||
| botId = COZE_EXTRACT_INFO_BOT_ID; | ||
| break; | ||
| case AIActionType.INSPECT_ELEMENT: | ||
| botId = COZE_INSPECT_ELEMENT_BOT_ID; | ||
| break; | ||
| default: | ||
| botId = COZE_AI_ACTION_BOT_ID; | ||
| } | ||
| const cozeMsg = transfromOpenAiArgsToCoze(msgs[1]); | ||
| const parseResult = await callCozeAi<T>({ | ||
| ...cozeMsg, | ||
| botId, | ||
| }); | ||
| return parseResult; | ||
| } | ||
|
|
||
| throw Error('Does not contain coze and openai environment variables'); | ||
| } |
Oops, something went wrong.