chore: update evalute

web-infra-dev · Dec 3, 2024 · f2778b6 · f2778b6
1 parent 9643dca
commit f2778b6
Show file tree

Hide file tree

Showing 15 changed files with 1,364 additions and 1,237 deletions.
diff --git a/.github/workflows/ai.yml b/.github/workflows/ai.yml
@@ -21,6 +21,7 @@ jobs:
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
+      MIDSCENE_MODEL_NAME: gpt-4o-2024-08-06
       MIDSCENE_DEBUG_AI_PROFILE: 1
 
     steps:

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -4,5 +4,5 @@
   },
   "editor.defaultFormatter": "biomejs.biome",
   "editor.formatOnSave": true,
-  "cSpell.words": ["AITEST", "aweme", "httpbin", "taobao"]
+  "cSpell.words": ["AITEST", "aweme", "httpbin", "iconfont", "taobao"]
 }
diff --git a/packages/midscene/src/ai-model/inspect.ts b/packages/midscene/src/ai-model/inspect.ts
@@ -29,7 +29,7 @@ export type AIArgs = [
 ];
 
 const liteContextConfig = {
-  filterEmptyContent: true,
+  filterNonTextContent: true,
   truncateTextLength: 100,
 };
 

diff --git a/packages/midscene/src/ai-model/prompt/util.ts b/packages/midscene/src/ai-model/prompt/util.ts
@@ -226,7 +226,7 @@ export async function describeUserPage<
   context: Omit<UIContext<ElementType>, 'describer'>,
   opt?: {
     truncateTextLength?: number;
-    filterEmptyContent?: boolean;
+    filterNonTextContent?: boolean;
   },
 ) {
   const { screenshotBase64 } = context;
@@ -254,7 +254,7 @@ export async function describeUserPage<
   const elementInfosDescription = cropFieldInformation(
     elementsInfo,
     opt?.truncateTextLength,
-    opt?.filterEmptyContent,
+    opt?.filterNonTextContent,
   );
 
   const contentList = elementInfosDescription
@@ -289,7 +289,7 @@ ${
 function cropFieldInformation(
   elementsInfo: BaseElement[],
   truncateTextLength = 20,
-  filterEmptyContent = false,
+  filterNonTextContent = false,
 ) {
   const elementInfosDescription: Array<Record<string, any>> = elementsInfo.map(
     (item) => {
@@ -300,8 +300,8 @@ function cropFieldInformation(
           const attributeVal = (attributes as any)[currentKey];
           if (currentKey === 'style' || currentKey === 'src') return res;
           if (currentKey === 'nodeType') {
-            // when filterEmptyContent is true, we don't need to keep the nodeType since they are all TEXT
-            if (!filterEmptyContent) {
+            // when filterNonTextContent is true, we don't need to keep the nodeType since they are all TEXT
+            if (!filterNonTextContent) {
               res[currentKey] = attributeVal.replace(/\sNode$/, '');
             }
           } else {
@@ -314,9 +314,11 @@ function cropFieldInformation(
 
       return {
         id,
-        ...(filterEmptyContent ? {} : { markerId: (item as any).indexId }),
+        ...(filterNonTextContent || tailorContent
+          ? {}
+          : { markerId: (item as any).indexId }),
         ...(tailorContent ? { content: tailorContent } : {}),
-        ...(Object.keys(tailorAttributes).length
+        ...(Object.keys(tailorAttributes).length && !tailorContent
           ? { attributes: tailorAttributes }
           : {}),
         rect: {
@@ -330,7 +332,7 @@ function cropFieldInformation(
     },
   );
 
-  if (filterEmptyContent) {
+  if (filterNonTextContent) {
     return elementInfosDescription.filter((item) => item.content);
   }
   return elementInfosDescription;

diff --git a/packages/midscene/src/insight/utils.ts b/packages/midscene/src/insight/utils.ts
@@ -14,7 +14,6 @@ import type {
   PartialInsightDumpFromSDK,
   Rect,
   UIContext,
-  UISection,
 } from '@/types';
 import {
   getLogDir,
@@ -118,30 +117,3 @@ export function shallowExpandIds<DataScheme extends object = {}>(
 
   return data;
 }
-
-/**
- * a fake parser that collects all text into a single section
- * It's useful for debugging and testing
- * @param context
- * @returns
- */
-export async function fakeParserCollectsTexts<P>(context: UIContext): Promise<{
-  [K in keyof P]: UISection;
-}> {
-  const { content } = context;
-  const section: UISection = {
-    name: 'all-texts',
-    description: 'all texts in the page',
-    sectionCharacteristics: 'all texts',
-    content,
-    rect: {
-      left: 0,
-      top: 0,
-      width: context.size.width,
-      height: context.size.height,
-    },
-  };
-  return {
-    'all-texts': section,
-  } as any;
-}
diff --git a/packages/midscene/tests/ai/evaluate/ai-data/inspect/taobao.json b/packages/midscene/tests/ai/evaluate/ai-data/inspect/taobao.json
@@ -27,25 +27,27 @@
       "response": [
         {
           "id": "294616",
-          "indexId": 94
+          "indexId": 96
         }
       ]
     },
     {
-      "prompt": "收藏店铺 ICON",
+      "prompt": "收藏的店ICON",
       "multi": false,
       "response": [
         {
-          "id": "201"
+          "id": "280588",
+          "indexId": 203
         }
       ]
     },
     {
-      "prompt": "右侧客服按钮",
+      "prompt": "右侧”官方客服“按钮文字",
       "multi": false,
       "response": [
         {
-          "id": "244"
+          "id": "284359",
+          "indexId": 247
         }
       ]
     }

diff --git a/packages/midscene/tests/ai/evaluate/inspect.test.ts b/packages/midscene/tests/ai/evaluate/inspect.test.ts
@@ -63,20 +63,27 @@ describe('ai inspect element', () => {
             path.join(__dirname, aiData.testDataPath),
           );
 
+          const { elementById } = await context.describer();
+
           const { aiResponse } = await runTestCases(
             aiData.testCases,
             context,
             async (testCase) => {
-              if (process.env.INSPECT_LOCATE_IN_PLANNING) {
+              if (runType === 'planning') {
                 // use planning to get quick answer to test element inspector
                 const res = await plan(`Tap this: ${testCase.description}`, {
                   context,
                 });
 
+                const matchedId = res.actions[0].locate?.id;
+                if (matchedId) {
+                  return {
+                    elements: [elementById(matchedId)],
+                  };
+                }
+
                 return {
-                  elements: res.actions[0].locate?.id
-                    ? [res.actions[0].locate]
-                    : [],
+                  elements: [],
                 };
               }